diff --git a/.github/workflows/cd-pipeline.yml b/.github/workflows/cd-pipeline.yml index c1a8100cba..e2385ec639 100644 --- a/.github/workflows/cd-pipeline.yml +++ b/.github/workflows/cd-pipeline.yml @@ -85,86 +85,6 @@ jobs: VIB_ENV_S3_ACCESS_KEY_ID: ${{ secrets.AWS_PUBLISH_ACCESS_KEY_ID }} VIB_ENV_S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_PUBLISH_SECRET_ACCESS_KEY }} VIB_ENV_S3_ROLE_ARN: ${{ secrets.AWS_PUBLISH_ROLE_ARN }} - update-index: - runs-on: ubuntu-latest - needs: - - vib-publish - name: Update charts index - steps: - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 - with: - path: ~/artifacts - # If we perform a checkout of the main branch, we will find conflicts with the submodules - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - with: - ref: 'index' - path: index - # The token is persisted in the local git config and enables scripts to run authenticated git commands. - token: ${{ secrets.BITNAMI_BOT_TOKEN }} - - name: Install helm - run: | - HELM_TARBALL="helm-v3.8.1-linux-amd64.tar.gz" - curl -SsLfO "https://get.helm.sh/${HELM_TARBALL}" && sudo tar xf "$HELM_TARBALL" --strip-components 1 -C /usr/local/bin - # Install file plugin - helm plugin add https://github.com/zoobab/helm_file_repo - - id: update-index - name: Fetch chart and update index - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PUBLISH_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PUBLISH_SECRET_ACCESS_KEY }} - AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_PUBLISH_ROLE_ARN }} - AWS_MAX_ATTEMPTS: 3 - AWS_DEFAULT_REGION: us-east-1 - run: | - # Configure AWS account - export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" $(aws sts assume-role --role-arn ${AWS_ASSUME_ROLE_ARN} --role-session-name GitHubCharts --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" --output text)) - # Extract chart release metadata from the publish report file - vib_publish_report_file=$(find ~/artifacts -name "report.json" -print -quit) - chart_name=$(jq -re '.actions|map(select(.action_id == "helm-publish"))[0] | .application.name' $vib_publish_report_file) - chart_version=$(jq -re '.actions|map(select(.action_id == "helm-publish"))[0] | .application.version' $vib_publish_report_file) - # Download published asset - mkdir download - aws s3 cp s3://${{ secrets.AWS_S3_BUCKET }}/bitnami/${chart_name}-${chart_version}.tgz download/ - - cd index - git config user.name "Bitnami Containers" - git config user.email "bitnami-bot@vmware.com" - - attempts=0 - max_attempts=5 - is_index_updated=0 - while [[ $attempts -lt $max_attempts && $is_index_updated -eq 0 ]]; do - attempts=$((attempts + 1)) - - # Pull changes from remote - git fetch origin index - current_commit_id=$(git rev-parse origin/index) - git reset --hard $(git commit-tree origin/index^{tree} -m "Update index.yaml") - - # Rebuild index - helm repo index --url https://charts.bitnami.com/bitnami --merge bitnami/index.yaml ../download - # Compare size of files - if [[ $(stat -c%s bitnami/index.yaml) -gt $(stat -c%s ../download/index.yaml) ]]; then - echo "New index.yaml file is shorter than the current one" - exit 1 - fi - # Adding tmp file as a helm repo - if ! helm repo add cache file://../download/ ; then - echo "New index.yaml file can't be indexed" - exit 1 - fi - - cp ../download/index.yaml bitnami/index.yaml - - # Push changes - git add bitnami/index.yaml && git commit --signoff --amend --no-edit - git push origin index --force-with-lease=index:${current_commit_id} && is_index_updated=1 || echo "Failed to push during attempt $attempts" - done - - if [[ $is_index_updated -ne 1 ]]; then - echo "Could not update the index after $max_attempts attempts" - exit 1 - fi update-deprecated-index: runs-on: ubuntu-latest needs: @@ -245,154 +165,3 @@ jobs: echo "Could not update the index after $max_attempts attempts" exit 1 fi - push-tag: - runs-on: ubuntu-latest - permissions: - contents: write - needs: - - get-chart - - vib-publish - name: Push tag - if: ${{ needs.get-chart.outputs.result == 'ok' }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - with: - path: charts - fetch-depth: 2 # to be able to obtain files changed in the latest commit - - id: push-tag - name: 'Push tag' - env: - CHART: ${{ needs.get-chart.outputs.chart }} - run: | - cd charts - # Get chart version and list of tags - chart_version="$(yq e '.version' bitnami/${CHART}/Chart.yaml)" - git fetch --tags - # If the tag does not exist, create and push it (this allows re-executing the job) - if ! git tag | grep ${CHART}/${chart_version}; then - git tag ${CHART}/${chart_version} - git push --tags - fi - push-promotion: - runs-on: ubuntu-latest - permissions: - contents: write - needs: - - get-chart - - update-index - name: Push Promotion files - if: ${{ needs.get-chart.outputs.result == 'ok' }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - with: - path: charts - - name: Install helm - run: | - HELM_TARBALL="helm-v3.8.1-linux-amd64.tar.gz" - curl -SsLfO "https://get.helm.sh/${HELM_TARBALL}" && sudo tar xf "$HELM_TARBALL" --strip-components 1 -C /usr/local/bin - - env: - CHART_NAME: ${{ needs.get-chart.outputs.chart }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PROMOTION_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PROMOTION_SECRET_ACCESS_KEY }} - AWS_PROMOTION_BUCKET: ${{ secrets.AWS_PROMOTION_BUCKET }} - AWS_MAX_ATTEMPTS: 3 - AWS_DEFAULT_REGION: us-east-1 - run: | - #!/bin/bash - chart="bitnami/${CHART_NAME}" - cd charts - # Get chart and app version - chart_version="$(yq e '.version' "${chart}/Chart.yaml")" - app_version="$(yq e '.appVersion' "${chart}/Chart.yaml")" - # Get image list (removing the registry) - helm dep build ${chart} - image_list="$(yq '.annotations.images' "${chart}/Chart.yaml" | yq '[ .[] | .image | sub("^[^/]+/", "") ] | tojson')" - for dependency in "${chart}/charts/"*.tgz; do - if [[ -s "$dependency" ]]; then - # Analyse all 'Chart.yaml' files in dependencies - while read -r chart_yaml_file; do - if [[ -n "${chart_yaml_file}" ]]; then - dependency_image_list="$(tar -xzO -f "$dependency" "$chart_yaml_file" | yq '.annotations.images' | yq '[ .[] | .image | sub("^[^/]+/", "") ] | tojson')" - image_list="$(jq -c --null-input --argjson arr1 "$image_list" --argjson arr2 "$dependency_image_list" '$arr1 + $arr2 | unique')" - fi - done < <(tar -tzf "$dependency" | grep -E "Chart.yaml$") - fi - done - # Build JSON files - release_date="$(date -u +"%Y/%m/%d")" - file_prefix="$(date -u +"%s%3N-${CHART_NAME}-${app_version}")" - json_template=$(cat << "EOF" - { - "platform_id": $platform_id, - "application": $app, - "external_id": "\($app):\($chart_version)", - "version": $app_version, - "bundled_os_version": $bundled_os_version, - "properties": { - "chart_url": "https://charts.bitnami.com/bitnami/\($app)-\($chart_version).tgz", - "containers": $image_list, - "github_repository": "bitnami/charts/tree/main/bitnami/\($app)", - } - } - EOF - ) - jq --null-input \ - --arg platform_id "bitnami-chart-debian-x64" \ - --arg app "${CHART_NAME}" \ - --arg app_version "${app_version}" \ - --arg chart_version "${chart_version}" \ - --arg bundled_os_version "12" \ - --argjson image_list "${image_list}" "${json_template}" > "${file_prefix}-bitnami-chart-debian-x64.json" - jq --null-input \ - --arg platform_id "vmware-chart-debian-x64" \ - --arg app "${CHART_NAME}" \ - --arg app_version "${app_version}" \ - --arg chart_version "${chart_version}" \ - --arg bundled_os_version "12" \ - --argjson image_list "${image_list}" "${json_template}" | jq '.properties += {"alias_platform_from": "bitnami-chart-debian-x64"}' > "${file_prefix}-vmware-chart-debian-x64.json" - # Upload files to the release bucket. - aws s3 cp "${file_prefix}-bitnami-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-bitnami-chart-debian-x64.json" - aws s3 cp "${file_prefix}-vmware-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-vmware-chart-debian-x64.json" - - # If the CD Pipeline does not succeed we should notify the interested agents - slack-notif: - runs-on: ubuntu-latest - needs: - - vib-publish - - update-index - - push-promotion - if: always() - name: Notify unsuccessful CD run - steps: - - name: Notify in Slack channel - if: ${{ needs.push-promotion.result != 'success' }} - uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 - with: - channel-id: ${{ secrets.CD_SLACK_CHANNEL_ID }} - payload: | - { - "attachments": [ - { - "color": "#CC0000", - "fallback": "Unsuccessful bitnami/charts CD pipeline", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "*Unsuccessful `bitnami/charts` CD pipeline*" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "The CD pipeline for <${{ github.event.head_commit.url }}|bitnami/charts@${{ github.event.head_commit.id }}> did not succeed. Check the related <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|action run> for more information." - } - } - ] - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.CD_SLACK_BOT_TOKEN }} diff --git a/.github/workflows/index-update.yml b/.github/workflows/index-update.yml new file mode 100644 index 0000000000..f1b84e17d9 --- /dev/null +++ b/.github/workflows/index-update.yml @@ -0,0 +1,169 @@ +name: '[Index] Sync index.yaml with OCI releases' +on: + schedule: + - cron: "*/30 * * * *" +# Remove all permissions by default. +permissions: {} +jobs: + find-new-releases: + runs-on: ubuntu-latest + name: Find new releases + outputs: + new-releases: ${{ steps.get-new-releases.outputs.new-releases }} + permissions: + contents: read + steps: + - id: checkout-repo + name: Checkout repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: index + path: index + - uses: oras-project/setup-oras@9c92598691bfef1424de2f8fae81941568f5889c + - id: get-oci-index + name: Get OCI index + run: | + oras pull registry-1.docker.io/bitnamicharts/charts-index:latest + cat charts-index.json | yq -P | yq eval '. | .entries[] |= .versions' > ./oci_index.yaml + - id: get-charts-index + name: Get Charts index + run: | + cp index/bitnami/index.yaml ./charts_index.yaml + - id: merge + name: Generate merged index + run: | + yq eval-all '. as $item ireduce ({}; . *+ $item )' charts_index.yaml oci_index.yaml > duplicates_index.yaml + yq eval '.entries[] |= unique_by(.name + .version)' duplicates_index.yaml > merged_index.yaml + - id: get-new-releases + name: Find new versions + run: | + yq eval '.entries[][] | .name + ":" + .version' charts_index.yaml |sort| uniq > charts_index_releases + yq eval '.entries[][] | .name + ":" + .version' merged_index.yaml | sort| uniq > merged_index_releases + new_releases="$(comm -13 charts_index_releases merged_index_releases | tr "\n" " " | sed 's/ $//')" + if [ -n "${new_releases}" ]; then + echo "Found new releases: ${new_releases}" + else + echo "No new releases detected" + fi + echo "new-releases=$new_releases" >> $GITHUB_OUTPUT + update-index-and-promotions: + runs-on: ubuntu-latest + needs: + - find-new-releases + name: Update index and push promotions + if: ${{ needs.find-new-releases.outputs.new-releases != '' }} + steps: + - name: Install helm + run: | + HELM_TARBALL="helm-v3.8.1-linux-amd64.tar.gz" + curl -SsLfO "https://get.helm.sh/${HELM_TARBALL}" && sudo tar xf "$HELM_TARBALL" --strip-components 1 -C /usr/local/bin + # Install file plugin + helm plugin add https://github.com/zoobab/helm_file_repo + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + with: + ref: 'index' + path: index + # The token is persisted in the local git config and enables scripts to run authenticated git commands. + token: ${{ secrets.BITNAMI_BOT_TOKEN }} + - id: update-index-and-promotions + name: Pull charts and update index + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PROMOTION_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PROMOTION_SECRET_ACCESS_KEY }} + AWS_PROMOTION_BUCKET: ${{ secrets.AWS_PROMOTION_BUCKET }} + AWS_MAX_ATTEMPTS: 3 + AWS_DEFAULT_REGION: us-east-1 + NEW_RELEASES: ${{ needs.find-new-releases.outputs.new-releases }} + run: | + # Promotions template + promotion_json_template=$(cat << "EOF" + { + "platform_id": $platform_id, + "application": $chart_name, + "external_id": "\($chart_name):\($chart_version)", + "version": $app_version, + "bundled_os_version": $bundled_os_version, + "properties": { + "chart_url": "oci://registry-1.docker.io/bitnamicharts/\($chart_name):\($chart_version)", + "containers": $image_list, + "github_repository": "bitnami/charts/tree/main/bitnami/\($chart_name)", + } + } + EOF + ) + + cd index + # Configure git + git config user.name "Bitnami Containers" + git config user.email "bitnami-bot@vmware.com" + + read -r -a new_releases_arr <<< $NEW_RELEASES + for release in "${new_releases_arr[@]}"; do + read -r -a release_arr <<< "$(tr ':' ' ' <<< "$release")" + chart_name="${release_arr[0]}" + chart_version="${release_arr[1]}" + + ## Update index + # Download published asset + mkdir ../download + helm pull "oci://registry-1.docker.io/bitnamicharts/${chart_name}" --version "${chart_version}" --destination ../download + # Rebuild index + helm repo index --url oci://registry-1.docker.io/bitnamicharts --merge bitnami/index.yaml ../download + # Replace .tgz in URL with OCI tag + sed -i "s|oci://registry-1.docker.io/bitnamicharts/$chart_name-$chart_version.tgz|oci://registry-1.docker.io/bitnamicharts/$chart_name:$chart_version|" ../download/index.yaml + + # Check index integrity + if [[ $(stat -c%s bitnami/index.yaml) -gt $(stat -c%s ../download/index.yaml) ]]; then + echo "New index.yaml file is shorter than the current one" + exit 1 + fi + # Check repo can be loaded + if ! helm repo add cache file://../download/ ; then + echo "New index.yaml file can't be used as a file" + exit 1 + else + # Remove the repo + helm repo remove cache + fi + cp ../download/index.yaml bitnami/index.yaml + + ## Build and push promotions + tar -xzf ../download/${chart_name}-${chart_version}.tgz -C ../download + image_list="[]" + # Get image list (removing the registry) + for chart_yaml in $(find "../download/${chart_name}" -name "Chart.yaml"); do + image_list_aux="$(yq '.annotations.images' "${chart_yaml}" | yq '[ .[] | .image | sub("^[^/]+/", "") ] | tojson')" + image_list="$(jq -c --null-input --argjson arr1 "$image_list" --argjson arr2 "$image_list_aux" '$arr1 + $arr2 | unique')" + done + app_version=$(yq '.appVersion' "../download/${chart_name}/Chart.yaml") + release_date="$(date -u +"%Y/%m/%d")" + file_prefix="$(date -u +"%s%3N-${chart_name}-${app_version}")" + # Build JSON files + jq --null-input \ + --arg platform_id "bitnami-chart-debian-x64" \ + --arg chart_name "${chart_name}" \ + --arg app_version "${app_version}" \ + --arg chart_version "${chart_version}" \ + --arg bundled_os_version "12" \ + --argjson image_list "${image_list}" "${promotion_json_template}" > "${file_prefix}-bitnami-chart-debian-x64.json" + jq --null-input \ + --arg platform_id "vmware-chart-debian-x64" \ + --arg chart_name "${chart_name}" \ + --arg app_version "${app_version}" \ + --arg chart_version "${chart_version}" \ + --arg bundled_os_version "12" \ + --argjson image_list "${image_list}" "${promotion_json_template}" | jq '.properties += {"alias_platform_from": "bitnami-chart-debian-x64"}' > "${file_prefix}-vmware-chart-debian-x64.json" + # Upload files to the release bucket + aws s3 cp "${file_prefix}-bitnami-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-bitnami-chart-debian-x64.json" + aws s3 cp "${file_prefix}-vmware-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-vmware-chart-debian-x64.json" + + # Remove chart files + rm -rf ../download + done + + # Avoid overriding index branch when remote commit does not match our checkout commit + current_commit_id=$(git rev-parse index) + + # Push changes + git add bitnami/index.yaml && git commit --signoff --amend --no-edit + git push origin index --force-with-lease=index:${current_commit_id} diff --git a/.github/workflows/push-tag.yml b/.github/workflows/push-tag.yml new file mode 100644 index 0000000000..3270accce6 --- /dev/null +++ b/.github/workflows/push-tag.yml @@ -0,0 +1,88 @@ +# Copyright Broadcom, Inc. All Rights Reserved. +# SPDX-License-Identifier: APACHE-2.0 + +name: '[CI/CD] Push tag' +on: # rebuild any PRs and main branch changes + push: + branches: + - main + paths: + - 'bitnami/**' + - '!**.md' +# Remove all permissions by default. +permissions: {} +jobs: + get-chart: + runs-on: ubuntu-latest + name: 'Get modified charts' + permissions: + contents: read + outputs: + chart: ${{ steps.get-chart.outputs.chart }} + result: ${{ steps.get-chart.outputs.result }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + path: charts + fetch-depth: 2 # to be able to obtain files changed in the latest commit + - id: get-chart + name: 'Get modified charts' + run: | + cd charts + files_changed="$(git show --pretty="" --name-only)" + # Adding || true to avoid "Process exited with code 1" errors + charts_dirs_changed="$(echo "$files_changed" | xargs dirname | grep -o "bitnami/[^/]*" | sort | uniq || true)" + # Using grep -c as a better alternative to wc -l when dealing with empty strings." + num_charts_changed="$(echo "$charts_dirs_changed" | grep -c "bitnami" || true)" + num_version_bumps="$(echo "$files_changed" | grep "bitnami/[^/]*/Chart.yaml" | xargs git show | grep -c "+version" || true)" + + if [[ "$num_charts_changed" -ne "$num_version_bumps" ]]; then + # Changes done in charts but version not bumped -> ERROR + charts_changed_str="$(echo ${charts_dirs_changed[@]})" + echo "error=Detected changes in charts without version bump in Chart.yaml. Charts changed: ${num_charts_changed} ${charts_changed_str}. Version bumps detected: ${num_version_bumps}" >> $GITHUB_OUTPUT + echo "result=fail" >> $GITHUB_OUTPUT + elif [[ "$num_charts_changed" -eq "1" ]]; then + # Changes done in only one chart -> OK + chart_name=$(echo "$charts_dirs_changed" | sed "s|bitnami/||g") + echo "chart=${chart_name}" >> $GITHUB_OUTPUT + echo "result=ok" >> $GITHUB_OUTPUT + else + # Changes done in more than chart -> FAIL + charts_changed_str="$(echo ${charts_dirs_changed[@]})" + echo "error=Changes detected in more than one chart directory: ${charts_changed_str}. The publish process will be stopped. Please create different commits for each chart." >> $GITHUB_OUTPUT + echo "result=fail" >> $GITHUB_OUTPUT + fi + - id: show-error + name: 'Show error' + if: ${{ steps.get-chart.outputs.result == 'fail' }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea + with: + script: | + core.setFailed('${{ steps.get-chart.outputs.error }}') + push-tag: + runs-on: ubuntu-latest + permissions: + contents: write + needs: + - get-chart + name: Push tag + if: ${{ needs.get-chart.outputs.result == 'ok' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + path: charts + fetch-depth: 2 # to be able to obtain files changed in the latest commit + - id: push-tag + name: 'Push tag' + env: + CHART: ${{ needs.get-chart.outputs.chart }} + run: | + cd charts + # Get chart version and list of tags + chart_version="$(yq e '.version' bitnami/${CHART}/Chart.yaml)" + git fetch --tags + # If the tag does not exist, create and push it (this allows re-executing the job) + if ! git tag | grep ${CHART}/${chart_version}; then + git tag ${CHART}/${chart_version} + git push --tags + fi