Bitnami/charts
2
0
Fork 0
mirror of https://github.com/bitnami/charts.git synced 2026-03-06 15:10:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

[github-actions] Update index.yaml to use OCI charts references (#30402)

Browse Source 
* [github-actions] Replace CD pipeline in favor of OCI releases

Signed-off-by: Miguel Ruiz <miguel.ruiz@broadcom.com>

* Apply suggestions from code review

Signed-off-by: Miguel Ruiz <miguel.ruiz@broadcom.com>

Co-authored-by: Fran Mulero <fmulero@vmware.com>
Signed-off-by: Miguel Ruiz <miguel.ruiz@broadcom.com>

---------

Signed-off-by: Miguel Ruiz <miguel.ruiz@broadcom.com>
Co-authored-by: Fran Mulero <fmulero@vmware.com>
This commit is contained in:
Miguel Ruiz
2024-11-18 08:56:51 +01:00
committed by GitHub
parent ec1cc794c1
commit bf0201c096
3 changed files with 257 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

231
.github/workflows/cd-pipeline.yml vendored
View File

@@ -85,86 +85,6 @@ jobs:
VIB_ENV_S3_ACCESS_KEY_ID: ${{ secrets.AWS_PUBLISH_ACCESS_KEY_ID }}
VIB_ENV_S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_PUBLISH_SECRET_ACCESS_KEY }}
VIB_ENV_S3_ROLE_ARN: ${{ secrets.AWS_PUBLISH_ROLE_ARN }}
update-index:
runs-on: ubuntu-latest
needs:
- vib-publish
name: Update charts index
steps:
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
with:
path: ~/artifacts
# If we perform a checkout of the main branch, we will find conflicts with the submodules
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
ref: 'index'
path: index
# The token is persisted in the local git config and enables scripts to run authenticated git commands.
token: ${{ secrets.BITNAMI_BOT_TOKEN }}
- name: Install helm
run: |
HELM_TARBALL="helm-v3.8.1-linux-amd64.tar.gz"
curl -SsLfO "https://get.helm.sh/${HELM_TARBALL}" && sudo tar xf "$HELM_TARBALL" --strip-components 1 -C /usr/local/bin
# Install file plugin
helm plugin add https://github.com/zoobab/helm_file_repo
- id: update-index
name: Fetch chart and update index
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PUBLISH_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PUBLISH_SECRET_ACCESS_KEY }}
AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_PUBLISH_ROLE_ARN }}
AWS_MAX_ATTEMPTS: 3
AWS_DEFAULT_REGION: us-east-1
run: |
# Configure AWS account
export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" $(aws sts assume-role --role-arn ${AWS_ASSUME_ROLE_ARN} --role-session-name GitHubCharts --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" --output text))
# Extract chart release metadata from the publish report file
vib_publish_report_file=$(find ~/artifacts -name "report.json" -print -quit)
chart_name=$(jq -re '.actions|map(select(.action_id == "helm-publish"))[0] | .application.name' $vib_publish_report_file)
chart_version=$(jq -re '.actions|map(select(.action_id == "helm-publish"))[0] | .application.version' $vib_publish_report_file)
# Download published asset
mkdir download
aws s3 cp s3://${{ secrets.AWS_S3_BUCKET }}/bitnami/${chart_name}-${chart_version}.tgz download/
cd index
git config user.name "Bitnami Containers"
git config user.email "bitnami-bot@vmware.com"
attempts=0
max_attempts=5
is_index_updated=0
while [[ $attempts -lt $max_attempts && $is_index_updated -eq 0 ]]; do
attempts=$((attempts + 1))
# Pull changes from remote
git fetch origin index
current_commit_id=$(git rev-parse origin/index)
git reset --hard $(git commit-tree origin/index^{tree} -m "Update index.yaml")
# Rebuild index
helm repo index --url https://charts.bitnami.com/bitnami --merge bitnami/index.yaml ../download
# Compare size of files
if [[ $(stat -c%s bitnami/index.yaml) -gt $(stat -c%s ../download/index.yaml) ]]; then
echo "New index.yaml file is shorter than the current one"
exit 1
fi
# Adding tmp file as a helm repo
if ! helm repo add cache file://../download/ ; then
echo "New index.yaml file can't be indexed"
exit 1
fi
cp ../download/index.yaml bitnami/index.yaml
# Push changes
git add bitnami/index.yaml && git commit --signoff --amend --no-edit
git push origin index --force-with-lease=index:${current_commit_id} && is_index_updated=1 || echo "Failed to push during attempt $attempts"
done
if [[ $is_index_updated -ne 1 ]]; then
echo "Could not update the index after $max_attempts attempts"
exit 1
fi
update-deprecated-index:
runs-on: ubuntu-latest
needs:
@@ -245,154 +165,3 @@ jobs:
echo "Could not update the index after $max_attempts attempts"
exit 1
fi
push-tag:
runs-on: ubuntu-latest
permissions:
contents: write
needs:
- get-chart
- vib-publish
name: Push tag
if: ${{ needs.get-chart.outputs.result == 'ok' }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
path: charts
fetch-depth: 2 # to be able to obtain files changed in the latest commit
- id: push-tag
name: 'Push tag'
env:
CHART: ${{ needs.get-chart.outputs.chart }}
run: |
cd charts
# Get chart version and list of tags
chart_version="$(yq e '.version' bitnami/${CHART}/Chart.yaml)"
git fetch --tags
# If the tag does not exist, create and push it (this allows re-executing the job)
if ! git tag | grep ${CHART}/${chart_version}; then
git tag ${CHART}/${chart_version}
git push --tags
fi
push-promotion:
runs-on: ubuntu-latest
permissions:
contents: write
needs:
- get-chart
- update-index
name: Push Promotion files
if: ${{ needs.get-chart.outputs.result == 'ok' }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
path: charts
- name: Install helm
run: |
HELM_TARBALL="helm-v3.8.1-linux-amd64.tar.gz"
curl -SsLfO "https://get.helm.sh/${HELM_TARBALL}" && sudo tar xf "$HELM_TARBALL" --strip-components 1 -C /usr/local/bin
- env:
CHART_NAME: ${{ needs.get-chart.outputs.chart }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PROMOTION_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PROMOTION_SECRET_ACCESS_KEY }}
AWS_PROMOTION_BUCKET: ${{ secrets.AWS_PROMOTION_BUCKET }}
AWS_MAX_ATTEMPTS: 3
AWS_DEFAULT_REGION: us-east-1
run: |
#!/bin/bash
chart="bitnami/${CHART_NAME}"
cd charts
# Get chart and app version
chart_version="$(yq e '.version' "${chart}/Chart.yaml")"
app_version="$(yq e '.appVersion' "${chart}/Chart.yaml")"
# Get image list (removing the registry)
helm dep build ${chart}
image_list="$(yq '.annotations.images' "${chart}/Chart.yaml" | yq '[ .[] | .image | sub("^[^/]+/", "") ] | tojson')"
for dependency in "${chart}/charts/"*.tgz; do
if [[ -s "$dependency" ]]; then
# Analyse all 'Chart.yaml' files in dependencies
while read -r chart_yaml_file; do
if [[ -n "${chart_yaml_file}" ]]; then
dependency_image_list="$(tar -xzO -f "$dependency" "$chart_yaml_file" | yq '.annotations.images' | yq '[ .[] | .image | sub("^[^/]+/", "") ] | tojson')"
image_list="$(jq -c --null-input --argjson arr1 "$image_list" --argjson arr2 "$dependency_image_list" '$arr1 + $arr2 | unique')"
fi
done < <(tar -tzf "$dependency" | grep -E "Chart.yaml$")
fi
done
# Build JSON files
release_date="$(date -u +"%Y/%m/%d")"
file_prefix="$(date -u +"%s%3N-${CHART_NAME}-${app_version}")"
json_template=$(cat << "EOF"
{
"platform_id": $platform_id,
"application": $app,
"external_id": "\($app):\($chart_version)",
"version": $app_version,
"bundled_os_version": $bundled_os_version,
"properties": {
"chart_url": "https://charts.bitnami.com/bitnami/\($app)-\($chart_version).tgz",
"containers": $image_list,
"github_repository": "bitnami/charts/tree/main/bitnami/\($app)",
}
}
EOF
)
jq --null-input \
--arg platform_id "bitnami-chart-debian-x64" \
--arg app "${CHART_NAME}" \
--arg app_version "${app_version}" \
--arg chart_version "${chart_version}" \
--arg bundled_os_version "12" \
--argjson image_list "${image_list}" "${json_template}" > "${file_prefix}-bitnami-chart-debian-x64.json"
jq --null-input \
--arg platform_id "vmware-chart-debian-x64" \
--arg app "${CHART_NAME}" \
--arg app_version "${app_version}" \
--arg chart_version "${chart_version}" \
--arg bundled_os_version "12" \
--argjson image_list "${image_list}" "${json_template}" | jq '.properties += {"alias_platform_from": "bitnami-chart-debian-x64"}' > "${file_prefix}-vmware-chart-debian-x64.json"
# Upload files to the release bucket.
aws s3 cp "${file_prefix}-bitnami-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-bitnami-chart-debian-x64.json"
aws s3 cp "${file_prefix}-vmware-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-vmware-chart-debian-x64.json"
# If the CD Pipeline does not succeed we should notify the interested agents
slack-notif:
runs-on: ubuntu-latest
needs:
- vib-publish
- update-index
- push-promotion
if: always()
name: Notify unsuccessful CD run
steps:
- name: Notify in Slack channel
if: ${{ needs.push-promotion.result != 'success' }}
uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0
with:
channel-id: ${{ secrets.CD_SLACK_CHANNEL_ID }}
payload: |
{
"attachments": [
{
"color": "#CC0000",
"fallback": "Unsuccessful bitnami/charts CD pipeline",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Unsuccessful `bitnami/charts` CD pipeline*"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "The CD pipeline for <${{ github.event.head_commit.url }}|bitnami/charts@${{ github.event.head_commit.id }}> did not succeed. Check the related <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|action run> for more information."
}
}
]
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.CD_SLACK_BOT_TOKEN }}

169
.github/workflows/index-update.yml vendored Normal file
View File

@@ -0,0 +1,169 @@
name: '[Index] Sync index.yaml with OCI releases'
on:
schedule:
- cron: "*/30 * * * *"
# Remove all permissions by default.
permissions: {}
jobs:
find-new-releases:
runs-on: ubuntu-latest
name: Find new releases
outputs:
new-releases: ${{ steps.get-new-releases.outputs.new-releases }}
permissions:
contents: read
steps:
- id: checkout-repo
name: Checkout repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
ref: index
path: index
- uses: oras-project/setup-oras@9c92598691bfef1424de2f8fae81941568f5889c
- id: get-oci-index
name: Get OCI index
run: |
oras pull registry-1.docker.io/bitnamicharts/charts-index:latest
cat charts-index.json | yq -P | yq eval '. | .entries[] |= .versions' > ./oci_index.yaml
- id: get-charts-index
name: Get Charts index
run: |
cp index/bitnami/index.yaml ./charts_index.yaml
- id: merge
name: Generate merged index
run: |
yq eval-all '. as $item ireduce ({}; . *+ $item )' charts_index.yaml oci_index.yaml > duplicates_index.yaml
yq eval '.entries[] |= unique_by(.name + .version)' duplicates_index.yaml > merged_index.yaml
- id: get-new-releases
name: Find new versions
run: |
yq eval '.entries[][] | .name + ":" + .version' charts_index.yaml |sort| uniq > charts_index_releases
yq eval '.entries[][] | .name + ":" + .version' merged_index.yaml | sort| uniq > merged_index_releases
new_releases="$(comm -13 charts_index_releases merged_index_releases | tr "\n" " " | sed 's/ $//')"
if [ -n "${new_releases}" ]; then
echo "Found new releases: ${new_releases}"
else
echo "No new releases detected"
fi
echo "new-releases=$new_releases" >> $GITHUB_OUTPUT
update-index-and-promotions:
runs-on: ubuntu-latest
needs:
- find-new-releases
name: Update index and push promotions
if: ${{ needs.find-new-releases.outputs.new-releases != '' }}
steps:
- name: Install helm
run: |
HELM_TARBALL="helm-v3.8.1-linux-amd64.tar.gz"
curl -SsLfO "https://get.helm.sh/${HELM_TARBALL}" && sudo tar xf "$HELM_TARBALL" --strip-components 1 -C /usr/local/bin
# Install file plugin
helm plugin add https://github.com/zoobab/helm_file_repo
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
with:
ref: 'index'
path: index
# The token is persisted in the local git config and enables scripts to run authenticated git commands.
token: ${{ secrets.BITNAMI_BOT_TOKEN }}
- id: update-index-and-promotions
name: Pull charts and update index
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PROMOTION_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PROMOTION_SECRET_ACCESS_KEY }}
AWS_PROMOTION_BUCKET: ${{ secrets.AWS_PROMOTION_BUCKET }}
AWS_MAX_ATTEMPTS: 3
AWS_DEFAULT_REGION: us-east-1
NEW_RELEASES: ${{ needs.find-new-releases.outputs.new-releases }}
run: |
# Promotions template
promotion_json_template=$(cat << "EOF"
{
"platform_id": $platform_id,
"application": $chart_name,
"external_id": "\($chart_name):\($chart_version)",
"version": $app_version,
"bundled_os_version": $bundled_os_version,
"properties": {
"chart_url": "oci://registry-1.docker.io/bitnamicharts/\($chart_name):\($chart_version)",
"containers": $image_list,
"github_repository": "bitnami/charts/tree/main/bitnami/\($chart_name)",
}
}
EOF
)
cd index
# Configure git
git config user.name "Bitnami Containers"
git config user.email "bitnami-bot@vmware.com"
read -r -a new_releases_arr <<< $NEW_RELEASES
for release in "${new_releases_arr[@]}"; do
read -r -a release_arr <<< "$(tr ':' ' ' <<< "$release")"
chart_name="${release_arr[0]}"
chart_version="${release_arr[1]}"
## Update index
# Download published asset
mkdir ../download
helm pull "oci://registry-1.docker.io/bitnamicharts/${chart_name}" --version "${chart_version}" --destination ../download
# Rebuild index
helm repo index --url oci://registry-1.docker.io/bitnamicharts --merge bitnami/index.yaml ../download
# Replace .tgz in URL with OCI tag
sed -i "s|oci://registry-1.docker.io/bitnamicharts/$chart_name-$chart_version.tgz|oci://registry-1.docker.io/bitnamicharts/$chart_name:$chart_version|" ../download/index.yaml
# Check index integrity
if [[ $(stat -c%s bitnami/index.yaml) -gt $(stat -c%s ../download/index.yaml) ]]; then
echo "New index.yaml file is shorter than the current one"
exit 1
fi
# Check repo can be loaded
if ! helm repo add cache file://../download/ ; then
echo "New index.yaml file can't be used as a file"
exit 1
else
# Remove the repo
helm repo remove cache
fi
cp ../download/index.yaml bitnami/index.yaml
## Build and push promotions
tar -xzf ../download/${chart_name}-${chart_version}.tgz -C ../download
image_list="[]"
# Get image list (removing the registry)
for chart_yaml in $(find "../download/${chart_name}" -name "Chart.yaml"); do
image_list_aux="$(yq '.annotations.images' "${chart_yaml}" | yq '[ .[] | .image | sub("^[^/]+/", "") ] | tojson')"
image_list="$(jq -c --null-input --argjson arr1 "$image_list" --argjson arr2 "$image_list_aux" '$arr1 + $arr2 | unique')"
done
app_version=$(yq '.appVersion' "../download/${chart_name}/Chart.yaml")
release_date="$(date -u +"%Y/%m/%d")"
file_prefix="$(date -u +"%s%3N-${chart_name}-${app_version}")"
# Build JSON files
jq --null-input \
--arg platform_id "bitnami-chart-debian-x64" \
--arg chart_name "${chart_name}" \
--arg app_version "${app_version}" \
--arg chart_version "${chart_version}" \
--arg bundled_os_version "12" \
--argjson image_list "${image_list}" "${promotion_json_template}" > "${file_prefix}-bitnami-chart-debian-x64.json"
jq --null-input \
--arg platform_id "vmware-chart-debian-x64" \
--arg chart_name "${chart_name}" \
--arg app_version "${app_version}" \
--arg chart_version "${chart_version}" \
--arg bundled_os_version "12" \
--argjson image_list "${image_list}" "${promotion_json_template}" | jq '.properties += {"alias_platform_from": "bitnami-chart-debian-x64"}' > "${file_prefix}-vmware-chart-debian-x64.json"
# Upload files to the release bucket
aws s3 cp "${file_prefix}-bitnami-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-bitnami-chart-debian-x64.json"
aws s3 cp "${file_prefix}-vmware-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-vmware-chart-debian-x64.json"
# Remove chart files
rm -rf ../download
done
# Avoid overriding index branch when remote commit does not match our checkout commit
current_commit_id=$(git rev-parse index)
# Push changes
git add bitnami/index.yaml && git commit --signoff --amend --no-edit
git push origin index --force-with-lease=index:${current_commit_id}

88
.github/workflows/push-tag.yml vendored Normal file
View File

@@ -0,0 +1,88 @@
# Copyright Broadcom, Inc. All Rights Reserved.
# SPDX-License-Identifier: APACHE-2.0
name: '[CI/CD] Push tag'
on: # rebuild any PRs and main branch changes
push:
branches:
- main
paths:
- 'bitnami/**'
- '!**.md'
# Remove all permissions by default.
permissions: {}
jobs:
get-chart:
runs-on: ubuntu-latest
name: 'Get modified charts'
permissions:
contents: read
outputs:
chart: ${{ steps.get-chart.outputs.chart }}
result: ${{ steps.get-chart.outputs.result }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
path: charts
fetch-depth: 2 # to be able to obtain files changed in the latest commit
- id: get-chart
name: 'Get modified charts'
run: |
cd charts
files_changed="$(git show --pretty="" --name-only)"
# Adding || true to avoid "Process exited with code 1" errors
charts_dirs_changed="$(echo "$files_changed" | xargs dirname | grep -o "bitnami/[^/]*" | sort | uniq || true)"
# Using grep -c as a better alternative to wc -l when dealing with empty strings."
num_charts_changed="$(echo "$charts_dirs_changed" | grep -c "bitnami" || true)"
num_version_bumps="$(echo "$files_changed" | grep "bitnami/[^/]*/Chart.yaml" | xargs git show | grep -c "+version" || true)"
if [[ "$num_charts_changed" -ne "$num_version_bumps" ]]; then
# Changes done in charts but version not bumped -> ERROR
charts_changed_str="$(echo ${charts_dirs_changed[@]})"
echo "error=Detected changes in charts without version bump in Chart.yaml. Charts changed: ${num_charts_changed} ${charts_changed_str}. Version bumps detected: ${num_version_bumps}" >> $GITHUB_OUTPUT
echo "result=fail" >> $GITHUB_OUTPUT
elif [[ "$num_charts_changed" -eq "1" ]]; then
# Changes done in only one chart -> OK
chart_name=$(echo "$charts_dirs_changed" | sed "s|bitnami/||g")
echo "chart=${chart_name}" >> $GITHUB_OUTPUT
echo "result=ok" >> $GITHUB_OUTPUT
else
# Changes done in more than chart -> FAIL
charts_changed_str="$(echo ${charts_dirs_changed[@]})"
echo "error=Changes detected in more than one chart directory: ${charts_changed_str}. The publish process will be stopped. Please create different commits for each chart." >> $GITHUB_OUTPUT
echo "result=fail" >> $GITHUB_OUTPUT
fi
- id: show-error
name: 'Show error'
if: ${{ steps.get-chart.outputs.result == 'fail' }}
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
with:
script: |
core.setFailed('${{ steps.get-chart.outputs.error }}')
push-tag:
runs-on: ubuntu-latest
permissions:
contents: write
needs:
- get-chart
name: Push tag
if: ${{ needs.get-chart.outputs.result == 'ok' }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
path: charts
fetch-depth: 2 # to be able to obtain files changed in the latest commit
- id: push-tag
name: 'Push tag'
env:
CHART: ${{ needs.get-chart.outputs.chart }}
run: |
cd charts
# Get chart version and list of tags
chart_version="$(yq e '.version' bitnami/${CHART}/Chart.yaml)"
git fetch --tags
# If the tag does not exist, create and push it (this allows re-executing the job)
if ! git tag | grep ${CHART}/${chart_version}; then
git tag ${CHART}/${chart_version}
git push --tags
fi