From 44e142a31e4c3e569264de6eb9fa748ee1c8f2ac Mon Sep 17 00:00:00 2001 From: Juan Ariza Toledano Date: Wed, 19 Feb 2025 14:14:41 +0100 Subject: [PATCH] ci: shellcheck on CI bash logic (#31980) --- .github/workflows/ci-pipeline.yml | 184 ++++++++++++++++-------------- 1 file changed, 100 insertions(+), 84 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index bacee95434..ce81aceb65 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -36,7 +36,7 @@ jobs: GITHUB_TOKEN: "${{ github.token }}" run: | # Using the Github API to detect the files changed as git merge-base stops working when the branch is behind - files_changed_data="$(gh api --paginate /repos/${GITHUB_REPOSITORY}/pulls/${PULL_REQUEST_NUMBER}/files)" + files_changed_data="$(gh api --paginate "/repos/${GITHUB_REPOSITORY}/pulls/${PULL_REQUEST_NUMBER}/files")" files_changed="$(echo "$files_changed_data" | jq -r '.[] | .filename')" # Adding || true to avoid "Process exited with code 1" errors charts_dirs_changed="$(echo "$files_changed" | xargs dirname | grep -o "bitnami/[^/]*" | sort | uniq || true)" @@ -47,33 +47,34 @@ jobs: if [[ $(curl -Lks "${PULL_REQUEST_URL}" | jq '.state | index("closed")') != *null* ]]; then # The PR for which this workflow run was launched is now closed -> SKIP - echo "error=The PR for which this workflow run was launched is now closed. The tests will be skipped." >> $GITHUB_OUTPUT - echo "result=skip" >> $GITHUB_OUTPUT + echo "error=The PR for which this workflow run was launched is now closed. The tests will be skipped." >> "$GITHUB_OUTPUT" + echo "result=skip" >> "$GITHUB_OUTPUT" elif [[ "$non_readme_files" -le "0" ]]; then # The only changes are .md files -> SKIP - echo "result=skip" >> $GITHUB_OUTPUT + echo "result=skip" >> "$GITHUB_OUTPUT" elif [[ "$num_charts_changed" -ne "$num_version_bumps" ]]; then # Changes done in charts but version not bumped -> ERROR - echo "error=Detected changes in charts without version bump in Chart.yaml. Charts changed: ${num_charts_changed}. Version bumps detected: ${num_version_bumps}" >> $GITHUB_OUTPUT - echo "result=fail" >> $GITHUB_OUTPUT + echo "error=Detected changes in charts without version bump in Chart.yaml. Charts changed: ${num_charts_changed}. Version bumps detected: ${num_version_bumps}" >> "$GITHUB_OUTPUT" + echo "result=fail" >> "$GITHUB_OUTPUT" elif [[ "$num_charts_changed" -eq "1" ]]; then # Changes done in only one chart -> OK - echo "result=ok" >> $GITHUB_OUTPUT + echo "result=ok" >> "$GITHUB_OUTPUT" # Extra output: chart name - chart_name=$(echo "$charts_dirs_changed" | sed "s|bitnami/||g") - echo "chart=${chart_name}" >> $GITHUB_OUTPUT + chart_name="${charts_dirs_changed//bitnami\/}" + echo "chart=${chart_name}" >> "$GITHUB_OUTPUT" # Extra output: values-updated - if [[ ${files_changed[@]} =~ "bitnami/${chart_name}/values.yaml" ]]; then - echo "values-updated=true" >> $GITHUB_OUTPUT + # shellcheck disable=SC2076 + if [[ "${files_changed[*]}" =~ "bitnami/${chart_name}/values.yaml" ]]; then + echo "values-updated=true" >> "$GITHUB_OUTPUT" fi elif [[ "$num_charts_changed" -le "0" ]]; then # Changes done in the bitnami/ folder but not inside a chart subfolder -> SKIP - echo "error=No changes detected in charts. The rest of the tests will be skipped." >> $GITHUB_OUTPUT - echo "result=skip" >> $GITHUB_OUTPUT + echo "error=No changes detected in charts. The rest of the tests will be skipped." >> "$GITHUB_OUTPUT" + echo "result=skip" >> "$GITHUB_OUTPUT" else # Changes done in more than chart -> SKIP - echo "error=Changes detected in more than one chart directory. It is strongly advised to change only one chart in a PR. The rest of the tests will be skipped." >> $GITHUB_OUTPUT - echo "result=skip" >> $GITHUB_OUTPUT + echo "error=Changes detected in more than one chart directory. It is strongly advised to change only one chart in a PR. The rest of the tests will be skipped." >> "$GITHUB_OUTPUT" + echo "result=skip" >> "$GITHUB_OUTPUT" fi # Using actions/github-scripts because using exit 1 in the script above would not provide any output # Source: https://github.community/t/no-output-on-process-completed-with-exit-code-1/123821/3 @@ -106,16 +107,17 @@ jobs: env: CHART: ${{ needs.get-chart.outputs.chart }} run: | + cd "${GITHUB_WORKSPACE}/charts" || exit 1 + hardcoded_images=() while read -r image; do if [[ -n "$image" && $image != {{*}} ]]; then hardcoded_images+=("${image}") fi - done <<< "$(grep --exclude "NOTES.txt" -REoh "\s*image:\s+[\"']*.+[\"']*\s*$" "charts/bitnami/${CHART}/templates" | sed "s/image: [\"']*//" | sed "s/[\"']*$//")" - + done <<< "$(grep --exclude "NOTES.txt" -REoh "\s*image:\s+[\"']*.+[\"']*\s*$" "bitnami/${CHART}/templates" | sed "s/image: [\"']*//" | sed "s/[\"']*$//")" echo "${hardcoded_images[@]}" if [[ ${#hardcoded_images[@]} -gt 0 ]] ; then - echo "error=Found hardcoded images in the chart templates: ${hardcoded_images[@]}" + echo "error=Found hardcoded images in the chart templates: ${hardcoded_images[*]}" exit 1 fi - id: check-image-warning-list @@ -123,42 +125,42 @@ jobs: env: CHART: ${{ needs.get-chart.outputs.chart }} run: | + cd "${GITHUB_WORKSPACE}/charts" || exit 1 + if [[ "$CHART" != "common" && "$CHART" != "fluentd" ]]; then - readarray -t tag_paths < <(yq e '.. | (path | join("."))' charts/bitnami/$CHART/values.yaml | grep -E '\.tag$' | sed 's/.tag$//g' | sort -u) - readarray -t registry_paths < <(yq e '.. | (path | join("."))' charts/bitnami/$CHART/values.yaml | grep '\.registry$' | sed 's/.registry$//g' | sort -u) + readarray -t tag_paths < <(yq e '.. | (path | join("."))' "bitnami/${CHART}/values.yaml" | grep -E '\.tag$' | sed 's/.tag$//g' | sort -u) + readarray -t registry_paths < <(yq e '.. | (path | join("."))' "bitnami/${CHART}/values.yaml" | grep '\.registry$' | sed 's/.registry$//g' | sort -u) # We assume that image objects are those that contain both keys 'tag' and 'registry' images_paths=() - for path in ${tag_paths[@]}; do + for path in "${tag_paths[@]}"; do if echo "${registry_paths[@]}" | grep -w -q "$path"; then - if [[ -n "$path" ]]; then - images_paths+=("$path") - fi + [[ -n "$path" ]] && images_paths+=("$path") fi done # Get the images defined in the image warning helper - readarray -d ' ' -t images_list_tmp < <(grep -E 'common.warnings.modifiedImages' charts/bitnami/$CHART/templates/NOTES.txt | sed -E 's/.*\(list (.+)\) "context".*/\1/' | sed 's/.Values.//g') + readarray -d ' ' -t images_list_tmp < <(grep -E 'common.warnings.modifiedImages' "bitnami/${CHART}/templates/NOTES.txt" | sed -E 's/.*\(list (.+)\) "context".*/\1/' | sed 's/.Values.//g') # Remove any empty element from the array images_list=() for i in "${images_list_tmp[@]}"; do - if echo $i | grep -q -E "\S+"; then + if echo "$i" | grep -q -E "\S+"; then images_list+=("$i") fi done # Compare the image objects and the image warning list if [[ ${#images_list[@]} -eq ${#images_paths[@]} ]]; then - for path in ${images_list[@]}; do - if ! echo ${images_paths[*]} | grep -w -q "$path"; then - echo "Found inconsistencies in the images warning list: '${images_list[@]}' should be equal to '${images_paths[@]}'" + for path in "${images_list[@]}"; do + if ! echo "${images_paths[*]}" | grep -w -q "$path"; then + echo "Found inconsistencies in the images warning list: '${images_list[*]}' should be equal to '${images_paths[*]}'" exit 1 fi done else - echo "Found inconsistencies in the images warning list: '${images_list[@]}' should be equal to '${images_paths[@]}'" - exit 1 + echo "Found inconsistencies in the images warning list: '${images_list[*]}' should be equal to '${images_paths[*]}'" + exit 1 fi fi update-pr: @@ -205,35 +207,40 @@ jobs: GITHUB_TOKEN: "${{ github.token }}" CHART: ${{ needs.get-chart.outputs.chart }} run: | - cd $GITHUB_WORKSPACE/upstream-charts + cd "${GITHUB_WORKSPACE}/upstream-charts" || exit 1 + # Get PR title using the API to avoid malicious string substitutions - pr_title="$(gh api /repos/${GITHUB_REPOSITORY}/pulls/${PULL_REQUEST_NUMBER} | jq -r '.title')" + pr_title="$(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${PULL_REQUEST_NUMBER}" | jq -r '.title')" # The generator needs the file to exist - chart_version="$(yq e '.version' $GITHUB_WORKSPACE/charts/bitnami/${CHART}/Chart.yaml)" - changelog_file="$GITHUB_WORKSPACE/charts/bitnami/${CHART}/CHANGELOG.md" - changelog_tmp="$GITHUB_WORKSPACE/charts/bitnami/${CHART}/CHANGELOG.md.tmp" + chart_version="$(yq e '.version' "${GITHUB_WORKSPACE}/charts/bitnami/${CHART}/Chart.yaml")" + changelog_file="${GITHUB_WORKSPACE}/charts/bitnami/${CHART}/CHANGELOG.md" + changelog_tmp="${GITHUB_WORKSPACE}/charts/bitnami/${CHART}/CHANGELOG.md.tmp" touch "$changelog_file" - npx conventional-changelog-cli -i ${changelog_file} -s -t ${CHART}/ -r 0 --commit-path bitnami/${CHART} + npx conventional-changelog-cli -i "$changelog_file" -s -t "${CHART}/" -r 0 --commit-path "bitnami/${CHART}" # The tool uses short sha to generate commit links. Sometimes, Github does not offer links with the short sha, so we change all commit links to use the full sha instead - for short_sha in $(grep -Eo "/commit/[a-z0-9]+" ${changelog_file} | awk -F/ '{print $3}'); do + for short_sha in $(grep -Eo "/commit/[a-z0-9]+" "$changelog_file" | awk -F/ '{print $3}'); do long_sha="$(git rev-list @ | grep "^$short_sha" | head -n 1)"; - sed -i "s%/commit/$short_sha%/commit/$long_sha%g" ${changelog_file}; + sed -i "s%/commit/$short_sha%/commit/$long_sha%g" "$changelog_file"; done - cd $GITHUB_WORKSPACE/charts + + cd "${GITHUB_WORKSPACE}/charts" || exit 1 # Remove unreleased section (includes all intermediate commits in the branch) and create future entry based on PR title # The unreleased section looks like this "## (YYYY-MM-DD)" whereas a released section looks like this "## 0.0.1 (YYYY-MM-DD)" # So we only need to find a released section to start printing in the awk script below - awk '/^##[^(]*[0-9]/ {flag=1} flag {print}' ${changelog_file} > ${changelog_tmp} + awk '/^##[^(]*[0-9]/ {flag=1} flag {print}' "$changelog_file" > "$changelog_tmp" # Remove extra newlines so the changelog file passes the markdown linter - sed -i -E -e '/^$/d' ${changelog_tmp} && sed -i -E -e 's/(##.*)/\n\1\n/g' ${changelog_tmp} + sed -i -E -e '/^$/d' "$changelog_tmp" && sed -i -E -e 's/(##.*)/\n\1\n/g' "$changelog_tmp" # Include h1 heading and add entry for the current version. There is no tag for the current version (this will be created once merged), so we need to manually add it. # We know the final squashed commit title, which will be the PR title. We cannot add a link to the commit in the main branch because it has not been # merged yet (this will be corrected once a new version regenerates the changelog). Instead, we add the PR url which contains the exact same information. - echo -e -n "# Changelog\n\n## $chart_version ($(date +'%Y-%m-%d'))\n\n* ${pr_title} ([#${PULL_REQUEST_NUMBER}](${PULL_REQUEST_URL}))\n" > ${changelog_file} - cat ${changelog_tmp} >> ${changelog_file} - rm ${changelog_tmp} + echo -e -n "# Changelog\n\n## $chart_version ($(date +'%Y-%m-%d'))\n\n* ${pr_title} ([#${PULL_REQUEST_NUMBER}](${PULL_REQUEST_URL}))\n" > "$changelog_file" + cat "$changelog_tmp" >> "$changelog_file" + rm "$changelog_tmp" + + # Commit all changes, if any if git status -s | grep "bitnami/${CHART}/CHANGELOG.md"; then - git add "bitnami/${CHART}/CHANGELOG.md" && git commit -m "Update CHANGELOG.md" --signoff + git add "bitnami/${CHART}/CHANGELOG.md" + git commit -m "Update CHANGELOG.md" --signoff fi - name: Install readme-generator-for-helm if: needs.get-chart.outputs.values-updated == 'true' @@ -244,40 +251,46 @@ jobs: env: CHART: ${{ needs.get-chart.outputs.chart }} run: | - exit_code=0 - cd $GITHUB_WORKSPACE/charts - echo "Validating README.md for bitnami/${CHART}" - # Validating *.registry parameters - while read line; do - echo "$line" | grep --quiet "\[default: \(REGISTRY_NAME\|\"\"\)\]" || exit_code=$? - done < <(grep "@param\s\+[A-Za-z\.]\+\.registry\s\+" "bitnami/${CHART}/values.yaml") - if [[ $exit_code -ne 0 ]]; then - echo "error=Please ensure all *.registry params include the [default: REGISTRY_NAME] modifier in the chart bitnami/${CHART}/values.yaml file" - exit "$exit_code" - fi - # Validating *.repository parameters - while read line; do - param=$(echo "$line" | awk '{print $3}') - # Checking if it's a image's registry-related param - registry_param=$(echo ${param} | sed 's/\.repository/\.registry/g') - grep --quiet "@param\s\+${registry_param}" "bitnami/${CHART}/values.yaml" && ( echo "$line" | grep --quiet "\[default: \(REPOSITORY_NAME/.*\|\"\"\)\]" || exit_code=$? ) - done < <(grep "@param\s\+[A-Za-z\.]\+\.repository\s\+" "bitnami/${CHART}/values.yaml") - if [[ $exit_code -ne 0 ]]; then - echo "error=Please ensure all *.repository params include the [default: REPOSITORY_NAME] modifier the in the chart bitnami/${CHART}/values.yaml file" - exit "$exit_code" - fi - # Validating *.tag parameters - ! grep --quiet "@param\s\+[A-Za-z\.]\+\.tag\s\+" "bitnami/${CHART}/values.yaml" || exit_code=$? - if [[ $exit_code -ne 0 ]]; then - echo "error=Please ensure all *.tag params are skipped (@skip) in the bitnami/${CHART}/values.yaml file" - exit "$exit_code" - fi - echo "Updating README.md for bitnami/${CHART}" - readme-generator --values "bitnami/${CHART}/values.yaml" --readme "bitnami/${CHART}/README.md" --schema "/tmp/schema.json" - # Commit all changes, if any - if git status -s | grep "bitnami/${CHART}"; then - git add "bitnami/${CHART}" && git commit -m "Update README.md with readme-generator-for-helm" --signoff - fi + exit_code=0 + cd "${GITHUB_WORKSPACE}/charts" || exit 1 + + echo "Validating README.md for bitnami/${CHART}" + + # Validating *.registry parameters + while read -r line; do + echo "$line" | grep --quiet "\[default: \(REGISTRY_NAME\|\"\"\)\]" || exit_code=$? + done < <(grep "@param\s\+[A-Za-z\.]\+\.registry\s\+" "bitnami/${CHART}/values.yaml") + if [[ $exit_code -ne 0 ]]; then + echo "error=Please ensure all *.registry params include the [default: REGISTRY_NAME] modifier in the chart bitnami/${CHART}/values.yaml file" + exit "$exit_code" + fi + + # Validating *.repository parameters + while read -r line; do + param=$(echo "$line" | awk '{print $3}') + # Checking if it's a image's registry-related param + registry_param="${param//.repository/.registry}" + grep --quiet "@param\s\+${registry_param}" "bitnami/${CHART}/values.yaml" && ( echo "$line" | grep --quiet "\[default: \(REPOSITORY_NAME/.*\|\"\"\)\]" || exit_code=$? ) + done < <(grep "@param\s\+[A-Za-z\.]\+\.repository\s\+" "bitnami/${CHART}/values.yaml") + if [[ $exit_code -ne 0 ]]; then + echo "error=Please ensure all *.repository params include the [default: REPOSITORY_NAME] modifier the in the chart bitnami/${CHART}/values.yaml file" + exit "$exit_code" + fi + + # Validating *.tag parameters + grep -v --quiet "@param\s\+[A-Za-z\.]\+\.tag\s\+" "bitnami/${CHART}/values.yaml" || exit_code=$? + if [[ $exit_code -ne 0 ]]; then + echo "error=Please ensure all *.tag params are skipped (@skip) in the bitnami/${CHART}/values.yaml file" + exit "$exit_code" + fi + echo "Updating README.md for bitnami/${CHART}" + readme-generator --values "bitnami/${CHART}/values.yaml" --readme "bitnami/${CHART}/README.md" --schema "/tmp/schema.json" + + # Commit all changes, if any + if git status -s | grep "bitnami/${CHART}"; then + git add "bitnami/${CHART}" + git commit -m "Update README.md with readme-generator-for-helm" --signoff + fi - id: update-crds name: 'Update CRDs' # To avoid malicious executions, only PRs performed by the bitnami-bot will perform the CRDs update @@ -285,12 +298,13 @@ jobs: env: CHART: ${{ needs.get-chart.outputs.chart }} run: | + cd "${GITHUB_WORKSPACE}/charts" || exit 1 + # Updating CRDs stored at 'bitnami/$CHART/crds', 'bitnami/$CHART/templates/crds', and "bitnami/${CHART}/charts/${CHART}-crds/crds" - cd $GITHUB_WORKSPACE/charts mapfile -t crd_files < <(find "bitnami/${CHART}/crds" "bitnami/${CHART}/templates/crds" "bitnami/${CHART}/charts/${CHART}-crds/crds" -name "*.yaml" -o -name "*.yml" 2>/dev/null || true) for file in "${crd_files[@]}"; do # Automatically update CRDs that use the '# Source' header - source_url_tpl="$(head -n 1 $file | grep -E "^# ?Source: ?" | sed -E 's|^# ?Source: ?||' || true)" + source_url_tpl="$(head -n 1 "$file" | grep -E "^# ?Source: ?" | sed -E 's|^# ?Source: ?||' || true)" if [[ -n "$source_url_tpl" ]]; then # Validate the second line of the CRD file includes the version of the CRD crd_version="$(head -n 2 $file | tail -n 1 | grep -E "^# ?Version: ?" | sed -E 's|^# ?Version: ?||' || true)" @@ -341,7 +355,7 @@ jobs: # Replace version placeholder, if present source_url=$(echo "$source_url_tpl" | sed "s/{version}/${APP_VERSION}/") # If the application version is newer, automatically update the CRD file - if [[ "$APP_VERSION" != "$crd_version" || "$skip_version" = true ]]; then + if [[ "$APP_VERSION" != "$crd_version" ]]; then if [[ "$USE_KUSTOMIZE" = "true" ]]; then kubectl kustomize "$source_url" > $file else @@ -360,9 +374,11 @@ jobs: echo "info=CRD file '$file' does not contain the '#Source' header. Skipping..." fi done + # Commit all changes, if any if git status -s | grep "bitnami/${CHART}"; then - git add "bitnami/${CHART}" && git commit -m "Update CRDs automatically" --signoff + git add "bitnami/${CHART}" + git commit -m "Update CRDs automatically" --signoff fi - id: update-pr name: Push changes