From e92a7b4847ad9edcca93757296c6f83251ba352b Mon Sep 17 00:00:00 2001 From: Fran Mulero Date: Wed, 2 Aug 2023 14:51:17 +0200 Subject: [PATCH] [containers/*] Remove CD pipelines (#43214) Remove CD pipelines Signed-off-by: Fran Mulero --- .github/workflows/cd-pipeline.yml | 133 ---------------------- .github/workflows/cd-prepare.yml | 177 ------------------------------ 2 files changed, 310 deletions(-) delete mode 100644 .github/workflows/cd-pipeline.yml delete mode 100644 .github/workflows/cd-prepare.yml diff --git a/.github/workflows/cd-pipeline.yml b/.github/workflows/cd-pipeline.yml deleted file mode 100644 index 6ded08630afd..000000000000 --- a/.github/workflows/cd-pipeline.yml +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright VMware, Inc. -# SPDX-License-Identifier: APACHE-2.0 - -name: '[CI/CD] CD Pipeline' -run-name: "${{ format('[CI/CD] CD Publish {0}',github.event.workflow_run.display_title) }}" -on: # rebuild any PRs and main branch changes - workflow_run: - workflows: - - '\[CI/CD\] CD Prepare' - types: - - 'completed' -# Remove all permissions by default. -permissions: {} -env: - CSP_API_URL: https://console.cloud.vmware.com - CSP_API_TOKEN: ${{ secrets.CSP_API_TOKEN }} - VIB_PUBLIC_URL: https://cp.bromelia.vmware.com -jobs: - get-metadata: - runs-on: ubuntu-latest - name: Get modified containers path - permissions: - actions: read - if: ${{ github.event.workflow_run.conclusion == 'success' }} - outputs: - packages_json_url: ${{ steps.get-artifacts.outputs.packages_json_url }} - containers: ${{ steps.get-artifacts.outputs.containers }} - steps: - - id: get-artifacts - name: Get modified containers path - env: - PREPARE_WORKFLOW_ARTIFACTS_URL: ${{ github.event.workflow_run.artifacts_url }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - curl -sSL -o artifacts.json -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -H "Authorization: Bearer $GITHUB_TOKEN" $PREPARE_WORKFLOW_ARTIFACTS_URL - publish_metadata_url="$(jq -r '.artifacts[] | select(.name == "publish-metadata.json") | .archive_download_url' artifacts.json)" - curl -sSL -o publish-metadata.json.zip -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -H "Authorization: Bearer $GITHUB_TOKEN" $publish_metadata_url - unzip publish-metadata.json.zip - containers=$(cat publish-metadata.json | jq -c) - packages_json_url="$(jq -r '.artifacts[] | select(.name == "packages.json") | .archive_download_url' artifacts.json)" - echo "packages_json_url=${packages_json_url}" >> $GITHUB_OUTPUT - echo "containers=${containers}" >> $GITHUB_OUTPUT - - vib-publish: - runs-on: ubuntu-latest - needs: get-metadata - name: VIB Publish - permissions: - contents: read - strategy: - fail-fast: false - max-parallel: 2 - matrix: - container: ${{ fromJSON(needs.get-metadata.outputs.containers) }} - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - # Full history is not required anymore - with: - ref: ${{ matrix.container.sha }} - fetch-depth: 1 - - id: get-registry-credentials - name: Get marketplace's registry credentials - run: | - csp_auth_token=$(curl -s -H 'Content-Type: application/x-www-form-urlencoded' -X POST -d "grant_type=refresh_token&api_token=${{ secrets.PROD_MARKETPLACE_API_TOKEN }}" https://console.cloud.vmware.com/csp/gateway/am/api/auth/api-tokens/authorize | jq -re .access_token) - repo_info=$(curl -s -X POST -H "Content-Type: application/json" -H "csp-auth-token:$csp_auth_token" -d '{"withCredentials":true, "storageType":"OCI"}' https://gtw.marketplace.cloud.vmware.com/api/v1/repositories/transient) - marketplace_user=$(echo "$repo_info" | jq -re .response.repodetails.username) - marketplace_passwd=$(echo "$repo_info" | jq -re .response.repodetails.token) - echo "::add-mask::${marketplace_user}" - echo "::add-mask::${marketplace_passwd}" - echo "marketplace_user=${marketplace_user}" >> $GITHUB_OUTPUT - echo "marketplace_passwd=${marketplace_passwd}" >> $GITHUB_OUTPUT - - uses: vmware-labs/vmware-image-builder-action@v0 - name: 'Publish ${{ matrix.container.name }}: ${{ matrix.container.tag }}' - with: - pipeline: ${{ matrix.container.dsl_path }}/vib-publish.json - env: - # Path with docker resources - VIB_ENV_PATH: "${{ matrix.container.path }}" - # Container name - VIB_ENV_CONTAINER: "${{ matrix.container.name }}" - VIB_ENV_CONTAINER_URL: "${{ matrix.container.sha_url }}" - VIB_ENV_PACKAGES_JSON_URL: "${{ needs.get-metadata.outputs.packages_json_url }}" - VIB_ENV_APP_VERSION: "${{ matrix.container.app_version }}" - VIB_ENV_OS_FLAVOUR: "${{ matrix.container.os_flavour }}" - VIB_ENV_GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - VIB_ENV_TAG: "${{ matrix.container.tag }}" - VIB_ENV_ROLLING_TAGS: "${{ toJSON(matrix.container.rolling_tags) }}" - VIB_ENV_REGISTRY_URL: "${{ secrets.PROD_MARKETPLACE_REGISTRY_URL }}" - VIB_ENV_REGISTRY_USERNAME: "${{ steps.get-registry-credentials.outputs.marketplace_user }}" - VIB_ENV_REGISTRY_PASSWORD: "${{ steps.get-registry-credentials.outputs.marketplace_passwd }}" - VIB_ENV_BOSSD_RELEASE_ID: "${{ secrets.BOSSD_RELEASE_ID }}" - - # If the CD Pipeline does not succeed we should notify the interested agents - slack-notif: - runs-on: ubuntu-latest - needs: - - vib-publish - if: always() - name: Notify unsuccessful CD run - steps: - - name: Notify in Slack channel - if: ${{ needs.vib-publish.result != 'success' }} - uses: slackapi/slack-github-action@v1.23.0 - with: - channel-id: ${{ secrets.CD_SLACK_CHANNEL_ID }} - payload: | - { - "attachments": [ - { - "color": "#CC0000", - "fallback": "Unsuccessful bitnami/containers CD pipeline", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "*Unsuccessful `bitnami/containers` CD pipeline*" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "The CD Pipeline for <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.event.workflow_run.head_commit.id }}|${{ github.repository }}@${{ github.event.workflow_run.head_commit.id }}> and launched by its CD Prepare <${{ github.event.workflow_run.html_url }}|workflow run> did not succeed. Check the related <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|CD Pipeline run> for more information." - } - } - ] - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.CD_SLACK_BOT_TOKEN }} diff --git a/.github/workflows/cd-prepare.yml b/.github/workflows/cd-prepare.yml deleted file mode 100644 index 3bdb8a973869..000000000000 --- a/.github/workflows/cd-prepare.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: '[CI/CD] CD Prepare' -run-name: "${{ github.event_name == 'workflow_dispatch' && format('Retrying SHA: {0}', inputs.sha) || '' }}" -on: # rebuild any PRs and main branch changes - workflow_dispatch: - inputs: - sha: - description: 'Commit to retry' - required: true - default: 'HEAD' - push: - branches: - - main - paths: - - 'bitnami/**' -# Remove all permissions by default. -permissions: {} -env: - CSP_API_URL: https://console.cloud.vmware.com - CSP_API_TOKEN: ${{ secrets.CSP_API_TOKEN }} - VIB_PUBLIC_URL: https://cp.bromelia.vmware.com -jobs: - prepare: - runs-on: ubuntu-latest - name: Retrieve metadata and auxiliary files - permissions: - contents: read - if: | - github.event_name == 'workflow_dispatch' || - (github.event.head_commit.author.username == 'bitnami-bot' && github.event.forced == false) - outputs: - result: ${{ steps.get-publish-metadata.outputs.result }} - containers: ${{ steps.get-publish-metadata.outputs.containers }} - steps: - - name: Install s3cmd - run: sudo apt-get install -y s3cmd - - name: Checkout Repository - uses: actions/checkout@v3 - # Full history is not required anymore - with: - ref: ${{github.event_name == 'workflow_dispatch' && inputs.sha || '' }} - fetch-depth: 1 - - id: get-publish-metadata - name: Get information about containers to publish - env: - GITHUB_REPO: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_COMMITS: ${{ github.event_name == 'workflow_dispatch' && format('[{{"id":"{0}"}}]', inputs.sha) || toJson(github.event.commits) }} - SHA_URL: ${{ format('{0}/tarball/{1}', github.event.repository.url, (github.event_name == 'workflow_dispatch' && inputs.sha || github.sha)) }} - SHA: ${{github.event_name == 'workflow_dispatch' && inputs.sha || '' }} - run: | - # Get all commits associated to the push - containers=() - while read -r commit; do - # Using the Github API to detect the files changed as git merge-base stops working when the branch is behind - url="https://api.github.com/repos/${GITHUB_REPO}/commits/${commit}" - files_changed_data=$(curl -s --header "authorization: Bearer ${GITHUB_TOKEN}" -X GET -G "$url") - files_changed="$(echo "$files_changed_data" | jq -r '.files[] | .filename')" - while read -r container; do - if [[ ! "${containers[*]}" =~ (^|[[:space:]])$container($|[[:space:]]) ]]; then - # Avoid duplicates - containers+=("${container}") - if [[ -d "${container}" ]]; then - tag="$(grep -oE "org.opencontainers.image.ref.name=\".+\"" "${container}/Dockerfile" | sed -nr "s|org.opencontainers.image.ref.name=\"(.+)\"|\1|p")" - if [[ -z "${tag}" ]]; then - echo "No tag found for: ${container}" - else - name="$(grep -oE "org.opencontainers.image.title=\".+\"" "${container}/Dockerfile" | sed -nr "s|org.opencontainers.image.title=\"(.+)\"|\1|p")" - app_version="$(grep -oE "org.opencontainers.image.version=\".+\"" "${container}/Dockerfile" | sed -nr "s|org.opencontainers.image.version=\"(.+)\"|\1|p")" - os_flavour_parsed="$(grep -oP "OS_FLAVOUR=\"\K[^\"]+" "${container}/Dockerfile" || true)" - os_flavour=${os_flavour_parsed:-scratch} - revision="${tag#"${app_version}-${os_flavour}-r"}" - rolling_tags="$(yq '.rolling-tags' "${container}/tags-info.yaml" -o json | jq -c)" - branch="$(echo "${container}" | awk -F '/' '{print $3}')" - dsl_path="${name}" - if [[ -d ".vib/${dsl_path}/${branch}" ]]; then - dsl_path="${dsl_path}/${branch}" - fi - # This is hack to avoid jq errors while getting the architectures - vib_publish="$(cat .vib/${dsl_path}/vib-publish.json | sed -e 's|{VIB_ENV_ROLLING_TAGS}|"${rolling_tags}"|')" - architectures="$(echo "${vib_publish}" | jq -cr '.phases.package.actions[] | select(.action_id == "container-image-package") | .params.architectures // ["linux/amd64"]')" - container_json=$(jq -n '{"name": $name, "path": $path, "os_flavour": $os_flavour, "branch": $branch, "app_version": $app_version, "revision": $revision, "sha": $sha, "sha_url": $sha_url, "dsl_path": $dsl_path, "tag": $tag, "rolling_tags": $rolling_tags, "architectures": $architectures}' \ - --arg name "$name" --arg path "$container" --arg os_flavour "$os_flavour" --arg branch "$branch" --arg app_version "$app_version" --arg revision "$revision" --arg sha "$SHA" --arg sha_url "$SHA_URL" --arg dsl_path "$dsl_path" --arg tag "$tag" --argjson rolling_tags "$rolling_tags" --argjson architectures "$architectures") - containers_json+=("${container_json}") - fi - fi - fi - done <<< "$(echo "$files_changed" | xargs dirname | grep -o "^bitnami/[^/]*/[^/]*/[^/]*" | sort | uniq || true)" - done <<< "$(echo "${GITHUB_COMMITS}" | jq -r '.[] | .id')" - - if [[ "${#containers[@]}" -le "0" ]]; then - echo "No changes detected in containers. The rest of the steps will be skipped." - echo "result=skip" >> $GITHUB_OUTPUT - else - publish_metadata=$(printf "%s\n" "${containers_json[@]}" | jq -s .) - echo "result=ok" >> $GITHUB_OUTPUT - echo "${publish_metadata}" > publish-metadata.json - fi - - name: Getting 3rd party packages for OSSPI - if: ${{ steps.get-publish-metadata.outputs.result == 'ok' }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - AWS_DEFAULT_REGION: us-east-1 - run: | - # - - # Example: 17.0.7-7-1 - # version: semver but it can have also '-' like the upper example. - # revision: number - COMPONENT_VERSION_REGEX="([0-9]+\.[0-9]+\.[0-9]+(-.+)?)-([0-9]+)" - while read -r container; do - # If it's set from outside, can be changed - image_name=$(echo "$container" | jq -cr '.tag') - image_path=$(echo "$container" | jq -cr '.path') - version=$(echo "$container" | jq -cr '.app_version') - containers_path=$(echo "$container" | jq -cr '.path') - components_json_file="${containers_path}/prebuildfs/opt/bitnami/.bitnami_components.json" - - # If the components.json file, so it seems has external packages - if [[ -f "$components_json_file" ]]; then - while read -r architecture; do - # Remove platform prefix. - arch="${architecture##*/}" - declare -A packages=() - # Iterating over the external components to get the involved elements - while read -r component_id; do - if [[ -n "$component_id" ]]; then - component_version_full=$(jq -cr '."'${component_id}'".version' "$components_json_file") - if [[ "$component_version_full" =~ $COMPONENT_VERSION_REGEX ]] && [[ "${#BASH_REMATCH[@]}" -ge 3 ]]; then - component_version=${BASH_REMATCH[1]} - component_distro=$(jq -cr '."'${component_id}'".distro' "$components_json_file") - # We will use the container arch instead of the component arch (component file has only amd64 references) - # component_arch=$(jq -cr '."'${component_id}'".arch' "$components_json_file") - component_arch=$arch - # Added "true" to avoid errors if compilation_recipe doesn't exists - compilation_recipe=$(s3cmd get "s3://${AWS_S3_BUCKET}/${component_id}/${component_version}/${component_distro}/${component_arch}/compilation-recipe.json" - 2>/dev/null || true) - # now getting each component to be reported - while read -r json_package; do - package_id="$(echo "$json_package" | jq -r '.id' )" - package_version="$(echo "$json_package" | jq -r '.version' )" - package_url="$(echo "$json_package" | jq -r '.source.upstreamSourceUrl')" - packages["$package_id"]="${package_version} ${package_url}" - done <<<"$(echo "$compilation_recipe" | jq -cr '.components[]')" - fi - fi - done <<<"$(jq -cr 'keys[]' "$components_json_file")" - - # Now creating the JSON file with the format required by the OSSPI Scan - osspi_packages="[]" - for package_id in "${!packages[@]}"; do - read -r -a version_url <<< "${packages["$package_id"]}" - http_url="${version_url[1]#git+}" - if [[ -z "$http_url" ]] - then - echo "[WARNING] The URL for ${package_id}:${version_url[0]} is missing in the recipe" - else - # Concat new package to osspi_packages array - osspi_packages="$(jq '. += [{"_unique_id": $uniqueID, "name": $name, "version": $version, "url": $url, "repository": "other"}]' --arg uniqueID "other:${package_id}:${version_url[0]}" --arg name "$package_id" --arg version "${version_url[0]}" --arg url "$http_url" <<< "$osspi_packages")" - fi - done - jq -n '{"packages": $packages, "server": { "scanner": "custom", "type": "Hub"}}' --argjson packages "${osspi_packages}" > "${image_path}/osspi-packages-${arch}.json" - s3cmd put "${image_path}/osspi-packages-${arch}.json" "s3://${AWS_S3_BUCKET}/${asset}/${version}/${os_flavour}/${arch}/${revision}/packages.json" - done <<<"$(echo "$container" | jq -cr '.architectures[]')" - else - echo "$image_name:$version doesn't have external components.json" - fi - done <<<"$(jq -c '.[]' publish-metadata.json)" - - uses: actions/upload-artifact@v3 - if: ${{ steps.get-publish-metadata.outputs.result == 'ok' }} - with: - name: publish-metadata.json - path: ./publish-metadata.json - - uses: actions/upload-artifact@v3 - if: ${{ steps.get-publish-metadata.outputs.result == 'ok' }} - with: - name: packages.json - path: ~/work/containers/**/osspi-packages*.json