mirror of
https://github.com/bitnami/containers.git
synced 2026-03-05 07:29:11 +08:00
[bitnami/containers] Add publish osspi feature (#21804)
[bitnami/containers] Add publish osspi feature #21804 Signed-off-by: Fran Mulero <fmulero@vmware.com>
This commit is contained in:
Fran Mulero
170
.github/workflows/cd-prepare.yml
vendored
Normal file
170
.github/workflows/cd-prepare.yml
vendored
Normal file
@@ -0,0 +1,170 @@
|
||||
name: '[CI/CD] CD Prepare'
|
||||
run-name: "${{ github.event_name == 'workflow_dispatch' && format('Retrying SHA: {0}', inputs.sha) || '' }}"
|
||||
on: # rebuild any PRs and main branch changes
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sha:
|
||||
description: 'Commit to retry'
|
||||
required: true
|
||||
default: 'HEAD'
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'bitnami/**'
|
||||
env:
|
||||
CSP_API_URL: https://console.cloud.vmware.com
|
||||
CSP_API_TOKEN: ${{ secrets.CSP_API_TOKEN }}
|
||||
VIB_PUBLIC_URL: https://cp.bromelia.vmware.com
|
||||
jobs:
|
||||
prepare:
|
||||
runs-on: ubuntu-latest
|
||||
name: Retrieve metadata and auxiliary files
|
||||
if: |
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event.head_commit.author.username == 'bitnami-bot' && github.event.forced == false)
|
||||
outputs:
|
||||
result: ${{ steps.get-publish-metadata.outputs.result }}
|
||||
containers: ${{ steps.get-publish-metadata.outputs.containers }}
|
||||
steps:
|
||||
- name: Install s3cmd
|
||||
run: sudo apt-get install -y s3cmd
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v3
|
||||
# Full history is not required anymore
|
||||
with:
|
||||
ref: ${{github.event_name == 'workflow_dispatch' && inputs.sha || '' }}
|
||||
fetch-depth: 1
|
||||
- id: get-publish-metadata
|
||||
name: Get information about containers to publish
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_COMMITS: ${{ github.event_name == 'workflow_dispatch' && format('[{{"id":"{0}"}}]', inputs.sha) || toJson(github.event.commits) }}
|
||||
SHA_URL: ${{ format('{0}/tarball/{1}', github.event.repository.url, (github.event_name == 'workflow_dispatch' && inputs.sha || github.sha)) }}
|
||||
SHA: ${{github.event_name == 'workflow_dispatch' && inputs.sha || '' }}
|
||||
run: |
|
||||
# Get all commits associated to the push
|
||||
containers=()
|
||||
while read -r commit; do
|
||||
# Using the Github API to detect the files changed as git merge-base stops working when the branch is behind
|
||||
url="https://api.github.com/repos/${GITHUB_REPO}/commits/${commit}"
|
||||
files_changed_data=$(curl -s --header "authorization: Bearer ${GITHUB_TOKEN}" -X GET -G "$url")
|
||||
files_changed="$(echo "$files_changed_data" | jq -r '.files[] | .filename')"
|
||||
while read -r container; do
|
||||
if [[ ! "${containers[*]}" =~ (^|[[:space:]])$container($|[[:space:]]) ]]; then
|
||||
# Avoid duplicates
|
||||
containers+=("${container}")
|
||||
if [[ -d "${container}" ]]; then
|
||||
tag="$(grep -oE "org.opencontainers.image.ref.name=\".+\"" "${container}/Dockerfile" | sed -nr "s|org.opencontainers.image.ref.name=\"(.+)\"|\1|p")"
|
||||
if [[ -z "${tag}" ]]; then
|
||||
echo "No tag found for: ${container}"
|
||||
else
|
||||
name="$(grep -oE "org.opencontainers.image.title=\".+\"" "${container}/Dockerfile" | sed -nr "s|org.opencontainers.image.title=\"(.+)\"|\1|p")"
|
||||
app_version="$(grep -oE "org.opencontainers.image.version=\".+\"" "${container}/Dockerfile" | sed -nr "s|org.opencontainers.image.version=\"(.+)\"|\1|p")"
|
||||
os_flavour="$(grep -oP "OS_FLAVOUR=\"\K[^\"]+" "${container}/Dockerfile")"
|
||||
revision="${tag#"${app_version}-${os_flavour}-r"}"
|
||||
rolling_tags="$(yq '.rolling-tags' "${container}/tags-info.yaml" -o json | jq -c)"
|
||||
branch="$(echo "${container}" | awk -F '/' '{print $3}')"
|
||||
dsl_path="${name}"
|
||||
if [[ -d ".vib/${dsl_path}/${branch}" ]]; then
|
||||
dsl_path="${dsl_path}/${branch}"
|
||||
fi
|
||||
container_json=$(jq -n '{"name": $name, "path": $path, "os_flavour": $os_flavour, "branch": $branch, "app_version": $app_version, "revision": $revision, "sha": $sha, "sha_url": $sha_url, "dsl_path": $dsl_path, "tag": $tag, "rolling_tags": $rolling_tags}' \
|
||||
--arg name "$name" --arg path "$container" --arg os_flavour "$os_flavour" --arg branch "$branch" --arg app_version "$app_version" --arg revision "$revision" --arg sha "$SHA" --arg sha_url "$SHA_URL" --arg dsl_path "$dsl_path" --arg tag "$tag" --argjson rolling_tags "$rolling_tags")
|
||||
containers_json+=("${container_json}")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done <<< "$(echo "$files_changed" | xargs dirname | grep -o "^bitnami/[^/]*/[^/]*/[^/]*" | sort | uniq || true)"
|
||||
done <<< "$(echo "${GITHUB_COMMITS}" | jq -r '.[] | .id')"
|
||||
|
||||
if [[ "${#containers[@]}" -le "0" ]]; then
|
||||
echo "No changes detected in containers. The rest of the steps will be skipped."
|
||||
echo "result=skip" >> $GITHUB_OUTPUT
|
||||
else
|
||||
publish_metadata=$(printf "%s\n" "${containers_json[@]}" | jq -s .)
|
||||
echo "result=ok" >> $GITHUB_OUTPUT
|
||||
echo "${publish_metadata}" > publish-metadata.json
|
||||
fi
|
||||
- name: Getting 3rd party packages for OSSPI
|
||||
if: ${{ steps.get-publish-metadata.outputs.result == 'ok' }}
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
run: |
|
||||
while read -r container; do
|
||||
# If it's set from outside, can be changed
|
||||
arch="${arch:-amd64}"
|
||||
image_name=$(echo "$container" | jq -cr '.tag')
|
||||
image_path=$(echo "$container" | jq -cr '.path')
|
||||
version=$(echo "$container" | jq -cr '.app_version')
|
||||
asset=$(echo "$container" | jq -cr '.name')
|
||||
os_flavour=$(echo "$container" | jq -cr '.os_flavour')
|
||||
revision=$(echo "$container" | jq -cr '.revision')
|
||||
|
||||
components_files=$(s3cmd ls -l "s3://$AWS_S3_BUCKET/$asset/$version/$os_flavour/$arch/$revision/" | grep -c "components.json")
|
||||
# If the components.json file, so it seems has external packages
|
||||
if [[ $components_files -gt 0 ]]; then
|
||||
s3cmd get -q "s3://${AWS_S3_BUCKET}/$asset/$version/$os_flavour/$arch/$revision/components.json" components.json
|
||||
|
||||
declare -A packages=()
|
||||
# Iterating over the external components to get the involved elements
|
||||
while read -r component_id; do
|
||||
component_version_full=$(jq -cr '."'${component_id}'".version' components.json)
|
||||
|
||||
#split by "-"
|
||||
component_parts=(${component_version_full//-/ })
|
||||
component_version=${component_parts[0]}
|
||||
compilation_recipe=$(s3cmd ls -l "s3://${AWS_S3_BUCKET}/$component_id/$component_version/$os_flavour/$arch/" | grep -c "compilation-recipe.json")
|
||||
# If the components.json file, so it seems has external packages
|
||||
if [[ $compilation_recipe -gt 0 ]]; then
|
||||
s3cmd get -q "s3://${AWS_S3_BUCKET}/$component_id/$component_version/$os_flavour/$arch/compilation-recipe.json" compilation-recipe.json
|
||||
else
|
||||
s3cmd get -q "s3://${AWS_S3_BUCKET}/$component_id/$component_version-${component_parts[1]}/$os_flavour/$arch/compilation-recipe.json" compilation-recipe.json
|
||||
fi
|
||||
# now getting each component to be reported
|
||||
while read -r json_package; do
|
||||
package_id=$(echo "$json_package" | jq -r '.id' )
|
||||
echo "${asset}: ${package_id}"
|
||||
package_version=$(echo "$json_package" | jq -r '.version' )
|
||||
package_url=$(echo "$json_package" | jq -r '.source.upstreamSourceUrl')
|
||||
packages[$package_id]="$package_version $package_url"
|
||||
done <<<"$(jq -cr '.components[]' compilation-recipe.json)"
|
||||
rm compilation-recipe.json
|
||||
done <<<"$(jq -cr 'keys[]' components.json)"
|
||||
|
||||
# Now creating the JSON file with the needed transformations
|
||||
declare -a json_packages=()
|
||||
for package_id in "${!packages[@]}"
|
||||
do
|
||||
values=(${packages[$package_id]// / })
|
||||
cleaned_url="${values[1]#git+}"
|
||||
if [ -z "$cleaned_url" ]
|
||||
then
|
||||
echo "[WARNING] The URL for ${package_id}:${values[0]} is missing in the recipe"
|
||||
else
|
||||
json_packages+=($(jq -n '{"_unique_id": $uniqueID, "name": $name, "version": $version, "url": $url, "repository": "other"}' --arg uniqueID "other:$package_id:${values[0]}" --arg name "$package_id" --arg version "${values[0]}" --arg url "$cleaned_url"))
|
||||
fi
|
||||
done
|
||||
JSON_ARRAY=$(printf "%s" "${json_packages[@]}" | jq -s)
|
||||
echo "${JSON_ARRAY}" > "${image_path}/packages.json"
|
||||
jq -n '{"packages": $packages, "server": { "scanner": "custom", "type": "Hub"}}' --argjson packages "$(cat ${image_path}/packages.json)" > "$image_path/osspi-packages.json"
|
||||
s3cmd put "${image_path}/packages.json" "s3://${AWS_S3_BUCKET}/${asset}/${version}/${os_flavour}/${arch}/${revision}/packages.json"
|
||||
rm components.json
|
||||
else
|
||||
echo "$image_name:$version doesn't have external components.json"
|
||||
fi
|
||||
done <<<"$(jq -c '.[]' publish-metadata.json)"
|
||||
- uses: actions/upload-artifact@v3
|
||||
if: ${{ steps.get-publish-metadata.outputs.result == 'ok' }}
|
||||
with:
|
||||
name: publish-metadata.json
|
||||
path: ./publish-metadata.json
|
||||
- uses: actions/upload-artifact@v3
|
||||
if: ${{ steps.get-publish-metadata.outputs.result == 'ok' }}
|
||||
with:
|
||||
name: packages.json
|
||||
path: ~/work/containers/**/osspi-packages.json
|
||||
Reference in New Issue
Block a user