mirror of
https://github.com/bitnami/containers.git
synced 2026-03-06 06:58:01 +08:00
[containers/*]Improve sync workflow (#12)
* Improve sync script to remove deprecated repos and include new ones Signed-off-by: Fran Mulero <fmulero@vmware.com> * Add sudo to install git-filter--repo Signed-off-by: Fran Mulero <fmulero@vmware.com> * Rename fetch script and Amend linter Signed-off-by: Bitnami Containers <bitnami-bot@vmware.com> * Sort results Signed-off-by: Bitnami Containers <bitnami-bot@vmware.com> Co-authored-by: Bitnami Containers <bitnami-bot@vmware.com>
This commit is contained in:
Fran Mulero
182
scripts/sync.sh
Executable file
182
scripts/sync.sh
Executable file
@@ -0,0 +1,182 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
#set -o xtrace # Uncomment this line for debugging purpose
|
||||
|
||||
TARGET_DIR="."
|
||||
|
||||
COMMIT_SHIFT="${1:-0}" # Used when you push commits manually
|
||||
CONTAINER="${2:-}" # Used when we want to sync a single container
|
||||
SKIP_COMMIT_ID="${3:-}" # Used in some cases when a patch does not apply because it was applied as part of a PR
|
||||
SPECIAL_FILES=("CONTRIBUTING.md" "CODE_OF_CONDUCT.md" "LICENSE.md" ".github") # Files to remove in new repos
|
||||
|
||||
queryRepos() {
|
||||
local page=0
|
||||
local repos="[]" # Initial empty JSON array
|
||||
local -r repos_per_page="100"
|
||||
|
||||
while [[ "$page" -gt -1 ]]; do
|
||||
# Query only the public repos since we won't add private containers to bitnami/containers
|
||||
page_repos="$(curl -sH 'Content-Type: application/json' -H 'Accept: application/json' "https://api.github.com/orgs/bitnami/repos?type=public&per_page=${repos_per_page}&page=${page}")"
|
||||
repos="$(jq -s 'reduce .[] as $x ([]; . + $x)' <(echo "$repos") <(echo "$page_repos"))"
|
||||
n_repos="$(jq length <<< "$page_repos")"
|
||||
if [[ "$n_repos" -lt "$repos_per_page" ]]; then
|
||||
page="-1"
|
||||
else
|
||||
page="$((page + 1))"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "$repos"
|
||||
}
|
||||
|
||||
getContainerRepos() {
|
||||
local -r repos="$(queryRepos)"
|
||||
local -r container_repos="$(jq -r '[ .[] | select(.name | test("bitnami-docker-.")) | select(.archived == false) ]' <<< "$repos")"
|
||||
local result=""
|
||||
while read -r repo_url; do
|
||||
result="${result} ${repo_url:42}"
|
||||
done < <(echo "$container_repos" | jq -r '.[].html_url' | uniq | sort)
|
||||
echo "$result"
|
||||
}
|
||||
|
||||
# Commits a directory
|
||||
gitConfigure() {
|
||||
git config --global user.name "Bitnami Containers"
|
||||
git config --global user.email "bitnami-bot@vmware.com"
|
||||
}
|
||||
|
||||
pushChanges() {
|
||||
git push origin "$(git rev-parse --abbrev-ref HEAD)"
|
||||
}
|
||||
|
||||
findCommitsToSync() {
|
||||
local origin_name="${1:?Missing origin name}"
|
||||
# Find the commit that doesn't have changes respect the container folder
|
||||
# Get the last commit message in the container folder
|
||||
local shift=$((COMMIT_SHIFT + 1))
|
||||
local -r last_commit_message="$(git log -n "$shift" --pretty=tformat:"%s" -- ./containers/"$origin_name" | tail -n 1)"
|
||||
# Search on the container origin repo the ID for the latest commit we have locally in the container folder
|
||||
local -r last_synced_commit_id="$(git rev-list "$origin_name"/master --grep="${last_commit_message}" | head -1)"
|
||||
local commits_to_sync=""
|
||||
local max=100 # If we need to sync more than 100 commits there must be something wrong since we run the job on a daily basis
|
||||
# Get all commits IDs on the origin
|
||||
local commits=()
|
||||
while IFS='' read -r line; do commits+=("$line"); done < <(git rev-list "${origin_name}/master" -- .)
|
||||
for commit in "${commits[@]}"; do
|
||||
if [[ "$commit" != "$last_synced_commit_id" ]] && [[ "$max" -gt "0" ]]; then
|
||||
actual_commit_id="$(plainCommit "${commit}")"
|
||||
if [[ "$commit" != "$SKIP_COMMIT_ID" ]] && [[ ! $commits_to_sync =~ (^|[[:space:]])$actual_commit_id($|[[:space:]]) ]]; then
|
||||
commits_to_sync="${actual_commit_id} ${commits_to_sync}"
|
||||
fi
|
||||
else
|
||||
# We reached the last commit synced
|
||||
break
|
||||
fi
|
||||
max=$((max - 1))
|
||||
done
|
||||
|
||||
[[ "$max" -eq "0" ]] && echo "Last commit not found into the original repo history" && return 1
|
||||
echo "$commits_to_sync"
|
||||
}
|
||||
|
||||
plainCommit() {
|
||||
local commit="${1:?Missing commit}"
|
||||
local result="$commit"
|
||||
actual_merge_commit_id="$(git log --pretty=%P -n 1 "$commit" | awk '{print $2}')"
|
||||
if [[ -n "$actual_merge_commit_id" ]]; then
|
||||
result="$(plainCommit "${actual_merge_commit_id}")"
|
||||
fi
|
||||
echo "$result"
|
||||
}
|
||||
|
||||
syncCommit() {
|
||||
local -r commit_id="${1:?Missing commit id}"
|
||||
local -r app="${2:?Missing app name}"
|
||||
local -r patch_file="$(git format-patch -1 "$commit_id")"
|
||||
# Apply patch
|
||||
git am --directory "containers/${app}" "$patch_file"
|
||||
rm -f "$patch_file"
|
||||
}
|
||||
|
||||
syncContainerCommits() {
|
||||
local -r name="${1:?Missing container name}"
|
||||
local -r repo_url="https://github.com/bitnami/bitnami-docker-${name}"
|
||||
(
|
||||
cd "$TARGET_DIR"
|
||||
# Fetch the old repo master
|
||||
git remote add --fetch "$name" "$repo_url"
|
||||
read -r -a commits_to_sync <<< "$(findCommitsToSync "$name")"
|
||||
if [[ "${#commits_to_sync[@]}" -eq "0" ]]; then
|
||||
echo "Nothing to sync for ${name}"
|
||||
else
|
||||
for commit in "${commits_to_sync[@]}"; do
|
||||
syncCommit "$commit" "$name"
|
||||
done
|
||||
fi
|
||||
git remote remove "$name"
|
||||
)
|
||||
}
|
||||
|
||||
syncNewContainer() {
|
||||
local -r container="${1:?Missing container name}"
|
||||
mkdir -p "${TARGET_DIR}/containers/${container}" # Create directory for the app
|
||||
# clone the repositoy outside of this one
|
||||
git clone "https://github.com/bitnami/bitnami-docker-${container}" "/tmp/${container}"
|
||||
cd "/tmp/${container}" || exit
|
||||
# Remove special files
|
||||
rm -rf "${SPECIAL_FILES[@]}" || true
|
||||
git add -A
|
||||
git commit -qm "Remove CONTRIBUTING.md CODE_OF_CONDUCT.md and LICENSE.md. Now these files are are in the root"
|
||||
cd - || exit
|
||||
# Rewrite history to remove special files.
|
||||
for file in "${SPECIAL_FILES[@]}"; do
|
||||
git filter-repo --quiet --source "/tmp/${container}" --target "/tmp/${container}" --invert-paths --force --path "${file}"
|
||||
done
|
||||
# Rewrite history to point files to containers/${container}
|
||||
git-filter-repo --quiet --source "/tmp/${container}" --target "/tmp/${container}" --to-subdirectory-filter "containers/${container}" --force
|
||||
|
||||
# Fetch the old repo and merge maintaining history
|
||||
git remote add --fetch "$container" "/tmp/${container}"
|
||||
git merge "${container}/master" --allow-unrelated-histories --no-log --no-ff -Xtheirs -qm "Merge bitnami-docker-${container} into bitnami/containers"
|
||||
git remote remove "$container"
|
||||
rm -Rf "/tmp/${container}"
|
||||
}
|
||||
|
||||
syncRepos() {
|
||||
gitConfigure # Configure Git client
|
||||
mkdir -p "$TARGET_DIR"
|
||||
|
||||
if [[ -z "$CONTAINER" ]]; then
|
||||
local -r repos="$(getContainerRepos)"
|
||||
# Sync changes
|
||||
for container in $repos; do
|
||||
if [[ -d "${TARGET_DIR}/containers/${container}" ]]; then
|
||||
echo "Syncing container: ${container}"
|
||||
syncContainerCommits "$container"
|
||||
else
|
||||
echo "Add new container: ${container}"
|
||||
syncNewContainer "$container"
|
||||
fi
|
||||
done
|
||||
# Clean deprecated
|
||||
cd "${TARGET_DIR}/containers" || exit
|
||||
for container in *; do
|
||||
if [[ ! $repos =~ (^|[[:space:]])$container($|[[:space:]]) ]]; then
|
||||
echo "Removing container: ${container}"
|
||||
rm -rf "${container}"
|
||||
git add "${container}"
|
||||
git commit -q -m "Remove deprecated container ${container}"
|
||||
fi
|
||||
done
|
||||
else
|
||||
syncContainerCommits "$CONTAINER"
|
||||
fi
|
||||
|
||||
pushChanges
|
||||
}
|
||||
|
||||
pip install git-filter-repo==2.34.0
|
||||
syncRepos
|
||||
Reference in New Issue
Block a user