mirror of
https://github.com/bitnami/containers.git
synced 2026-03-21 15:29:05 +08:00
2.2.3-debian-10-r47 release
This commit is contained in:
@@ -11,12 +11,12 @@ COPY prebuildfs /
|
|||||||
# Install required system packages and dependencies
|
# Install required system packages and dependencies
|
||||||
RUN install_packages acl ca-certificates curl gzip libbsd0 libbz2-1.0 libc6 libcom-err2 libcurl4 libedit2 libffi6 libgcc1 libgcrypt20 libgmp10 libgnutls30 libgpg-error0 libgssapi-krb5-2 libhogweed4 libicu63 libidn2-0 libk5crypto3 libkeyutils1 libkrb5-3 libkrb5support0 libldap-2.4-2 liblzma5 libmariadb3 libncursesw6 libnettle6 libnghttp2-14 libp11-kit0 libpsl5 libreadline7 librtmp1 libsasl2-2 libsqlite3-0 libssh2-1 libssl1.1 libstdc++6 libtasn1-6 libtinfo6 libunistring2 libuuid1 libxml2 libxslt1.1 netbase procps tar zlib1g
|
RUN install_packages acl ca-certificates curl gzip libbsd0 libbz2-1.0 libc6 libcom-err2 libcurl4 libedit2 libffi6 libgcc1 libgcrypt20 libgmp10 libgnutls30 libgpg-error0 libgssapi-krb5-2 libhogweed4 libicu63 libidn2-0 libk5crypto3 libkeyutils1 libkrb5-3 libkrb5support0 libldap-2.4-2 liblzma5 libmariadb3 libncursesw6 libnettle6 libnghttp2-14 libp11-kit0 libpsl5 libreadline7 librtmp1 libsasl2-2 libsqlite3-0 libssh2-1 libssl1.1 libstdc++6 libtasn1-6 libtinfo6 libunistring2 libuuid1 libxml2 libxslt1.1 netbase procps tar zlib1g
|
||||||
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "wait-for-port" "1.0.1-6" --checksum 845ff207415b5730cadd45aa3c757ca45a5c9e73bc54c24aa481ddca24eec35d
|
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "wait-for-port" "1.0.1-6" --checksum 845ff207415b5730cadd45aa3c757ca45a5c9e73bc54c24aa481ddca24eec35d
|
||||||
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "python" "3.8.12-12" --checksum c081f7f9663d4d64e7f26d6e6a77b3860ee1f39c309bd564f801c39192042d2f
|
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "python" "3.8.12-30" --checksum 5aa3fd465e5e0d6fe86e984121ffb2c02a34b44a8c9cd9cce9b18d3181534b68
|
||||||
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "postgresql-client" "10.19.0-1" --checksum 53e64e567d4563efaea05f4e9e9eead3a3fbec1213a05528a5eeec2d2f8e8a43
|
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "postgresql-client" "10.19.0-30" --checksum 58714b132871182d2ab826fa678330c978260cbe42277f65152d400b29885c2f
|
||||||
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "ini-file" "1.4.1-3" --checksum 26f2ba99a02266d6f3857a3a11db7ed441d76a2fbc1066a17d4b0b17b047e01d
|
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "ini-file" "1.4.1-3" --checksum 26f2ba99a02266d6f3857a3a11db7ed441d76a2fbc1066a17d4b0b17b047e01d
|
||||||
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "git" "2.35.1-0" --checksum 2566d89a20e9794698ad0868e8ea0b73cf27343d552c00c5278379c47aa62d29
|
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "git" "2.35.1-0" --checksum 2566d89a20e9794698ad0868e8ea0b73cf27343d552c00c5278379c47aa62d29
|
||||||
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "gosu" "1.14.0-3" --checksum 276ab5a0be4b05e136ec468d62c8f9cc4f40d9664c55f01f16a9f1209ba16980
|
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "gosu" "1.14.0-3" --checksum 276ab5a0be4b05e136ec468d62c8f9cc4f40d9664c55f01f16a9f1209ba16980
|
||||||
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "airflow-scheduler" "2.2.3-4" --checksum ae1dd33f99a6dada2253c82199727455177b9daf4fb4435f6c310b831c530f88
|
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "airflow-scheduler" "2.2.3-5" --checksum ae1dd33f99a6dada2253c82199727455177b9daf4fb4435f6c310b831c530f88
|
||||||
RUN apt-get update && apt-get upgrade -y && \
|
RUN apt-get update && apt-get upgrade -y && \
|
||||||
rm -r /var/lib/apt/lists /var/cache/apt/archives
|
rm -r /var/lib/apt/lists /var/cache/apt/archives
|
||||||
RUN chmod g+rwX /opt/bitnami
|
RUN chmod g+rwX /opt/bitnami
|
||||||
@@ -25,7 +25,7 @@ COPY rootfs /
|
|||||||
RUN /opt/bitnami/scripts/airflow-scheduler/postunpack.sh
|
RUN /opt/bitnami/scripts/airflow-scheduler/postunpack.sh
|
||||||
ENV AIRFLOW_HOME="/opt/bitnami/airflow" \
|
ENV AIRFLOW_HOME="/opt/bitnami/airflow" \
|
||||||
BITNAMI_APP_NAME="airflow-scheduler" \
|
BITNAMI_APP_NAME="airflow-scheduler" \
|
||||||
BITNAMI_IMAGE_VERSION="2.2.3-debian-10-r46" \
|
BITNAMI_IMAGE_VERSION="2.2.3-debian-10-r47" \
|
||||||
LD_LIBRARY_PATH="/opt/bitnami/python/lib/:/opt/bitnami/airflow/venv/lib/python3.8/site-packages/numpy.libs/:$LD_LIBRARY_PATH" \
|
LD_LIBRARY_PATH="/opt/bitnami/python/lib/:/opt/bitnami/airflow/venv/lib/python3.8/site-packages/numpy.libs/:$LD_LIBRARY_PATH" \
|
||||||
LIBNSS_WRAPPER_PATH="/opt/bitnami/common/lib/libnss_wrapper.so" \
|
LIBNSS_WRAPPER_PATH="/opt/bitnami/common/lib/libnss_wrapper.so" \
|
||||||
LNAME="airflow" \
|
LNAME="airflow" \
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
"digest": "ae1dd33f99a6dada2253c82199727455177b9daf4fb4435f6c310b831c530f88",
|
"digest": "ae1dd33f99a6dada2253c82199727455177b9daf4fb4435f6c310b831c530f88",
|
||||||
"distro": "debian-10",
|
"distro": "debian-10",
|
||||||
"type": "NAMI",
|
"type": "NAMI",
|
||||||
"version": "2.2.3-4"
|
"version": "2.2.3-5"
|
||||||
},
|
},
|
||||||
"git": {
|
"git": {
|
||||||
"arch": "amd64",
|
"arch": "amd64",
|
||||||
@@ -29,17 +29,17 @@
|
|||||||
},
|
},
|
||||||
"postgresql-client": {
|
"postgresql-client": {
|
||||||
"arch": "amd64",
|
"arch": "amd64",
|
||||||
"digest": "53e64e567d4563efaea05f4e9e9eead3a3fbec1213a05528a5eeec2d2f8e8a43",
|
"digest": "58714b132871182d2ab826fa678330c978260cbe42277f65152d400b29885c2f",
|
||||||
"distro": "debian-10",
|
"distro": "debian-10",
|
||||||
"type": "NAMI",
|
"type": "NAMI",
|
||||||
"version": "10.19.0-1"
|
"version": "10.19.0-30"
|
||||||
},
|
},
|
||||||
"python": {
|
"python": {
|
||||||
"arch": "amd64",
|
"arch": "amd64",
|
||||||
"digest": "c081f7f9663d4d64e7f26d6e6a77b3860ee1f39c309bd564f801c39192042d2f",
|
"digest": "5aa3fd465e5e0d6fe86e984121ffb2c02a34b44a8c9cd9cce9b18d3181534b68",
|
||||||
"distro": "debian-10",
|
"distro": "debian-10",
|
||||||
"type": "NAMI",
|
"type": "NAMI",
|
||||||
"version": "3.8.12-12"
|
"version": "3.8.12-30"
|
||||||
},
|
},
|
||||||
"wait-for-port": {
|
"wait-for-port": {
|
||||||
"arch": "amd64",
|
"arch": "amd64",
|
||||||
|
|||||||
@@ -57,15 +57,12 @@ unset airflow_scheduler_env_vars
|
|||||||
export AIRFLOW_BASE_DIR="${BITNAMI_ROOT_DIR}/airflow"
|
export AIRFLOW_BASE_DIR="${BITNAMI_ROOT_DIR}/airflow"
|
||||||
export AIRFLOW_HOME="${AIRFLOW_BASE_DIR}"
|
export AIRFLOW_HOME="${AIRFLOW_BASE_DIR}"
|
||||||
export AIRFLOW_BIN_DIR="${AIRFLOW_BASE_DIR}/venv/bin"
|
export AIRFLOW_BIN_DIR="${AIRFLOW_BASE_DIR}/venv/bin"
|
||||||
export AIRFLOW_VOLUME_DIR="/bitnami/airflow"
|
|
||||||
export AIRFLOW_DATA_DIR="${AIRFLOW_BASE_DIR}/data"
|
|
||||||
export AIRFLOW_LOGS_DIR="${AIRFLOW_BASE_DIR}/logs"
|
export AIRFLOW_LOGS_DIR="${AIRFLOW_BASE_DIR}/logs"
|
||||||
export AIRFLOW_SCHEDULER_LOGS_DIR="${AIRFLOW_LOGS_DIR}/scheduler"
|
export AIRFLOW_SCHEDULER_LOGS_DIR="${AIRFLOW_LOGS_DIR}/scheduler"
|
||||||
export AIRFLOW_LOG_FILE="${AIRFLOW_LOGS_DIR}/airflow-scheduler.log"
|
export AIRFLOW_LOG_FILE="${AIRFLOW_LOGS_DIR}/airflow-scheduler.log"
|
||||||
export AIRFLOW_CONF_FILE="${AIRFLOW_BASE_DIR}/airflow.cfg"
|
export AIRFLOW_CONF_FILE="${AIRFLOW_BASE_DIR}/airflow.cfg"
|
||||||
export AIRFLOW_TMP_DIR="${AIRFLOW_BASE_DIR}/tmp"
|
export AIRFLOW_TMP_DIR="${AIRFLOW_BASE_DIR}/tmp"
|
||||||
export AIRFLOW_PID_FILE="${AIRFLOW_TMP_DIR}/airflow-scheduler.pid"
|
export AIRFLOW_PID_FILE="${AIRFLOW_TMP_DIR}/airflow-scheduler.pid"
|
||||||
export AIRFLOW_DATA_TO_PERSIST="$AIRFLOW_DATA_DIR"
|
|
||||||
export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags"
|
export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags"
|
||||||
|
|
||||||
# System users (when running with a privileged user)
|
# System users (when running with a privileged user)
|
||||||
|
|||||||
@@ -15,14 +15,11 @@ set -o pipefail
|
|||||||
. /opt/bitnami/scripts/libfs.sh
|
. /opt/bitnami/scripts/libfs.sh
|
||||||
. /opt/bitnami/scripts/libos.sh
|
. /opt/bitnami/scripts/libos.sh
|
||||||
|
|
||||||
for dir in "$AIRFLOW_VOLUME_DIR" "$AIRFLOW_BASE_DIR" "$AIRFLOW_DATA_DIR"; do
|
ensure_dir_exists "$AIRFLOW_BASE_DIR"
|
||||||
ensure_dir_exists "$dir"
|
|
||||||
done
|
|
||||||
|
|
||||||
# Ensure the needed directories exist with write permissions
|
# Ensure the needed directories exist with write permissions
|
||||||
for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do
|
for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do
|
||||||
ensure_dir_exists "$dir"
|
ensure_dir_exists "$dir"
|
||||||
configure_permissions_ownership "$dir" -d "775" -f "664" -g "root"
|
configure_permissions_ownership "$dir" -d "775" -f "664" -g "root"
|
||||||
done
|
done
|
||||||
|
|
||||||
chmod -R g+rwX /bitnami "$AIRFLOW_VOLUME_DIR" "$AIRFLOW_BASE_DIR"
|
chmod -R g+rwX "$AIRFLOW_BASE_DIR"
|
||||||
|
|||||||
@@ -95,7 +95,7 @@ airflow_initialize() {
|
|||||||
info "Initializing Airflow ..."
|
info "Initializing Airflow ..."
|
||||||
|
|
||||||
# Change permissions if running as root
|
# Change permissions if running as root
|
||||||
for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do
|
for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do
|
||||||
ensure_dir_exists "$dir"
|
ensure_dir_exists "$dir"
|
||||||
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
|
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
|
||||||
done
|
done
|
||||||
@@ -108,73 +108,27 @@ airflow_initialize() {
|
|||||||
info "Configuration file found, loading configuration"
|
info "Configuration file found, loading configuration"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Check if Airflow has already been initialized and persisted in a previous run
|
info "Trying to connect to the database server"
|
||||||
local -r app_name="airflow"
|
airflow_wait_for_postgresql_connection
|
||||||
local -a postgresql_remote_execute_args=("$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER" "$AIRFLOW_DATABASE_NAME" "$AIRFLOW_DATABASE_USERNAME" "$AIRFLOW_DATABASE_PASSWORD")
|
# Check if the Airflow database has been already initialized
|
||||||
if ! is_app_initialized "$app_name"; then
|
if ! debug_execute airflow db check-migrations; then
|
||||||
# Delete pid file
|
# Delete pid file
|
||||||
rm -f "$AIRFLOW_PID_FILE"
|
rm -f "$AIRFLOW_PID_FILE"
|
||||||
|
|
||||||
# Initialize database
|
# Initialize database
|
||||||
info "Trying to connect to the database server"
|
|
||||||
airflow_wait_for_postgresql_connection "${postgresql_remote_execute_args[@]}"
|
|
||||||
info "Populating database"
|
info "Populating database"
|
||||||
airflow_execute_command "initdb" "db init"
|
debug_execute airflow db init
|
||||||
|
|
||||||
airflow_create_admin_user
|
airflow_create_admin_user
|
||||||
|
|
||||||
airflow_create_pool
|
airflow_create_pool
|
||||||
|
|
||||||
info "Persisting Airflow installation"
|
|
||||||
persist_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
|
|
||||||
else
|
else
|
||||||
# Restore persisted data
|
|
||||||
info "Restoring persisted Airflow installation"
|
|
||||||
restore_persisted_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
|
|
||||||
|
|
||||||
info "Trying to connect to the database server"
|
|
||||||
airflow_wait_for_postgresql_connection "${postgresql_remote_execute_args[@]}"
|
|
||||||
|
|
||||||
# Upgrade database
|
# Upgrade database
|
||||||
airflow_execute_command "upgradedb" "db upgrade"
|
info "Upgrading database schema"
|
||||||
|
debug_execute airflow db upgrade
|
||||||
# Change the permissions after restoring the persisted data in case we are root
|
|
||||||
for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR"; do
|
|
||||||
ensure_dir_exists "$dir"
|
|
||||||
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
|
|
||||||
done
|
|
||||||
true # Avoid return false when I am not root
|
true # Avoid return false when I am not root
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
########################
|
|
||||||
# Executes airflow command
|
|
||||||
# Globals:
|
|
||||||
# AIRFLOW_*
|
|
||||||
# Arguments:
|
|
||||||
# None
|
|
||||||
# Returns:
|
|
||||||
# None
|
|
||||||
#########################
|
|
||||||
airflow_execute_command() {
|
|
||||||
local oldCommand="${1?Missing old command}"
|
|
||||||
local newCommand="${2?Missing new command}"
|
|
||||||
local flags="${3:-}"
|
|
||||||
|
|
||||||
# The commands can contain more than one argument. Convert them to an array
|
|
||||||
IFS=' ' read -ra oldCommand <<<"$oldCommand"
|
|
||||||
IFS=' ' read -ra newCommand <<<"$newCommand"
|
|
||||||
|
|
||||||
# Execute commands depending on the version
|
|
||||||
command=("${oldCommand[@]}")
|
|
||||||
[[ "${BITNAMI_IMAGE_VERSION:0:1}" == "2" ]] && command=("${newCommand[@]}")
|
|
||||||
|
|
||||||
# Add flags if provided
|
|
||||||
[[ -n "$flags" ]] && IFS=' ' read -ra flags <<<"$flags" && command+=("${flags[@]}")
|
|
||||||
|
|
||||||
debug "Executing ${AIRFLOW_BIN_DIR}/airflow ${command[*]}"
|
|
||||||
debug_execute "${AIRFLOW_BIN_DIR}/airflow" "${command[@]}"
|
|
||||||
}
|
|
||||||
|
|
||||||
########################
|
########################
|
||||||
# Generate Airflow conf file
|
# Generate Airflow conf file
|
||||||
@@ -187,7 +141,7 @@ airflow_execute_command() {
|
|||||||
#########################
|
#########################
|
||||||
airflow_generate_config() {
|
airflow_generate_config() {
|
||||||
# Generate Airflow default files
|
# Generate Airflow default files
|
||||||
airflow_execute_command "version" "version"
|
debug_execute airflow version
|
||||||
|
|
||||||
# Setup Airflow base URL
|
# Setup Airflow base URL
|
||||||
airflow_configure_base_url
|
airflow_configure_base_url
|
||||||
@@ -414,28 +368,16 @@ airflow_configure_celery_executor() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
########################
|
########################
|
||||||
# Wait until the database is accessible with the currently-known credentials
|
# Wait until the database is accessible
|
||||||
# Globals:
|
# Globals:
|
||||||
# *
|
# None
|
||||||
# Arguments:
|
# Arguments:
|
||||||
# $1 - database host
|
# None
|
||||||
# $2 - database port
|
|
||||||
# $3 - database name
|
|
||||||
# $4 - database username
|
|
||||||
# $5 - database user password (optional)
|
|
||||||
# Returns:
|
# Returns:
|
||||||
# true if the database connection succeeded, false otherwise
|
# true if the database connection succeeded, false otherwise
|
||||||
#########################
|
#########################
|
||||||
airflow_wait_for_postgresql_connection() {
|
airflow_wait_for_postgresql_connection() {
|
||||||
local -r db_host="${1:?missing database host}"
|
if ! retry_while "debug_execute airflow db check"; then
|
||||||
local -r db_port="${2:?missing database port}"
|
|
||||||
local -r db_name="${3:?missing database name}"
|
|
||||||
local -r db_user="${4:?missing database user}"
|
|
||||||
local -r db_pass="${5:-}"
|
|
||||||
check_postgresql_connection() {
|
|
||||||
echo "SELECT 1" | postgresql_remote_execute "$db_host" "$db_port" "$db_name" "$db_user" "$db_pass"
|
|
||||||
}
|
|
||||||
if ! retry_while "check_postgresql_connection"; then
|
|
||||||
error "Could not connect to the database"
|
error "Could not connect to the database"
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
@@ -450,7 +392,7 @@ airflow_wait_for_postgresql_connection() {
|
|||||||
#########################
|
#########################
|
||||||
airflow_create_admin_user() {
|
airflow_create_admin_user() {
|
||||||
info "Creating Airflow admin user"
|
info "Creating Airflow admin user"
|
||||||
airflow_execute_command "create_user" "users create" "-r Admin -u ${AIRFLOW_USERNAME} -e ${AIRFLOW_EMAIL} -p ${AIRFLOW_PASSWORD} -f ${AIRFLOW_FIRSTNAME} -l ${AIRFLOW_LASTNAME}"
|
debug_execute airflow users create -r "Admin" -u "$AIRFLOW_USERNAME" -e "$AIRFLOW_EMAIL" -p "$AIRFLOW_PASSWORD" -f "$AIRFLOW_FIRSTNAME" -l "$AIRFLOW_LASTNAME"
|
||||||
}
|
}
|
||||||
|
|
||||||
########################
|
########################
|
||||||
@@ -463,7 +405,7 @@ airflow_create_admin_user() {
|
|||||||
airflow_create_pool() {
|
airflow_create_pool() {
|
||||||
if [[ -n "$AIRFLOW_POOL_NAME" ]] && [[ -n "$AIRFLOW_POOL_SIZE" ]] && [[ -n "$AIRFLOW_POOL_DESC" ]]; then
|
if [[ -n "$AIRFLOW_POOL_NAME" ]] && [[ -n "$AIRFLOW_POOL_SIZE" ]] && [[ -n "$AIRFLOW_POOL_DESC" ]]; then
|
||||||
info "Creating Airflow pool"
|
info "Creating Airflow pool"
|
||||||
airflow_execute_command "pool" "pool" "-s ${AIRFLOW_POOL_NAME} ${AIRFLOW_POOL_SIZE} ${AIRFLOW_POOL_DESC}"
|
debug_execute airflow pool -s "$AIRFLOW_POOL_NAME" "$AIRFLOW_POOL_SIZE" "$AIRFLOW_POOL_DESC"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ airflow_scheduler_validate() {
|
|||||||
#########################
|
#########################
|
||||||
airflow_scheduler_initialize() {
|
airflow_scheduler_initialize() {
|
||||||
# Change permissions if running as root
|
# Change permissions if running as root
|
||||||
for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_SCHEDULER_LOGS_DIR" "$AIRFLOW_DATA_DIR"; do
|
for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_SCHEDULER_LOGS_DIR"; do
|
||||||
ensure_dir_exists "$dir"
|
ensure_dir_exists "$dir"
|
||||||
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
|
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
|
||||||
done
|
done
|
||||||
@@ -60,29 +60,13 @@ airflow_scheduler_initialize() {
|
|||||||
info "Configuration file found, loading configuration"
|
info "Configuration file found, loading configuration"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Check if Airflow has already been initialized and persisted in a previous run
|
info "Trying to connect to the database server"
|
||||||
local -r app_name="airflow"
|
airflow_wait_for_postgresql_connection
|
||||||
local -a postgresql_remote_execute_args=("$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER" "$AIRFLOW_DATABASE_NAME" "$AIRFLOW_DATABASE_USERNAME" "$AIRFLOW_DATABASE_PASSWORD")
|
# Change the permissions after restoring the persisted data in case we are root
|
||||||
if ! is_app_initialized "$app_name"; then
|
for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR"; do
|
||||||
airflow_wait_for_postgresql_connection "${postgresql_remote_execute_args[@]}"
|
ensure_dir_exists "$dir"
|
||||||
|
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
|
||||||
info "Persisting Airflow installation"
|
done
|
||||||
persist_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
|
|
||||||
else
|
|
||||||
# Check database connection
|
|
||||||
airflow_wait_for_postgresql_connection "${postgresql_remote_execute_args[@]}"
|
|
||||||
|
|
||||||
# Restore persisted data
|
|
||||||
info "Restoring persisted Airflow installation"
|
|
||||||
restore_persisted_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
|
|
||||||
|
|
||||||
# Change the permissions after restoring the persisted data in case we are root
|
|
||||||
for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR"; do
|
|
||||||
ensure_dir_exists "$dir"
|
|
||||||
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
|
|
||||||
done
|
|
||||||
true # Avoid return false when I am not root
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Wait for airflow webserver to be available
|
# Wait for airflow webserver to be available
|
||||||
airflow_scheduler_wait_for_webserver "$AIRFLOW_WEBSERVER_HOST" "$AIRFLOW_WEBSERVER_PORT_NUMBER"
|
airflow_scheduler_wait_for_webserver "$AIRFLOW_WEBSERVER_HOST" "$AIRFLOW_WEBSERVER_PORT_NUMBER"
|
||||||
@@ -103,7 +87,7 @@ airflow_scheduler_initialize() {
|
|||||||
#########################
|
#########################
|
||||||
airflow_scheduler_generate_config() {
|
airflow_scheduler_generate_config() {
|
||||||
# Generate Airflow default files
|
# Generate Airflow default files
|
||||||
airflow_execute_command "version" "version"
|
debug_execute airflow version
|
||||||
|
|
||||||
# Configure Airflow Hostname
|
# Configure Airflow Hostname
|
||||||
[[ -n "$AIRFLOW_HOSTNAME_CALLABLE" ]] && airflow_conf_set "core" "hostname_callable" "$AIRFLOW_HOSTNAME_CALLABLE"
|
[[ -n "$AIRFLOW_HOSTNAME_CALLABLE" ]] && airflow_conf_set "core" "hostname_callable" "$AIRFLOW_HOSTNAME_CALLABLE"
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ You can find the default credentials and available configuration options in the
|
|||||||
Learn more about the Bitnami tagging policy and the difference between rolling tags and immutable tags [in our documentation page](https://docs.bitnami.com/tutorials/understand-rolling-tags-containers/).
|
Learn more about the Bitnami tagging policy and the difference between rolling tags and immutable tags [in our documentation page](https://docs.bitnami.com/tutorials/understand-rolling-tags-containers/).
|
||||||
|
|
||||||
|
|
||||||
* [`2`, `2-debian-10`, `2.2.3`, `2.2.3-debian-10-r46`, `latest` (2/debian-10/Dockerfile)](https://github.com/bitnami/bitnami-docker-airflow-scheduler/blob/2.2.3-debian-10-r46/2/debian-10/Dockerfile)
|
* [`2`, `2-debian-10`, `2.2.3`, `2.2.3-debian-10-r47`, `latest` (2/debian-10/Dockerfile)](https://github.com/bitnami/bitnami-docker-airflow-scheduler/blob/2.2.3-debian-10-r47/2/debian-10/Dockerfile)
|
||||||
|
|
||||||
Subscribe to project updates by watching the [bitnami/airflow GitHub repo](https://github.com/bitnami/bitnami-docker-airflow-scheduler).
|
Subscribe to project updates by watching the [bitnami/airflow GitHub repo](https://github.com/bitnami/bitnami-docker-airflow-scheduler).
|
||||||
|
|
||||||
@@ -93,10 +93,9 @@ If you want to run the application manually instead of using `docker-compose`, t
|
|||||||
bitnami/redis:latest
|
bitnami/redis:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
4. Create volumes for Airflow persistence and launch the container
|
4. Launch the Apache Airflow Scheduler web container
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ docker volume create --name airflow_data
|
|
||||||
$ docker run -d --name airflow -p 8080:8080 \
|
$ docker run -d --name airflow -p 8080:8080 \
|
||||||
-e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \
|
-e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \
|
||||||
-e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \
|
-e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \
|
||||||
@@ -109,14 +108,12 @@ If you want to run the application manually instead of using `docker-compose`, t
|
|||||||
-e AIRFLOW_USERNAME=user \
|
-e AIRFLOW_USERNAME=user \
|
||||||
-e AIRFLOW_EMAIL=user@example.com \
|
-e AIRFLOW_EMAIL=user@example.com \
|
||||||
--net airflow-tier \
|
--net airflow-tier \
|
||||||
--volume airflow_data:/bitnami \
|
|
||||||
bitnami/airflow:latest
|
bitnami/airflow:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
5. Create volumes for Apache Airflow Scheduler persistence and launch the container
|
5. Launch the Apache Airflow Scheduler scheduler container
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ docker volume create --name airflow_scheduler_data
|
|
||||||
$ docker run -d --name airflow-scheduler \
|
$ docker run -d --name airflow-scheduler \
|
||||||
-e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \
|
-e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \
|
||||||
-e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \
|
-e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \
|
||||||
@@ -126,14 +123,12 @@ If you want to run the application manually instead of using `docker-compose`, t
|
|||||||
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
|
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
|
||||||
-e AIRFLOW_LOAD_EXAMPLES=yes \
|
-e AIRFLOW_LOAD_EXAMPLES=yes \
|
||||||
--net airflow-tier \
|
--net airflow-tier \
|
||||||
--volume airflow_scheduler_data:/bitnami \
|
|
||||||
bitnami/airflow-scheduler:latest
|
bitnami/airflow-scheduler:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
6. Create volumes for Airflow Worker persistence and launch the container
|
6. Launch the Apache Airflow Scheduler worker container
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ docker volume create --name airflow_worker_data
|
|
||||||
$ docker run -d --name airflow-worker \
|
$ docker run -d --name airflow-worker \
|
||||||
-e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \
|
-e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \
|
||||||
-e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \
|
-e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \
|
||||||
@@ -142,7 +137,6 @@ If you want to run the application manually instead of using `docker-compose`, t
|
|||||||
-e AIRFLOW_DATABASE_USERNAME=bn_airflow \
|
-e AIRFLOW_DATABASE_USERNAME=bn_airflow \
|
||||||
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
|
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
|
||||||
--net airflow-tier \
|
--net airflow-tier \
|
||||||
--volume airflow_worker_data:/bitnami \
|
|
||||||
bitnami/airflow-worker:latest
|
bitnami/airflow-worker:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -150,11 +144,9 @@ Access your application at `http://your-ip:8080`
|
|||||||
|
|
||||||
### Persisting your application
|
### Persisting your application
|
||||||
|
|
||||||
If you remove the container all your data and configurations will be lost, and the next time you run the image the database will be reinitialized. To avoid this loss of data, you should mount a volume that will persist even after the container is removed.
|
The Bitnami Airflow container relies on the PostgreSQL database & Redis to persist the data. This means that Airflow does not persist anything. To avoid loss of data, you should mount volumes for persistence of [PostgreSQL data](https://github.com/bitnami/bitnami-docker-mariadb#persisting-your-database) and [Redis(TM) data](https://github.com/bitnami/bitnami-docker-redis#persisting-your-database)
|
||||||
|
|
||||||
For persistence you should mount a volume at the `/bitnami` path. Additionally you should mount volumes for persistence of [PostgreSQL data](https://github.com/bitnami/bitnami-docker-mariadb#persisting-your-database) and [Redis(TM) data](https://github.com/bitnami/bitnami-docker-mariadb#persisting-your-database)
|
The above examples define docker volumes namely `postgresql_data`, and `redis_data`. The Airflow application state will persist as long as these volumes are not removed.
|
||||||
|
|
||||||
The above examples define docker volumes namely `postgresql_data`, `redis_data`, `airflow_data`, `airflow_scheduler_data` and `airflow_worker_data`. The Apache Airflow Scheduler application state will persist as long as these volumes are not removed.
|
|
||||||
|
|
||||||
To avoid inadvertent removal of these volumes you can [mount host directories as data volumes](https://docs.docker.com/engine/tutorials/dockervolumes/). Alternatively you can make use of volume plugins to host the volume data.
|
To avoid inadvertent removal of these volumes you can [mount host directories as data volumes](https://docs.docker.com/engine/tutorials/dockervolumes/). Alternatively you can make use of volume plugins to host the volume data.
|
||||||
|
|
||||||
@@ -172,13 +164,13 @@ services:
|
|||||||
- POSTGRESQL_USERNAME=bn_airflow
|
- POSTGRESQL_USERNAME=bn_airflow
|
||||||
- POSTGRESQL_PASSWORD=bitnami1
|
- POSTGRESQL_PASSWORD=bitnami1
|
||||||
volumes:
|
volumes:
|
||||||
- /path/to/airflow-persistence:/bitnami
|
- /path/to/postgresql-persistence:/bitnami
|
||||||
redis:
|
redis:
|
||||||
image: 'bitnami/redis:latest'
|
image: 'bitnami/redis:latest'
|
||||||
environment:
|
environment:
|
||||||
- ALLOW_EMPTY_PASSWORD=yes
|
- ALLOW_EMPTY_PASSWORD=yes
|
||||||
volumes:
|
volumes:
|
||||||
- /path/to/airflow-persistence:/bitnami
|
- /path/to/redis-persistence:/bitnami
|
||||||
airflow-worker:
|
airflow-worker:
|
||||||
image: bitnami/airflow-worker:latest
|
image: bitnami/airflow-worker:latest
|
||||||
environment:
|
environment:
|
||||||
@@ -189,8 +181,6 @@ services:
|
|||||||
- AIRFLOW_DATABASE_USERNAME=bn_airflow
|
- AIRFLOW_DATABASE_USERNAME=bn_airflow
|
||||||
- AIRFLOW_DATABASE_PASSWORD=bitnami1
|
- AIRFLOW_DATABASE_PASSWORD=bitnami1
|
||||||
- AIRFLOW_LOAD_EXAMPLES=yes
|
- AIRFLOW_LOAD_EXAMPLES=yes
|
||||||
volumes:
|
|
||||||
- /path/to/airflow-persistence:/bitnami
|
|
||||||
airflow-scheduler:
|
airflow-scheduler:
|
||||||
image: bitnami/airflow-scheduler:latest
|
image: bitnami/airflow-scheduler:latest
|
||||||
environment:
|
environment:
|
||||||
@@ -201,8 +191,6 @@ services:
|
|||||||
- AIRFLOW_DATABASE_USERNAME=bn_airflow
|
- AIRFLOW_DATABASE_USERNAME=bn_airflow
|
||||||
- AIRFLOW_DATABASE_PASSWORD=bitnami1
|
- AIRFLOW_DATABASE_PASSWORD=bitnami1
|
||||||
- AIRFLOW_LOAD_EXAMPLES=yes
|
- AIRFLOW_LOAD_EXAMPLES=yes
|
||||||
volumes:
|
|
||||||
- /path/to/airflow-persistence:/bitnami
|
|
||||||
airflow:
|
airflow:
|
||||||
image: bitnami/airflow:latest
|
image: bitnami/airflow:latest
|
||||||
environment:
|
environment:
|
||||||
@@ -217,8 +205,6 @@ services:
|
|||||||
- AIRFLOW_EMAIL=user@example.com
|
- AIRFLOW_EMAIL=user@example.com
|
||||||
ports:
|
ports:
|
||||||
- '8080:8080'
|
- '8080:8080'
|
||||||
volumes:
|
|
||||||
- /path/to/airflow-persistence:/bitnami
|
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Mount host directories as data volumes using the Docker command line
|
#### Mount host directories as data volumes using the Docker command line
|
||||||
@@ -251,7 +237,7 @@ services:
|
|||||||
bitnami/redis:latest
|
bitnami/redis:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
4. Create the Airflow container with host volumes
|
4. Create the Airflow container
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ docker run -d --name airflow -p 8080:8080 \
|
$ docker run -d --name airflow -p 8080:8080 \
|
||||||
@@ -266,11 +252,10 @@ services:
|
|||||||
-e AIRFLOW_USERNAME=user \
|
-e AIRFLOW_USERNAME=user \
|
||||||
-e AIRFLOW_EMAIL=user@example.com \
|
-e AIRFLOW_EMAIL=user@example.com \
|
||||||
--net airflow-tier \
|
--net airflow-tier \
|
||||||
--volume /path/to/airflow-persistence:/bitnami \
|
|
||||||
bitnami/airflow:latest
|
bitnami/airflow:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
5. Create the Apache Airflow Scheduler container with host volumes
|
5. Create the Apache Airflow Scheduler container
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ docker run -d --name airflow-scheduler \
|
$ docker run -d --name airflow-scheduler \
|
||||||
@@ -282,11 +267,10 @@ services:
|
|||||||
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
|
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
|
||||||
-e AIRFLOW_LOAD_EXAMPLES=yes \
|
-e AIRFLOW_LOAD_EXAMPLES=yes \
|
||||||
--net airflow-tier \
|
--net airflow-tier \
|
||||||
--volume /path/to/airflow-scheduler-persistence:/bitnami \
|
|
||||||
bitnami/airflow-scheduler:latest
|
bitnami/airflow-scheduler:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
6. Create the Airflow Worker container with host volumes
|
6. Create the Airflow Worker container
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ docker run -d --name airflow-worker \
|
$ docker run -d --name airflow-worker \
|
||||||
@@ -297,7 +281,6 @@ services:
|
|||||||
-e AIRFLOW_DATABASE_USERNAME=bn_airflow \
|
-e AIRFLOW_DATABASE_USERNAME=bn_airflow \
|
||||||
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
|
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
|
||||||
--net airflow-tier \
|
--net airflow-tier \
|
||||||
--volume /path/to/airflow-worker-persistence:/bitnami \
|
|
||||||
bitnami/airflow-worker:latest
|
bitnami/airflow-worker:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user