2.2.3-debian-10-r47 release

This commit is contained in:
Bitnami Bot
2022-02-09 12:47:57 +00:00
parent cbe4ed982d
commit cce893ee82
7 changed files with 46 additions and 143 deletions

View File

@@ -11,12 +11,12 @@ COPY prebuildfs /
# Install required system packages and dependencies # Install required system packages and dependencies
RUN install_packages acl ca-certificates curl gzip libbsd0 libbz2-1.0 libc6 libcom-err2 libcurl4 libedit2 libffi6 libgcc1 libgcrypt20 libgmp10 libgnutls30 libgpg-error0 libgssapi-krb5-2 libhogweed4 libicu63 libidn2-0 libk5crypto3 libkeyutils1 libkrb5-3 libkrb5support0 libldap-2.4-2 liblzma5 libmariadb3 libncursesw6 libnettle6 libnghttp2-14 libp11-kit0 libpsl5 libreadline7 librtmp1 libsasl2-2 libsqlite3-0 libssh2-1 libssl1.1 libstdc++6 libtasn1-6 libtinfo6 libunistring2 libuuid1 libxml2 libxslt1.1 netbase procps tar zlib1g RUN install_packages acl ca-certificates curl gzip libbsd0 libbz2-1.0 libc6 libcom-err2 libcurl4 libedit2 libffi6 libgcc1 libgcrypt20 libgmp10 libgnutls30 libgpg-error0 libgssapi-krb5-2 libhogweed4 libicu63 libidn2-0 libk5crypto3 libkeyutils1 libkrb5-3 libkrb5support0 libldap-2.4-2 liblzma5 libmariadb3 libncursesw6 libnettle6 libnghttp2-14 libp11-kit0 libpsl5 libreadline7 librtmp1 libsasl2-2 libsqlite3-0 libssh2-1 libssl1.1 libstdc++6 libtasn1-6 libtinfo6 libunistring2 libuuid1 libxml2 libxslt1.1 netbase procps tar zlib1g
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "wait-for-port" "1.0.1-6" --checksum 845ff207415b5730cadd45aa3c757ca45a5c9e73bc54c24aa481ddca24eec35d RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "wait-for-port" "1.0.1-6" --checksum 845ff207415b5730cadd45aa3c757ca45a5c9e73bc54c24aa481ddca24eec35d
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "python" "3.8.12-12" --checksum c081f7f9663d4d64e7f26d6e6a77b3860ee1f39c309bd564f801c39192042d2f RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "python" "3.8.12-30" --checksum 5aa3fd465e5e0d6fe86e984121ffb2c02a34b44a8c9cd9cce9b18d3181534b68
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "postgresql-client" "10.19.0-1" --checksum 53e64e567d4563efaea05f4e9e9eead3a3fbec1213a05528a5eeec2d2f8e8a43 RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "postgresql-client" "10.19.0-30" --checksum 58714b132871182d2ab826fa678330c978260cbe42277f65152d400b29885c2f
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "ini-file" "1.4.1-3" --checksum 26f2ba99a02266d6f3857a3a11db7ed441d76a2fbc1066a17d4b0b17b047e01d RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "ini-file" "1.4.1-3" --checksum 26f2ba99a02266d6f3857a3a11db7ed441d76a2fbc1066a17d4b0b17b047e01d
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "git" "2.35.1-0" --checksum 2566d89a20e9794698ad0868e8ea0b73cf27343d552c00c5278379c47aa62d29 RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "git" "2.35.1-0" --checksum 2566d89a20e9794698ad0868e8ea0b73cf27343d552c00c5278379c47aa62d29
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "gosu" "1.14.0-3" --checksum 276ab5a0be4b05e136ec468d62c8f9cc4f40d9664c55f01f16a9f1209ba16980 RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "gosu" "1.14.0-3" --checksum 276ab5a0be4b05e136ec468d62c8f9cc4f40d9664c55f01f16a9f1209ba16980
RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "airflow-scheduler" "2.2.3-4" --checksum ae1dd33f99a6dada2253c82199727455177b9daf4fb4435f6c310b831c530f88 RUN . /opt/bitnami/scripts/libcomponent.sh && component_unpack "airflow-scheduler" "2.2.3-5" --checksum ae1dd33f99a6dada2253c82199727455177b9daf4fb4435f6c310b831c530f88
RUN apt-get update && apt-get upgrade -y && \ RUN apt-get update && apt-get upgrade -y && \
rm -r /var/lib/apt/lists /var/cache/apt/archives rm -r /var/lib/apt/lists /var/cache/apt/archives
RUN chmod g+rwX /opt/bitnami RUN chmod g+rwX /opt/bitnami
@@ -25,7 +25,7 @@ COPY rootfs /
RUN /opt/bitnami/scripts/airflow-scheduler/postunpack.sh RUN /opt/bitnami/scripts/airflow-scheduler/postunpack.sh
ENV AIRFLOW_HOME="/opt/bitnami/airflow" \ ENV AIRFLOW_HOME="/opt/bitnami/airflow" \
BITNAMI_APP_NAME="airflow-scheduler" \ BITNAMI_APP_NAME="airflow-scheduler" \
BITNAMI_IMAGE_VERSION="2.2.3-debian-10-r46" \ BITNAMI_IMAGE_VERSION="2.2.3-debian-10-r47" \
LD_LIBRARY_PATH="/opt/bitnami/python/lib/:/opt/bitnami/airflow/venv/lib/python3.8/site-packages/numpy.libs/:$LD_LIBRARY_PATH" \ LD_LIBRARY_PATH="/opt/bitnami/python/lib/:/opt/bitnami/airflow/venv/lib/python3.8/site-packages/numpy.libs/:$LD_LIBRARY_PATH" \
LIBNSS_WRAPPER_PATH="/opt/bitnami/common/lib/libnss_wrapper.so" \ LIBNSS_WRAPPER_PATH="/opt/bitnami/common/lib/libnss_wrapper.so" \
LNAME="airflow" \ LNAME="airflow" \

View File

@@ -4,7 +4,7 @@
"digest": "ae1dd33f99a6dada2253c82199727455177b9daf4fb4435f6c310b831c530f88", "digest": "ae1dd33f99a6dada2253c82199727455177b9daf4fb4435f6c310b831c530f88",
"distro": "debian-10", "distro": "debian-10",
"type": "NAMI", "type": "NAMI",
"version": "2.2.3-4" "version": "2.2.3-5"
}, },
"git": { "git": {
"arch": "amd64", "arch": "amd64",
@@ -29,17 +29,17 @@
}, },
"postgresql-client": { "postgresql-client": {
"arch": "amd64", "arch": "amd64",
"digest": "53e64e567d4563efaea05f4e9e9eead3a3fbec1213a05528a5eeec2d2f8e8a43", "digest": "58714b132871182d2ab826fa678330c978260cbe42277f65152d400b29885c2f",
"distro": "debian-10", "distro": "debian-10",
"type": "NAMI", "type": "NAMI",
"version": "10.19.0-1" "version": "10.19.0-30"
}, },
"python": { "python": {
"arch": "amd64", "arch": "amd64",
"digest": "c081f7f9663d4d64e7f26d6e6a77b3860ee1f39c309bd564f801c39192042d2f", "digest": "5aa3fd465e5e0d6fe86e984121ffb2c02a34b44a8c9cd9cce9b18d3181534b68",
"distro": "debian-10", "distro": "debian-10",
"type": "NAMI", "type": "NAMI",
"version": "3.8.12-12" "version": "3.8.12-30"
}, },
"wait-for-port": { "wait-for-port": {
"arch": "amd64", "arch": "amd64",

View File

@@ -57,15 +57,12 @@ unset airflow_scheduler_env_vars
export AIRFLOW_BASE_DIR="${BITNAMI_ROOT_DIR}/airflow" export AIRFLOW_BASE_DIR="${BITNAMI_ROOT_DIR}/airflow"
export AIRFLOW_HOME="${AIRFLOW_BASE_DIR}" export AIRFLOW_HOME="${AIRFLOW_BASE_DIR}"
export AIRFLOW_BIN_DIR="${AIRFLOW_BASE_DIR}/venv/bin" export AIRFLOW_BIN_DIR="${AIRFLOW_BASE_DIR}/venv/bin"
export AIRFLOW_VOLUME_DIR="/bitnami/airflow"
export AIRFLOW_DATA_DIR="${AIRFLOW_BASE_DIR}/data"
export AIRFLOW_LOGS_DIR="${AIRFLOW_BASE_DIR}/logs" export AIRFLOW_LOGS_DIR="${AIRFLOW_BASE_DIR}/logs"
export AIRFLOW_SCHEDULER_LOGS_DIR="${AIRFLOW_LOGS_DIR}/scheduler" export AIRFLOW_SCHEDULER_LOGS_DIR="${AIRFLOW_LOGS_DIR}/scheduler"
export AIRFLOW_LOG_FILE="${AIRFLOW_LOGS_DIR}/airflow-scheduler.log" export AIRFLOW_LOG_FILE="${AIRFLOW_LOGS_DIR}/airflow-scheduler.log"
export AIRFLOW_CONF_FILE="${AIRFLOW_BASE_DIR}/airflow.cfg" export AIRFLOW_CONF_FILE="${AIRFLOW_BASE_DIR}/airflow.cfg"
export AIRFLOW_TMP_DIR="${AIRFLOW_BASE_DIR}/tmp" export AIRFLOW_TMP_DIR="${AIRFLOW_BASE_DIR}/tmp"
export AIRFLOW_PID_FILE="${AIRFLOW_TMP_DIR}/airflow-scheduler.pid" export AIRFLOW_PID_FILE="${AIRFLOW_TMP_DIR}/airflow-scheduler.pid"
export AIRFLOW_DATA_TO_PERSIST="$AIRFLOW_DATA_DIR"
export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags" export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags"
# System users (when running with a privileged user) # System users (when running with a privileged user)

View File

@@ -15,14 +15,11 @@ set -o pipefail
. /opt/bitnami/scripts/libfs.sh . /opt/bitnami/scripts/libfs.sh
. /opt/bitnami/scripts/libos.sh . /opt/bitnami/scripts/libos.sh
for dir in "$AIRFLOW_VOLUME_DIR" "$AIRFLOW_BASE_DIR" "$AIRFLOW_DATA_DIR"; do ensure_dir_exists "$AIRFLOW_BASE_DIR"
ensure_dir_exists "$dir"
done
# Ensure the needed directories exist with write permissions # Ensure the needed directories exist with write permissions
for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do
ensure_dir_exists "$dir" ensure_dir_exists "$dir"
configure_permissions_ownership "$dir" -d "775" -f "664" -g "root" configure_permissions_ownership "$dir" -d "775" -f "664" -g "root"
done done
chmod -R g+rwX /bitnami "$AIRFLOW_VOLUME_DIR" "$AIRFLOW_BASE_DIR" chmod -R g+rwX "$AIRFLOW_BASE_DIR"

View File

@@ -95,7 +95,7 @@ airflow_initialize() {
info "Initializing Airflow ..." info "Initializing Airflow ..."
# Change permissions if running as root # Change permissions if running as root
for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do
ensure_dir_exists "$dir" ensure_dir_exists "$dir"
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir" am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
done done
@@ -108,73 +108,27 @@ airflow_initialize() {
info "Configuration file found, loading configuration" info "Configuration file found, loading configuration"
fi fi
# Check if Airflow has already been initialized and persisted in a previous run info "Trying to connect to the database server"
local -r app_name="airflow" airflow_wait_for_postgresql_connection
local -a postgresql_remote_execute_args=("$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER" "$AIRFLOW_DATABASE_NAME" "$AIRFLOW_DATABASE_USERNAME" "$AIRFLOW_DATABASE_PASSWORD") # Check if the Airflow database has been already initialized
if ! is_app_initialized "$app_name"; then if ! debug_execute airflow db check-migrations; then
# Delete pid file # Delete pid file
rm -f "$AIRFLOW_PID_FILE" rm -f "$AIRFLOW_PID_FILE"
# Initialize database # Initialize database
info "Trying to connect to the database server"
airflow_wait_for_postgresql_connection "${postgresql_remote_execute_args[@]}"
info "Populating database" info "Populating database"
airflow_execute_command "initdb" "db init" debug_execute airflow db init
airflow_create_admin_user airflow_create_admin_user
airflow_create_pool airflow_create_pool
info "Persisting Airflow installation"
persist_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
else else
# Restore persisted data
info "Restoring persisted Airflow installation"
restore_persisted_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
info "Trying to connect to the database server"
airflow_wait_for_postgresql_connection "${postgresql_remote_execute_args[@]}"
# Upgrade database # Upgrade database
airflow_execute_command "upgradedb" "db upgrade" info "Upgrading database schema"
debug_execute airflow db upgrade
# Change the permissions after restoring the persisted data in case we are root
for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR"; do
ensure_dir_exists "$dir"
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
done
true # Avoid return false when I am not root true # Avoid return false when I am not root
fi fi
} }
########################
# Executes airflow command
# Globals:
# AIRFLOW_*
# Arguments:
# None
# Returns:
# None
#########################
airflow_execute_command() {
local oldCommand="${1?Missing old command}"
local newCommand="${2?Missing new command}"
local flags="${3:-}"
# The commands can contain more than one argument. Convert them to an array
IFS=' ' read -ra oldCommand <<<"$oldCommand"
IFS=' ' read -ra newCommand <<<"$newCommand"
# Execute commands depending on the version
command=("${oldCommand[@]}")
[[ "${BITNAMI_IMAGE_VERSION:0:1}" == "2" ]] && command=("${newCommand[@]}")
# Add flags if provided
[[ -n "$flags" ]] && IFS=' ' read -ra flags <<<"$flags" && command+=("${flags[@]}")
debug "Executing ${AIRFLOW_BIN_DIR}/airflow ${command[*]}"
debug_execute "${AIRFLOW_BIN_DIR}/airflow" "${command[@]}"
}
######################## ########################
# Generate Airflow conf file # Generate Airflow conf file
@@ -187,7 +141,7 @@ airflow_execute_command() {
######################### #########################
airflow_generate_config() { airflow_generate_config() {
# Generate Airflow default files # Generate Airflow default files
airflow_execute_command "version" "version" debug_execute airflow version
# Setup Airflow base URL # Setup Airflow base URL
airflow_configure_base_url airflow_configure_base_url
@@ -414,28 +368,16 @@ airflow_configure_celery_executor() {
} }
######################## ########################
# Wait until the database is accessible with the currently-known credentials # Wait until the database is accessible
# Globals: # Globals:
# * # None
# Arguments: # Arguments:
# $1 - database host # None
# $2 - database port
# $3 - database name
# $4 - database username
# $5 - database user password (optional)
# Returns: # Returns:
# true if the database connection succeeded, false otherwise # true if the database connection succeeded, false otherwise
######################### #########################
airflow_wait_for_postgresql_connection() { airflow_wait_for_postgresql_connection() {
local -r db_host="${1:?missing database host}" if ! retry_while "debug_execute airflow db check"; then
local -r db_port="${2:?missing database port}"
local -r db_name="${3:?missing database name}"
local -r db_user="${4:?missing database user}"
local -r db_pass="${5:-}"
check_postgresql_connection() {
echo "SELECT 1" | postgresql_remote_execute "$db_host" "$db_port" "$db_name" "$db_user" "$db_pass"
}
if ! retry_while "check_postgresql_connection"; then
error "Could not connect to the database" error "Could not connect to the database"
return 1 return 1
fi fi
@@ -450,7 +392,7 @@ airflow_wait_for_postgresql_connection() {
######################### #########################
airflow_create_admin_user() { airflow_create_admin_user() {
info "Creating Airflow admin user" info "Creating Airflow admin user"
airflow_execute_command "create_user" "users create" "-r Admin -u ${AIRFLOW_USERNAME} -e ${AIRFLOW_EMAIL} -p ${AIRFLOW_PASSWORD} -f ${AIRFLOW_FIRSTNAME} -l ${AIRFLOW_LASTNAME}" debug_execute airflow users create -r "Admin" -u "$AIRFLOW_USERNAME" -e "$AIRFLOW_EMAIL" -p "$AIRFLOW_PASSWORD" -f "$AIRFLOW_FIRSTNAME" -l "$AIRFLOW_LASTNAME"
} }
######################## ########################
@@ -463,7 +405,7 @@ airflow_create_admin_user() {
airflow_create_pool() { airflow_create_pool() {
if [[ -n "$AIRFLOW_POOL_NAME" ]] && [[ -n "$AIRFLOW_POOL_SIZE" ]] && [[ -n "$AIRFLOW_POOL_DESC" ]]; then if [[ -n "$AIRFLOW_POOL_NAME" ]] && [[ -n "$AIRFLOW_POOL_SIZE" ]] && [[ -n "$AIRFLOW_POOL_DESC" ]]; then
info "Creating Airflow pool" info "Creating Airflow pool"
airflow_execute_command "pool" "pool" "-s ${AIRFLOW_POOL_NAME} ${AIRFLOW_POOL_SIZE} ${AIRFLOW_POOL_DESC}" debug_execute airflow pool -s "$AIRFLOW_POOL_NAME" "$AIRFLOW_POOL_SIZE" "$AIRFLOW_POOL_DESC"
fi fi
} }

View File

@@ -47,7 +47,7 @@ airflow_scheduler_validate() {
######################### #########################
airflow_scheduler_initialize() { airflow_scheduler_initialize() {
# Change permissions if running as root # Change permissions if running as root
for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_SCHEDULER_LOGS_DIR" "$AIRFLOW_DATA_DIR"; do for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_SCHEDULER_LOGS_DIR"; do
ensure_dir_exists "$dir" ensure_dir_exists "$dir"
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir" am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
done done
@@ -60,29 +60,13 @@ airflow_scheduler_initialize() {
info "Configuration file found, loading configuration" info "Configuration file found, loading configuration"
fi fi
# Check if Airflow has already been initialized and persisted in a previous run info "Trying to connect to the database server"
local -r app_name="airflow" airflow_wait_for_postgresql_connection
local -a postgresql_remote_execute_args=("$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER" "$AIRFLOW_DATABASE_NAME" "$AIRFLOW_DATABASE_USERNAME" "$AIRFLOW_DATABASE_PASSWORD") # Change the permissions after restoring the persisted data in case we are root
if ! is_app_initialized "$app_name"; then for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR"; do
airflow_wait_for_postgresql_connection "${postgresql_remote_execute_args[@]}" ensure_dir_exists "$dir"
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
info "Persisting Airflow installation" done
persist_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
else
# Check database connection
airflow_wait_for_postgresql_connection "${postgresql_remote_execute_args[@]}"
# Restore persisted data
info "Restoring persisted Airflow installation"
restore_persisted_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
# Change the permissions after restoring the persisted data in case we are root
for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR"; do
ensure_dir_exists "$dir"
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
done
true # Avoid return false when I am not root
fi
# Wait for airflow webserver to be available # Wait for airflow webserver to be available
airflow_scheduler_wait_for_webserver "$AIRFLOW_WEBSERVER_HOST" "$AIRFLOW_WEBSERVER_PORT_NUMBER" airflow_scheduler_wait_for_webserver "$AIRFLOW_WEBSERVER_HOST" "$AIRFLOW_WEBSERVER_PORT_NUMBER"
@@ -103,7 +87,7 @@ airflow_scheduler_initialize() {
######################### #########################
airflow_scheduler_generate_config() { airflow_scheduler_generate_config() {
# Generate Airflow default files # Generate Airflow default files
airflow_execute_command "version" "version" debug_execute airflow version
# Configure Airflow Hostname # Configure Airflow Hostname
[[ -n "$AIRFLOW_HOSTNAME_CALLABLE" ]] && airflow_conf_set "core" "hostname_callable" "$AIRFLOW_HOSTNAME_CALLABLE" [[ -n "$AIRFLOW_HOSTNAME_CALLABLE" ]] && airflow_conf_set "core" "hostname_callable" "$AIRFLOW_HOSTNAME_CALLABLE"

View File

@@ -37,7 +37,7 @@ You can find the default credentials and available configuration options in the
Learn more about the Bitnami tagging policy and the difference between rolling tags and immutable tags [in our documentation page](https://docs.bitnami.com/tutorials/understand-rolling-tags-containers/). Learn more about the Bitnami tagging policy and the difference between rolling tags and immutable tags [in our documentation page](https://docs.bitnami.com/tutorials/understand-rolling-tags-containers/).
* [`2`, `2-debian-10`, `2.2.3`, `2.2.3-debian-10-r46`, `latest` (2/debian-10/Dockerfile)](https://github.com/bitnami/bitnami-docker-airflow-scheduler/blob/2.2.3-debian-10-r46/2/debian-10/Dockerfile) * [`2`, `2-debian-10`, `2.2.3`, `2.2.3-debian-10-r47`, `latest` (2/debian-10/Dockerfile)](https://github.com/bitnami/bitnami-docker-airflow-scheduler/blob/2.2.3-debian-10-r47/2/debian-10/Dockerfile)
Subscribe to project updates by watching the [bitnami/airflow GitHub repo](https://github.com/bitnami/bitnami-docker-airflow-scheduler). Subscribe to project updates by watching the [bitnami/airflow GitHub repo](https://github.com/bitnami/bitnami-docker-airflow-scheduler).
@@ -93,10 +93,9 @@ If you want to run the application manually instead of using `docker-compose`, t
bitnami/redis:latest bitnami/redis:latest
``` ```
4. Create volumes for Airflow persistence and launch the container 4. Launch the Apache Airflow Scheduler web container
```console ```console
$ docker volume create --name airflow_data
$ docker run -d --name airflow -p 8080:8080 \ $ docker run -d --name airflow -p 8080:8080 \
-e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \
-e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \ -e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \
@@ -109,14 +108,12 @@ If you want to run the application manually instead of using `docker-compose`, t
-e AIRFLOW_USERNAME=user \ -e AIRFLOW_USERNAME=user \
-e AIRFLOW_EMAIL=user@example.com \ -e AIRFLOW_EMAIL=user@example.com \
--net airflow-tier \ --net airflow-tier \
--volume airflow_data:/bitnami \
bitnami/airflow:latest bitnami/airflow:latest
``` ```
5. Create volumes for Apache Airflow Scheduler persistence and launch the container 5. Launch the Apache Airflow Scheduler scheduler container
```console ```console
$ docker volume create --name airflow_scheduler_data
$ docker run -d --name airflow-scheduler \ $ docker run -d --name airflow-scheduler \
-e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \
-e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \ -e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \
@@ -126,14 +123,12 @@ If you want to run the application manually instead of using `docker-compose`, t
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
-e AIRFLOW_LOAD_EXAMPLES=yes \ -e AIRFLOW_LOAD_EXAMPLES=yes \
--net airflow-tier \ --net airflow-tier \
--volume airflow_scheduler_data:/bitnami \
bitnami/airflow-scheduler:latest bitnami/airflow-scheduler:latest
``` ```
6. Create volumes for Airflow Worker persistence and launch the container 6. Launch the Apache Airflow Scheduler worker container
```console ```console
$ docker volume create --name airflow_worker_data
$ docker run -d --name airflow-worker \ $ docker run -d --name airflow-worker \
-e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \
-e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \ -e AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08= \
@@ -142,7 +137,6 @@ If you want to run the application manually instead of using `docker-compose`, t
-e AIRFLOW_DATABASE_USERNAME=bn_airflow \ -e AIRFLOW_DATABASE_USERNAME=bn_airflow \
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
--net airflow-tier \ --net airflow-tier \
--volume airflow_worker_data:/bitnami \
bitnami/airflow-worker:latest bitnami/airflow-worker:latest
``` ```
@@ -150,11 +144,9 @@ Access your application at `http://your-ip:8080`
### Persisting your application ### Persisting your application
If you remove the container all your data and configurations will be lost, and the next time you run the image the database will be reinitialized. To avoid this loss of data, you should mount a volume that will persist even after the container is removed. The Bitnami Airflow container relies on the PostgreSQL database & Redis to persist the data. This means that Airflow does not persist anything. To avoid loss of data, you should mount volumes for persistence of [PostgreSQL data](https://github.com/bitnami/bitnami-docker-mariadb#persisting-your-database) and [Redis(TM) data](https://github.com/bitnami/bitnami-docker-redis#persisting-your-database)
For persistence you should mount a volume at the `/bitnami` path. Additionally you should mount volumes for persistence of [PostgreSQL data](https://github.com/bitnami/bitnami-docker-mariadb#persisting-your-database) and [Redis(TM) data](https://github.com/bitnami/bitnami-docker-mariadb#persisting-your-database) The above examples define docker volumes namely `postgresql_data`, and `redis_data`. The Airflow application state will persist as long as these volumes are not removed.
The above examples define docker volumes namely `postgresql_data`, `redis_data`, `airflow_data`, `airflow_scheduler_data` and `airflow_worker_data`. The Apache Airflow Scheduler application state will persist as long as these volumes are not removed.
To avoid inadvertent removal of these volumes you can [mount host directories as data volumes](https://docs.docker.com/engine/tutorials/dockervolumes/). Alternatively you can make use of volume plugins to host the volume data. To avoid inadvertent removal of these volumes you can [mount host directories as data volumes](https://docs.docker.com/engine/tutorials/dockervolumes/). Alternatively you can make use of volume plugins to host the volume data.
@@ -172,13 +164,13 @@ services:
- POSTGRESQL_USERNAME=bn_airflow - POSTGRESQL_USERNAME=bn_airflow
- POSTGRESQL_PASSWORD=bitnami1 - POSTGRESQL_PASSWORD=bitnami1
volumes: volumes:
- /path/to/airflow-persistence:/bitnami - /path/to/postgresql-persistence:/bitnami
redis: redis:
image: 'bitnami/redis:latest' image: 'bitnami/redis:latest'
environment: environment:
- ALLOW_EMPTY_PASSWORD=yes - ALLOW_EMPTY_PASSWORD=yes
volumes: volumes:
- /path/to/airflow-persistence:/bitnami - /path/to/redis-persistence:/bitnami
airflow-worker: airflow-worker:
image: bitnami/airflow-worker:latest image: bitnami/airflow-worker:latest
environment: environment:
@@ -189,8 +181,6 @@ services:
- AIRFLOW_DATABASE_USERNAME=bn_airflow - AIRFLOW_DATABASE_USERNAME=bn_airflow
- AIRFLOW_DATABASE_PASSWORD=bitnami1 - AIRFLOW_DATABASE_PASSWORD=bitnami1
- AIRFLOW_LOAD_EXAMPLES=yes - AIRFLOW_LOAD_EXAMPLES=yes
volumes:
- /path/to/airflow-persistence:/bitnami
airflow-scheduler: airflow-scheduler:
image: bitnami/airflow-scheduler:latest image: bitnami/airflow-scheduler:latest
environment: environment:
@@ -201,8 +191,6 @@ services:
- AIRFLOW_DATABASE_USERNAME=bn_airflow - AIRFLOW_DATABASE_USERNAME=bn_airflow
- AIRFLOW_DATABASE_PASSWORD=bitnami1 - AIRFLOW_DATABASE_PASSWORD=bitnami1
- AIRFLOW_LOAD_EXAMPLES=yes - AIRFLOW_LOAD_EXAMPLES=yes
volumes:
- /path/to/airflow-persistence:/bitnami
airflow: airflow:
image: bitnami/airflow:latest image: bitnami/airflow:latest
environment: environment:
@@ -217,8 +205,6 @@ services:
- AIRFLOW_EMAIL=user@example.com - AIRFLOW_EMAIL=user@example.com
ports: ports:
- '8080:8080' - '8080:8080'
volumes:
- /path/to/airflow-persistence:/bitnami
``` ```
#### Mount host directories as data volumes using the Docker command line #### Mount host directories as data volumes using the Docker command line
@@ -251,7 +237,7 @@ services:
bitnami/redis:latest bitnami/redis:latest
``` ```
4. Create the Airflow container with host volumes 4. Create the Airflow container
```console ```console
$ docker run -d --name airflow -p 8080:8080 \ $ docker run -d --name airflow -p 8080:8080 \
@@ -266,11 +252,10 @@ services:
-e AIRFLOW_USERNAME=user \ -e AIRFLOW_USERNAME=user \
-e AIRFLOW_EMAIL=user@example.com \ -e AIRFLOW_EMAIL=user@example.com \
--net airflow-tier \ --net airflow-tier \
--volume /path/to/airflow-persistence:/bitnami \
bitnami/airflow:latest bitnami/airflow:latest
``` ```
5. Create the Apache Airflow Scheduler container with host volumes 5. Create the Apache Airflow Scheduler container
```console ```console
$ docker run -d --name airflow-scheduler \ $ docker run -d --name airflow-scheduler \
@@ -282,11 +267,10 @@ services:
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
-e AIRFLOW_LOAD_EXAMPLES=yes \ -e AIRFLOW_LOAD_EXAMPLES=yes \
--net airflow-tier \ --net airflow-tier \
--volume /path/to/airflow-scheduler-persistence:/bitnami \
bitnami/airflow-scheduler:latest bitnami/airflow-scheduler:latest
``` ```
6. Create the Airflow Worker container with host volumes 6. Create the Airflow Worker container
```console ```console
$ docker run -d --name airflow-worker \ $ docker run -d --name airflow-worker \
@@ -297,7 +281,6 @@ services:
-e AIRFLOW_DATABASE_USERNAME=bn_airflow \ -e AIRFLOW_DATABASE_USERNAME=bn_airflow \
-e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \
--net airflow-tier \ --net airflow-tier \
--volume /path/to/airflow-worker-persistence:/bitnami \
bitnami/airflow-worker:latest bitnami/airflow-worker:latest
``` ```