[bitnami/airflow] Split init containers (#30155)

This commit is contained in:
Juan Ariza Toledano
2024-10-31 15:55:05 +01:00
committed by GitHub
parent 95f4ee6bce
commit 4945ae6dd9
7 changed files with 156 additions and 98 deletions
+5 -1
View File
@@ -1,8 +1,12 @@
# Changelog
## 21.0.1 (2024-10-31)
* [bitnami/airflow] Split init containers ([#30155](https://github.com/bitnami/charts/pull/30155))
## 21.0.0 (2024-10-31)
* [bitnami/airflow] Use a single Airflow image ([#30137](https://github.com/bitnami/charts/pull/30137))
* [bitnami/airflow] Use a single Airflow image (#30137) ([e9aed1b](https://github.com/bitnami/charts/commit/e9aed1bfed27dddaff454760ef3318d81e3bb05a)), closes [#30137](https://github.com/bitnami/charts/issues/30137)
## <small>20.0.2 (2024-10-30)</small>
+1 -1
View File
@@ -39,4 +39,4 @@ maintainers:
name: airflow
sources:
- https://github.com/bitnami/charts/tree/main/bitnami/airflow
version: 21.0.0
version: 21.0.1
@@ -46,11 +46,10 @@ create folders or volume names
{{- end -}}
{{/*
Returns an init-container that loads DAGs and/or plugins from a ConfigMap or Git repositories
Returns shared structure between load-dags and load-plugins init containers
*/}}
{{- define "airflow.defaultInitContainers.loadDAGsPlugins" -}}
- name: load-dags-plugins
image: {{ include "airflow.image" . }}
{{- define "airflow.defaultInitContainers.shared" -}}
- image: {{ include "airflow.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- if .Values.defaultInitContainers.loadDAGsPlugins.containerSecurityContext.enabled }}
securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.defaultInitContainers.loadDAGsPlugins.containerSecurityContext "context" .) | nindent 4 }}
@@ -65,28 +64,6 @@ Returns an init-container that loads DAGs and/or plugins from a ConfigMap or Git
{{- else }}
command: ["/bin/bash"]
{{- end }}
{{- if .Values.defaultInitContainers.loadDAGsPlugins.args }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.defaultInitContainers.loadDAGsPlugins.args "context" .) | nindent 4 }}
{{- else }}
args:
- -ec
- |
. /opt/bitnami/scripts/libfs.sh
{{- if .Values.dags.enabled }}
{{- range .Values.dags.repositories }}
is_dir_empty "/dags/{{ include "airflow.dagsPlugins.repository.name" . }}" && git clone {{ .repository }} --depth 1 --branch {{ .branch }} /dags/{{ include "airflow.dagsPlugins.repository.name" . }}
{{- end }}
{{- end }}
{{- if .Values.plugins.enabled }}
{{- range .Values.plugins.repositories }}
is_dir_empty "/plugins/{{ include "airflow.dagsPlugins.repository.name" . }}" && git clone {{ .repository }} --depth 1 --branch {{ .branch }} /plugins/{{ include "airflow.dagsPlugins.repository.name" . }}
{{- end }}
{{- end }}
{{- if not (empty .Values.dags.existingConfigmap) }}
cp /configmap/* /dags/external
{{- end }}
{{- end }}
{{- if .Values.defaultInitContainers.loadDAGsPlugins.extraEnvVars }}
env: {{- include "common.tplvalues.render" (dict "value" .Values.defaultInitContainers.loadDAGsPlugins.extraEnvVars "context" .) | nindent 4 }}
{{- end }}
@@ -111,6 +88,16 @@ Returns an init-container that loads DAGs and/or plugins from a ConfigMap or Git
- name: empty-dir
mountPath: /.ssh
subPath: ssh-dir
{{- if .Values.defaultInitContainers.loadDAGsPlugins.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.defaultInitContainers.loadDAGsPlugins.extraVolumeMounts "context" $) | nindent 4 }}
{{- end }}
{{- end -}}
{{/*
Returns an init-container that loads DAGs from a ConfigMap or Git repositories
*/}}
{{- define "airflow.defaultInitContainers.loadDAGs" -}}
{{ include "airflow.defaultInitContainers.shared" . }}
{{- if not (empty .Values.dags.existingConfigmap) }}
- name: external-dags
mountPath: /configmap
@@ -120,22 +107,52 @@ Returns an init-container that loads DAGs and/or plugins from a ConfigMap or Git
mountPath: /dags
subPath: app-dags-dir
{{- end }}
{{- if not (empty .Values.plugins.repositories) }}
- name: empty-dir
mountPath: /plugins
subPath: app-plugins-dir
{{- if .Values.defaultInitContainers.loadDAGsPlugins.args }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.defaultInitContainers.loadDAGsPlugins.args "context" .) | nindent 4 }}
{{- else }}
args:
- -ec
- |
. /opt/bitnami/scripts/libfs.sh
{{- range .Values.dags.repositories }}
is_dir_empty "/dags/{{ include "airflow.dagsPlugins.repository.name" . }}" && git clone {{ .repository }} --depth 1 --branch {{ .branch }} /dags/{{ include "airflow.dagsPlugins.repository.name" . }}
{{- end }}
{{- if .Values.defaultInitContainers.loadDAGsPlugins.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.defaultInitContainers.loadDAGsPlugins.extraVolumeMounts "context" $) | nindent 4 }}
{{- if not (empty .Values.dags.existingConfigmap) }}
cp /configmap/* /dags/external
{{- end }}
{{- end }}
name: load-dags
{{- end -}}
{{/*
Returns a sidecar that syncs DAGs and/or plugins from Git repositories
Returns an init-container that loads plugins from Git repositories
*/}}
{{- define "airflow.defaultSidecars.syncDAGsPlugins" -}}
- name: sync-dags-plugins
image: {{ include "airflow.image" . }}
{{- define "airflow.defaultInitContainers.loadPlugins" -}}
{{ include "airflow.defaultInitContainers.shared" . }}
- name: empty-dir
mountPath: /plugins
subPath: app-plugins-dir
{{- if .Values.defaultInitContainers.loadDAGsPlugins.args }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.defaultInitContainers.loadDAGsPlugins.args "context" .) | nindent 4 }}
{{- else }}
args:
- -ec
- |
. /opt/bitnami/scripts/libfs.sh
{{- range .Values.plugins.repositories }}
is_dir_empty "/plugins/{{ include "airflow.dagsPlugins.repository.name" . }}" && git clone {{ .repository }} --depth 1 --branch {{ .branch }} /plugins/{{ include "airflow.dagsPlugins.repository.name" . }}
{{- end }}
{{- end }}
name: load-plugins
{{- end -}}
{{/*
Returns shared structure between sync-dags and sync-plugins sidecars
*/}}
{{- define "airflow.defaultSidecars.shared" -}}
- image: {{ include "airflow.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- if .Values.defaultSidecars.syncDAGsPlugins.containerSecurityContext.enabled }}
securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.defaultSidecars.syncDAGsPlugins.containerSecurityContext "context" .) | nindent 4 }}
@@ -150,27 +167,6 @@ Returns a sidecar that syncs DAGs and/or plugins from Git repositories
{{- else }}
command: ["/bin/bash"]
{{- end }}
{{- if .Values.defaultSidecars.syncDAGsPlugins.args }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.defaultSidecars.syncDAGsPlugins.args "context" .) | nindent 4 }}
{{- else }}
args:
- -ec
- |
while true; do
{{- if .Values.dags.enabled }}
{{- range .Values.dags.repositories }}
cd /dags/{{ include "airflow.dagsPlugins.repository.name" . }} && git pull origin {{ .branch }} || true
ls -la /dags/{{ include "airflow.dagsPlugins.repository.name" . }}
{{- end }}
{{- end }}
{{- if .Values.plugins.enabled }}
{{- range .Values.plugins.repositories }}
cd /plugins/{{ include "airflow.dagsPlugins.repository.name" . }} && git pull origin {{ .branch }} || true
{{- end }}
{{- end }}
sleep {{ default "60" .Values.defaultSidecars.syncDAGsPlugins.interval }}
done
{{- end }}
{{- if .Values.defaultSidecars.syncDAGsPlugins.extraEnvVars }}
env: {{- include "common.tplvalues.render" (dict "value" .Values.defaultSidecars.syncDAGsPlugins.extraEnvVars "context" .) | nindent 4 }}
{{- end }}
@@ -195,26 +191,63 @@ Returns a sidecar that syncs DAGs and/or plugins from Git repositories
- name: empty-dir
mountPath: /.ssh
subPath: ssh-dir
{{- if not (empty .Values.dags.repositories) }}
- name: empty-dir
mountPath: /dags
subPath: app-dags-dir
{{- end }}
{{- if not (empty .Values.plugins.repositories) }}
- name: empty-dir
mountPath: /plugins
subPath: app-plugins-dir
{{- end }}
{{- if .Values.defaultSidecars.syncDAGsPlugins.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.defaultSidecars.syncDAGsPlugins.extraVolumeMounts "context" $) | nindent 4 }}
{{- end }}
{{- end -}}
{{/*
Returns the volume mounts to use on Airflow containers to mount custom DAGs and plugins
Returns a sidecar that syncs DAGs from Git repositories
*/}}
{{- define "airflow.dagsPlugins.volumeMounts" -}}
{{- if .Values.dags.enabled }}
{{- define "airflow.defaultSidecars.syncDAGs" -}}
{{ include "airflow.defaultSidecars.shared" . }}
- name: empty-dir
mountPath: /dags
subPath: app-dags-dir
{{- if .Values.defaultSidecars.syncDAGsPlugins.args }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.defaultSidecars.syncDAGsPlugins.args "context" .) | nindent 4 }}
{{- else }}
args:
- -ec
- |
while true; do
{{- range .Values.dags.repositories }}
cd /dags/{{ include "airflow.dagsPlugins.repository.name" . }} && git pull origin {{ .branch }} || true
{{- end }}
sleep {{ default "60" .Values.defaultSidecars.syncDAGsPlugins.interval }}
done
{{- end }}
name: sync-dags
{{- end -}}
{{/*
Returns a sidecar that syncs plugins from Git repositories
*/}}
{{- define "airflow.defaultSidecars.syncPlugins" -}}
{{ include "airflow.defaultSidecars.shared" . }}
- name: empty-dir
mountPath: /plugins
subPath: app-plugins-dir
{{- if .Values.defaultSidecars.syncDAGsPlugins.args }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.defaultSidecars.syncDAGsPlugins.args "context" .) | nindent 4 }}
{{- else }}
args:
- -ec
- |
while true; do
{{- range .Values.plugins.repositories }}
cd /plugins/{{ include "airflow.dagsPlugins.repository.name" . }} && git pull origin {{ .branch }} || true
{{- end }}
sleep {{ default "60" .Values.defaultSidecars.syncDAGsPlugins.interval }}
done
{{- end }}
name: sync-plugins
{{- end -}}
{{/*
Returns the volume mounts to use on Airflow containers to mount custom DAGs
*/}}
{{- define "airflow.dags.volumeMounts" -}}
{{- if not (empty .Values.dags.existingConfigmap) }}
- name: empty-dir
mountPath: /opt/bitnami/airflow/dags/external
@@ -229,8 +262,12 @@ Returns the volume mounts to use on Airflow containers to mount custom DAGs and
subPath: app-dags-dir/{{ include "airflow.dagsPlugins.repository.name" . }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.plugins.enabled }}
{{- end -}}
{{/*
Returns the volume mounts to use on Airflow containers to mount custom plugins
*/}}
{{- define "airflow.plugins.volumeMounts" -}}
{{- range .Values.plugins.repositories }}
- name: empty-dir
mountPath: /opt/bitnami/airflow/plugins/git_{{ include "airflow.dagsPlugins.repository.name" . }}
@@ -239,6 +276,5 @@ Returns the volume mounts to use on Airflow containers to mount custom DAGs and
{{- else }}
subPath: app-plugins-dir/{{ include "airflow.dagsPlugins.repository.name" . }}
{{- end }}
{{- end }}
{{- end }}
{{- end -}}
@@ -70,8 +70,11 @@ spec:
{{- end }}
initContainers:
{{- include "airflow.defaultInitContainers.createDefaultConfig" . | nindent 8 }}
{{- if or .Values.dags.enabled .Values.plugins.enabled }}
{{- include "airflow.defaultInitContainers.loadDAGsPlugins" . | nindent 8 }}
{{- if and .Values.dags.enabled }}
{{- include "airflow.defaultInitContainers.loadDAGs" . | nindent 8 }}
{{- end }}
{{- if .Values.plugins.enabled }}
{{- include "airflow.defaultInitContainers.loadPlugins" . | nindent 8 }}
{{- end }}
{{- if .Values.initContainers }}
{{- include "common.tplvalues.render" (dict "value" .Values.initContainers "context" $) | trim | nindent 8 }}
@@ -219,8 +222,11 @@ spec:
mountPath: /opt/bitnami/airflow/pod_template.yaml
subPath: pod_template.yaml
{{- end }}
{{- if or .Values.dags.enabled .Values.plugins.enabled }}
{{- include "airflow.dagsPlugins.volumeMounts" . | nindent 12 }}
{{- if and .Values.dags.enabled }}
{{- include "airflow.dags.volumeMounts" . | nindent 12 }}
{{- end }}
{{- if .Values.plugins.enabled }}
{{- include "airflow.plugins.volumeMounts" . | nindent 12 }}
{{- end }}
{{- if .Values.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }}
@@ -228,8 +234,11 @@ spec:
{{- if .Values.scheduler.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.scheduler.extraVolumeMounts "context" $) | nindent 12 }}
{{- end }}
{{- if or .Values.dags.enabled .Values.plugins.enabled }}
{{- include "airflow.defaultSidecars.syncDAGsPlugins" . | nindent 8 }}
{{- if and .Values.dags.enabled }}
{{- include "airflow.defaultSidecars.syncDAGs" . | nindent 8 }}
{{- end }}
{{- if .Values.plugins.enabled }}
{{- include "airflow.defaultSidecars.syncPlugins" . | nindent 8 }}
{{- end }}
{{- if .Values.sidecars }}
{{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | trim | nindent 8 }}
+6 -11
View File
@@ -69,8 +69,8 @@ spec:
{{- end }}
initContainers:
{{- include "airflow.defaultInitContainers.createDefaultConfig" . | nindent 8 }}
{{- if or .Values.dags.enabled .Values.plugins.enabled }}
{{- include "airflow.defaultInitContainers.loadDAGsPlugins" . | nindent 8 }}
{{- if .Values.plugins.enabled }}
{{- include "airflow.defaultInitContainers.loadPlugins" . | nindent 8 }}
{{- end }}
{{- if .Values.initContainers }}
{{- include "common.tplvalues.render" (dict "value" .Values.initContainers "context" $) | trim | nindent 8 }}
@@ -269,8 +269,8 @@ spec:
mountPath: {{ .Values.ldap.tls.certificatesMountPath }}
readOnly: true
{{- end }}
{{- if or .Values.dags.enabled .Values.plugins.enabled }}
{{- include "airflow.dagsPlugins.volumeMounts" . | nindent 12 }}
{{- if .Values.plugins.enabled }}
{{- include "airflow.plugins.volumeMounts" . | nindent 12 }}
{{- end }}
{{- if .Values.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }}
@@ -278,8 +278,8 @@ spec:
{{- if .Values.web.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.web.extraVolumeMounts "context" $) | nindent 12 }}
{{- end }}
{{- if or .Values.dags.enabled .Values.plugins.enabled }}
{{- include "airflow.defaultSidecars.syncDAGsPlugins" . | nindent 8 }}
{{- if .Values.plugins.enabled }}
{{- include "airflow.defaultSidecars.syncPlugins" . | nindent 8 }}
{{- end }}
{{- if .Values.sidecars }}
{{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | trim | nindent 8 }}
@@ -290,11 +290,6 @@ spec:
volumes:
- name: empty-dir
emptyDir: {}
{{- if and .Values.dags.enabled .Values.dags.existingConfigmap }}
- name: external-dags
configMap:
name: {{ tpl .Values.dags.existingConfigmap $ }}
{{- end }}
{{- if or .Values.configuration .Values.existingConfigmap }}
- name: custom-configuration-file
configMap:
@@ -74,8 +74,11 @@ spec:
{{- end }}
initContainers:
{{- include "airflow.defaultInitContainers.createDefaultConfig" . | nindent 8 }}
{{- if or .Values.dags.enabled .Values.plugins.enabled }}
{{- include "airflow.defaultInitContainers.loadDAGsPlugins" . | nindent 8 }}
{{- if .Values.dags.enabled }}
{{- include "airflow.defaultInitContainers.loadDAGs" . | nindent 8 }}
{{- end }}
{{- if .Values.plugins.enabled }}
{{- include "airflow.defaultInitContainers.loadPlugins" . | nindent 8 }}
{{- end }}
{{- if .Values.initContainers }}
{{- include "common.tplvalues.render" (dict "value" .Values.initContainers "context" $) | trim | nindent 8 }}
@@ -216,8 +219,11 @@ spec:
- name: empty-dir
mountPath: /opt/bitnami/airflow/webserver_config.py
subPath: app-default-conf-dir/webserver_config.py
{{- if or .Values.dags.enabled .Values.plugins.enabled }}
{{- include "airflow.dagsPlugins.volumeMounts" . | nindent 12 }}
{{- if .Values.dags.enabled }}
{{- include "airflow.dags.volumeMounts" . | nindent 12 }}
{{- end }}
{{- if .Values.plugins.enabled }}
{{- include "airflow.plugins.volumeMounts" . | nindent 12 }}
{{- end }}
{{- if .Values.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }}
@@ -225,8 +231,11 @@ spec:
{{- if .Values.worker.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeMounts "context" $) | nindent 12 }}
{{- end }}
{{- if or .Values.dags.enabled .Values.plugins.enabled }}
{{- include "airflow.defaultSidecars.syncDAGsPlugins" . | nindent 8 }}
{{- if .Values.dags.enabled }}
{{- include "airflow.defaultSidecars.syncDAGs" . | nindent 8 }}
{{- end }}
{{- if .Values.plugins.enabled }}
{{- include "airflow.defaultSidecars.syncPlugins" . | nindent 8 }}
{{- end }}
{{- if .Values.sidecars }}
{{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | trim | nindent 8 }}
+5
View File
@@ -396,6 +396,7 @@ extraVolumeMounts: []
extraVolumes: []
## @section Airflow web parameters
##
web:
## @param web.baseUrl URL used to access to Airflow web ui
##
@@ -709,7 +710,9 @@ web:
##
ingressNSMatchLabels: {}
ingressNSPodMatchLabels: {}
## @section Airflow scheduler parameters
##
scheduler:
## @param scheduler.replicaCount Number of scheduler replicas
##
@@ -1013,7 +1016,9 @@ scheduler:
##
ingressNSMatchLabels: {}
ingressNSPodMatchLabels: {}
## @section Airflow worker parameters
##
worker:
## @param worker.command Override default container command (useful when using custom images)
##