[bitnami/airflow] Add sidecar container to update git repository periodically

Signed-off-by: juan131 <juan@bitnami.com>

bitnami/airflow/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: airflow
-version: 0.0.4
+version: 0.1.0
 appVersion: 1.10.3
 description: Apache Airflow is a platform to programmatically author, schedule and monitor workflows.
 keywords:

bitnami/airflow/README.md

@@ -79,7 +79,8 @@ The following tables lists the configurable parameters of the Kafka chart and th
 | `airflow.loadExamples`                    | Switch to load some Airflow examples                                                        | `true`                                                       |
 | `airflow.cloneDagFilesFromGit.enabled`    | Enable in order to download DAG files from git repository.                                  | `false`                                                      |
 | `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from                                                    | `nil`                                                        |
-| `airflow.cloneDagFilesFromGit.revision`   | Revision from repository to checkout                                                        | `nil`                                                        |
+| `airflow.cloneDagFilesFromGit.branch`     | Branch from repository to checkout                                                          | `nil`                                                        |
+| `airflow.cloneDagFilesFromGit.interval`   | Interval to pull the repository on sidecar container                                        | `nil`                                                        |
 | `airflow.baseUrl`                         | URL used to access to airflow web ui                                                        | `nil`                                                        |
 | `airflow.worker.port`                     | Airflow Worker port                                                                         | `8793`                                                       |
 | `airflow.worker.replicas`                 | Number of Airflow Worker replicas                                                           | `2`                                                          |
@@ -188,12 +189,12 @@ You can manually create a config map containing all your DAG files and then pass
 
 ### Option 3: Get your DAG files from a git repository
 
-You can store all your DAG files on a GitHub repository and then clone to the Airflow pods with an initContainer. In order to do that, you can deploy airflow with the following options:
+You can store all your DAG files on a GitHub repository and then clone to the Airflow pods with an initContainer. The repository will be periodically updated using a sidecar container. In order to do that, you can deploy airflow with the following options:
 
 ```console
 helm install --name my-release bitnami/airflow \
              --set airflow.cloneDagFilesFromGit.enabled=true \
              --set airflow.cloneDagFilesFromGit.repository=https://github.com/USERNAME/REPOSITORY \
-             --set airflow.cloneDagFilesFromGit.revision=master
-
-```
+             --set airflow.cloneDagFilesFromGit.branch=master
+             --set airflow.cloneDagFilesFromGit.interval=3600
+```

bitnami/airflow/templates/NOTES.txt

@@ -75,4 +75,6 @@ host. To configure Airflow with the URL of your service:
   echo User:    {{ .Values.airflow.auth.username }}
   echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode)
 
-  {{- end  }}
+{{- end  }}
+
+{{ include "airflow.validateValues" . }}

bitnami/airflow/templates/_helpers.tpl

@@ -146,7 +146,6 @@ Also, we can't use a single if because lazy evaluation is not an option
 {{- end -}}
 {{- end -}}
 
-
 {{/*
 Return the proper Docker Image Registry Secret Names
 */}}
@@ -233,4 +232,32 @@ Get the secret name
 {{- else -}}
 {{- printf "%s" (include "airflow.fullname" .) -}}
 {{- end -}}
-{{- end -}}
+{{- end -}}
+
+{{/*
+Compile all warnings into a single message, and call fail.
+*/}}
+{{- define "airflow.validateValues" -}}
+{{- $messages := list -}}
+{{- $messages := append $messages (include "airflow.validateValues.cloneDagFilesFromGit" .) -}}
+{{- $messages := without $messages "" -}}
+{{- $message := join "\n" $messages -}}
+
+{{- if $message -}}
+{{-   printf "\nVALUES VALIDATION:\n%s" $message | fail -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Airflow - "airflow.cloneDagFilesFromGit.repository" must be provided when "airflow.cloneDagFilesFromGit.enabled" is "true" */}}
+{{- define "airflow.validateValues.cloneDagFilesFromGit" -}}
+{{- if and .Values.airflow.cloneDagFilesFromGit.enabled (empty .Values.airflow.cloneDagFilesFromGit.repository) -}}
+airflow: airflow.cloneDagFilesFromGit.repository
+    The repository must be provided when enabling downloading DAG files
+    from git repository (--set airflow.cloneDagFilesFromGit.repository="xxx")
+{{- end -}}
+{{- if and .Values.airflow.cloneDagFilesFromGit.enabled (empty .Values.airflow.cloneDagFilesFromGit.branch) -}}
+airflow: airflow.cloneDagFilesFromGit.branch
+    The branch must be provided when enabling downloading DAG files
+    from git repository (--set airflow.cloneDagFilesFromGit.branch="xxx")
+{{- end -}}
+{{- end -}}

bitnami/airflow/templates/deployment-scheduler.yaml

@@ -50,12 +50,32 @@ spec:
       - name: git-clone-repository
         image: "{{ template "git.image" . }}"
         imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
-        command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}']
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
         volumeMounts:
         - name: git-cloned-dag-files
           mountPath: /dags
-      {{- end }}
       containers:
+      - name: clone-repo-sidecar
+        image: "{{ template "git.image" . }}"
+        imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          while true; do
+              cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
+              sleep {{ default "3600" .Values.airflow.cloneDagFilesFromGit.interval }}
+          done
+        volumeMounts:
+        - name: git-cloned-dag-files
+          mountPath: /dags
+      {{- else }}
+      containers:
+      {{- end }}
       - name: airflow-scheduler
         image: {{ template "airflow.schedulerImage" . }}
         imagePullPolicy: {{ .Values.schedulerImage.pullPolicy | quote }}
@@ -187,4 +207,4 @@ spec:
       - name: custom-configuration-file
         configMap:
           name: {{ .Values.airflow.configurationConfigMap }}
-      {{- end }}
+      {{- end }}

bitnami/airflow/templates/deployment-web.yaml

@@ -50,12 +50,32 @@ spec:
       - name: git-clone-repository
         image: "{{ template "git.image" . }}"
         imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
-        command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}']
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
         volumeMounts:
         - name: git-cloned-dag-files
           mountPath: /dags
-      {{- end }}
       containers:
+      - name: clone-repo-sidecar
+        image: "{{ template "git.image" . }}"
+        imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          while true; do
+              cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
+              sleep {{ default "3600" .Values.airflow.cloneDagFilesFromGit.interval }}
+          done
+        volumeMounts:
+        - name: git-cloned-dag-files
+          mountPath: /dags
+      {{- else }}
+      containers:
+      {{- end }}
       - name: airflow-web
         image: {{ template "airflow.image" . }}
         imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
@@ -217,4 +237,4 @@ spec:
       - name: custom-configuration-file
         configMap:
           name: {{ .Values.airflow.configurationConfigMap }}
-      {{- end }}
+      {{- end }}

bitnami/airflow/templates/statefulset-worker.yaml

@@ -54,12 +54,32 @@ spec:
       - name: git-clone-repository
         image: "{{ template "git.image" . }}"
         imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
-        command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}']
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
         volumeMounts:
         - name: git-cloned-dag-files
           mountPath: /dags
-      {{- end }}
       containers:
+      - name: clone-repo-sidecar
+        image: "{{ template "git.image" . }}"
+        imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          while true; do
+              cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
+              sleep {{ default "3600" .Values.airflow.cloneDagFilesFromGit.interval }}
+          done
+        volumeMounts:
+        - name: git-cloned-dag-files
+          mountPath: /dags
+      {{- else }}
+      containers:
+      {{- end }}
       - name: airflow-worker
         image: "{{ template "airflow.workerImage" . }}"
         imagePullPolicy: "{{ .Values.workerImage.pullPolicy }}"
@@ -202,4 +222,4 @@ spec:
       - name: custom-configuration-file
         configMap:
           name: {{ .Values.airflow.configurationConfigMap }}
-      {{- end }}
+      {{- end }}

bitnami/airflow/values-production.yaml

@@ -120,8 +120,9 @@ airflow:
   ##
   cloneDagFilesFromGit:
     enabled: false
-    repository:
-    revision:
+    # repository:
+    # branch:
+    # interval:
   ## URL used to access to airflow web ui
   ##
   baseUrl: http://airflow.local
@@ -354,4 +355,4 @@ metrics:
 
   ## Metrics exporter pod Annotation and Labels
   # podAnnotations: {}
-  # podLabels: {}
+  # podLabels: {}

bitnami/airflow/values.yaml

@@ -120,8 +120,9 @@ airflow:
   ##
   cloneDagFilesFromGit:
     enabled: true
-    repository: https://github.com/tompizmor/sample-airflow-dag
-    revision: master
+    # repository:
+    # branch:
+    # interval:
   ## URL used to access to airflow web ui
   ##
   # baseUrl: