[bitnami/airflow] Add support for plugins loaded from a git repo (#1898)

* added support for loading plugins from a git repository * mount plugins from git in scheduler and worker as well * added missing values to values-production.yaml
2026-03-11 07:17:45 +08:00 · 2020-02-11 08:46:40 +01:00
parent 5eda19cb34
commit f3cbda95e4
8 changed files with 164 additions and 14 deletions
--- a/bitnami/airflow/Chart.yaml
+++ b/bitnami/airflow/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 name: airflow
-version: 4.2.1
+version: 4.3.0
 appVersion: 1.10.9
 description: Apache Airflow is a platform to programmatically author, schedule and monitor workflows.
 keywords:
--- a/bitnami/airflow/README.md
+++ b/bitnami/airflow/README.md
@@ -81,11 +81,15 @@ The following tables lists the configurable parameters of the Kafka chart and th
 | `airflow.configurationConfigMap`          | Name of an existing config map containing the Airflow config file                                    | `nil`                                                        |
 | `airflow.dagsConfigMap`                   | Name of an existing config map containing all the DAGs files you want to load in Airflow.            | `nil`                                                        |
 | `airflow.loadExamples`                    | Switch to load some Airflow examples                                                                 | `true`                                                       |
+| `airflow.gitSyncInterval`                 | Interval (in seconds) to pull the git repository containing the plugins and/or DAG files             | `60`                                                         |
 | `airflow.cloneDagFilesFromGit.enabled`    | Enable in order to download DAG files from git repository.                                           | `false`                                                      |
 | `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from                                                             | `nil`                                                        |
 | `airflow.cloneDagFilesFromGit.branch`     | Branch from repository to checkout                                                                   | `nil`                                                        |
-| `airflow.cloneDagFilesFromGit.interval`   | Interval to pull the repository on sidecar container                                                 | `nil`                                                        |
 | `airflow.cloneDagFilesFromGit.path`       | Path to a folder in the repository containing DAGs. If not set, all DAGS from the repo are loaded.   | `nil`                                                        |
+| `airflow.clonePluginsFromGit.enabled`     | Enable in order to download plugins from git repository.                                             | `false`                                                      |
+| `airflow.clonePluginsFromGit.repository`  | Repository where download plugins from                                                               | `nil`                                                        |
+| `airflow.clonePluginsFromGit.branch`      | Branch from repository to checkout                                                                   | `nil`                                                        |
+| `airflow.clonePluginsFromGit.path`        | Path to a folder in the repository containing the plugins.                                           | `nil`                                                        |
 | `airflow.baseUrl`                         | URL used to access to airflow web ui                                                                 | `nil`                                                        |
 | `airflow.worker.port`                     | Airflow Worker port                                                                                  | `8793`                                                       |
 | `airflow.worker.replicas`                 | Number of Airflow Worker replicas                                                                    | `2`                                                          |
@@ -186,24 +190,28 @@ Bitnami will release a new chart updating its containers if a new version of the
 This chart includes a `values-production.yaml` file where you can find some parameters oriented to production configuration in comparison to the regular `values.yaml`. You can use this file instead of the default one.

 - URL used to access to airflow web ui:
+
 ```diff
 - # airflow.baseUrl:
 + airflow.baseUrl: http://airflow.local
 ```

 - Number of Airflow Worker replicas:
+
 ```diff
 - airflow.worker.replicas: 1
 + airflow.worker.replicas: 3
 ```

 - Force users to specify a password:
+
 ```diff
 - airflow.auth.forcePassword: false
 + airflow.auth.forcePassword: true
 ```

 - Enable ingress controller resource:
+
 ```diff
 - ingress.enabled: false
 + ingress.enabled: true
@@ -235,7 +243,17 @@ You can store all your DAG files on a GitHub repository and then clone to the Ai
 airflow.cloneDagFilesFromGit.enabled=true
 airflow.cloneDagFilesFromGit.repository=https://github.com/USERNAME/REPOSITORY
 airflow.cloneDagFilesFromGit.branch=master
-airflow.cloneDagFilesFromGit.interval=60
+```
+
+### Loading Plugins
+
+You can load plugins into the chart by specifying a git repository containing the plugin files. The repository will be periodically updated using a sidecar container. In order to do that, you can deploy airflow with the following options:
+
+```console
+airflow.clonePluginsFromGit.enabled=true
+airflow.clonePluginsFromGit.repository=https://github.com/teamclairvoyant/airflow-rest-api-plugin.git
+airflow.clonePluginsFromGit.branch=v1.0.9-branch
+airflow.clonePluginsFromGit.path=plugins
 ```

 ## Persistence
--- a/bitnami/airflow/templates/_helpers.tpl
+++ b/bitnami/airflow/templates/_helpers.tpl
@@ -266,6 +266,23 @@ airflow: airflow.cloneDagFilesFromGit.branch
 {{- end -}}
 {{- end -}}

+{{/* Validate values of Airflow - "airflow.clonePluginsFromGit.repository" must be provided when "airflow.clonePluginsFromGit.enabled" is "true" */}}
+{{- define "airflow.validateValues.clonePluginsFromGit.repository" -}}
+{{- if and .Values.airflow.clonePluginsFromGit.enabled (empty .Values.airflow.clonePluginsFromGit.repository) -}}
+airflow: airflow.clonePluginsFromGit.repository
+    The repository must be provided when enabling downloading plugins
+    from git repository (--set airflow.clonePluginsFromGit.repository="xxx")
+{{- end -}}
+{{- end -}}
+{{/* Validate values of Airflow - "airflow.clonePluginsFromGit.branch" must be provided when "airflow.clonePluginsFromGit.enabled" is "true" */}}
+{{- define "airflow.validateValues.clonePluginsFromGit.branch" -}}
+{{- if and .Values.airflow.clonePluginsFromGit.enabled (empty .Values.airflow.clonePluginsFromGit.branch) -}}
+airflow: airflow.clonePluginsFromGit.branch
+    The branch must be provided when enabling downloading plugins
+    from git repository (--set airflow.clonePluginsFromGit.branch="xxx")
+{{- end -}}
+{{- end -}}
+
 {{/* Check if there are rolling tags in the images */}}
 {{- define "airflow.checkRollingTags" -}}
 {{- if and (contains "bitnami/" .Values.image.repository) (not (.Values.image.tag | toString | regexFind "-r\\d+$|sha256:")) }}
--- a/bitnami/airflow/templates/deployment-scheduler.yaml
+++ b/bitnami/airflow/templates/deployment-scheduler.yaml
@@ -45,7 +45,7 @@ spec:
        fsGroup: {{ .Values.securityContext.fsGroup }}
        runAsUser: {{ .Values.securityContext.runAsUser }}
      {{- end }}
-      {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+      {{- if or .Values.airflow.cloneDagFilesFromGit.enabled .Values.airflow.clonePluginsFromGit.enabled }}
      initContainers:
        - name: git-clone-repository
          image: "{{ template "git.image" . }}"
@@ -54,10 +54,21 @@ spec:
            - /bin/bash
            - -ec
            - |
-              git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+                git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- end }}
+              {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                git clone {{ .Values.airflow.clonePluginsFromGit.repository }} --branch {{ .Values.airflow.clonePluginsFromGit.branch }} /plugins
+              {{- end }}
          volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
            - name: git-cloned-dag-files
              mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
      containers:
        - name: git-repo-syncer
          image: "{{ template "git.image" . }}"
@@ -67,12 +78,23 @@ spec:
            - -ec
            - |
              while true; do
+                  {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
                  cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
-                  sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }}
+                  {{- end }}
+                  {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                  cd /plugins && git pull origin {{ .Values.airflow.clonePluginsFromGit.branch }}
+                  {{- end }}
+                  sleep {{ default "60" .Values.airflow.gitSyncInterval }}
              done
          volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
            - name: git-cloned-dag-files
              mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
      {{- else }}
      containers:
      {{- end }}
@@ -188,6 +210,13 @@ spec:
              subPath: {{ .Values.airflow.cloneDagFilesFromGit.path }}
              {{- end }}
          {{- end }}
+          {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /opt/bitnami/airflow/plugins
+              {{- if .Values.airflow.clonePluginsFromGit.path }}
+              subPath: {{ .Values.airflow.clonePluginsFromGit.path }}
+              {{- end }}
+          {{- end }}
          {{- if .Values.airflow.configurationConfigMap }}
            - name: custom-configuration-file
              mountPath: /opt/bitnami/airflow/airflow.cfg
@@ -209,6 +238,10 @@ spec:
        - name: git-cloned-dag-files
          emptyDir: {}
      {{- end }}
+      {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+        - name: git-cloned-plugins
+          emptyDir: {}
+      {{- end }}
      {{- if .Values.airflow.configurationConfigMap }}
        - name: custom-configuration-file
          configMap:
--- a/bitnami/airflow/templates/deployment-web.yaml
+++ b/bitnami/airflow/templates/deployment-web.yaml
@@ -45,7 +45,7 @@ spec:
        fsGroup: {{ .Values.securityContext.fsGroup }}
        runAsUser: {{ .Values.securityContext.runAsUser }}
      {{- end }}
-      {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+      {{- if or .Values.airflow.cloneDagFilesFromGit.enabled .Values.airflow.clonePluginsFromGit.enabled }}
      initContainers:
        - name: git-clone-repository
          image: "{{ template "git.image" . }}"
@@ -54,10 +54,21 @@ spec:
            - /bin/bash
            - -ec
            - |
-              git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+                git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- end }}
+              {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                git clone {{ .Values.airflow.clonePluginsFromGit.repository }} --branch {{ .Values.airflow.clonePluginsFromGit.branch }} /plugins
+              {{- end }}
          volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
            - name: git-cloned-dag-files
              mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
      containers:
        - name: git-repo-syncer
          image: "{{ template "git.image" . }}"
@@ -67,12 +78,23 @@ spec:
            - -ec
            - |
              while true; do
+                  {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
                  cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
-                  sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }}
+                  {{- end }}
+                  {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                  cd /plugins && git pull origin {{ .Values.airflow.clonePluginsFromGit.branch }}
+                  {{- end }}
+                  sleep {{ default "60" .Values.airflow.gitSyncInterval }}
              done
          volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
            - name: git-cloned-dag-files
              mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
      {{- else }}
      containers:
      {{- end }}
@@ -218,6 +240,13 @@ spec:
              subPath: {{ .Values.airflow.cloneDagFilesFromGit.path }}
              {{- end }}
            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /opt/bitnami/airflow/plugins
+              {{- if .Values.airflow.clonePluginsFromGit.path }}
+              subPath: {{ .Values.airflow.clonePluginsFromGit.path }}
+              {{- end }}
+            {{- end }}
            {{- if .Values.airflow.configurationConfigMap }}
            - name: custom-configuration-file
              mountPath: /opt/bitnami/airflow/airflow.cfg
@@ -244,6 +273,10 @@ spec:
        - name: git-cloned-dag-files
          emptyDir: {}
        {{- end }}
+        {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+        - name: git-cloned-plugins
+          emptyDir: {}
+        {{- end }}
        {{- if .Values.airflow.configurationConfigMap }}
        - name: custom-configuration-file
          configMap:
--- a/bitnami/airflow/templates/statefulset-worker.yaml
+++ b/bitnami/airflow/templates/statefulset-worker.yaml
@@ -46,7 +46,7 @@ spec:
      {{- if .Values.affinity }}
      affinity: {{- toYaml .Values.affinity | nindent 8 }}
      {{- end }}
-      {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+      {{- if or .Values.airflow.cloneDagFilesFromGit.enabled .Values.airflow.clonePluginsFromGit.enabled }}
      initContainers:
        - name: git-clone-repository
          image: "{{ template "git.image" . }}"
@@ -55,10 +55,21 @@ spec:
            - /bin/bash
            - -ec
            - |
-              git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+                git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- end }}
+              {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                git clone {{ .Values.airflow.clonePluginsFromGit.repository }} --branch {{ .Values.airflow.clonePluginsFromGit.branch }} /plugins
+              {{- end }}
          volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
            - name: git-cloned-dag-files
              mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
      containers:
        - name: git-repo-syncer
          image: "{{ template "git.image" . }}"
@@ -68,12 +79,23 @@ spec:
            - -ec
            - |
              while true; do
+                  {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
                  cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
-                  sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }}
+                  {{- end }}
+                  {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                  cd /plugins && git pull origin {{ .Values.airflow.clonePluginsFromGit.branch }}
+                  {{- end }}
+                  sleep {{ default "60" .Values.airflow.gitSyncInterval }}
              done
          volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
            - name: git-cloned-dag-files
              mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
      {{- else }}
      containers:
      {{- end }}
@@ -201,6 +223,13 @@ spec:
              subPath: {{ .Values.airflow.cloneDagFilesFromGit.path }}
              {{- end }}
            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /opt/bitnami/airflow/plugins
+              {{- if .Values.airflow.clonePluginsFromGit.path }}
+              subPath: {{ .Values.airflow.clonePluginsFromGit.path }}
+              {{- end }}
+            {{- end }}
            {{- if .Values.airflow.configurationConfigMap }}
            - name: custom-configuration-file
              mountPath: /opt/bitnami/airflow/airflow.cfg
@@ -221,6 +250,10 @@ spec:
      - name: git-cloned-dag-files
        emptyDir: {}
      {{- end }}
+      {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+      - name: git-cloned-plugins
+        emptyDir: {}
+      {{- end }}
      {{- if .Values.airflow.configurationConfigMap }}
      - name: custom-configuration-file
        configMap:
--- a/bitnami/airflow/values-production.yaml
+++ b/bitnami/airflow/values-production.yaml
@@ -125,13 +125,21 @@ airflow:
  ## Airflow generic configuration
  ##
  loadExamples: false
+  
+  ## Interval to pull the git repository containing the plugins and/or DAG files
+  #
+  gitSyncInterval: 60  
  ## Enable in order to download DAG files from git repository.
  ##
  cloneDagFilesFromGit:
    enabled: false
    # repository:
    # branch:
-    # interval:
+    # path:
+  clonePluginsFromGit:
+    enabled: false
+    # repository:
+    # branch:
    # path:
  ## URL used to access to airflow web ui
  ##
--- a/bitnami/airflow/values.yaml
+++ b/bitnami/airflow/values.yaml
@@ -125,13 +125,21 @@ airflow:
  ## Airflow generic configuration
  ##
  loadExamples: false
+
+  ## Interval to pull the git repository containing the plugins and/or DAG files
+  #
+  gitSyncInterval: 60
  ## Enable in order to download DAG files from git repository.
  ##
  cloneDagFilesFromGit:
    enabled: false
    # repository:
    # branch:
-    # interval:
+    # path:
+  clonePluginsFromGit:
+    enabled: false
+    # repository:
+    # branch:
    # path:
  ## URL used to access to airflow web ui
  ##