Update NOTES.txt and other fixes

bitnami/airflow/README.md

@@ -80,9 +80,10 @@ The following tables lists the configurable parameters of the Kafka chart and th
 | `airflow.cloneDagFilesFromGit.enabled`    | Enable in order to download DAG files from git repository.                                  | `false`                                                 |
 | `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from                                                    | `nil`                                                   |
 | `airflow.cloneDagFilesFromGit.revision`   | Revision from repository to checkout                                                        | `nil`                                                   |
-| `airflow.baseUrl`                     | URL used to access to airflow web ui                                                        | `nil`                                                   |
+| `airflow.baseUrl`                         | URL used to access to airflow web ui                                                        | `nil`                                                   |
 | `airflow.worker.port`                     | Airflow Worker port                                                                         | `8793`                                                  |
 | `airflow.worker.replicas`                 | Number of Airflow Worker replicas                                                           | `2`                                                     |
+| `airflow.auth.forcePassword`              | Force users to specify a password                                                           | `false`                                                 |
 | `airflow.auth.airflowUsername`            | Username to access web UI                                                                   | `user`                                                  |
 | `airflow.auth.airflowPassword`            | Password to access web UI                                                                   | `nil`                                                   |
 | `airflow.auth.fernetKey`                  | Fernet key to secure connections                                                            | `nil`                                                   |

bitnami/airflow/requirements.lock

@@ -1,9 +1,9 @@
 dependencies:
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
-  version: 3.18.2
+  version: 3.18.3
 - name: redis
   repository: https://charts.bitnami.com/bitnami
-  version: 6.4.5
+  version: 7.0.1
-digest: sha256:f8531d43c9fce57c6c4c396d76bc752cc0065783bb18cba3b6bae50558e1a473
+digest: sha256:cf902ebdc6a871b1414ad64c95abb2fecf188b6cb0209bb91995620f84ca35ca
-generated: 2019-04-23T11:12:57.062256486+02:00
+generated: 2019-05-02T11:16:02.72367469+02:00

bitnami/airflow/requirements.yaml

@@ -4,6 +4,6 @@ dependencies:
   repository: https://charts.bitnami.com/bitnami
   condition: postgresql.enabled
 - name: redis
-  version: 6.x.x
+  version: 7.x.x
   repository: https://charts.bitnami.com/bitnami
   condition: redis.enabled

bitnami/airflow/templates/NOTES.txt

@@ -1,22 +1,78 @@
+{{- if not .Values.airflow.baseUrl -}}
+###############################################################################
+### ERROR: You did not provide an external URL in your 'helm install' call ###
+###############################################################################
+
+This deployment will be incomplete until you configure Airflow with a resolvable
+host. To configure Airflow with the URL of your service:
+
+{{- if .Values.ingress.enabled }}
+
+1. Get the Airflow URL indicated on the Ingress Rule and associate it to your cluster external IP:
+
+   export CLUSTER_IP=$(minikube ip) # On Minikube. Use: `kubectl cluster-info` on others K8s clusters
+   export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath='{.spec.rules[0].host}')
+   echo "Airflow URL: http://$HOSTNAME/"
+   echo "$CLUSTER_IP  $HOSTNAME" | sudo tee -a /etc/hosts
+
+{{- else }}
 1. Get the Airflow URL by running:
 
-{{- if eq .Values.service.type "ClusterIP" }}
+  {{- if contains "NodePort" .Values.service.type }}
+
+   export APP_HOST=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+   export APP_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "airflow.fullname" . }})
+
+  {{- else if contains "ClusterIP" .Values.service.type }}
+
+  export APP_HOST=127.0.0.1
+  export APP_PORT=8080
+
+  {{- else if contains "LoadBalancer" .Values.service.type }}
+
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+        Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "airflow.fullname" . }}'
+
+  export APP_HOST=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} --template "{{ "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}" }}")
+  export APP_PORT=80
+  {{- end }}
+  export APP_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode)
+  export APP_DATABASE_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.postgresql.fullname" . }} -o jsonpath="{.data.postgresql-password}" | base64 --decode)
+  export APP_REDIS_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.redis.secretName" . }} -o jsonpath="{.data.redis-password}" | base64 --decode)
+
+{{- end }}
+2. Complete your Airflow deployment by running:
+
+  helm upgrade {{ .Release.Name }} bitnami/{{ .Chart.Name }} \
+    --set service.type={{ .Values.service.type }},airflow.baseUrl=http://$APP_HOST:$APP_PORT,airflow.auth.airflowPassword=$APP_PASSWORD,postgresql.postgresqlPassword=$APP_DATABASE_PASSWORD,redis.password=$APP_REDIS_PASSWORD{{- if .Values.global }}{{- if .Values.global.imagePullSecrets }},global.imagePullSecrets={{ .Values.global.imagePullSecrets }}{{- end }}{{- end }}
+
+{{- else -}}
+
+1. Get the Airflow URL by running:
+
+{{- if .Values.ingress.enabled }}
+
+  echo URL  : {{ .Values.airflow.baseUrl }}
+
+{{- else if eq .Values.service.type "ClusterIP" }}
 
   echo URL  : http://127.0.0.1:{{ default "8080" .Values.service.port }}
   kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "airflow.fullname" . }} {{ default "8080" .Values.service.port }}:8080
 
-{{- else if eq .Values.service.type "NodePort" }}
+{{- else if eq .Values.servicePort "nodePort" }}
-  export APP_BASE_URL=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  export APP_HOST=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
   export APP_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "airflow.fullname" . }})
 
-  echo URL  : $APP_BASE_URL
+  echo URL  : http://$APP_HOST:$APP_PORT
 
 {{- else }}
 
-  echo URL  : http://{{ .Values.airflow.baseUrl }}
+  echo URL  : {{ .Values.airflow.baseUrl }}
 {{- end }}
 
 2. Get your Airflow login credentials by running:
 
   echo User:    {{ .Values.airflow.auth.airflowUsername }}
-  echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode)
+  echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode)
+
+  {{- end  }}

bitnami/airflow/templates/deployment-web.yaml

@@ -141,7 +141,7 @@ spec:
         - name: AIRFLOW_WEBSERVER_HOST
           value: {{ template "airflow.fullname" . }}
           {{- if .Values.airflow.baseUrl }}
-        - name: AIRFLOW_BASE_URl
+        - name: AIRFLOW_BASE_URL
           value: {{ .Values.airflow.baseUrl }}
           {{- end }}
         - name: AIRFLOW_LOAD_EXAMPLES

bitnami/airflow/templates/secrets.yaml

@@ -12,12 +12,16 @@ type: Opaque
 data:
   {{- if .Values.airflow.auth.airflowPassword }}
   airflow-password: {{ .Values.airflow.auth.airflowPassword | b64enc | quote }}
-  {{ else }}
+  {{ else if (not .Values.airflow.auth.forcePassword) }}
   airflow-password: {{ randAlphaNum 10 | b64enc | quote }}
+  {{ else }}
+  airflow-password: {{ required "An Airflow Password is required!" .Values.airflow.auth.airflowPassword }}
   {{- end }}
   {{- if .Values.airflow.auth.fernetKey }}
   airflow-fernetKey: {{ .Values.airflow.auth.fernetKey | b64enc | quote }}
-  {{ else }}
+  {{ else if (not .Values.airflow.auth.forcePassword) }}
   airflow-fernetKey: {{ randAlphaNum 32 | b64enc | b64enc | quote }}
+  {{ else }}
+  airflow-fernetKey: {{ required "An Airflow Fernet Key is required!" .Values.airflow.auth.fernetKey }}
   {{- end }}
 {{- end }}

bitnami/airflow/templates/svc-headless-worker.yaml

@@ -11,9 +11,9 @@ spec:
   type: ClusterIP
   clusterIP: None
   ports:
-  - name: airflow-worker
+  - name: worker
     port: 8793
-    targetPort: airflow-worker
+    targetPort: worker
   selector:
     app.kubernetes.io/name: {{ include "airflow.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}

bitnami/airflow/values-production.yaml

@@ -0,0 +1,323 @@
+## Global Docker image parameters
+## Please, note that this will override the image parameters, including dependencies, configured to use the global value
+## Current available global Docker image parameters: imageRegistry and imagePullSecrets
+##
+# global:
+#   imageRegistry: myRegistryName
+#   imagePullSecrets:
+#     - myRegistryKeySecretName
+
+## Bitnami Airflow image version
+## ref: https://hub.docker.com/r/bitnami/airflow/tags/
+##
+image:
+  registry: docker.io
+  repository: bitnami/airflow
+  tag: 1.10.3
+  ## Specify a imagePullPolicy
+  ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+  ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+  ##
+  pullPolicy: Always
+  ## Optionally specify an array of imagePullSecrets.
+  ## Secrets must be manually created in the namespace.
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ##
+  # pullSecrets:
+  #   - myRegistryKeySecretName
+
+  ## Set to true if you would like to see extra information on logs
+  ## It turns BASH and NAMI debugging in minideb
+  ## ref:  https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging
+  debug: false
+
+## Bitnami Airflow Scheduler image version
+## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/
+##
+schedulerImage:
+  registry: docker.io
+  repository: bitnami/airflow-scheduler
+  tag: 1.10.3
+  ## Specify a imagePullPolicy
+  ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+  ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+  ##
+  pullPolicy: Always
+  ## Optionally specify an array of imagePullSecrets.
+  ## Secrets must be manually created in the namespace.
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ##
+  # pullSecrets:
+  #   - myRegistryKeySecretName
+
+  ## Set to true if you would like to see extra information on logs
+  ## It turns BASH and NAMI debugging in minideb
+  ## ref:  https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging
+  debug: false
+
+## Bitnami Airflow Worker image version
+## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/
+##
+workerImage:
+  registry: docker.io
+  repository: bitnami/airflow-worker
+  tag: 1.10.3
+  ## Specify a imagePullPolicy
+  ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+  ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+  ##
+  pullPolicy: Always
+  ## Optionally specify an array of imagePullSecrets.
+  ## Secrets must be manually created in the namespace.
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ##
+  # pullSecrets:
+  #   - myRegistryKeySecretName
+
+  ## Set to true if you would like to see extra information on logs
+  ## It turns BASH and NAMI debugging in minideb
+  ## ref:  https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging
+  debug: false
+
+## Bitnami git image version
+## ref: https://hub.docker.com/r/bitnami/git/tags/
+##
+git:
+  registry: docker.io
+  repository: bitnami/git
+  tag: latest
+  pullPolicy: IfNotPresent
+  ## Optionally specify an array of imagePullSecrets.
+  ## Secrets must be manually created in the namespace.
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ##
+  # pullSecrets:
+  #   - myRegistryKeySecretName
+
+## StatefulSet controller supports automated updates. There are two valid update strategies: RollingUpdate and OnDelete
+## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets
+##
+updateStrategy: RollingUpdate
+
+## Partition update strategy
+## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions
+##
+# rollingUpdatePartition:
+
+## Airflow Components configuration
+##
+airflow:
+  ## Name of an existing config map containing the Airflow config file
+  ##
+  # airflowConfigurationConfigMap:
+  ## Name of an existing config map containing all the DAGs files you want to load in Airflow.
+  ##
+  # dagsConfigMap:
+  ## Airflow generic configuration
+  ##
+  loadExamples: false
+  ## Enable in order to download DAG files from git repository.
+  ##
+  cloneDagFilesFromGit:
+    enabled: false
+    repository:
+    revision:
+  ## URL used to access to airflow web ui
+  ##
+  baseUrl: http://airflow.local
+  ## Airflow worker component configuration
+  worker:
+    port: 8793
+    replicas: 3
+
+  auth:
+    ## Option to force users to specify a password. That is required for 'helm upgrade' to work properly.
+    ## If it is not forced, a random password will be generated.
+    ##
+    forcePassword: true
+    ## Specify username to acccess web UI
+    ##
+    airflowUsername: user
+    ## Specify password to acccess web UI
+    ##
+    # airflowPassword:
+    ## Specify a fernet key to secure airflow connections
+    ## Fernet key must be 32 url-safe base64-encoded bytes.
+    ## More info at:
+    ## - https://airflow.readthedocs.io/en/stable/howto/secure-connections.html
+    ## - https://bcb.github.io/airflow/fernet-key
+    ##
+    # fernetKey:
+    ## Use existing secret (ignores previous airflowPassword and fernetKey)
+    ##
+    # existingSecret:
+
+  ## All the parameters from the configuration file can be overwritten by using environment variables with this format:
+  ## AIRFLOW__{SECTION}__{KEY}. Note the double underscores. More info at https://airflow.readthedocs.io/en/stable/howto/set-config.html
+  ##
+  # extraEnvVars:
+  #   - name: AIRFLOW__SMTP__SMTP_HOST
+  #     value: "localhost"
+  #   - name: AIRFLOW__SMTP__SMTP_PORT
+  #     value: "25"
+  #   - name: AIRFLOW__SMTP__USER
+  #     value: ""
+
+## Kubernetes Security Context
+## https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+##
+securityContext:
+  enabled: true
+  fsGroup: 1001
+  runAsUser: 1001
+
+## Kubernetes configuration
+## For minikube, set this to NodePort, elsewhere use LoadBalancer
+##
+service:
+  type: ClusterIP
+  port: 8080
+
+  ## Specify the NodePort value for the LoadBalancer and NodePort service types.
+  ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
+  ##
+  # nodePort:
+
+  ## Use loadBalancerIP to request a specific static IP,
+  # loadBalancerIP:
+
+  ## Service annotations done as key:value pairs
+  annotations:
+
+## Configure the ingress resource that allows you to access the
+## Airflow installation. Set up the URL
+## ref: http://kubernetes.io/docs/user-guide/ingress/
+##
+ingress:
+  ## Set to true to enable ingress record generation
+  enabled: true
+
+  ## Set this to true in order to add the corresponding annotations for cert-manager
+  certManager: false
+
+  ## Ingress annotations done as key:value pairs
+  ## For a full list of possible ingress annotations, please see
+  ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/annotations.md
+  ##
+  ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set
+  ## If certManager is set to true, annotation kubernetes.io/tls-acme: "true" will automatically be set
+  annotations:
+  #  kubernetes.io/ingress.class: nginx
+
+  ## The list of hostnames to be covered with this ingress record.
+  ## Most likely this will be just one host, but in the event more hosts are needed, this is an array
+  hosts:
+  - name: airflow.local
+    path: /
+
+  ## The tls configuration for the ingress
+  ## see: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls
+  tls:
+  - hosts:
+      - airflow.local
+    secretName: airflow.local-tls
+
+  secrets:
+  ## If you're providing your own certificates, please use this to add the certificates as secrets
+  ## key and certificate should start with -----BEGIN CERTIFICATE----- or
+  ## -----BEGIN RSA PRIVATE KEY-----
+  ##
+  ## name should line up with a tlsSecret set further up
+  ## If you're using cert-manager, this is unneeded, as it will create the secret for you if it is not set
+  ##
+  ## It is also possible to create and manage the certificates outside of this helm chart
+  ## Please see README.md for more information
+  # - name: airflow.local-tls
+  #   key:
+  #   certificate:
+
+## Node labels for pod assignment
+## Ref: https://kubernetes.io/docs/user-guide/node-selection/
+##
+nodeSelector: {}
+
+## Tolerations for pod assignment
+## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+##
+tolerations: []
+
+## Affinity for pod assignment
+## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+##
+affinity: {}
+
+## Configure resource requests and limits
+## ref: http://kubernetes.io/docs/user-guide/compute-resources/
+##
+resources:
+#  limits:
+#    cpu: 200m
+#    memory: 1Gi
+#  requests:
+#    memory: 256Mi
+#    cpu: 250m
+
+## Configure extra options for liveness and readiness probes
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes)
+livenessProbe:
+  enabled: true
+  initialDelaySeconds: 180
+  periodSeconds: 20
+  timeoutSeconds: 5
+  failureThreshold: 6
+  successThreshold: 1
+
+readinessProbe:
+  enabled: true
+  initialDelaySeconds: 30
+  periodSeconds: 10
+  timeoutSeconds: 5
+  failureThreshold: 6
+  successThreshold: 1
+
+##
+## PostgreSQL chart configuration
+##
+## https://github.com/helm/charts/blob/master/stable/postgresql/values.yaml
+##
+postgresql:
+  enabled: true
+  postgresqlUsername: bn_airflow
+  #postgresqlPassword:
+  postgresqlDatabase: bitnami_airflow
+
+externalDatabase:
+  ## All of these values are only used when postgresql.enabled is set to false
+  ## Database host
+  host: localhost
+  ## non-root Username for Airflow Database
+  user: bn_airflow
+  ## Database password
+  password: ""
+  ## Database name
+  database: bitnami_airflow
+  ## Database port number
+  port: 5432
+
+##
+## Redis chart configuration
+##
+## https://github.com/helm/charts/blob/master/stable/redis/values.yaml
+##
+redis:
+  enabled: true
+  # password: ""
+
+externalRedis:
+  ## All of these values are only used when redis.enabled is set to false
+  ## Redis host
+  host: localhost
+  ## Redis port number
+  port: 6379
+  ## Redis password
+  password: ""

bitnami/airflow/values.yaml

@@ -128,9 +128,13 @@ airflow:
   ## Airflow worker component configuration
   worker:
     port: 8793
-    replicas: 2
+    replicas: 1
 
   auth:
+    ## Option to force users to specify a password. That is required for 'helm upgrade' to work properly.
+    ## If it is not forced, a random password will be generated.
+    ##
+    forcePassword: false
     ## Specify username to acccess web UI
     ##
     airflowUsername: user
@@ -138,7 +142,10 @@ airflow:
     ##
     # airflowPassword:
     ## Specify a fernet key to secure airflow connections
-    ## More info at: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html
+    ## Fernet key must be 32 url-safe base64-encoded bytes.
+    ## More info at:
+    ## - https://airflow.readthedocs.io/en/stable/howto/secure-connections.html
+    ## - https://bcb.github.io/airflow/fernet-key
     ##
     # fernetKey:
     ## Use existing secret (ignores previous airflowPassword and fernetKey)
@@ -304,6 +311,7 @@ externalDatabase:
 ##
 redis:
   enabled: true
+  # password: ""
 
 externalRedis:
   ## All of these values are only used when redis.enabled is set to false