From 865f21847041d78b34edf6d679505f696ba74c3d Mon Sep 17 00:00:00 2001 From: tompizmor Date: Wed, 15 May 2019 09:16:24 +0200 Subject: [PATCH] [bitnami/kakfa] Add option to use existingClaim Signed-off-by: tompizmor --- bitnami/kafka/Chart.yaml | 2 +- bitnami/kafka/README.md | 215 ++++++++++++----------- bitnami/kafka/templates/statefulset.yaml | 18 +- bitnami/kafka/values-production.yaml | 6 + bitnami/kafka/values.yaml | 6 + 5 files changed, 134 insertions(+), 113 deletions(-) diff --git a/bitnami/kafka/Chart.yaml b/bitnami/kafka/Chart.yaml index 69a4479e8b..7e8faed5eb 100644 --- a/bitnami/kafka/Chart.yaml +++ b/bitnami/kafka/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: kafka -version: 2.1.0 +version: 2.2.0 appVersion: 2.2.0 description: Apache Kafka is a distributed streaming platform. keywords: diff --git a/bitnami/kafka/README.md b/bitnami/kafka/README.md index 139ca067d7..600b97544b 100644 --- a/bitnami/kafka/README.md +++ b/bitnami/kafka/README.md @@ -45,113 +45,114 @@ The command removes all the Kubernetes components associated with the chart and The following tables lists the configurable parameters of the Kafka chart and their default values. -| Parameter | Description | Default | -| -------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | -| `global.imageRegistry` | Global Docker image registry | `nil` | -| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | -| `image.registry` | Kafka image registry | `docker.io` | -| `image.repository` | Kafka Image name | `bitnami/kafka` | -| `image.tag` | Kafka Image tag | `{VERSION}` | -| `image.pullPolicy` | Kafka image pull policy | `Always` | -| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | -| `image.debug` | Specify if debug values should be set | `false` | -| `updateStrategy` | Update strategy for the stateful set | `RollingUpdate` | -| `rollingUpdatePartition` | Partition update strategy | `nil` | -| `podDisruptionBudget.maxUnavailable` | Max number of pods down simultaneously | `1` | -| `replicaCount` | Number of Kafka nodes | `1` | -| `config` | Configuration file for Kafka | `nil` | -| `allowPlaintextListener` | Allow to use the PLAINTEXT listener | `true` | -| `listeners` | The address the socket server listens on. | `nil` | -| `advertisedListeners` | Hostname and port the broker will advertise to producers and consumers. | `nil` | -| `brokerId` | ID of the Kafka node | `-1` | -| `deleteTopicEnable` | Switch to enable topic deletion or not. | `false` | -| `heapOpts` | Kafka's Java Heap size. | `-Xmx1024m -Xms1024m` | -| `logFlushIntervalMessages` | The number of messages to accept before forcing a flush of data to disk. | `10000` | -| `logFlushIntervalMs` | The maximum amount of time a message can sit in a log before we force a flush. | `1000` | -| `logRetentionBytes` | A size-based retention policy for logs. | `_1073741824` | -| `logRetentionCheckIntervalMs` | The interval at which log segments are checked to see if they can be deleted. | `300000` | -| `logRetentionHours` | The minimum age of a log file to be eligible for deletion due to age. | `168` | -| `logSegmentBytes` | The maximum size of a log segment file. When this size is reached a new log segment will be created. | `_1073741824` | -| `logMessageFormatVersion` | Logging message format version. | `` | -| `logsDirs` | A comma separated list of directories under which to store log files. | `/opt/bitnami/kafka/data` | -| `maxMessageBytes` | The largest record batch size allowed by Kafka. | `1000012` | -| `defaultReplicationFactor` | Default replication factors for automatically created topics | `1` | -| `offsetsTopicReplicationFactor` | The replication factor for the offsets topic | `1` | -| `transactionStateLogReplicationFactor `| The replication factor for the transaction topic | `1` | -| `transactionStateLogMinIsr ` | Overridden min.insync.replicas config for the transaction topic | `1` | -| `numIoThreads` | The number of threads doing disk I/O. | `8` | -| `numNetworkThreads` | The number of threads handling network requests. | `3` | -| `numPartitions` | The default number of log partitions per topic. | `1` | -| `numRecoveryThreadsPerDataDir` | The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. | `1` | -| `socketReceiveBufferBytes` | The receive buffer (SO_RCVBUF) used by the socket server. | `102400` | -| `socketRequestMaxBytes` | The maximum size of a request that the socket server will accept (protection against OOM). | `_104857600` | -| `socketSendBufferBytes` | The send buffer (SO_SNDBUF) used by the socket server. | `102400` | -| `zookeeperConnectionTimeoutMs` | Timeout in ms for connecting to zookeeper. | `6000` | -| `sslEndpointIdentificationAlgorithm` | The endpoint identification algorithm to validate server hostname using server certificate. | `https` | -| `auth.enabled` | Switch to enable the kafka authentication. | `false` | -| `auth.existingSecret` | Name of the existing secret containing credentials for brokerUser, interBrokerUser and zookeeperUser. | `nil` | -| `auth.certificatesSecret` | Name of the existing secret containing the certificate files that will be used by Kafka. | `nil` | -| `auth.certificatesPassword` | Password for the above certificates if they are password protected. | `nil` | -| `auth.brokerUser` | Kafka client user. | `user` | -| `auth.brokerPassword` | Kafka client password. | `nil` | -| `auth.interBrokerUser` | Kafka inter broker communication user | `admin` | -| `auth.interBrokerPassword` | Kafka inter broker communication password. | `nil` | -| `auth.zookeeperUser` | Kafka Zookeeper user. | `nil` | -| `auth.zookeeperPassword` | Kafka Zookeeper password. | `nil` | -| `securityContext.enabled` | Enable security context | `true` | -| `securityContext.fsGroup` | Group ID for the container | `1001` | -| `securityContext.runAsUser` | User ID for the container | `1001` | -| `service.type` | Kubernetes Service type | `ClusterIP` | -| `service.port` | Kafka port | `9092` | -| `service.nodePort` | Kubernetes Service nodePort | `nil` | -| `service.loadBalancerIP` | loadBalancerIP for Kafka Service | `nil` | -| `service.annotations` | Service annotations | `` | -| `persistence.enabled` | Enable persistence using PVC | `true` | -| `persistence.storageClass` | PVC Storage Class for Kafka volume | `nil` | -| `persistence.accessMode` | PVC Access Mode for Kafka volume | `ReadWriteOnce` | -| `persistence.size` | PVC Storage Request for Kafka volume | `8Gi` | -| `persistence.annotations` | Annotations for the PVC | `{}` | -| `nodeSelector` | Node labels for pod assignment | `{}` | -| `tolerations` | Toleration labels for pod assignment | `[]` | -| `affinity` | Map of node/pod affinities | `{}` | -| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | -| `livenessProbe.enabled` | would you like a livessProbed to be enabled | `true` | -| `livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 30 | -| `livenessProbe.periodSeconds` | How often to perform the probe | 10 | -| `livenessProbe.timeoutSeconds` | When the probe times out | 5 | -| `livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | -| `livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | -| `readinessProbe.enabled` | would you like a readinessProbe to be enabled | `true` | -| `readinessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 5 | -| `readinessProbe.periodSeconds` | How often to perform the probe | 10 | -| `readinessProbe.timeoutSeconds` | When the probe times out | 5 | -| `readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | -| `readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | -| `metrics.kafka.enabled` | Whether or not to create a separate Kafka exporter | `false` | -| `metrics.kafka.image.registry` | Kafka exporter image registry | `docker.io` | -| `metrics.kafka.image.repository` | Kafka exporter image name | `danielqsj/kafka-exporter` | -| `metrics.kafka.image.tag` | Kafka exporter image tag | `v1.0.1` | -| `metrics.kafka.image.pullPolicy` | Kafka exporter image pull policy | `Always` | -| `metrics.kafka.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | -| `metrics.kafka.interval` | Interval that Prometheus scrapes Kafka metrics when using Prometheus Operator | `10s` | -| `metrics.kafka.port` | Kafka Exporter Port which exposes metrics in Prometheus format for scraping | `9308` | -| `metrics.kafka.resources` | Allows setting resource limits for kafka-exporter pod | `{}` | -| `metrics.jmx.resources` | Allows setting resource limits for jmx sidecar container | `{}` | -| `metrics.jmx.enabled` | Whether or not to expose JMX metrics to Prometheus | `false` | -| `metrics.jmx.image.registry` | JMX exporter image registry | `docker.io` | -| `metrics.jmx.image.repository` | JMX exporter image name | `solsson/kafka-prometheus-jmx-exporter@sha256` | -| `metrics.jmx.image.tag` | JMX exporter image tag | `a23062396cd5af1acdf76512632c20ea6be76885dfc20cd9ff40fb23846557e8` | -| `metrics.jmx.image.pullPolicy` | JMX exporter image pull policy | `Always` | -| `metrics.jmx.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | -| `metrics.jmx.interval` | Interval that Prometheus scrapes JMX metrics when using Prometheus Operator | `10s` | -| `metrics.jmx.exporterPort` | JMX Exporter Port which exposes metrics in Prometheus format for scraping | `5556` | -| `metrics.jmx.configMap.enabled` | Enable the default ConfigMap for JMX | `true` | -| `metrics.jmx.configMap.overrideConfig` | Allows config file to be generated by passing values to ConfigMap | `{}` | -| `metrics.jmx.configMap.overrideName` | Allows setting the name of the ConfigMap to be used | `""` | -| `metrics.jmx.jmxPort` | The jmx port which JMX style metrics are exposed (note: these are not scrapeable by Prometheus) | `5555` | -| `metrics.jmx.whitelistObjectNames` | Allows setting which JMX objects you want to expose to via JMX stats to JMX Exporter | (see `values.yaml`) | -| `zookeeper.enabled` | Switch to enable or disable the Zookeeper helm chart | `true` | -| `externalZookeeper.servers` | Server or list of external zookeeper servers to use. | `nil` | +| Parameter | Description | Default | +| --------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.registry` | Kafka image registry | `docker.io` | +| `image.repository` | Kafka Image name | `bitnami/kafka` | +| `image.tag` | Kafka Image tag | `{VERSION}` | +| `image.pullPolicy` | Kafka image pull policy | `Always` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.debug` | Specify if debug values should be set | `false` | +| `updateStrategy` | Update strategy for the stateful set | `RollingUpdate` | +| `rollingUpdatePartition` | Partition update strategy | `nil` | +| `podDisruptionBudget.maxUnavailable` | Max number of pods down simultaneously | `1` | +| `replicaCount` | Number of Kafka nodes | `1` | +| `config` | Configuration file for Kafka | `nil` | +| `allowPlaintextListener` | Allow to use the PLAINTEXT listener | `true` | +| `listeners` | The address the socket server listens on. | `nil` | +| `advertisedListeners` | Hostname and port the broker will advertise to producers and consumers. | `nil` | +| `brokerId` | ID of the Kafka node | `-1` | +| `deleteTopicEnable` | Switch to enable topic deletion or not. | `false` | +| `heapOpts` | Kafka's Java Heap size. | `-Xmx1024m -Xms1024m` | +| `logFlushIntervalMessages` | The number of messages to accept before forcing a flush of data to disk. | `10000` | +| `logFlushIntervalMs` | The maximum amount of time a message can sit in a log before we force a flush. | `1000` | +| `logRetentionBytes` | A size-based retention policy for logs. | `_1073741824` | +| `logRetentionCheckIntervalMs` | The interval at which log segments are checked to see if they can be deleted. | `300000` | +| `logRetentionHours` | The minimum age of a log file to be eligible for deletion due to age. | `168` | +| `logSegmentBytes` | The maximum size of a log segment file. When this size is reached a new log segment will be created. | `_1073741824` | +| `logMessageFormatVersion` | Logging message format version. | `` | +| `logsDirs` | A comma separated list of directories under which to store log files. | `/opt/bitnami/kafka/data` | +| `maxMessageBytes` | The largest record batch size allowed by Kafka. | `1000012` | +| `defaultReplicationFactor` | Default replication factors for automatically created topics | `1` | +| `offsetsTopicReplicationFactor` | The replication factor for the offsets topic | `1` | +| `transactionStateLogReplicationFactor ` | The replication factor for the transaction topic | `1` | +| `transactionStateLogMinIsr ` | Overridden min.insync.replicas config for the transaction topic | `1` | +| `numIoThreads` | The number of threads doing disk I/O. | `8` | +| `numNetworkThreads` | The number of threads handling network requests. | `3` | +| `numPartitions` | The default number of log partitions per topic. | `1` | +| `numRecoveryThreadsPerDataDir` | The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. | `1` | +| `socketReceiveBufferBytes` | The receive buffer (SO_RCVBUF) used by the socket server. | `102400` | +| `socketRequestMaxBytes` | The maximum size of a request that the socket server will accept (protection against OOM). | `_104857600` | +| `socketSendBufferBytes` | The send buffer (SO_SNDBUF) used by the socket server. | `102400` | +| `zookeeperConnectionTimeoutMs` | Timeout in ms for connecting to zookeeper. | `6000` | +| `sslEndpointIdentificationAlgorithm` | The endpoint identification algorithm to validate server hostname using server certificate. | `https` | +| `auth.enabled` | Switch to enable the kafka authentication. | `false` | +| `auth.existingSecret` | Name of the existing secret containing credentials for brokerUser, interBrokerUser and zookeeperUser. | `nil` | +| `auth.certificatesSecret` | Name of the existing secret containing the certificate files that will be used by Kafka. | `nil` | +| `auth.certificatesPassword` | Password for the above certificates if they are password protected. | `nil` | +| `auth.brokerUser` | Kafka client user. | `user` | +| `auth.brokerPassword` | Kafka client password. | `nil` | +| `auth.interBrokerUser` | Kafka inter broker communication user | `admin` | +| `auth.interBrokerPassword` | Kafka inter broker communication password. | `nil` | +| `auth.zookeeperUser` | Kafka Zookeeper user. | `nil` | +| `auth.zookeeperPassword` | Kafka Zookeeper password. | `nil` | +| `securityContext.enabled` | Enable security context | `true` | +| `securityContext.fsGroup` | Group ID for the container | `1001` | +| `securityContext.runAsUser` | User ID for the container | `1001` | +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | Kafka port | `9092` | +| `service.nodePort` | Kubernetes Service nodePort | `nil` | +| `service.loadBalancerIP` | loadBalancerIP for Kafka Service | `nil` | +| `service.annotations` | Service annotations | `` | +| `persistence.enabled` | Enable persistence using PVC | `true` | +| `persistence.existingClaim` | Provide an existing `PersistentVolumeClaim`, the value is evaluated as a template. | `nil` | +| `persistence.storageClass` | PVC Storage Class for Kafka volume | `nil` | +| `persistence.accessMode` | PVC Access Mode for Kafka volume | `ReadWriteOnce` | +| `persistence.size` | PVC Storage Request for Kafka volume | `8Gi` | +| `persistence.annotations` | Annotations for the PVC | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Map of node/pod affinities | `{}` | +| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `livenessProbe.enabled` | would you like a livessProbed to be enabled | `true` | +| `livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 30 | +| `livenessProbe.periodSeconds` | How often to perform the probe | 10 | +| `livenessProbe.timeoutSeconds` | When the probe times out | 5 | +| `livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `readinessProbe.enabled` | would you like a readinessProbe to be enabled | `true` | +| `readinessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 5 | +| `readinessProbe.periodSeconds` | How often to perform the probe | 10 | +| `readinessProbe.timeoutSeconds` | When the probe times out | 5 | +| `readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `metrics.kafka.enabled` | Whether or not to create a separate Kafka exporter | `false` | +| `metrics.kafka.image.registry` | Kafka exporter image registry | `docker.io` | +| `metrics.kafka.image.repository` | Kafka exporter image name | `danielqsj/kafka-exporter` | +| `metrics.kafka.image.tag` | Kafka exporter image tag | `v1.0.1` | +| `metrics.kafka.image.pullPolicy` | Kafka exporter image pull policy | `Always` | +| `metrics.kafka.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `metrics.kafka.interval` | Interval that Prometheus scrapes Kafka metrics when using Prometheus Operator | `10s` | +| `metrics.kafka.port` | Kafka Exporter Port which exposes metrics in Prometheus format for scraping | `9308` | +| `metrics.kafka.resources` | Allows setting resource limits for kafka-exporter pod | `{}` | +| `metrics.jmx.resources` | Allows setting resource limits for jmx sidecar container | `{}` | +| `metrics.jmx.enabled` | Whether or not to expose JMX metrics to Prometheus | `false` | +| `metrics.jmx.image.registry` | JMX exporter image registry | `docker.io` | +| `metrics.jmx.image.repository` | JMX exporter image name | `solsson/kafka-prometheus-jmx-exporter@sha256` | +| `metrics.jmx.image.tag` | JMX exporter image tag | `a23062396cd5af1acdf76512632c20ea6be76885dfc20cd9ff40fb23846557e8` | +| `metrics.jmx.image.pullPolicy` | JMX exporter image pull policy | `Always` | +| `metrics.jmx.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `metrics.jmx.interval` | Interval that Prometheus scrapes JMX metrics when using Prometheus Operator | `10s` | +| `metrics.jmx.exporterPort` | JMX Exporter Port which exposes metrics in Prometheus format for scraping | `5556` | +| `metrics.jmx.configMap.enabled` | Enable the default ConfigMap for JMX | `true` | +| `metrics.jmx.configMap.overrideConfig` | Allows config file to be generated by passing values to ConfigMap | `{}` | +| `metrics.jmx.configMap.overrideName` | Allows setting the name of the ConfigMap to be used | `""` | +| `metrics.jmx.jmxPort` | The jmx port which JMX style metrics are exposed (note: these are not scrapeable by Prometheus) | `5555` | +| `metrics.jmx.whitelistObjectNames` | Allows setting which JMX objects you want to expose to via JMX stats to JMX Exporter | (see `values.yaml`) | +| `zookeeper.enabled` | Switch to enable or disable the Zookeeper helm chart | `true` | +| `externalZookeeper.servers` | Server or list of external zookeeper servers to use. | `nil` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, diff --git a/bitnami/kafka/templates/statefulset.yaml b/bitnami/kafka/templates/statefulset.yaml index 09988054d7..fa26c5aea3 100644 --- a/bitnami/kafka/templates/statefulset.yaml +++ b/bitnami/kafka/templates/statefulset.yaml @@ -272,14 +272,25 @@ spec: secretName: {{ required "A secret containinig the Kafka JKS certificates is required when authentication in enabled" .Values.auth.certificatesSecret }} defaultMode: 256 {{ end }} -{{- if .Values.persistence.enabled }} +{{- if and .Values.persistence.enabled .Values.persistence.existingClaim }} + - name: data + persistentVolumeClaim: +{{- with .Values.persistence.existingClaim }} + claimName: {{ tpl . $ }} +{{- end }} +{{- else if not .Values.persistence.enabled }} + - name: data + emptyDir: {} +{{- else if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} volumeClaimTemplates: - metadata: name: data + {{- with .Values.persistence.annotations }} annotations: - {{- range $key, $value := .Values.persistence.annotations }} + {{- range $key, $value := . }} {{ $key }}: {{ $value }} {{- end }} + {{- end }} spec: accessModes: {{- range .Values.persistence.accessModes }} @@ -295,7 +306,4 @@ spec: storageClassName: "{{ .Values.persistence.storageClass }}" {{- end }} {{- end }} -{{- else }} - - name: data - emptyDir: {} {{- end }} diff --git a/bitnami/kafka/values-production.yaml b/bitnami/kafka/values-production.yaml index 9aee4e8694..4c9fb87053 100644 --- a/bitnami/kafka/values-production.yaml +++ b/bitnami/kafka/values-production.yaml @@ -232,6 +232,12 @@ service: ## persistence: enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template, so, for example, the name can depend on .Release or .Chart + ## + # existingClaim: + # storageClass: "-" accessModes: - ReadWriteOnce diff --git a/bitnami/kafka/values.yaml b/bitnami/kafka/values.yaml index f076248c16..53176390b1 100644 --- a/bitnami/kafka/values.yaml +++ b/bitnami/kafka/values.yaml @@ -232,6 +232,12 @@ service: ## persistence: enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template, so, for example, the name can depend on .Release or .Chart + ## + # existingClaim: + # storageClass: "-" accessModes: - ReadWriteOnce