From 3a97a5232f8a4e3ca4071e27c317179b4b78055c Mon Sep 17 00:00:00 2001 From: Miguel Ruiz Date: Thu, 11 May 2023 13:55:02 +0200 Subject: [PATCH] [bitnami/kafka] Fix issue with Kafka where Zookeeper mode could not be used + minor fixes (#16558) * [bitnami/kafka] Fix issue with Kafka where Zookeeper mode could not be used + minor fixes Signed-off-by: Miguel Ruiz * Revert values.yaml changes Signed-off-by: Miguel Ruiz --------- Signed-off-by: Miguel Ruiz --- bitnami/kafka/templates/_helpers.tpl | 6 +-- .../templates/networkpolicy-ingress.yaml | 2 +- .../kafka/templates/scripts-configmap.yaml | 6 +-- bitnami/kafka/templates/statefulset.yaml | 54 +++++++++---------- bitnami/kafka/templates/svc-headless.yaml | 2 +- 5 files changed, 33 insertions(+), 37 deletions(-) diff --git a/bitnami/kafka/templates/_helpers.tpl b/bitnami/kafka/templates/_helpers.tpl index 1b48201892..73c1b332da 100644 --- a/bitnami/kafka/templates/_helpers.tpl +++ b/bitnami/kafka/templates/_helpers.tpl @@ -519,8 +519,8 @@ kafka: .Values.auth.tls.existingSecrets {{/* Validate values of Kafka provisioning - keyPasswordSecretKey, keystorePasswordSecretKey or truststorePasswordSecretKey must not be used without passwordsSecret */}} {{- define "kafka.validateValues.tlsPasswords" -}} -{{- if and (include "kafka.client.tlsEncryption" .) (not .Values.auth.tls.passwordsSecret) }} -{{- if or .Values.auth.tls.keyPasswordSecretKey .Values.auth.tls.keystorePasswordSecretKey .Values.auth.tls.truststorePasswordSecretKey }} +{{- if and (include "kafka.client.tlsEncryption" .) (not .Values.provisioning.auth.tls.passwordsSecret) }} +{{- if or .Values.provisioning.auth.tls.keyPasswordSecretKey .Values.provisioning.auth.tls.keystorePasswordSecretKey .Values.provisioning.auth.tls.truststorePasswordSecretKey }} kafka: auth.tls.keyPasswordSecretKey,auth.tls.keystorePasswordSecretKey,auth.tls.truststorePasswordSecretKey auth.tls.keyPasswordSecretKey,auth.tls.keystorePasswordSecretKey,auth.tls.truststorePasswordSecretKey must not be used without passwordsSecret setted. @@ -552,4 +552,4 @@ kafka: Kraft mode .Values.kraft.controllerQuorumVoters must not be empty if .Values.kraft.enabled set to true and .Values.kraft.processRoles does not contain "controller". If you deploy brokers without controllers you have to define external controllers with .Values.kraft.controllerQuorumVoters {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/bitnami/kafka/templates/networkpolicy-ingress.yaml b/bitnami/kafka/templates/networkpolicy-ingress.yaml index 258dcabb6e..fa38240955 100644 --- a/bitnami/kafka/templates/networkpolicy-ingress.yaml +++ b/bitnami/kafka/templates/networkpolicy-ingress.yaml @@ -41,7 +41,7 @@ spec: {{- if .Values.externalAccess.enabled }} - ports: - port: {{ .Values.containerPorts.external }} - {{- if .Values.externalAccess.from }} + {{- if .Values.networkPolicy.externalAccess.from }} from: {{- include "common.tplvalues.render" ( dict "value" .Values.networkPolicy.externalAccess.from "context" $ ) | nindent 8 }} {{- end }} {{- end }} diff --git a/bitnami/kafka/templates/scripts-configmap.yaml b/bitnami/kafka/templates/scripts-configmap.yaml index b5191ee1b9..dd491479a8 100644 --- a/bitnami/kafka/templates/scripts-configmap.yaml +++ b/bitnami/kafka/templates/scripts-configmap.yaml @@ -78,7 +78,7 @@ data: ID="${MY_POD_NAME#"{{ $fullname }}-"}" # If process.roles is not set at all, it is assumed to be in ZooKeeper mode. # https://kafka.apache.org/documentation/#kraft_role - + if [[ -f "{{ .Values.logsDirs | splitList "," | first }}/meta.properties" ]]; then if [[ $KAFKA_CFG_PROCESS_ROLES == "" ]]; then export KAFKA_CFG_BROKER_ID="$(grep "broker.id" "{{ .Values.logsDirs | splitList "," | first }}/meta.properties" | awk -F '=' '{print $2}')" @@ -97,8 +97,8 @@ data: node_id={{ .Values.minId }} pod_id=0 while : - do - VOTERS="${VOTERS}$node_id@{{ include "common.names.fullname" . }}-$pod_id.{{ include "common.names.fullname" . }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ .Values.containerPorts.controller }}" + do + VOTERS="${VOTERS}$node_id@{{ include "common.names.fullname" . }}-$pod_id.{{ include "common.names.fullname" . }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ .Values.service.ports.controller }}" node_id=$(( $node_id + 1 )) pod_id=$(( $pod_id + 1 )) if [[ $pod_id -ge {{ .Values.replicaCount }} ]]; then diff --git a/bitnami/kafka/templates/statefulset.yaml b/bitnami/kafka/templates/statefulset.yaml index 5fbc171872..12f9a6e16a 100644 --- a/bitnami/kafka/templates/statefulset.yaml +++ b/bitnami/kafka/templates/statefulset.yaml @@ -185,21 +185,19 @@ spec: - name: KAFKA_INTER_BROKER_LISTENER_NAME value: {{ .Values.interBrokerListenerName | quote }} - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + {{- $securityProtocolMap := list }} {{- if .Values.listenerSecurityProtocolMap }} - value: {{ .Values.listenerSecurityProtocolMap | quote }} - {{- else if .Values.externalAccess.enabled }} - {{- if .Values.kraft.enabled }} - value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }},CONTROLLER:{{ $controllerProtocol }},EXTERNAL:{{ $externalClientProtocol }}" - {{- else }} - value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }},EXTERNAL:{{ $externalClientProtocol }}" - {{- end}} + {{- $securityProtocolMap = append $securityProtocolMap .Values.listenerSecurityProtocolMap }} {{- else }} - {{- if .Values.kraft.enabled }} - value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }},CONTROLLER:{{ $controllerProtocol }}" - {{- else }} - value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }}" - {{- end }} + {{- $securityProtocolMap = append $securityProtocolMap (printf "INTERNAL:%s,CLIENT:%s" $interBrokerProtocol $clientProtocol) }} + {{- if .Values.kraft.enabled }} + {{- $securityProtocolMap = append $securityProtocolMap (printf "CONTROLLER:%s" $controllerProtocol) }} + {{- end}} + {{- if .Values.externalAccess.enabled }} + {{- $securityProtocolMap = append $securityProtocolMap (printf "EXTERNAL:%s" $externalClientProtocol) }} {{- end }} + {{- end }} + value: {{ join "," $securityProtocolMap | quote }} {{- if or ($clientProtocol | regexFind "SASL") ($externalClientProtocol | regexFind "SASL") ($interBrokerProtocol | regexFind "SASL") .Values.auth.sasl.jaas.zookeeperUser }} - name: KAFKA_CFG_SASL_ENABLED_MECHANISMS value: {{ upper .Values.auth.sasl.mechanisms | quote }} @@ -207,21 +205,19 @@ spec: value: {{ upper .Values.auth.sasl.interBrokerMechanism | quote }} {{- end }} - name: KAFKA_CFG_LISTENERS + {{- $listeners := list }} {{- if .Values.listeners }} - value: {{ join "," .Values.listeners }} - {{- else if .Values.externalAccess.enabled }} - {{- if .Values.kraft.enabled }} - value: "INTERNAL://:{{ .Values.containerPorts.internal }},CLIENT://:{{ .Values.containerPorts.client }},CONTROLLER://:{{ .Values.containerPorts.controller }},EXTERNAL://:{{ .Values.containerPorts.external }}" - {{- else }} - value: "INTERNAL://:{{ .Values.containerPorts.internal }},CLIENT://:{{ .Values.containerPorts.client }},EXTERNAL://:{{ .Values.containerPorts.external }}" - {{- end }} + {{- $listeners = .Values.listeners }} {{- else }} - {{- if .Values.kraft.enabled }} - value: "INTERNAL://:{{ .Values.containerPorts.internal }},CLIENT://:{{ .Values.containerPorts.client }},CONTROLLER://:{{ .Values.containerPorts.controller }}" - {{- else }} - value: "INTERNAL://:{{ .Values.containerPorts.internal }},CLIENT://:{{ .Values.containerPorts.client }}" - {{- end }} + {{- $listeners = append $listeners (printf "INTERNAL://:%d,CLIENT://:%d" (int .Values.containerPorts.internal) (int .Values.containerPorts.client)) }} + {{- if .Values.kraft.enabled }} + {{- $listeners = append $listeners (printf "CONTROLLER://:%d" (int .Values.containerPorts.controller)) }} + {{- end}} + {{- if .Values.externalAccess.enabled }} + {{- $listeners = append $listeners (printf "EXTERNAL://:%d" (int .Values.containerPorts.external)) }} {{- end }} + {{- end }} + value: {{ join "," $listeners | quote }} {{- if .Values.externalAccess.enabled }} {{- if .Values.externalAccess.autoDiscovery.enabled }} - name: SHARED_FILE @@ -371,15 +367,15 @@ spec: value: {{ .Values.allowEveryoneIfNoAclFound | quote }} - name: KAFKA_CFG_SUPER_USERS value: {{ .Values.superUsers | quote }} + - name: KAFKA_ENABLE_KRAFT + value: {{ ternary "true" "false" .Values.kraft.enabled | quote }} {{- if .Values.kraft.enabled }} - name: KAFKA_KRAFT_CLUSTER_ID - value: {{ .Values.kraft.clusterId | quote }} + value: {{ .Values.kraft.clusterId | quote }} - name: KAFKA_CFG_PROCESS_ROLES - value: {{ .Values.kraft.processRoles | quote }} + value: {{ .Values.kraft.processRoles | quote }} - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES - value: {{ .Values.kraft.controllerListenerNames | quote }} - - name: KAFKA_ENABLE_KRAFT - value: "true" + value: {{ .Values.kraft.controllerListenerNames | quote }} {{- if .Values.kraft.controllerQuorumVoters }} - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ .Values.kraft.controllerQuorumVoters}} diff --git a/bitnami/kafka/templates/svc-headless.yaml b/bitnami/kafka/templates/svc-headless.yaml index 8611308dab..e2f0eeb5ff 100644 --- a/bitnami/kafka/templates/svc-headless.yaml +++ b/bitnami/kafka/templates/svc-headless.yaml @@ -36,7 +36,7 @@ spec: {{- if and .Values.kraft.enabled (contains "controller" .Values.kraft.processRoles) }} - name: tcp-controller protocol: TCP - port: {{ .Values.containerPorts.controller }} + port: {{ .Values.service.ports.controller }} targetPort: kafka-ctlr {{- end }} selector: {{- include "common.labels.matchLabels" . | nindent 4 }}