From 3bbab7d4876edd35da13a3ae4df85ebc6c6edec0 Mon Sep 17 00:00:00 2001 From: gkulkarni32 <74709839+gkulkarni32@users.noreply.github.com> Date: Wed, 3 Mar 2021 19:08:19 +0530 Subject: [PATCH] bitnami/dataplatform-bp1 addition of observability framework (#5602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updated metrics and values schema json * updated chart version * Update bitnami/dataplatform-bp1/values.yaml Co-authored-by: Marcos Bjoerkelund * Update bitnami/dataplatform-bp1/Chart.yaml Co-authored-by: Carlos Rodríguez Hernández * Update bitnami/dataplatform-bp1/Chart.yaml Co-authored-by: Carlos Rodríguez Hernández * fixed typo and added wavefront in README * updated schema json * updated version * updated solr exporter parameter * Update bitnami/dataplatform-bp1/README.md Co-authored-by: Miguel Ángel Cabrera Miñagorri * updated spark master/worker annotation * updated spark ports * updated Chart.lock file * Remove trailing spaces Co-authored-by: Marcos Bjoerkelund Co-authored-by: Carlos Rodríguez Hernández Co-authored-by: Miguel Ángel Cabrera Miñagorri --- bitnami/dataplatform-bp1/Chart.lock | 17 +- bitnami/dataplatform-bp1/Chart.yaml | 15 +- bitnami/dataplatform-bp1/README.md | 54 +++++- bitnami/dataplatform-bp1/values.schema.json | 124 ++++++++++++ bitnami/dataplatform-bp1/values.yaml | 198 ++++++++++++++++++++ 5 files changed, 392 insertions(+), 16 deletions(-) create mode 100644 bitnami/dataplatform-bp1/values.schema.json diff --git a/bitnami/dataplatform-bp1/Chart.lock b/bitnami/dataplatform-bp1/Chart.lock index 50a800889a..017aa94f72 100644 --- a/bitnami/dataplatform-bp1/Chart.lock +++ b/bitnami/dataplatform-bp1/Chart.lock @@ -1,18 +1,21 @@ dependencies: - name: kafka repository: https://charts.bitnami.com/bitnami - version: 12.7.5 + version: 12.9.5 - name: spark repository: https://charts.bitnami.com/bitnami - version: 5.1.2 + version: 5.2.2 - name: solr repository: https://charts.bitnami.com/bitnami - version: 0.1.0 + version: 0.2.2 - name: zookeeper repository: https://charts.bitnami.com/bitnami - version: 6.4.0 + version: 6.5.1 +- name: wavefront + repository: https://charts.bitnami.com/bitnami + version: 1.3.0 - name: common repository: https://charts.bitnami.com/bitnami - version: 1.3.9 -digest: sha256:8326e46b3915a947b127b901f8de190dfa55fd36a9c5d3723b38971f5b35c2e3 -generated: "2021-02-11T11:47:44.576918765Z" + version: 1.4.1 +digest: sha256:4286b91b09fe77c150e5f4af6f0042d0a8b7cd4b265e8078997deab8bc9e22ba +generated: "2021-03-03T18:58:26.048231+05:30" diff --git a/bitnami/dataplatform-bp1/Chart.yaml b/bitnami/dataplatform-bp1/Chart.yaml index 47e44011b1..6bf9e22a15 100644 --- a/bitnami/dataplatform-bp1/Chart.yaml +++ b/bitnami/dataplatform-bp1/Chart.yaml @@ -19,6 +19,10 @@ dependencies: name: zookeeper repository: https://charts.bitnami.com/bitnami version: 6.x.x + - condition: wavefront.enabled + name: wavefront + repository: https://charts.bitnami.com/bitnami + version: 1.x.x - name: common repository: https://charts.bitnami.com/bitnami tags: @@ -26,13 +30,16 @@ dependencies: version: 1.x.x description: OCTO Data platform Kafka-Spark-Solr Helm Chart engine: gotpl -home: https://github.com/bitnami/charts/tree/master/bitnami/dataplatform +home: https://github.com/bitnami/charts/tree/master/bitnami/dataplatform-bp1 +icon: https://bitnami.com/assets/stacks/dataplatform-bp1/img/dataplatform-bp1-stack-220x234.png keywords: - dataplatform - kafka - spark - solr - zookeeper + - wavefront + - observability - apache maintainers: - email: containers@bitnami.com @@ -47,4 +54,8 @@ sources: - https://github.com/bitnami/bitnami-docker-solr - https://zookeeper.apache.org/ - https://github.com/bitnami/bitnami-docker-zookeeper -version: 0.1.0 + - https://github.com/bitnami/bitnami-docker-wavefront-kubernetes-collector + - https://github.com/bitnami/bitnami-docker-wavefront-proxy + - https://github.com/wavefrontHQ/wavefront-collector-for-kubernetes + - https://github.com/wavefrontHQ/wavefront-proxy +version: 0.2.0 diff --git a/bitnami/dataplatform-bp1/README.md b/bitnami/dataplatform-bp1/README.md index e94edf5ece..b09bffa676 100644 --- a/bitnami/dataplatform-bp1/README.md +++ b/bitnami/dataplatform-bp1/README.md @@ -16,6 +16,7 @@ Helm chart blueprint for this data platform deployment, covering: - Pod placement rules – Affinity rules to ensure placement diversity to prevent single point of failures and optimize load distribution - Pod resource sizing rules – Optimized Pod and JVM sizing settings for optimal performance and efficient resource usage - Default settings to ensure Pod access security +- Optional Wavefront Observability framework configuration In addition to the Pod resource optimizations, this blueprint is validated and tested to provide Kubernetes node count and sizing recommendations [(see Kubernetes Cluster Requirements)](#kubernetes-cluster-requirements) to facilitate cloud platform capacity planning. The goal is optimize the number of required Kubernetes nodes in order to optimize server resource usage and, at the same time, ensuring runtime and resource diversity. @@ -40,6 +41,8 @@ The "Small" size data platform in default configuration deploys the following: 3. Solr with 2 nodes using the zookeeper deployed above 4. Spark with 1 Master and 2 worker nodes +The data platform can be optionally deployed with the wavefront observability framework. In that case, the wavefront collectors will be set up as a DaemonSet to collect the Kubernetes cluster metrics to enable runtime feed into the Tanzu Observability service. It will also be pre-configured to scrape the metrics from the Prometheus endpoint that each application (Kafka/Spark/Solr) emits the metrics to. + Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. ## Prerequisites @@ -116,6 +119,12 @@ Parameters below are set as per the recommended values, they can be overwritten | `kafka.resources.limits` | The resources limits for Kafka containers | `{}` | | `kafka.resources.requests` | The requested resources for Kafka containers for a small kubernetes cluster | Kafka pods Resource requests set for optimal resource usage | | `kafka.affinity` | Affinity for pod assignment | Kafka pods affinity rules set for best possible resiliency (evaluated as a template) | +| `kafka.metrics.kafka.enabled` | Whether or not to create a standalone Kafka exporter to expose Kafka metrics | `false` | +| `kafka.metrics.kafka.resources.requests` | Kafka Exporter container resource requests | Kafka Exporter pod resource requests set for optimal resource usage | +| `kafka.metrics.kafka.service.port` | Kafka Exporter Prometheus port | `9308` | +| `kafka.metrics.jmx.enabled` | Whether or not to expose JMX metrics to Prometheus | `false` | +| `kafka.metrics.jmx.resources.requests` | JMX Exporter container resource requests | JMX exporter pod resource requests set for best possible resiliency | +| `kafka.metrics.jmx.service.port` | JMX Exporter Prometheus port | `5556` | | `kafka.zookeeper.enabled` | Switch to enable or disable the Zookeeper helm chart | `false` Common Zookeeper deployment used for kafka and solr | | `kafka.externalZookeeper.servers` | Server or list of external Zookeeper servers to use | Zookeeper installed as a subchart to be used | @@ -133,6 +142,10 @@ Parameters below are set as per the recommended values, they can be overwritten | `solr.javaMem` | Java memory options to pass to the Solr container | Solr Java Heap size set for optimal resource usage | | `solr.heap` | Java Heap options to pass to the solr container | `nil` | | `solr.affinity` | Affinity for Solr pods assignment | Solr pods Affinity rules set for best possible resiliency (evaluated as a template) | +| `solr.exporter.enabled` | Start a prometheus exporter | `false` | +| `solr.exporter.port` | Solr exporter port | `9983` | +| `solr.exporter.resources` | Exporter resource requests/limit | Solr exporter pod resource requests for optimal resource usage size | +| `solr.exporter.affinity` | Affinity for Solr pods assignment | Solr exporter pod affinity rules for best possible resiliency (evaluated as a template) | | `solr.zookeeper.enabled` | Enable Zookeeper deployment. Needed for Solr cloud. | `false` common zookeeper used between kafka and solr | | `solr.externalZookeeper.servers` | Servers for an already existing Zookeeper. | Zookeeper installed as a subchart to be used | @@ -143,12 +156,35 @@ Parameters below are set as per the recommended values, they can be overwritten | Parameter | Description | Default | |:----------------------------|:-------------------------------------------------|:--------------------------------------------------------------------------------------------| | `spark.enabled` | Switch to enable or disable the Spark helm chart | `true` | +| `spark.master.webPort` | Specify the port where the web interface will listen on the master | `8080` | | `spark.master.affinity` | Spark master affinity for pod assignment | Spark master pod Affinity rules set for best possible resiliency (evaluated as a template) | | `spark.master.resources` | CPU/Memory resource requests/limits for Master | Spark master pods resource requests set for optimal resource usage | -| `spark.worker.javaOptions` | Set options for the JVM in the form `-Dx=y` | No default | | `spark.worker.replicaCount` | Set the number of workers | `2` | +| `spark.worker.webPort` | Specify the port where the web interface will listen on the worker | `8081` | +| `spark.worker.javaOptions` | Set options for the JVM in the form `-Dx=y` | No default | | `spark.worker.affinity` | Spark worker affinity for pod assignment | Spark worker pods Affinity rules set for best possible resiliency (evaluated as a template) | | `spark.worker.resources` | CPU/Memory resource requests/limits for worker | Spark worker pods resource requests set for optimal resource usage | +| `spark.metrics.enabled` | Start a side-car prometheus exporter | `false` | +| `spark.metrics.masterAnnotations` | Annotations for enabling prometheus to access the metrics endpoint of the master nodes | `{prometheus.io/scrape: "true", prometheus.io/path: "/metrics/", prometheus.io/port: "8080"}` | +| `spark.metrics.workerAnnotations` | Annotations for enabling prometheus to access the metrics endpoint of the worker nodes | `{prometheus.io/scrape: "true", prometheus.io/path: "/metrics/", prometheus.io/port: "8081"}` | +| `spark.metrics.resources.requests` | The requested resources for the metrics exporter container | Spark exporter container resource requests for optimal resource usage size| + +### Wavefront chart parameters + +| Parameter | Description | Default | +|--------------------------------------------|------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `wavefront.enabled` | Switch to enable or disable the Wavefront helm chart | `true` | +| `wavefront.clusterName` | Unique name for the Kubernetes cluster (required) | `KUBERNETES_CLUSTER_NAME` | +| `wavefront.wavefront.url` | Wavefront URL for your cluster (required) | `https://YOUR_CLUSTER.wavefront.com` | +| `wavefront.wavefront.token` | Wavefront API Token (required) | `YOUR_API_TOKEN` | +| `wavefront.wavefront.existingSecret` | Name of an existing secret containing the token | `nil` | +| `wavefront.collector.discovery.enabled` | Rules based and Prometheus endpoints auto-discovery | `true` | +| `wavefront.collector.discovery.enableRuntimeConfigs` | Enable runtime discovery rules | `true` | +| `wavefront.collector.discovery.config` | Configuration for rules based auto-discovery | Data Platform components pods discovery config | +| `wavefront.collector.resources.limits` | The resources limits for the collector container | `{}` | +| `wavefront.collector.resources.requests` | The requested resources for the collector container | Collectors pods resource requests for optimal resource usage size | +| `wavefront.proxy.resources.limits` | The resources limits for the proxy container | `{}` | +| `wavefront.proxy.resources.requests` | The requested resources for the proxy container | Proxy pods resource resource requests for optimal resource usage size | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, @@ -160,16 +196,20 @@ $ helm install my-release \ The above command deploys the data platform with Kafka with 3 nodes (replicas). -In case you need to deploy the data platform skipping any component, you can specify the 'enabled' parameter using the `--set .enabled=false` argument to `helm install`. For Example, +In case you need to deploy the data platform with Wavefront Observability Framework for all the applications (Kafka/Spark/Solr) in the data platform, you can specify the 'enabled' parameter using the `--set .metrics.enabled=true` argument to `helm install`. For Solr, the parameter is `solr.exporter.enabled=true` For Example, ```console -$ helm install my-release \ - --set solr.enabled=false \ - bitnami/dataplatform-bp1 +$ helm install my-release bitnami/dataplatform-bp1 \ + --set kafka.metrics.kafka.enabled=true \ + --set kafka.metrics.jmx.enabled=true \ + --set spark.metrics.enabled=true \ + --set solr.exporter.enabled=true \ + --set wavefront.enabled=true \ + --set wavefront.clusterName= \ + --set wavefront.wavefront.url=https://.wavefront.com \ + --set wavefront.wavefront.token= ``` -The above command deploys the data platform without Solr. - Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, ```console diff --git a/bitnami/dataplatform-bp1/values.schema.json b/bitnami/dataplatform-bp1/values.schema.json new file mode 100644 index 0000000000..5e00fbaebb --- /dev/null +++ b/bitnami/dataplatform-bp1/values.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "kafka": { + "type": "object", + "title": "Kafka Metrics Details", + "form": true, + "properties": { + "metrics": { + "type": "object", + "properties": { + "kafka": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable metrics for Kafka", + "description": "Whether to enable metrics for Kafka. Switch this off in case Kafka metrics are not needed" + } + } + }, + "jmx": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable JMX metrics for Kafka", + "description": "Whether to enable JMX metrics for Kafka. Switch this off in case Kafka JMX metrics are not needed" + } + } + } + } + } + } + }, + "spark": { + "type": "object", + "title": "Spark Metrics Details", + "form": true, + "properties": { + "metrics": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable metrics for Spark", + "description": "Whether to enable metrics for Spark. Switch this off in case Spark metrics are not needed" + } + } + } + } + }, + "solr": { + "type": "object", + "title": "Solr Metrics Details", + "form": true, + "properties": { + "exporter": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable metrics for Solr", + "description": "Whether to enable metrics for Solr. Switch this off in case Solr metrics are not needed" + } + } + } + } + }, + "wavefront": { + "type": "object", + "title": "Wavefront Observability Details", + "form": true, + "properties": { + "enabled": { + "type": "boolean", + "title": "Deploy Wavefront observability for the data platform cluster including configuration for metrics exporters for all the components", + "form": true, + "description": "Whether to deploy a Wavefront observability as part of data platform. Switch this off in case Wavefront is not needed" + }, + "clusterName": { + "type": "string", + "title": "Provide the kubernetes cluster name to be configured in wavefront", + "form": true, + "hidden": { + "value": false, + "path": "wavefront/enabled" + }, + "description": "Kubernetes cluster name to be configured in wavefront, if not provided it will be defaulted to KUBERNETES_CLUSTER_NAME" + }, + "wavefront": { + "type": "object", + "properties": { + "url": { + "type": "string", + "title": "Provide the wavefront cluster url to be configured", + "form": true, + "hidden": { + "value": false, + "path": "wavefront/enabled" + }, + "description": "Wavefront cluster url to be configured to export the metrics to" + }, + "token": { + "type": "string", + "title": "Provide the wavefront user API token to be used for the cluster url provided above", + "form": true, + "hidden": { + "value": false, + "path": "wavefront/enabled" + }, + "description": "Wavefront user API token to be used for the cluster url provided above" + } + } + } + } + } + } +} diff --git a/bitnami/dataplatform-bp1/values.yaml b/bitnami/dataplatform-bp1/values.yaml index 113b079500..b0729be005 100644 --- a/bitnami/dataplatform-bp1/values.yaml +++ b/bitnami/dataplatform-bp1/values.yaml @@ -9,6 +9,7 @@ # storageClass: myStorageClass zookeeper: + enabled: true replicaCount: 3 ## Size in MB for the Java Heap options (Xmx and XMs). This env var is ignored if Xmx an Xms are configured via JVMFLAGS ## @@ -71,6 +72,46 @@ kafka: values: - zookeeper topologyKey: "kubernetes.io/hostname" + ## Prometheus Exporters / Metrics + ## + metrics: + ## Prometheus Kafka Exporter: exposes complimentary metrics to JMX Exporter + ## + kafka: + enabled: false + ## Prometheus Kafka Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: {} + requests: + cpu: 100m + memory: 128Mi + ## Service configuration + ## + service: + ## Kafka Exporter Prometheus port to be used in wavefront configuration + ## + port: 9308 + + ## Prometheus JMX Exporter: exposes the majority of Kafkas metrics + ## + jmx: + enabled: false + ## Prometheus JMX Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: {} + requests: + cpu: 100m + memory: 128Mi + ## Service configuration + ## + service: + ## JMX Exporter Prometheus port + ## + port: 5556 zookeeper: enabled: false ## This value is only used when zookeeper.enabled is set to false. @@ -88,6 +129,9 @@ spark: ## Spark master specific configuration ## master: + ## Spark container ports + ## + webPort: 8080 resources: ## Recommended values for cpu and memory requests ## @@ -111,6 +155,9 @@ spark: ## worker: replicaCount: 2 + ## Spark container ports + ## + webPort: 8081 ## Anti affinity rules set for resiliency ## affinity: @@ -131,12 +178,32 @@ spark: requests: cpu: 250m memory: 5120Mi + ## Metrics configuration + ## + metrics: + enabled: false + ## Annotations for the Prometheus metrics on master nodes + ## + masterAnnotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics/' + prometheus.io/port: '8080' + ## Annotations for the Prometheus metrics on worker nodes + ## + workerAnnotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics/' + prometheus.io/port: '8081' ## Solr Subchart parameters ## solr: enabled: true replicaCount: 2 + ## Exporter deployment does not work with authentication enabled- Bug + ## + authentication: + enabled: false ## Java memory options recommended value ## @@ -161,6 +228,30 @@ solr: requests: cpu: 250m memory: 5120Mi + ## Configuration for the solr prometheus exporter + ## + exporter: + enabled: false + ## Solr exporter port + ## + port: 9983 + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - solr + topologyKey: "kubernetes.io/hostname" + resources: + limits: {} + requests: + cpu: 100m + memory: 128Mi zookeeper: enabled: false ## This value is only used when zookeeper.enabled is set to false @@ -170,3 +261,110 @@ solr: ## servers: - '{{ .Release.Name }}-zookeeper' + +## Wavefront Subchart parameters +## +wavefront: + enabled: false + ## This is a unique name for the cluster (required) + ## All metrics will receive a `cluster` tag with this value + ## + clusterName: KUBERNETES_CLUSTER_NAME + ## Wavefront URL (cluster) and API Token (required) + ## + wavefront: + url: https://YOUR_CLUSTER.wavefront.com + token: YOUR_API_TOKEN + ## Name of an existing secret containing the token + ## + existingSecret: + ## Wavefront Collector is responsible to get all Kubernetes metrics from your cluster. + ## It will capture Kubernetes resources metrics available from the kubelets, + ## as well as auto-discovery capabilities. + ## + collector: + ## Rules based discovery configuration + ## Ref: https://github.com/wavefrontHQ/wavefront-kubernetes-collector/blob/master/docs/discovery.md + ## + resources: + limits: {} + requests: + cpu: 200m + memory: 10Mi + discovery: + enabled: true + ## Whether to enable runtime discovery configurations + ## Ref: https://github.com/wavefrontHQ/wavefront-collector-for-kubernetes/blob/master/docs/discovery.md#runtime-configurations + ## + enableRuntimeConfigs: true + ## Can be used to add additional discovery rules + ## + config: + ## auto-discover kafka-exporter + ## + - name: kafka-discovery + type: prometheus + selectors: + images: + - '*bitnami/kafka-exporter*' + port: 9308 + path: /metrics + scheme: http + prefix: kube.kafka. + + ## auto-discover jmx exporter + ## + - name: kafka-jmx-discovery + type: prometheus + selectors: + images: + - '*bitnami/jmx-exporter*' + port: 5556 + path: /metrics + scheme: http + prefix: kube.kafkajmx. + + ## auto-discover solr + ## + - name: solr-discovery + type: prometheus + selectors: + images: + - '*bitnami/solr*' + port: 9983 + path: /metrics + scheme: http + prefix: kube.solr. + + ## auto-discover spark + ## + - name: spark-worker-discovery + type: prometheus + selectors: + images: + - '*bitnami/spark*' + port: 8081 + path: /metrics/ + scheme: http + prefix: kube.spark. + + ## auto-discover spark + ## + - name: spark-master-discovery + type: prometheus + selectors: + images: + - '*bitnami/spark*' + port: 8080 + path: /metrics/ + scheme: http + prefix: kube.spark. + proxy: + ## Wavefront Proxy resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: {} + requests: + cpu: 100m + memory: 5Gi