[bitnami/dremio] feat: 🎉 Add chart (#27453)

Description of the change

This PR adds the chart for Dremio OSS, based on the upstream dremio chart. Features:

 - BItnami standards
 - Full configuration using the upstream Dremio configuration
 - Use of indirect environment variables in configuration
 - Master coordinator, Coordinator and Executor (with multiple engines)
 - TLS for the web interface (the only one allowed in Dremio OSS)


---------

Signed-off-by: Javier Salmeron Garcia <jsalmeron@vmware.com>
Signed-off-by: Bitnami Containers <bitnami-bot@vmware.com>
Co-authored-by: Bitnami Containers <bitnami-bot@vmware.com>

.vib/dremio/cypress/cypress.config.js

@@ -0,0 +1,15 @@
+module.exports = {
+  env: {
+    username: 'vib_user',
+    password: 'ComplicatedPassword!1234',
+    postgresql_host: 'postgresql',
+    postgresql_user: 'postgres',
+    postgresql_db: 'vib_test',
+    postgresql_password: 'password123',
+  },
+  defaultCommandTimeout: 30000,
+  e2e: {
+    setupNodeEvents(on, config) {},
+    baseUrl: 'http://localhost',
+  },
+}

.vib/dremio/cypress/cypress/e2e/dremio.cy.js

@@ -0,0 +1,32 @@
+/*
+ * Copyright Broadcom, Inc. All Rights Reserved.
+ * SPDX-License-Identifier: APACHE-2.0
+ */
+
+/// <reference types="cypress" />
+import {
+  random,
+} from '../support/utils';
+
+it('Allows to import a PostgreSQL database and perform a query', () => {
+  cy.login();
+  // Go to the collections page
+  cy.contains('Add Source').click();
+  cy.contains('PostgreSQL').click();
+  cy.fixture('database').then((d) => {
+    cy.get('#name').should('be.enabled').clear({force: true}).type(`${d.database.name}${random}`);
+    // Cannot use # because of the middle dot
+    cy.get('[id="config.hostname"]').should('be.enabled').clear({force: true}).type(Cypress.env('postgresql_host'));
+    cy.get('[id="config.databaseName"]').should('be.enabled').clear({force: true}).type(Cypress.env('postgresql_db'));
+    cy.get('[id="config.username"]').should('be.enabled').clear({force: true}).type(Cypress.env('postgresql_user'));
+    cy.get('[id="config.password"]').should('be.enabled').clear({force: true}).type(Cypress.env('postgresql_password'));
+    cy.contains('Save').click();
+    cy.get(`[href*="source/${d.database.name}${random}"]`);
+    // Does not have text as it is an icon, so we need to use the href selector
+    cy.get('[href*="new_query"]').click();
+    // Ensure the editor is fully loaded
+    cy.contains(`${d.database.name}${random}`).click();
+    cy.contains('public').click();
+    cy.contains(d.database.expectedRes);
+  });
+});

.vib/dremio/cypress/cypress/fixtures/database.json

@@ -0,0 +1,6 @@
+{
+  "database": {
+    "name": "vib_psql",
+    "expectedRes": "cars"
+  }
+}

.vib/dremio/cypress/cypress/support/commands.js

@@ -0,0 +1,46 @@
+/*
+ * Copyright Broadcom, Inc. All Rights Reserved.
+ * SPDX-License-Identifier: APACHE-2.0
+ */
+
+const COMMAND_DELAY = 2000;
+
+for (const command of ['click']) {
+  Cypress.Commands.overwrite(command, (originalFn, ...args) => {
+    const origVal = originalFn(...args);
+
+    return new Promise((resolve) => {
+      setTimeout(() => {
+        resolve(origVal);
+      }, COMMAND_DELAY);
+    });
+  });
+}
+
+Cypress.Commands.add(
+  'login',
+  (username = Cypress.env('username'), password = Cypress.env('password')) => {
+     cy.visit('/login');
+     cy.get('#userName').should('be.enabled').clear({force: true}).type(username);
+     cy.get('#password').should('be.enabled').clear({force: true}).type(password);
+     cy.contains('Log In').click();
+     cy.contains('Datasets');
+   }
+);
+
+Cypress.on('uncaught:exception', (err) => {
+  if (err.message.includes('Cannot read properties of')) {
+    return false;
+  }
+  if (err.message.includes('Normalize accepts')) {
+    return false;
+  }
+  // We expect an error "Failed to execute 'observe' on 'IntersectionObserver'"
+  // during the installation of a template so we add an exception
+  if (err.message.includes("Failed to execute 'observe' on 'IntersectionObserver'")) {
+    return false;
+  }
+
+  // we still want to ensure there are no other unexpected
+  // errors, so we let them fail the test
+})

.vib/dremio/cypress/cypress/support/e2e.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright Broadcom, Inc. All Rights Reserved.
+ * SPDX-License-Identifier: APACHE-2.0
+ */
+
+// ***********************************************************
+// This example support/index.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands';
+
+// Alternatively you can use CommonJS syntax:
+// require('./commands')

.vib/dremio/cypress/cypress/support/utils.js

@@ -0,0 +1,8 @@
+/*
+ * Copyright Broadcom, Inc. All Rights Reserved.
+ * SPDX-License-Identifier: APACHE-2.0
+ */
+
+/// <reference types="cypress" />
+
+export let random = (Math.random() + 1).toString(36).substring(7);

.vib/dremio/goss/goss.yaml

@@ -0,0 +1,108 @@
+# Copyright Broadcom, Inc. All Rights Reserved.
+# SPDX-License-Identifier: APACHE-2.0
+
+http:
+  http://127.0.0.1:{{ .Vars.dremio.containerPorts.web }}:
+    status: 200
+    body:
+      - /dremio/
+  http://127.0.0.1:{{ .Vars.dremio.containerPorts.web }}/apiv2/login:
+    status: 200
+    method: POST
+    request-headers:
+      - "Content-Type: application/json"
+    request-body: '{"userName": "{{ .Vars.dremio.auth.username }}", "password": "{{ .Vars.dremio.auth.password }}"}'
+    body:
+      - /{{ .Vars.dremio.auth.username }}/
+      - /{{ .Vars.dremio.auth.email }}/
+      - /{{ .Vars.dremio.auth.firstName }}/
+      - /{{ .Vars.dremio.auth.lastName }}/
+  http://dremio:{{ .Vars.service.ports.web }}:
+    status: 200
+    body:
+      - /dremio/
+  http://dremio:{{ .Vars.service.ports.web }}/apiv2/login:
+    status: 200
+    method: POST
+    request-headers:
+      - "Content-Type: application/json"
+    request-body: '{"userName": "{{ .Vars.dremio.auth.username }}", "password": "{{ .Vars.dremio.auth.password }}"}'
+    body:
+      - /{{ .Vars.dremio.auth.username }}/
+      - /{{ .Vars.dremio.auth.email }}/
+      - /{{ .Vars.dremio.auth.firstName }}/
+      - /{{ .Vars.dremio.auth.lastName }}/
+  {{- if .Vars.metrics.enabled }}
+  http://127.0.0.1:{{ .Vars.metrics.containerPorts.metrics }}/metrics:
+    status: 200
+    timeout: 30000
+    body:
+      - /metrics_com_dremio/
+  {{- /* Not testing the body output because it is causing a CancelTimeoutError from time to time (probably because of the resource preset we are using) */}}
+  http://dremio-metrics:{{ .Vars.metrics.service.ports.metrics }}/metrics:
+    status: 200
+    timeout: 30000
+  {{- end }}
+addr:
+  tcp://dremio-master-coordinator-0.dremio-headless:{{ .Vars.dremio.containerPorts.liveness }}:
+    reachable: true
+    timeout: 10000
+  tcp://127.0.0.1:{{ .Vars.dremio.containerPorts.conduit }}:
+    reachable: true
+    timeout: 10000
+  {{- if .Vars.metrics.enabled }}
+  tcp://127.0.0.1:{{ .Vars.dremio.containerPorts.jmx }}:
+    reachable: true
+    timeout: 10000
+  {{- end }}
+  tcp://dremio-coordinator-0.dremio-headless:{{ .Vars.dremio.containerPorts.client }}:
+    reachable: true
+    timeout: 10000
+  tcp://dremio-coordinator-0.dremio-headless:{{ .Vars.dremio.containerPorts.fabric }}:
+    reachable: true
+    timeout: 10000
+  tcp://dremio:{{ .Vars.service.ports.client }}:
+    reachable: true
+    timeout: 10000
+  tcp://dremio:{{ .Vars.service.ports.flight }}:
+    reachable: true
+    timeout: 10000
+file:
+  {{ .Vars.masterCoordinator.persistence.mountPath }}:
+    exists: true
+    filetype: directory
+    mode: '2775'
+    owner: root
+  /opt/bitnami/dremio/conf/dremio.conf:
+    exists: true
+    contents:
+      - /services.coordinator.web.tokens.cache.size.*{{ .Vars.masterCoordinator.dremioConf.configOverrides.services.coordinator.web.tokens.cache.size }}/
+      - /services.conduit.ssl.trustStorePassword.*{{ .Vars.masterCoordinator.dremioConf.secretConfigOverrides.services.conduit.ssl.trustStorePassword }}/
+      - /services.fabric.memory.reservation.*{{ .Vars.dremio.dremioConf.configOverrides.services.fabric.memory.reservation }}/
+      - /services.coordinator.web.ssl.trustStorePassword.*{{ .Vars.dremio.dremioConf.secretConfigOverrides.services.coordinator.web.ssl.trustStorePassword }}/
+      - /services.web-admin.port.*{{ .Vars.dremio.containerPorts.liveness }}/
+      - /services.coordinator.web.port.*{{ .Vars.dremio.containerPorts.web }}/
+      - /services.conduit.port.*{{ .Vars.dremio.containerPorts.conduit }}/
+      - /services.fabric.port.*{{ .Vars.dremio.containerPorts.fabric }}/
+      - /services.flight.port.*{{ .Vars.dremio.containerPorts.flight }}/
+  /opt/bitnami/dremio/conf/core-site.xml:
+    exists: true
+    contents:
+      - /fs.s3a.endpoint.region/
+      - /us-east-1/
+  /opt/bitnami/dremio/conf/vib.conf:
+    exists: true
+    contents:
+      - /IT WORKS/
+  /opt/bitnami/dremio/conf/vib_secret.conf:
+    exists: true
+    contents:
+      - /IT SECRETLY WORKS/
+command:
+  {{- $uid := .Vars.masterCoordinator.containerSecurityContext.runAsUser }}
+  {{- $gid := .Vars.masterCoordinator.podSecurityContext.fsGroup }}
+  check-user-info:
+    # The UID and GID should always be either the one specified as vars (always a bigger number that the default)
+    # or the one randomly defined by openshift (larger values). Otherwise, the chart is still using the default value.
+    exec: if [ $(id -u) -lt {{ $uid }} ] || [ $(id -G | awk '{print $2}') -lt {{ $gid }} ]; then exit 1; fi
+    exit-status: 0

.vib/dremio/runtime-parameters.yaml

@@ -0,0 +1,231 @@
+masterCoordinator:
+  podSecurityContext:
+    fsGroup: 1002
+  containerSecurityContext:
+    runAsUser: 1002
+  persistence:
+    enabled: true
+    mountPath: /bitnami/dremio/data
+  # Add specific overrides to dremio.conf
+  dremioConf:
+    configOverrides:
+      services:
+        coordinator:
+          web:
+            tokens:
+              cache:
+                size: 120
+    secretConfigOverrides:
+      services:
+        conduit:
+          ssl:
+            trustStorePassword: "test"
+  # Add extra files to the configmap
+    extraFiles:
+      vib.conf: |
+        IT WORKS
+    extraSecretFiles:
+      vib_secret.conf: |
+        IT SECRETLY WORKS
+dremio:
+  containerPorts:
+    web: 9048
+    client: 31011
+    flight: 32011
+    fabric: 45678
+    conduit: 45679
+    jmx: 50001
+    liveness: 44350
+  # Add common overrides to dremio.conf
+  dremioConf:
+    configOverrides:
+      services:
+        fabric:
+          memory:
+            reservation: 150M
+    secretConfigOverrides:
+      services:
+        coordinator:
+          web:
+            ssl:
+              trustStorePassword: "password123"
+  # Add extra configuration to core-site.xml
+  coreSite:
+    appendConfiguration: |
+      <property>
+        <name>fs.s3a.endpoint.region</name>
+        <value>us-east-1</value>
+      </property>
+  auth:
+    enabled: true
+    username: vib_user
+    password: "ComplicatedPassword!1234"
+    firstName: VIBFIRST
+    lastName: VIBNAME
+    email: vib_email@example.com
+usePasswordFile: true
+service:
+  type: LoadBalancer
+  ports:
+    web: 80
+    client: 41010
+    flight: 42010
+metrics:
+  enabled: true
+  service:
+    ports:
+      metrics: 41881
+  containerPorts:
+    metrics: 41884
+# We deploy a PostgreSQL instance so we can use it in Dremio as a data source. We create a database
+# that we can verify in the Cypress test
+extraDeploy:
+  - |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: postgresql
+      labels:
+        app.kubernetes.io/instance: postgresql
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/name: postgresql
+        app.kubernetes.io/version: 16.3.0
+        helm.sh/chart: postgresql-15.5.6
+    type: Opaque
+    stringData:
+      postgres-password: "password123"
+  - |
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: postgresql
+      labels:
+        app.kubernetes.io/instance: postgresql
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/name: postgresql
+        app.kubernetes.io/version: 16.3.0
+        helm.sh/chart: postgresql-15.5.6
+        app.kubernetes.io/component: primary
+    spec:
+      type: ClusterIP
+      sessionAffinity: None
+      ports:
+        - name: tcp-postgresql
+          port: 5432
+          targetPort: tcp-postgresql
+          nodePort: null
+      selector:
+        app.kubernetes.io/instance: postgresql
+        app.kubernetes.io/name: postgresql
+        app.kubernetes.io/component: primary
+  - |
+    apiVersion: apps/v1
+    kind: StatefulSet
+    metadata:
+      name: postgresql
+      labels:
+        app.kubernetes.io/instance: postgresql
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/name: postgresql
+        app.kubernetes.io/version: 16.3.0
+        helm.sh/chart: postgresql-15.5.6
+        app.kubernetes.io/component: primary
+    spec:
+      replicas: 1
+      updateStrategy:
+        rollingUpdate: {}
+        type: RollingUpdate
+      selector:
+        matchLabels:
+          app.kubernetes.io/instance: postgresql
+          app.kubernetes.io/name: postgresql
+          app.kubernetes.io/component: primary
+      template:
+        metadata:
+          name: postgresql
+          labels:
+            app.kubernetes.io/instance: postgresql
+            app.kubernetes.io/managed-by: Helm
+            app.kubernetes.io/name: postgresql
+            app.kubernetes.io/version: 16.3.0
+            helm.sh/chart: postgresql-15.5.6
+            app.kubernetes.io/component: primary
+        spec:
+          # Using the same security context as the master-coordinator to avoid issues with Openshift
+          securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.masterCoordinator.podSecurityContext "context" $) | nindent 8 }}
+          containers:
+            - name: postgresql
+              image: docker.io/bitnami/postgresql:16.3.0-debian-12-r14
+              imagePullPolicy: "IfNotPresent"
+              # Using the same security context as the master-coordinator to avoid issues with Openshift
+              securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.masterCoordinator.containerSecurityContext "context" $) | nindent 12 }}
+              env:
+                - name: POSTGRES_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: postgresql
+                      key: postgres-password
+              ports:
+                - name: tcp-postgresql
+                  containerPort: 5432
+              volumeMounts:
+                - name: empty-dir
+                  mountPath: /tmp
+                  subPath: tmp-dir
+                - name: empty-dir
+                  mountPath: /opt/bitnami/postgresql/conf
+                  subPath: app-conf-dir
+                - name: empty-dir
+                  mountPath: /opt/bitnami/postgresql/tmp
+                  subPath: app-tmp-dir
+                - name: custom-init-scripts
+                  mountPath: /docker-entrypoint-initdb.d/
+                - name: dshm
+                  mountPath: /dev/shm
+                - name: data
+                  mountPath: /bitnami/postgresql
+          volumes:
+            - name: empty-dir
+              emptyDir: {}
+            - name: custom-init-scripts
+              configMap:
+                name: postgresql-init-scripts
+            - name: dshm
+              emptyDir:
+                medium: Memory
+      volumeClaimTemplates:
+        - apiVersion: v1
+          kind: PersistentVolumeClaim
+          metadata:
+            name: data
+          spec:
+            accessModes:
+              - "ReadWriteOnce"
+            resources:
+              requests:
+                storage: "8Gi"
+  - |
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: postgresql-init-scripts
+      labels:
+        app.kubernetes.io/instance: postgresql
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/name: postgresql
+        app.kubernetes.io/version: 16.3.0
+        helm.sh/chart: postgresql-15.5.6
+    data:
+      add_data.sql: |
+        CREATE DATABASE vib_test;
+        \c vib_test;
+        CREATE TABLE cars (
+          brand VARCHAR(255),
+          model VARCHAR(255),
+          year INT
+        );
+        INSERT INTO cars (brand, model, year)
+        VALUES
+          ('Toyota', 'Camry', 2020),
+          ('Honda', 'Civic', 2019),
+          ('Ford', 'Mustang', 2018);

.vib/dremio/vib-action.config

@@ -0,0 +1 @@
+verification-mode=SERIAL

.vib/dremio/vib-publish.json

@@ -0,0 +1,38 @@
+{
+  "phases": {
+    "package": {
+      "context": {
+        "resources": {
+          "url": "{SHA_ARCHIVE}",
+          "path": "/bitnami/dremio"
+        }
+      },
+      "actions": [
+        {
+          "action_id": "helm-package"
+        },
+        {
+          "action_id": "helm-lint"
+        }
+      ]
+    },
+    "publish": {
+      "actions": [
+        {
+          "action_id": "helm-publish",
+          "params": {
+            "repository": {
+              "kind": "S3",
+              "url": "{VIB_ENV_S3_URL}",
+              "authn": {
+                "access_key_id": "{VIB_ENV_S3_ACCESS_KEY_ID}",
+                "secret_access_key": "{VIB_ENV_S3_SECRET_ACCESS_KEY}",
+                "role": "{VIB_ENV_S3_ROLE_ARN}"
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
+}

.vib/dremio/vib-verify.json

@@ -0,0 +1,61 @@
+{
+  "phases": {
+    "package": {
+      "context": {
+        "resources": {
+          "url": "{SHA_ARCHIVE}",
+          "path": "/bitnami/dremio"
+        }
+      },
+      "actions": [
+        {
+          "action_id": "helm-package"
+        },
+        {
+          "action_id": "helm-lint"
+        }
+      ]
+    },
+    "verify": {
+      "context": {
+        "resources": {
+          "url": "{SHA_ARCHIVE}",
+          "path": "/bitnami/dremio"
+        },
+        "target_platform": {
+          "target_platform_id": "{VIB_ENV_TARGET_PLATFORM}",
+          "size": {
+            "name": "L4"
+          }
+        }
+      },
+      "actions": [
+        {
+          "action_id": "goss",
+          "params": {
+            "resources": {
+              "path": "/.vib"
+            },
+            "tests_file": "dremio/goss/goss.yaml",
+            "vars_file": "dremio/runtime-parameters.yaml",
+            "remote": {
+              "pod": {
+                "workload": "sts-dremio-master-coordinator"
+              }
+            }
+          }
+        },
+        {
+          "action_id": "cypress",
+          "params": {
+            "resources": {
+              "path": "/.vib/dremio/cypress"
+            },
+            "endpoint": "lb-dremio-http-web",
+            "app_protocol": "HTTP"
+          }
+        }
+      ]
+    }
+  }
+}