{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}

{{- if .Values.worker.enabled }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: {{ include "deepspeed.v0.worker.fullname" . }}
  namespace: {{ include "common.names.namespace" . | quote }}
  {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.labels .Values.commonLabels ) "context" . ) }}
  labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }}
    app.kubernetes.io/part-of: deepspeed
    app.kubernetes.io/component: worker
  {{- if or .Values.worker.annotations .Values.commonAnnotations }}
  {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.annotations .Values.commonAnnotations ) "context" . ) }}
  annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
  {{- end }}
spec:
  replicas: {{ .Values.worker.replicaCount }}
  podManagementPolicy: {{ .Values.worker.podManagementPolicy | quote }}
  serviceName: {{ printf "%s-headless" (include "deepspeed.v0.worker.fullname" .) }}
  {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }}
  selector:
    matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
      app.kubernetes.io/part-of: deepspeed
      app.kubernetes.io/component: worker
  {{- if .Values.worker.updateStrategy }}
  updateStrategy: {{- toYaml .Values.worker.updateStrategy | nindent 4 }}
  {{- end }}
  template:
    metadata:
      annotations:
        checksum/ssh-private-key: {{ include (print $.Template.BasePath "/ssh-keys-secret.yaml") . | sha256sum }}
        checksum/ssh-server-conf: {{ include (print $.Template.BasePath "/worker/ssh-server-configmap.yaml") . | sha256sum }}
        {{- if (include "deepspeed.v0.source.createConfigMap" .) }}
        checksum/source: {{ include (print $.Template.BasePath "/client/source-configmap.yaml") . | sha256sum }}
        {{- end }}
        {{- if or .Values.worker.podAnnotations .Values.commonAnnotations }}
        {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podAnnotations .Values.commonAnnotations ) "context" . ) }}
        {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 8 }}
        {{- end }}
      labels: {{- include "common.labels.standard" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }}
        app.kubernetes.io/part-of: deepspeed
        app.kubernetes.io/component: worker
    spec:
      serviceAccountName: {{ template "deepspeed.v0.worker.serviceAccountName" . }}
      {{- include "deepspeed.v0.imagePullSecrets" . | nindent 6 }}
      automountServiceAccountToken: {{ .Values.worker.automountServiceAccountToken }}
      {{- if .Values.worker.hostAliases }}
      hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.worker.hostAliases "context" $) | nindent 8 }}
      {{- end }}
      {{- if .Values.worker.affinity }}
      affinity: {{- include "common.tplvalues.render" (dict "value" .Values.worker.affinity "context" $) | nindent 8 }}
      {{- else }}
      affinity:
        podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAffinityPreset "component" "worker" "customLabels" $podLabels "context" $) | nindent 10 }}
        podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAntiAffinityPreset "component" "worker" "customLabels" $podLabels "context" $) | nindent 10 }}
        nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.worker.nodeAffinityPreset.type "key" .Values.worker.nodeAffinityPreset.key "values" .Values.worker.nodeAffinityPreset.values) | nindent 10 }}
      {{- end }}
      {{- if .Values.worker.nodeSelector }}
      nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.worker.nodeSelector "context" $) | nindent 8 }}
      {{- end }}
      {{- if .Values.worker.runtimeClassName }}
      runtimeClassName: {{ .Values.worker.runtimeClassName | quote }}
      {{- end }}
      {{- if .Values.worker.tolerations }}
      tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.tolerations "context" .) | nindent 8 }}
      {{- end }}
      {{- if .Values.worker.priorityClassName }}
      priorityClassName: {{ .Values.worker.priorityClassName | quote }}
      {{- end }}
      {{- if .Values.worker.schedulerName }}
      schedulerName: {{ .Values.worker.schedulerName | quote }}
      {{- end }}
      {{- if .Values.worker.podSecurityContext.enabled }}
      securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.podSecurityContext "context" $) | nindent 8 }}
      {{- end }}
      {{- if .Values.worker.topologySpreadConstraints }}
      topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.worker.topologySpreadConstraints "context" .) | nindent 8 }}
      {{- end }}
      {{- if .Values.worker.terminationGracePeriodSeconds }}
      terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
      {{- end }}
      initContainers:
        {{- if .Values.worker.enableDefaultInitContainers }}
        {{- include "deepspeed.v0.ssh.serverInitContainer" . | nindent 8 }}
        {{- end }}
        {{- if eq .Values.source.type "git" }}
        {{- include "deespeed.git.cloneInitContainer" (dict "component" "worker" "context" $) | nindent 8 }}
        {{- end }}
        {{- if and .Values.volumePermissions.enabled .Values.client.persistence.enabled }}
        {{- include "deepspeed.v0.volumePermissionsInitContainer" (dict "component" "worker" "context" $) | nindent 8 }}
        {{- end }}
        {{- if .Values.worker.initContainers }}
        {{- include "common.tplvalues.render" (dict "value" .Values.worker.initContainers "context" $) | nindent 8 }}
        {{- end }}
      containers:
        - name: deepspeed
          image: {{ include "deepspeed.v0.image" . }}
          imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
          {{- if .Values.worker.containerSecurityContext.enabled }}
          securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | nindent 12 }}
          {{- end }}
          {{- if .Values.diagnosticMode.enabled }}
          command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
          {{- else if .Values.worker.command }}
          command: {{- include "common.tplvalues.render" (dict "value" .Values.worker.command "context" $) | nindent 12 }}
          {{- else }}
          command:
            - bash
          {{- end }}
          {{- if .Values.diagnosticMode.enabled }}
          args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
          {{- else if .Values.worker.args }}
          args: {{- include "common.tplvalues.render" (dict "value" .Values.worker.args "context" $) | nindent 12 }}
          {{- else }}
          args:
            - -ec
            - |
              #!/bin/bash
              [[ -f "/opt/bitnami/scripts/deepspeed/entrypoint.sh" ]] && source "/opt/bitnami/scripts/deepspeed/entrypoint.sh"
              ## sshd requires the absolute path
              SSHD_PATH="$(command -v sshd)"
              exec "$SSHD_PATH" -D -h "/bitnami/ssh/server-private-key/ssh_host_rsa_key" -e
          {{- end }}
          ports:
            - name: tcp-ssh
              containerPort: {{ .Values.worker.containerPorts.ssh }}
              protocol: TCP
          env:
            - name: HOME
              value: /home/deepspeed
            {{- if .Values.worker.extraEnvVars }}
            {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVars "context" $) | nindent 12 }}
            {{- end }}
          envFrom:
            {{- if .Values.worker.extraEnvVarsCM }}
            - configMapRef:
                name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsCM "context" $) }}
            {{- end }}
            {{- if .Values.worker.extraEnvVarsSecret }}
            - secretRef:
                name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsSecret "context" $) }}
            {{- end }}
          {{- if .Values.worker.resources }}
          resources: {{- toYaml .Values.worker.resources | nindent 12 }}
          {{- else if ne .Values.worker.resourcesPreset "none" }}
          resources: {{- include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | nindent 12 }}
          {{- end }}
          {{- if not .Values.diagnosticMode.enabled }}
          {{- if .Values.worker.customLivenessProbe }}
          livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customLivenessProbe "context" $) | nindent 12 }}
          {{- else if .Values.worker.livenessProbe.enabled }}
          livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.livenessProbe "enabled") "context" $) | nindent 12 }}
            exec:
              command:
                - pgrep
                - -f
                - sshd
          {{- end }}
          {{- if .Values.worker.customReadinessProbe }}
          readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customReadinessProbe "context" $) | nindent 12 }}
          {{- else if .Values.worker.readinessProbe.enabled }}
          readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.readinessProbe "enabled") "context" $) | nindent 12 }}
            tcpSocket:
              port: tcp-ssh
          {{- end }}
          {{- if .Values.worker.customStartupProbe }}
          startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customStartupProbe "context" $) | nindent 12 }}
          {{- else if .Values.worker.startupProbe.enabled }}
          startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.startupProbe "enabled") "context" $) | nindent 12 }}
            tcpSocket:
              port: tcp-ssh
          {{- end }}
          {{- end }}
          {{- if .Values.worker.lifecycleHooks }}
          lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.worker.lifecycleHooks "context" $) | nindent 12 }}
          {{- end }}
          volumeMounts:
            - name: ssh-server-config
              mountPath: /etc/ssh/sshd_config.d
            - name: empty-dir
              mountPath: /etc/ssh/sshd_config
              subPath: sshd-conf-dir/sshd_config
            # As we are disabling host key verification from the client, we can
            # auto-generate the worker private key
            - name: empty-dir
              mountPath: /bitnami/ssh/server-private-key
              subPath: app-worker-private-key-dir
            # We need to use /home because of the permission
            # limitations of sshd
            - name: empty-dir
              mountPath: /home
              subPath: home-dir
            - name: empty-dir
              mountPath: /tmp
              subPath: tmp-dir
            - name: empty-dir
              mountPath: /run
              subPath: run-dir
            - name: empty-dir
              mountPath: /home/deepspeed/.cache
              subPath: app-cache-dir
            - name: empty-dir
              mountPath: /home/deepspeed/.local
              subPath: app-local-dir
            - name: source
              mountPath: /app
            {{- if .Values.worker.extraVolumeMounts }}
            {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeMounts "context" $) | nindent 12 }}
            {{- end }}
        {{- if .Values.worker.sidecars }}
        {{- include "common.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
        {{- end }}
      volumes:
        - name: empty-dir
          emptyDir: {}
        # Depending on the application, the source needs to be in the worker host
        - name: source
          {{- if or (.Values.source.existingConfigMap) (include "deepspeed.v0.source.createConfigMap" .) }}
          configMap:
            name: {{ include "deepspeed.v0.source.configMapName" . }}
          {{- else }}
          emptyDir: {}
          {{- end }}
        # We need the client private key for generating the public key and updating the authorized_keys file
        - name: ssh-client-private-key
          secret:
            secretName: {{ include "deepspeed.v0.ssh.keySecretName" . }}
        # SSH Server configuration
        - name: ssh-server-config
          configMap:
            name: {{ include "deepspeed.v0.ssh.serverConfigMapName" . }}
        {{- if .Values.worker.extraVolumes }}
        {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumes "context" $) | nindent 8 }}
        {{- end }}
  {{- if not .Values.worker.persistence.enabled }}
        - name: data
          emptyDir: {}
  {{- else }}
  volumeClaimTemplates:
    - metadata:
        name: data
        {{- if .Values.worker.persistence.annotations }}
        annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.persistence.annotations "context" $) | nindent 10 }}
        {{- end }}
        {{- if .Values.worker.persistence.labels }}
        labels: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.persistence.labels "context" $) | nindent 10 }}
        {{- end }}
      spec:
        accessModes:
        {{- range .Values.worker.persistence.accessModes }}
          - {{ . | quote }}
        {{- end }}
        resources:
          requests:
            storage: {{ .Values.worker.persistence.size | quote }}
        {{- if .Values.worker.persistence.selector }}
        selector: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.persistence.selector "context" $) | nindent 10 }}
        {{- end }}
        {{ include "common.storage.class" (dict "persistence" .Values.worker.persistence "global" .Values.global) }}
  {{- end }}
{{- end }}
