Add TSDB support to grafana-loki chart (#22471)

* Add TSDB support to grafana-loki chart

TSDB is the recommended Loki index since version 2.8. This configuration is based on an example configuration from https://grafana.com/docs/loki/latest/operations/storage/tsdb/

Signed-off-by: pahanik <prispav@gmail.com>

* Version bump and correct formatting

Signed-off-by: pahanik <prispav@gmail.com>

* Set proper effective date for TSDB
Signed-off-by: Pavlo Prystupa <prispav@gmail.com>

Signed-off-by: pahanik <prispav@gmail.com>

* Change effective date
Signed-off-by: pahanik <prispav@gmail.com>

Signed-off-by: pahanik <prispav@gmail.com>

* Remove comments

Signed-off-by: pahanik <prispav@gmail.com>

* Keep the existing dir structure

Co-authored-by: Andrés Bono <andresbonojimenez@gmail.com>
Signed-off-by: pahanik <prispav@gmail.com>

* Set future date for TSDB switch

Signed-off-by: pahanik <prispav@gmail.com>

* Enable 2 weeks retention period

Signed-off-by: pahanik <prispav@gmail.com>

* Test checks

Signed-off-by: pahanik <prispav@gmail.com>

* Return default compactor values

Signed-off-by: pahanik <prispav@gmail.com>

* Version bump
Signed-off-by: pahanik <prispav@gmail.com>

* Remove odd line

Signed-off-by: pahanik <prispav@gmail.com>

* Add note about switch to TSDB and corresponding changes
Signed-off-by: pahanik <prispav@gmail.com>

---------

Signed-off-by: pahanik <prispav@gmail.com>
Signed-off-by: Andrés Bono <andresbono@vmware.com>
Signed-off-by: Carlos Rodríguez Hernández <carlosrh@vmware.com>
Co-authored-by: Andrés Bono <andresbonojimenez@gmail.com>
Co-authored-by: Andrés Bono <andresbono@vmware.com>
Co-authored-by: Carlos Rodríguez Hernández <carlosrh@vmware.com>

bitnami/grafana-loki/Chart.yaml

@@ -57,4 +57,4 @@ maintainers:
 name: grafana-loki
 sources:
 - https://github.com/bitnami/charts/tree/main/bitnami/grafana-loki
-version: 2.17.2
+version: 2.17.3

bitnami/grafana-loki/templates/NOTES.txt

@@ -100,6 +100,10 @@ Installed components:
 {{- end }}
 {{- end }}
 
+Grafana Loki is switched from BoltDB to the TSDB store on 2024-03-12. A two-week retention period is introduced. memcachedindexqueries.enabled will be off after 2024-03-26. Info:
+https://grafana.com/docs/loki/latest/operations/storage/tsdb/
+https://grafana.com/docs/loki/latest/operations/storage/retention/
+
 {{- include "grafana-loki.checkRollingTags" . }}
 {{- include "grafana-loki.validateValues" . }}
 {{- include "common.warnings.resources" (dict "sections" (list "compactor" "distributor" "gateway" "indexGateway" "ingester" "promtail" "querier" "queryFrontend" "queryScheduler" "ruler" "tableManager" "volumePermissions") "context" $) }}

bitnami/grafana-loki/values.yaml

@@ -122,6 +122,7 @@ loki:
         dir: {{ .Values.loki.dataDir }}/wal
 
     limits_config:
+      retention_period: 336h
       enforce_metric_name: false
       reject_old_samples: true
       reject_old_samples_max_age: 168h
@@ -137,6 +138,13 @@ loki:
         index:
           prefix: index_
           period: 24h
+      - from: 2024-03-12
+        index:
+          period: 24h
+          prefix: index_
+        object_store: filesystem
+        schema: v12
+        store: tsdb
 
     storage_config:
       boltdb_shipper:
@@ -160,6 +168,20 @@ loki:
           addresses: dns+{{ include "grafana-loki.memcached-index-queries.host" . }}
           service: http
         {{- end }}
+      tsdb_shipper:
+        active_index_directory: {{ .Values.loki.dataDir }}/loki/tsdb-index
+        cache_location: {{ .Values.loki.dataDir }}/loki/tsdb-cache
+        {{- if .Values.indexGateway.enabled }}
+        index_gateway_client:
+          server_address: {{ (printf "dns:///%s:9095" (include "grafana-loki.index-gateway.fullname" .)) }}
+        {{- end }}
+        shared_store: filesystem
+
+    query_scheduler:
+      max_outstanding_requests_per_tenant: 32768
+
+    querier:
+      max_concurrent: 16
 
     chunk_store_config:
       max_look_back_period: 0s
@@ -216,7 +238,12 @@ loki:
       tail_proxy_url: http://{{ include "grafana-loki.querier.fullname" . }}:{{ .Values.querier.service.ports.http }}
 
     compactor:
+      working_directory: {{ .Values.loki.dataDir }}/loki/retention
       shared_store: filesystem
+      compaction_interval: 10m
+      retention_enabled: true
+      retention_delete_delay: 2h
+      retention_delete_worker_count: 150
 
     ruler:
       storage: