feat: migrate logs to Loki in Grafana Cloud

chart/README.md

@@ -15,7 +15,7 @@ Helm chart to deploy all different types OTEL Collectors for infrastructure tele
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | apps | object | `{"container-logs":{"collector":"container-logs.yaml","enable":true,"spec":{"env":[{"name":"KUBE_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}}],"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"daemonset","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"container-logs-collector"},"podSecurityContext":{"runAsNonRoot":false},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"resources":{"limits":{"cpu":1,"memory":"1Gi"},"requests":{"cpu":"250m","memory":"256Mi"}},"securityContext":{},"serviceAccount":"otel-container-logs-sa","volumeMounts":[{"mountPath":"/var/log/pods","name":"varlogpods","readOnly":true},{"mountPath":"/var/lib/docker/containers","name":"varlibdockercontainers","readOnly":true}],"volumes":[{"hostPath":{"path":"/var/log/pods"},"name":"varlogpods"},{"hostPath":{"path":"/var/lib/docker/containers"},"name":"varlibdockercontainers"}]}},"k8s-cluster":{"collector":"k8s-cluster.yaml","enable":true,"spec":{"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"deployment","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"cluster-metrics-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"50m","memory":"256Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-k8scluster-sa"}},"k8s-events":{"collector":"k8s-events.yaml","enable":true,"spec":{"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"deployment","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"cluster-events-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"50m","memory":"256Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-k8sevents-sa"}},"kubelet-stats":{"collector":"kubelet-stats-node-name.yaml","enable":true,"spec":{"env":[{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"KUBE_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"NODE_IP","valueFrom":{"fieldRef":{"fieldPath":"status.hostIP"}}}],"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"daemonset","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"kubelet-stats-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"50m","memory":"128Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-kubelet-sa"}},"otlp":{"collector":"otlp.yaml","enable":true,"spec":{"env":[{"name":"KUBE_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}}],"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"daemonset","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"otlp-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"50m","memory":"128Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-otlp-sa"}},"target-allocator":{"collector":"ta.yaml","enable":true,"spec":{"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"statefulset","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"target-allocator-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-ta-sa","targetAllocator":{"enabled":true,"prometheusCR":{"enabled":true},"serviceAccount":"otel-collector-ta-sa"}}}}` | Dictionary of collectors to deploy. Key is the name of the collector, while the value is the configuration for the collector. This has 2 sub keys: `collector` which is the actual [collector configuration](https://opentelemetry.io/docs/collector/configuration/), and `spec`, which is the [operator's configuration](https://github.com/open-telemetry/opentelemetry-operator/blob/main/docs/api.md#opentelemetrycollectorspec) for the collector. |
-| auth | object | `{"external":{"enable":true,"policy":{"creation":"Owner","deletion":"Retain"},"secretStore":{"kind":"SecretStore","name":"doppler-silicon"}},"internal":{"enable":false,"o2":"sometoken"},"o2remote":"MANUAL_O2_TOKEN","secretName":"o2-cloud-secrets"}` | Auth configuration for the collectors |
+| auth | object | `{"external":{"enable":true,"policy":{"creation":"Owner","deletion":"Retain"},"secretStore":{"kind":"SecretStore","name":"doppler-silicon"}},"internal":{"enable":false,"loki":{"token":"sometoken","user":"someuser"},"o2":"sometoken"},"remote":{"loki":{"token":"MANUAL_LOKI_TOKEN","user":"MANUAL_LOKI_USER"},"o2":"MANUAL_O2_TOKEN"},"secretName":"o2-cloud-secrets"}` | Auth configuration for the collectors |
 | auth.external | object | `{"enable":true,"policy":{"creation":"Owner","deletion":"Retain"},"secretStore":{"kind":"SecretStore","name":"doppler-silicon"}}` | Use external auth for the collectors |
 | auth.external.enable | bool | `true` | Enable external auth |
 | auth.external.policy | object | `{"creation":"Owner","deletion":"Retain"}` | External Secret Policy |
@@ -24,10 +24,16 @@ Helm chart to deploy all different types OTEL Collectors for infrastructure tele
 | auth.external.secretStore | object | `{"kind":"SecretStore","name":"doppler-silicon"}` | Secret Store to use for secrets |
 | auth.external.secretStore.kind | string | `"SecretStore"` | Kind of the secret store, either `ClusterSecretStore` or `SecretStore` |
 | auth.external.secretStore.name | string | `"doppler-silicon"` | Name of the secret store |
-| auth.internal | object | `{"enable":false,"o2":"sometoken"}` | Use internal auth for the collectors (hard coded password) |
+| auth.internal | object | `{"enable":false,"loki":{"token":"sometoken","user":"someuser"},"o2":"sometoken"}` | Use internal auth for the collectors (hard coded password) |
 | auth.internal.enable | bool | `false` | Enable internal auth |
+| auth.internal.loki | object | `{"token":"sometoken","user":"someuser"}` | Grafana Cloud Loki plaintext user |
+| auth.internal.loki.token | string | `"sometoken"` | Grafana Cloud Loki plaintext token |
+| auth.internal.loki.user | string | `"someuser"` | Grafana Cloud Loki plaintext user |
 | auth.internal.o2 | string | `"sometoken"` | OpenObserve plaintext token |
-| auth.o2remote | string | `"MANUAL_O2_TOKEN"` | Remote OpenObserve Token |
+| auth.remote | object | `{"loki":{"token":"MANUAL_LOKI_TOKEN","user":"MANUAL_LOKI_USER"},"o2":"MANUAL_O2_TOKEN"}` | Remote Tokens |
+| auth.remote.loki.token | string | `"MANUAL_LOKI_TOKEN"` | Grafana Cloud Loki Token |
+| auth.remote.loki.user | string | `"MANUAL_LOKI_USER"` | Grafana Cloud Loki User |
+| auth.remote.o2 | string | `"MANUAL_O2_TOKEN"` | OpenObserve Token |
 | auth.secretName | string | `"o2-cloud-secrets"` | Name of the secret to use for the collector |
 | cluster | string | `"opal"` | Cluster the operators are deployed to |
 | configMapName | string | `"otel-common-config-map"` | Name of the common config map to propagate to all collectors |
@@ -54,6 +60,7 @@ Helm chart to deploy all different types OTEL Collectors for infrastructure tele
 | kubelet.serviceAccount.name | string | `"otel-collector-kubelet-sa"` | Name of the service account |
 | kubelet.vclusterCompatibility | bool | `false` | VCluster Compatibility |
 | landscape | string | `"entei"` | Landscape the operator is deployed to |
+| lokiEndpoint | string | `"https://logs-prod-020.grafana.net/loki/api/v1/push"` | Grafana Cloud Loki Endpoint |
 | o2Endpoint | string | `"https://api.openobserve.ai/api/atomicloud_MwvsSHPiOT9uFdn/"` | Open Observe Endpoint |
 | otlp.serviceAccount | object | `{"create":true,"name":"otel-collector-otlp-sa"}` | Service account for OTLP |
 | otlp.serviceAccount.create | bool | `true` | Enable creation of the service account |

chart/collectors/container-logs.yaml

@@ -155,21 +155,24 @@ processors:
         - 'resource.attributes["drop_log"] == "true"'
 
 exporters:
-  otlphttp/openobserve:
-    endpoint: ${env:O2_ENDPOINT}
-    headers:
-      Authorization: Basic ${env:O2_AUTH}
-
+  loki:
+    endpoint: ${env:LOKI_ENDPOINT}
+    auth:
+      authenticator: basicauth/loki
 extensions:
   health_check:
   pprof:
   zpages:
     endpoint: 0.0.0.0:55679
+  basicauth/loki:
+    client_auth:
+      username: ${env:LOKI_USER}
+      password: ${env:LOKI_TOKEN}
 
 service:
-  extensions: [ health_check, pprof, zpages ]
+  extensions: [ health_check, pprof, zpages, basicauth/loki ]
   pipelines:
     logs:
       receivers: [ filelog ]
       processors: [ k8sattributes, resource, attributes, batch ]
-      exporters: [ otlphttp/openobserve ]
+      exporters: [ loki ]

chart/collectors/k8s-events.yaml

@@ -2,17 +2,20 @@ receivers:
   k8s_events:
 
 exporters:
-  otlphttp/openobserve_k8s_events:
-    endpoint: ${env:O2_ENDPOINT}
-    headers:
-      Authorization: Basic ${env:O2_AUTH}
-      stream-name: k8s_events
+  loki:
+    endpoint: ${env:LOKI_ENDPOINT}
+    auth:
+      authenticator: basicauth/loki
 
 extensions:
   health_check:
   pprof:
   zpages:
     endpoint: 0.0.0.0:55679
+  basicauth/loki:
+    client_auth:
+      username: ${env:LOKI_USER}
+      password: ${env:LOKI_TOKEN}
 
 processors:
   batch:
@@ -31,9 +34,9 @@ processors:
         value: ${env:K8S_CLUSTER}
 
 service:
-  extensions: [ health_check, pprof, zpages ]
+  extensions: [ health_check, pprof, zpages, basicauth/loki ]
   pipelines:
     logs:
       receivers: [ k8s_events ]
       processors: [ batch, attributes, resource ]
-      exporters: [ otlphttp/openobserve_k8s_events ]
+      exporters: [ loki ]

chart/collectors/otlp.yaml

@@ -89,20 +89,27 @@ exporters:
     endpoint: ${env:O2_ENDPOINT}
     headers:
       Authorization: Basic ${env:O2_AUTH}
+  loki:
+    endpoint: ${env:LOKI_ENDPOINT}
+    auth:
+      authenticator: basicauth/loki
 
 extensions:
   health_check:
   pprof:
   zpages:
     endpoint: 0.0.0.0:55679
-
+  basicauth/loki:
+    client_auth:
+      username: ${env:LOKI_USER}
+      password: ${env:LOKI_TOKEN}
 service:
-  extensions: [ health_check, pprof, zpages ]
+  extensions: [ health_check, pprof, zpages, basicauth/loki ]
   pipelines:
     logs:
       receivers: [ otlp ]
       processors: [ k8sattributes, resource, attributes, batch ]
-      exporters: [ otlphttp/openobserve ]
+      exporters: [ loki ]
     traces:
       receivers: [ otlp ]
       processors: [ k8sattributes, resource, attributes, batch ]

chart/templates/otel-common-config.yaml

@@ -9,3 +9,5 @@ data:
   K8S_CLUSTER: "{{ .Values.cluster }}"
   TA_ENDPOINT: "{{ .Values.taEndpoint }}"
   O2_ENDPOINT: "{{ .Values.o2Endpoint }}"
+  LOKI_ENDPOINT: "{{ .Values.lokiEndpoint }}"
+

chart/templates/o2_tokens.yaml → chart/templates/secrets.yaml

@@ -18,7 +18,13 @@ spec:
   data:
     - secretKey: O2_AUTH
       remoteRef:
-        key: {{ .Values.auth.o2remote }}
+        key: {{ .Values.auth.remote.o2 }}
+    - secretKey: LOKI_USER
+      remoteRef:
+        key: {{ .Values.auth.remote.loki.user }}
+    - secretKey: LOKI_TOKEN
+      remoteRef:
+        key: {{ .Values.auth.remote.loki.token }}
 ---
 {{- end }}
 {{- if .Values.auth.internal.enable }}
@@ -31,4 +37,6 @@ metadata:
     {{- toYaml .Values.secretAnnotation | nindent 4 }}
 data:
   O2_AUTH: {{ b64enc .Values.auth.internal.o2 }}
+  LOKI_USER: {{ b64enc .Values.auth.internal.loki.user }}
+  LOKI_TOKEN: {{ b64enc .Values.auth.internal.loki.token }}
 {{- end }}

chart/values.yaml

@@ -20,6 +20,9 @@ cluster: opal
 # -- Open Observe Endpoint
 o2Endpoint: "https://api.openobserve.ai/api/atomicloud_MwvsSHPiOT9uFdn/"
 
+# -- Grafana Cloud Loki Endpoint
+lokiEndpoint: "https://logs-prod-020.grafana.net/loki/api/v1/push"
+
 # -- Name of the common config map to propagate to all collectors
 configMapName: &configMapName otel-common-config-map
 
@@ -120,14 +123,28 @@ secretAnnotation:
 auth:
   # -- Name of the secret to use for the collector
   secretName: &secretName o2-cloud-secrets
-  # -- Remote OpenObserve Token
-  o2remote: MANUAL_O2_TOKEN
+  # -- Remote Tokens
+  remote:
+    # -- OpenObserve Token
+    o2: MANUAL_O2_TOKEN
+    loki:
+      # -- Grafana Cloud Loki User
+      user: MANUAL_LOKI_USER
+      # -- Grafana Cloud Loki Token
+      token: MANUAL_LOKI_TOKEN
+
   # -- Use internal auth for the collectors (hard coded password)
   internal:
     # -- Enable internal auth
     enable: false
     # -- OpenObserve plaintext token
     o2: sometoken
+    # -- Grafana Cloud Loki plaintext user
+    loki:
+      # -- Grafana Cloud Loki plaintext user
+      user: someuser
+      # -- Grafana Cloud Loki plaintext token
+      token: sometoken
   # -- Use external auth for the collectors
   external:
     # -- Enable external auth