betelgeusebytes/k8s/observability-stack/06-alloy-config.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: alloy-config
  namespace: observability
data:
  config.alloy: |
    // Logging configuration
    logging {
      level  = "info"
      format = "logfmt"
    }

    // Discover Kubernetes pods for log collection
    discovery.kubernetes "pods" {
      role = "pod"
    }

    // Discover Kubernetes nodes
    discovery.kubernetes "nodes" {
      role = "node"
    }

    // Relabel pods for log collection
    discovery.relabel "pod_logs" {
      targets = discovery.kubernetes.pods.targets

      // Only scrape pods with logs
      rule {
        source_labels = ["__meta_kubernetes_pod_container_name"]
        action        = "keep"
        regex         = ".+"
      }

      // Set the log path
      rule {
        source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
        target_label  = "__path__"
        separator     = "/"
        replacement   = "/var/log/pods/*$1/*.log"
      }

      // Set namespace label
      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        target_label  = "namespace"
      }

      // Set pod name label
      rule {
        source_labels = ["__meta_kubernetes_pod_name"]
        target_label  = "pod"
      }

      // Set container name label
      rule {
        source_labels = ["__meta_kubernetes_pod_container_name"]
        target_label  = "container"
      }

      // Set node name label
      rule {
        source_labels = ["__meta_kubernetes_pod_node_name"]
        target_label  = "node"
      }

      // Copy all pod labels
      rule {
        action = "labelmap"
        regex  = "__meta_kubernetes_pod_label_(.+)"
      }
    }

    // Read logs from discovered pods
    loki.source.kubernetes "pod_logs" {
      targets    = discovery.relabel.pod_logs.output
      forward_to = [loki.process.pod_logs.receiver]
    }

    // Process and enrich logs
    loki.process "pod_logs" {
      forward_to = [loki.write.local.receiver]

      // Parse JSON logs
      stage.json {
        expressions = {
          level   = "level",
          message = "message",
          timestamp = "timestamp",
        }
      }

      // Extract log level
      stage.labels {
        values = {
          level = "",
        }
      }

      // Add cluster label
      stage.static_labels {
        values = {
          cluster = "betelgeuse-k8s",
        }
      }
    }

    // Write logs to Loki
    loki.write "local" {
      endpoint {
        url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
      }
    }

    // OpenTelemetry receiver for traces
    otelcol.receiver.otlp "default" {
      grpc {
        endpoint = "0.0.0.0:4317"
      }

      http {
        endpoint = "0.0.0.0:4318"
      }

      output {
        traces  = [otelcol.exporter.otlp.tempo.input]
        metrics = [otelcol.exporter.prometheus.metrics.input]
      }
    }

    // Export traces to Tempo
    otelcol.exporter.otlp "tempo" {
      client {
        endpoint = "tempo.observability.svc.cluster.local:4317"
        tls {
          insecure = true
        }
      }
    }

    // Export OTLP metrics to Prometheus
    otelcol.exporter.prometheus "metrics" {
      forward_to = [prometheus.remote_write.local.receiver]
    }

    // Remote write to Prometheus
    prometheus.remote_write "local" {
      endpoint {
        url = "http://prometheus.observability.svc.cluster.local:9090/api/v1/write"
      }
    }

    // Scrape local metrics (Alloy's own metrics)
    prometheus.scrape "alloy" {
      targets = [{
        __address__ = "localhost:12345",
      }]
      forward_to = [prometheus.remote_write.local.receiver]
    }