Wire Forge-native release infrastructure

nixos/modules/burrow-observability.nix

@@ -0,0 +1,302 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.burrow.observability;
+  otelCollectorPackage =
+    if builtins.hasAttr "opentelemetry-collector-contrib" pkgs then
+      pkgs."opentelemetry-collector-contrib"
+    else
+      pkgs.opentelemetry-collector;
+  garageCfg = config.services.burrow.garage;
+  headscaleCfg = config.services.burrow.headscale;
+  grafanaHttpPort = config.services.grafana.settings.server.http_port or 3000;
+  grafanaHttpAddress = config.services.grafana.settings.server.http_addr or "127.0.0.1";
+  tailscaleExporterEnabled = cfg.tailscaleExporterEnvironmentFile != null;
+in
+{
+  options.services.burrow.observability = {
+    enable = lib.mkEnableOption "Burrow Prometheus, OpenTelemetry, Jaeger, and Grafana wiring";
+
+    prometheusListenAddress = lib.mkOption {
+      type = lib.types.str;
+      default = "127.0.0.1";
+      description = "Address used by the local Prometheus server.";
+    };
+
+    prometheusPort = lib.mkOption {
+      type = lib.types.port;
+      default = 9090;
+      description = "Port used by the local Prometheus server.";
+    };
+
+    otelGrpcEndpoint = lib.mkOption {
+      type = lib.types.str;
+      default = "127.0.0.1:4317";
+      description = "Local OTLP gRPC endpoint accepted by the OpenTelemetry collector.";
+    };
+
+    otelHttpEndpoint = lib.mkOption {
+      type = lib.types.str;
+      default = "127.0.0.1:4318";
+      description = "Local OTLP HTTP endpoint accepted by the OpenTelemetry collector.";
+    };
+
+    otelPrometheusEndpoint = lib.mkOption {
+      type = lib.types.str;
+      default = "127.0.0.1:9464";
+      description = "Prometheus endpoint where the OpenTelemetry collector exports its own metrics.";
+    };
+
+    jaegerImage = lib.mkOption {
+      type = lib.types.str;
+      default = "docker.io/jaegertracing/all-in-one:1.57";
+      description = "OCI image used for the local Jaeger all-in-one service.";
+    };
+
+    jaegerUiPort = lib.mkOption {
+      type = lib.types.port;
+      default = 16686;
+      description = "Local Jaeger UI and query API port.";
+    };
+
+    jaegerOtlpGrpcPort = lib.mkOption {
+      type = lib.types.port;
+      default = 14317;
+      description = "Local remapped OTLP gRPC port forwarded to Jaeger.";
+    };
+
+    jaegerOtlpHttpPort = lib.mkOption {
+      type = lib.types.port;
+      default = 14318;
+      description = "Local remapped OTLP HTTP port forwarded to Jaeger.";
+    };
+
+    jaegerAdminPort = lib.mkOption {
+      type = lib.types.port;
+      default = 14269;
+      description = "Local Jaeger admin and metrics port.";
+    };
+
+    tailscaleExporterEnvironmentFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = "Optional environment file for the Prometheus Tailscale exporter OAuth credentials.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable (lib.mkMerge [
+    {
+      virtualisation.podman.enable = true;
+
+      services.grafana = {
+        settings = {
+          metrics = {
+            enabled = true;
+            disable_total_stats = true;
+          };
+        };
+        provision = {
+          enable = true;
+          datasources.settings = {
+            apiVersion = 1;
+            prune = true;
+            datasources = [
+              {
+                name = "Prometheus";
+                uid = "prometheus";
+                type = "prometheus";
+                access = "proxy";
+                url = "http://${cfg.prometheusListenAddress}:${toString cfg.prometheusPort}";
+                isDefault = true;
+                jsonData = {
+                  timeInterval = "15s";
+                  httpMethod = "POST";
+                };
+              }
+              {
+                name = "Jaeger";
+                uid = "jaeger";
+                type = "jaeger";
+                access = "proxy";
+                url = "http://127.0.0.1:${toString cfg.jaegerUiPort}";
+              }
+            ];
+          };
+        };
+      };
+
+      services.prometheus = {
+        enable = true;
+        listenAddress = cfg.prometheusListenAddress;
+        port = cfg.prometheusPort;
+        retentionTime = "30d";
+        checkConfig = "syntax-only";
+        exporters = {
+          node = {
+            enable = true;
+            listenAddress = "127.0.0.1";
+          };
+          systemd = {
+            enable = true;
+            listenAddress = "127.0.0.1";
+          };
+        };
+        scrapeConfigs = [
+          {
+            job_name = "prometheus";
+            static_configs = [
+              { targets = [ "${cfg.prometheusListenAddress}:${toString cfg.prometheusPort}" ]; }
+            ];
+          }
+          {
+            job_name = "node";
+            static_configs = [
+              { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ]; }
+            ];
+          }
+          {
+            job_name = "systemd";
+            static_configs = [
+              { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.systemd.port}" ]; }
+            ];
+          }
+          {
+            job_name = "grafana";
+            metrics_path = "/metrics";
+            static_configs = [
+              { targets = [ "${grafanaHttpAddress}:${toString grafanaHttpPort}" ]; }
+            ];
+          }
+          {
+            job_name = "headscale";
+            metrics_path = "/metrics";
+            static_configs = [
+              { targets = [ "${headscaleCfg.metricsListenAddress}:${toString headscaleCfg.metricsPort}" ]; }
+            ];
+          }
+          {
+            job_name = "otel-collector";
+            static_configs = [
+              { targets = [ cfg.otelPrometheusEndpoint ]; }
+            ];
+          }
+          {
+            job_name = "jaeger";
+            metrics_path = "/metrics";
+            static_configs = [
+              { targets = [ "127.0.0.1:${toString cfg.jaegerAdminPort}" ]; }
+            ];
+          }
+        ] ++ lib.optionals garageCfg.enable [
+          {
+            job_name = "garage";
+            metrics_path = "/metrics";
+            static_configs = [
+              { targets = [ "${garageCfg.adminListenAddress}:${toString garageCfg.adminPort}" ]; }
+            ];
+          }
+        ] ++ lib.optionals tailscaleExporterEnabled [
+          {
+            job_name = "tailscale";
+            static_configs = [
+              { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.tailscale.port}" ]; }
+            ];
+          }
+        ];
+      };
+
+      services.opentelemetry-collector = {
+        enable = true;
+        package = otelCollectorPackage;
+        settings = {
+          receivers = {
+            otlp = {
+              protocols = {
+                grpc.endpoint = cfg.otelGrpcEndpoint;
+                http.endpoint = cfg.otelHttpEndpoint;
+              };
+            };
+          };
+          processors = {
+            resource.attributes = [
+              {
+                key = "service.namespace";
+                value = "burrow";
+                action = "upsert";
+              }
+              {
+                key = "deployment.environment";
+                value = config.networking.hostName;
+                action = "upsert";
+              }
+            ];
+            batch = { };
+          };
+          exporters = {
+            "otlp/jaeger" = {
+              endpoint = "127.0.0.1:${toString cfg.jaegerOtlpGrpcPort}";
+              tls.insecure = true;
+            };
+            prometheus.endpoint = cfg.otelPrometheusEndpoint;
+          };
+          service.pipelines = {
+            traces = {
+              receivers = [ "otlp" ];
+              processors = [
+                "resource"
+                "batch"
+              ];
+              exporters = [ "otlp/jaeger" ];
+            };
+            metrics = {
+              receivers = [ "otlp" ];
+              processors = [
+                "resource"
+                "batch"
+              ];
+              exporters = [ "prometheus" ];
+            };
+          };
+        };
+      };
+
+      virtualisation.oci-containers.containers.burrow-jaeger = {
+        image = cfg.jaegerImage;
+        autoStart = true;
+        environment = {
+          COLLECTOR_OTLP_ENABLED = "true";
+          SPAN_STORAGE_TYPE = "badger";
+          BADGER_EPHEMERAL = "false";
+          BADGER_DIRECTORY_VALUE = "/badger/data";
+          BADGER_DIRECTORY_KEY = "/badger/key";
+        };
+        volumes = [
+          "burrow-jaeger-badger:/badger"
+        ];
+        ports = [
+          "127.0.0.1:${toString cfg.jaegerUiPort}:16686"
+          "127.0.0.1:${toString cfg.jaegerOtlpGrpcPort}:4317"
+          "127.0.0.1:${toString cfg.jaegerOtlpHttpPort}:4318"
+          "127.0.0.1:${toString cfg.jaegerAdminPort}:14269"
+        ];
+        extraOptions = [ "--pull=always" ];
+      };
+
+      systemd.services.opentelemetry-collector.after = [ "podman-burrow-jaeger.service" ];
+      systemd.services.opentelemetry-collector.wants = [ "podman-burrow-jaeger.service" ];
+
+      environment.sessionVariables = {
+        OTEL_EXPORTER_OTLP_ENDPOINT = "http://${cfg.otelGrpcEndpoint}";
+        OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
+      };
+    }
+
+    (lib.mkIf tailscaleExporterEnabled {
+      services.prometheus.exporters.tailscale = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        environmentFile = cfg.tailscaleExporterEnvironmentFile;
+      };
+    })
+  ]);
+}