Add service health monitoring: postgres, node, cAdvisor exporters + dashboard

Exporters:
- postgres_exporter: DB connections, transactions, cache hit ratio, query stats
- node_exporter: host CPU, memory, disk, network
- cAdvisor: per-container CPU and memory usage

PostgreSQL:
- Enable pg_stat_statements for query-level performance tracking
- Track index scans vs sequential scans, cache hit ratio

Dashboard (service-health.json):
- DB: connections, size, transactions/s, slow queries, cache hit ratio, index usage
- Host: disk gauge, CPU, memory, network I/O, disk I/O
- BRouter: request latency p50/p95/p99, container CPU + memory
- All containers: CPU and memory comparison

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

infrastructure/Caddyfile

@@ -1,3 +1,10 @@
+{
+	servers {
+		metrics
+	}
+	admin 0.0.0.0:2019
+}
+
 (security_headers) {
 	header {
 		Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"

infrastructure/docker-compose.yml

@@ -78,12 +78,52 @@ services:
       POSTGRES_DB: trails
     volumes:
       - pgdata:/var/lib/postgresql/data
+    command:
+      - "postgres"
+      - "-c"
+      - "shared_preload_libraries=pg_stat_statements"
+      - "-c"
+      - "pg_stat_statements.track=all"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U trails"]
       interval: 5s
       timeout: 5s
       retries: 5
 
+  postgres-exporter:
+    image: prometheuscommunity/postgres-exporter:latest
+    restart: unless-stopped
+    environment:
+      DATA_SOURCE_NAME: postgresql://trails:${POSTGRES_PASSWORD:-trails}@postgres:5432/trails?sslmode=disable
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    pid: host
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/rootfs"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    restart: unless-stopped
+    privileged: true
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+
   prometheus:
     image: prom/prometheus:latest
     restart: unless-stopped

infrastructure/grafana/dashboards/service-health.json

@@ -0,0 +1,187 @@
+{
+  "dashboard": {
+    "title": "Service Health",
+    "uid": "trails-service-health",
+    "timezone": "browser",
+    "refresh": "30s",
+    "panels": [
+      {
+        "title": "PostgreSQL — Active Connections",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+        "targets": [
+          { "expr": "pg_stat_activity_count{datname=\"trails\"}", "legendFormat": "{{state}}" }
+        ]
+      },
+      {
+        "title": "PostgreSQL — Database Size",
+        "type": "stat",
+        "gridPos": { "h": 8, "w": 6, "x": 12, "y": 0 },
+        "targets": [
+          { "expr": "pg_database_size_bytes{datname=\"trails\"}", "legendFormat": "trails" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "bytes" } }
+      },
+      {
+        "title": "PostgreSQL — Transactions/sec",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 6, "x": 18, "y": 0 },
+        "targets": [
+          { "expr": "rate(pg_stat_database_xact_commit{datname=\"trails\"}[5m])", "legendFormat": "commits/s" },
+          { "expr": "rate(pg_stat_database_xact_rollback{datname=\"trails\"}[5m])", "legendFormat": "rollbacks/s" }
+        ]
+      },
+      {
+        "title": "PostgreSQL — Slowest Queries (pg_stat_statements)",
+        "type": "table",
+        "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 },
+        "targets": [
+          {
+            "expr": "topk(10, pg_stat_statements_mean_time_seconds{datname=\"trails\"})",
+            "legendFormat": "{{query}}",
+            "format": "table",
+            "instant": true
+          }
+        ]
+      },
+      {
+        "title": "PostgreSQL — Cache Hit Ratio",
+        "type": "gauge",
+        "gridPos": { "h": 8, "w": 6, "x": 0, "y": 16 },
+        "targets": [
+          { "expr": "pg_stat_database_blks_hit{datname=\"trails\"} / (pg_stat_database_blks_hit{datname=\"trails\"} + pg_stat_database_blks_read{datname=\"trails\"}) * 100", "legendFormat": "hit %" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 90 }, { "color": "green", "value": 99 }] } } }
+      },
+      {
+        "title": "PostgreSQL — Index Scans vs Sequential Scans",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 18, "x": 6, "y": 16 },
+        "targets": [
+          { "expr": "sum(rate(pg_stat_user_tables_idx_scan[5m]))", "legendFormat": "Index scans/s" },
+          { "expr": "sum(rate(pg_stat_user_tables_seq_scan[5m]))", "legendFormat": "Sequential scans/s" }
+        ]
+      },
+      {
+        "title": "Host — Disk Usage",
+        "type": "gauge",
+        "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 },
+        "targets": [
+          { "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100", "legendFormat": "Used %" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 85 }] } } }
+      },
+      {
+        "title": "Host — CPU Usage",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 },
+        "targets": [
+          { "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", "legendFormat": "CPU %" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "percent", "max": 100 } }
+      },
+      {
+        "title": "Host — Memory Usage",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 },
+        "targets": [
+          { "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", "legendFormat": "Used" },
+          { "expr": "node_memory_MemAvailable_bytes", "legendFormat": "Available" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "bytes" } }
+      },
+      {
+        "title": "Host — Network I/O",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
+        "targets": [
+          { "expr": "rate(node_network_receive_bytes_total{device!~\"lo|veth.*|br.*|docker.*\"}[5m])", "legendFormat": "{{device}} rx" },
+          { "expr": "-rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|br.*|docker.*\"}[5m])", "legendFormat": "{{device}} tx" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "Bps" } }
+      },
+      {
+        "title": "Host — Disk I/O",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
+        "targets": [
+          { "expr": "rate(node_disk_read_bytes_total[5m])", "legendFormat": "{{device}} read" },
+          { "expr": "-rate(node_disk_written_bytes_total[5m])", "legendFormat": "{{device}} write" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "Bps" } }
+      },
+      {
+        "title": "BRouter — Request Latency (from Planner proxy)",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
+        "targets": [
+          { "expr": "histogram_quantile(0.50, rate(brouter_request_duration_seconds_bucket[5m]))", "legendFormat": "p50" },
+          { "expr": "histogram_quantile(0.95, rate(brouter_request_duration_seconds_bucket[5m]))", "legendFormat": "p95" },
+          { "expr": "histogram_quantile(0.99, rate(brouter_request_duration_seconds_bucket[5m]))", "legendFormat": "p99" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "s" } }
+      },
+      {
+        "title": "Container — CPU Usage (all containers)",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
+        "targets": [
+          { "expr": "rate(container_cpu_usage_seconds_total{name=~\"trails-cool.*\"}[5m]) * 100", "legendFormat": "{{name}}" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "percent" } }
+      },
+      {
+        "title": "Container — Memory Usage (all containers)",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 },
+        "targets": [
+          { "expr": "container_memory_usage_bytes{name=~\"trails-cool.*\"}", "legendFormat": "{{name}}" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "bytes" } }
+      },
+      {
+        "title": "BRouter — Container Resources",
+        "type": "stat",
+        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 },
+        "targets": [
+          { "expr": "container_memory_usage_bytes{name=~\".*brouter.*\"}", "legendFormat": "Memory" },
+          { "expr": "rate(container_cpu_usage_seconds_total{name=~\".*brouter.*\"}[5m]) * 100", "legendFormat": "CPU %" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "bytes" } }
+      },
+      {
+        "title": "Caddy — Request Rate by Host",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 },
+        "targets": [
+          { "expr": "sum(rate(caddy_http_requests_total[5m])) by (server)", "legendFormat": "{{server}}" }
+        ]
+      },
+      {
+        "title": "Caddy — Response Status Codes",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 },
+        "targets": [
+          { "expr": "sum(rate(caddy_http_responses_total[5m])) by (code)", "legendFormat": "{{code}}" }
+        ]
+      },
+      {
+        "title": "Caddy — Request Duration by Host",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 64 },
+        "targets": [
+          { "expr": "histogram_quantile(0.95, sum(rate(caddy_http_request_duration_seconds_bucket[5m])) by (le, server))", "legendFormat": "p95 {{server}}" }
+        ],
+        "fieldConfig": { "defaults": { "unit": "s" } }
+      },
+      {
+        "title": "Caddy — Active Connections",
+        "type": "timeseries",
+        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 64 },
+        "targets": [
+          { "expr": "caddy_http_requests_in_flight", "legendFormat": "in-flight" }
+        ]
+      }
+    ]
+  }
+}

infrastructure/prometheus/prometheus.yml

@@ -13,6 +13,18 @@ scrape_configs:
     static_configs:
       - targets: ["planner:3001"]
 
+  - job_name: "postgres"
+    static_configs:
+      - targets: ["postgres-exporter:9187"]
+
+  - job_name: "node"
+    static_configs:
+      - targets: ["node-exporter:9100"]
+
+  - job_name: "cadvisor"
+    static_configs:
+      - targets: ["cadvisor:8080"]
+
   - job_name: "caddy"
     static_configs:
       - targets: ["caddy:2019"]