Add service health monitoring: postgres, node, cAdvisor exporters + dashboard

Exporters:
- postgres_exporter: DB connections, transactions, cache hit ratio, query stats
- node_exporter: host CPU, memory, disk, network
- cAdvisor: per-container CPU and memory usage

PostgreSQL:
- Enable pg_stat_statements for query-level performance tracking
- Track index scans vs sequential scans, cache hit ratio

Dashboard (service-health.json):
- DB: connections, size, transactions/s, slow queries, cache hit ratio, index usage
- Host: disk gauge, CPU, memory, network I/O, disk I/O
- BRouter: request latency p50/p95/p99, container CPU + memory
- All containers: CPU and memory comparison

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

infrastructure/Caddyfile

@@ -1,3 +1,10 @@
+{
+	servers {
+		metrics
+	}
+	admin 0.0.0.0:2019
+}
+
 (security_headers) {
 	header {
 		Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"

infrastructure/docker-compose.yml

@@ -78,12 +78,52 @@ services:
       POSTGRES_DB: trails
     volumes:
       - pgdata:/var/lib/postgresql/data
+    command:
+      - "postgres"
+      - "-c"
+      - "shared_preload_libraries=pg_stat_statements"
+      - "-c"
+      - "pg_stat_statements.track=all"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U trails"]
       interval: 5s
       timeout: 5s
       retries: 5
 
+  postgres-exporter:
+    image: prometheuscommunity/postgres-exporter:latest
+    restart: unless-stopped
+    environment:
+      DATA_SOURCE_NAME: postgresql://trails:${POSTGRES_PASSWORD:-trails}@postgres:5432/trails?sslmode=disable
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    pid: host
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/rootfs"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    restart: unless-stopped
+    privileged: true
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+
   prometheus:
     image: prom/prometheus:latest
     restart: unless-stopped
@@ -114,6 +154,8 @@ services:
       GF_AUTH_GITHUB_ALLOWED_ORGANIZATIONS: trails-cool
       GF_AUTH_GITHUB_SCOPES: user:email,read:org
       GF_AUTH_DISABLE_LOGIN_FORM: "true"
+      GF_AUTH_GITHUB_ROLE_ATTRIBUTE_PATH: "contains(groups[*], '@trails-cool/owners') && 'GrafanaAdmin' || 'Editor'"
+      GF_AUTH_GITHUB_ALLOW_ASSIGN_GRAFANA_ADMIN: "true"
     volumes:
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
       - ./grafana/dashboards:/var/lib/grafana/dashboards:ro

infrastructure/grafana/dashboards/infrastructure.json

@@ -1,45 +1,87 @@
 {
-  "dashboard": {
-    "title": "Infrastructure",
-    "uid": "trails-infra",
-    "timezone": "browser",
-    "refresh": "30s",
-    "panels": [
-      {
-        "title": "Process Memory (RSS)",
-        "type": "timeseries",
-        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
-        "targets": [
-          { "expr": "process_resident_memory_bytes", "legendFormat": "{{job}}" }
-        ],
-        "fieldConfig": { "defaults": { "unit": "bytes" } }
+  "title": "Infrastructure",
+  "uid": "trails-infra",
+  "timezone": "browser",
+  "refresh": "30s",
+  "panels": [
+    {
+      "title": "Process Memory (RSS)",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
       },
-      {
-        "title": "CPU Usage",
-        "type": "timeseries",
-        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
-        "targets": [
-          { "expr": "rate(process_cpu_seconds_total[5m])", "legendFormat": "{{job}}" }
-        ],
-        "fieldConfig": { "defaults": { "unit": "percentunit" } }
-      },
-      {
-        "title": "Event Loop Lag",
-        "type": "timeseries",
-        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-        "targets": [
-          { "expr": "nodejs_eventloop_lag_seconds", "legendFormat": "{{job}}" }
-        ],
-        "fieldConfig": { "defaults": { "unit": "s" } }
-      },
-      {
-        "title": "Open File Descriptors",
-        "type": "timeseries",
-        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-        "targets": [
-          { "expr": "process_open_fds", "legendFormat": "{{job}}" }
-        ]
+      "targets": [
+        {
+          "expr": "process_resident_memory_bytes",
+          "legendFormat": "{{job}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        }
       }
-    ]
-  }
-}
+    },
+    {
+      "title": "CPU Usage",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "targets": [
+        {
+          "expr": "rate(process_cpu_seconds_total[5m])",
+          "legendFormat": "{{job}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        }
+      }
+    },
+    {
+      "title": "Event Loop Lag",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "targets": [
+        {
+          "expr": "nodejs_eventloop_lag_seconds",
+          "legendFormat": "{{job}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      }
+    },
+    {
+      "title": "Open File Descriptors",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "targets": [
+        {
+          "expr": "process_open_fds",
+          "legendFormat": "{{job}}"
+        }
+      ]
+    }
+  ]
+}

infrastructure/grafana/dashboards/overview.json

@@ -1,57 +1,121 @@
 {
-  "dashboard": {
-    "title": "trails.cool Overview",
-    "uid": "trails-overview",
-    "timezone": "browser",
-    "refresh": "30s",
-    "panels": [
-      {
-        "title": "Request Rate",
-        "type": "timeseries",
-        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
-        "targets": [
-          {
-            "expr": "sum(rate(http_request_duration_seconds_count[5m])) by (job)",
-            "legendFormat": "{{job}}"
-          }
-        ]
+  "title": "trails.cool Overview",
+  "uid": "trails-overview",
+  "timezone": "browser",
+  "refresh": "30s",
+  "panels": [
+    {
+      "title": "Request Rate",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
       },
-      {
-        "title": "Error Rate",
-        "type": "timeseries",
-        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
-        "targets": [
-          {
-            "expr": "sum(rate(http_request_duration_seconds_count{status=~\"5..\"}[5m])) / sum(rate(http_request_duration_seconds_count[5m])) * 100",
-            "legendFormat": "Error %"
-          }
-        ],
-        "fieldConfig": {
-          "defaults": { "unit": "percent", "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 5 }] } }
+      "targets": [
+        {
+          "expr": "sum(rate(http_request_duration_seconds_count[5m])) by (job)",
+          "legendFormat": "{{job}}"
         }
+      ]
+    },
+    {
+      "title": "Error Rate",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
       },
-      {
-        "title": "Latency p50 / p95 / p99",
-        "type": "timeseries",
-        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-        "targets": [
-          { "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
-          { "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
-          { "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
-        ],
-        "fieldConfig": { "defaults": { "unit": "s" } }
-      },
-      {
-        "title": "Health Status",
-        "type": "stat",
-        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-        "targets": [
-          { "expr": "up", "legendFormat": "{{job}}" }
-        ],
-        "fieldConfig": {
-          "defaults": { "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }] }
+      "targets": [
+        {
+          "expr": "sum(rate(http_request_duration_seconds_count{status=~\"5..\"}[5m])) / sum(rate(http_request_duration_seconds_count[5m])) * 100",
+          "legendFormat": "Error %"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 5
+              }
+            ]
+          }
         }
       }
-    ]
-  }
-}
+    },
+    {
+      "title": "Latency p50 / p95 / p99",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p50"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p95"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p99"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      }
+    },
+    {
+      "title": "Health Status",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "targets": [
+        {
+          "expr": "up",
+          "legendFormat": "{{job}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": {
+                  "text": "DOWN",
+                  "color": "red"
+                },
+                "1": {
+                  "text": "UP",
+                  "color": "green"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

infrastructure/grafana/dashboards/planner.json

@@ -1,40 +1,81 @@
 {
-  "dashboard": {
-    "title": "Planner",
-    "uid": "trails-planner",
-    "timezone": "browser",
-    "refresh": "30s",
-    "panels": [
-      {
-        "title": "Active Sessions",
-        "type": "stat",
-        "gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 },
-        "targets": [{ "expr": "planner_active_sessions", "legendFormat": "Sessions" }]
+  "title": "Planner",
+  "uid": "trails-planner",
+  "timezone": "browser",
+  "refresh": "30s",
+  "panels": [
+    {
+      "title": "Active Sessions",
+      "type": "stat",
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 0,
+        "y": 0
       },
-      {
-        "title": "Connected Clients",
-        "type": "stat",
-        "gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 },
-        "targets": [{ "expr": "planner_connected_clients", "legendFormat": "Clients" }]
+      "targets": [
+        {
+          "expr": "planner_active_sessions",
+          "legendFormat": "Sessions"
+        }
+      ]
+    },
+    {
+      "title": "Connected Clients",
+      "type": "stat",
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 6,
+        "y": 0
       },
-      {
-        "title": "BRouter Latency",
-        "type": "timeseries",
-        "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
-        "targets": [
-          { "expr": "histogram_quantile(0.50, rate(brouter_request_duration_seconds_bucket[5m]))", "legendFormat": "p50" },
-          { "expr": "histogram_quantile(0.95, rate(brouter_request_duration_seconds_bucket[5m]))", "legendFormat": "p95" }
-        ],
-        "fieldConfig": { "defaults": { "unit": "s" } }
+      "targets": [
+        {
+          "expr": "planner_connected_clients",
+          "legendFormat": "Clients"
+        }
+      ]
+    },
+    {
+      "title": "BRouter Latency",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 0
       },
-      {
-        "title": "Request Rate by Route",
-        "type": "timeseries",
-        "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
-        "targets": [
-          { "expr": "sum(rate(http_request_duration_seconds_count{job=\"planner\"}[5m])) by (route)", "legendFormat": "{{route}}" }
-        ]
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, rate(brouter_request_duration_seconds_bucket[5m]))",
+          "legendFormat": "p50"
+        },
+        {
+          "expr": "histogram_quantile(0.95, rate(brouter_request_duration_seconds_bucket[5m]))",
+          "legendFormat": "p95"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
       }
-    ]
-  }
-}
+    },
+    {
+      "title": "Request Rate by Route",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 6
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(http_request_duration_seconds_count{job=\"planner\"}[5m])) by (route)",
+          "legendFormat": "{{route}}"
+        }
+      ]
+    }
+  ]
+}

infrastructure/grafana/dashboards/service-health.json

@@ -0,0 +1,443 @@
+{
+  "title": "Service Health",
+  "uid": "trails-service-health",
+  "timezone": "browser",
+  "refresh": "30s",
+  "panels": [
+    {
+      "title": "PostgreSQL \u2014 Active Connections",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "targets": [
+        {
+          "expr": "pg_stat_activity_count{datname=\"trails\"}",
+          "legendFormat": "{{state}}"
+        }
+      ]
+    },
+    {
+      "title": "PostgreSQL \u2014 Database Size",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "targets": [
+        {
+          "expr": "pg_database_size_bytes{datname=\"trails\"}",
+          "legendFormat": "trails"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        }
+      }
+    },
+    {
+      "title": "PostgreSQL \u2014 Transactions/sec",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "targets": [
+        {
+          "expr": "rate(pg_stat_database_xact_commit{datname=\"trails\"}[5m])",
+          "legendFormat": "commits/s"
+        },
+        {
+          "expr": "rate(pg_stat_database_xact_rollback{datname=\"trails\"}[5m])",
+          "legendFormat": "rollbacks/s"
+        }
+      ]
+    },
+    {
+      "title": "PostgreSQL \u2014 Slowest Queries (pg_stat_statements)",
+      "type": "table",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "targets": [
+        {
+          "expr": "topk(10, pg_stat_statements_mean_time_seconds{datname=\"trails\"})",
+          "legendFormat": "{{query}}",
+          "format": "table",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "title": "PostgreSQL \u2014 Cache Hit Ratio",
+      "type": "gauge",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 16
+      },
+      "targets": [
+        {
+          "expr": "pg_stat_database_blks_hit{datname=\"trails\"} / (pg_stat_database_blks_hit{datname=\"trails\"} + pg_stat_database_blks_read{datname=\"trails\"}) * 100",
+          "legendFormat": "hit %"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 90
+              },
+              {
+                "color": "green",
+                "value": 99
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "title": "PostgreSQL \u2014 Index Scans vs Sequential Scans",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 18,
+        "x": 6,
+        "y": 16
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(pg_stat_user_tables_idx_scan[5m]))",
+          "legendFormat": "Index scans/s"
+        },
+        {
+          "expr": "sum(rate(pg_stat_user_tables_seq_scan[5m]))",
+          "legendFormat": "Sequential scans/s"
+        }
+      ]
+    },
+    {
+      "title": "Host \u2014 Disk Usage",
+      "type": "gauge",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 24
+      },
+      "targets": [
+        {
+          "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100",
+          "legendFormat": "Used %"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 85
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "title": "Host \u2014 CPU Usage",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 24
+      },
+      "targets": [
+        {
+          "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+          "legendFormat": "CPU %"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "max": 100
+        }
+      }
+    },
+    {
+      "title": "Host \u2014 Memory Usage",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 24
+      },
+      "targets": [
+        {
+          "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
+          "legendFormat": "Used"
+        },
+        {
+          "expr": "node_memory_MemAvailable_bytes",
+          "legendFormat": "Available"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        }
+      }
+    },
+    {
+      "title": "Host \u2014 Network I/O",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "targets": [
+        {
+          "expr": "rate(node_network_receive_bytes_total{device!~\"lo|veth.*|br.*|docker.*\"}[5m])",
+          "legendFormat": "{{device}} rx"
+        },
+        {
+          "expr": "-rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|br.*|docker.*\"}[5m])",
+          "legendFormat": "{{device}} tx"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps"
+        }
+      }
+    },
+    {
+      "title": "Host \u2014 Disk I/O",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "targets": [
+        {
+          "expr": "rate(node_disk_read_bytes_total[5m])",
+          "legendFormat": "{{device}} read"
+        },
+        {
+          "expr": "-rate(node_disk_written_bytes_total[5m])",
+          "legendFormat": "{{device}} write"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps"
+        }
+      }
+    },
+    {
+      "title": "BRouter \u2014 Request Latency (from Planner proxy)",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 40
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, rate(brouter_request_duration_seconds_bucket[5m]))",
+          "legendFormat": "p50"
+        },
+        {
+          "expr": "histogram_quantile(0.95, rate(brouter_request_duration_seconds_bucket[5m]))",
+          "legendFormat": "p95"
+        },
+        {
+          "expr": "histogram_quantile(0.99, rate(brouter_request_duration_seconds_bucket[5m]))",
+          "legendFormat": "p99"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      }
+    },
+    {
+      "title": "Container \u2014 CPU Usage (all containers)",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 40
+      },
+      "targets": [
+        {
+          "expr": "rate(container_cpu_usage_seconds_total{name=~\"trails-cool.*\"}[5m]) * 100",
+          "legendFormat": "{{name}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent"
+        }
+      }
+    },
+    {
+      "title": "Container \u2014 Memory Usage (all containers)",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 48
+      },
+      "targets": [
+        {
+          "expr": "container_memory_usage_bytes{name=~\"trails-cool.*\"}",
+          "legendFormat": "{{name}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        }
+      }
+    },
+    {
+      "title": "BRouter \u2014 Container Resources",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 48
+      },
+      "targets": [
+        {
+          "expr": "container_memory_usage_bytes{name=~\".*brouter.*\"}",
+          "legendFormat": "Memory"
+        },
+        {
+          "expr": "rate(container_cpu_usage_seconds_total{name=~\".*brouter.*\"}[5m]) * 100",
+          "legendFormat": "CPU %"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        }
+      }
+    },
+    {
+      "title": "Caddy \u2014 Request Rate by Host",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 56
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(caddy_http_requests_total[5m])) by (server)",
+          "legendFormat": "{{server}}"
+        }
+      ]
+    },
+    {
+      "title": "Caddy \u2014 Response Status Codes",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 56
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(caddy_http_responses_total[5m])) by (code)",
+          "legendFormat": "{{code}}"
+        }
+      ]
+    },
+    {
+      "title": "Caddy \u2014 Request Duration by Host",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 64
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(caddy_http_request_duration_seconds_bucket[5m])) by (le, server))",
+          "legendFormat": "p95 {{server}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      }
+    },
+    {
+      "title": "Caddy \u2014 Active Connections",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 64
+      },
+      "targets": [
+        {
+          "expr": "caddy_http_requests_in_flight",
+          "legendFormat": "in-flight"
+        }
+      ]
+    }
+  ]
+}

infrastructure/prometheus/prometheus.yml

@@ -13,6 +13,18 @@ scrape_configs:
     static_configs:
       - targets: ["planner:3001"]
 
+  - job_name: "postgres"
+    static_configs:
+      - targets: ["postgres-exporter:9187"]
+
+  - job_name: "node"
+    static_configs:
+      - targets: ["node-exporter:9100"]
+
+  - job_name: "cadvisor"
+    static_configs:
+      - targets: ["cadvisor:8080"]
+
   - job_name: "caddy"
     static_configs:
       - targets: ["caddy:2019"]