From 37fb5fa1c942fde89e5325bf29bc645b4b31c93d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= <hi+github@ullrich.is>
Date: Sun, 12 Apr 2026 16:00:00 +0200
Subject: [PATCH] Add Journal dashboard, fix Caddy metrics, and improve alerts

Fix blind spot where Caddy 502 errors were invisible: the dashboards and
alerts queried caddy_http_response_duration_seconds (upstream responses only),
missing 502s that Caddy generates itself. Switch to
caddy_http_request_duration_seconds (server-level, all responses).

Add Journal Grafana dashboard with: 502 rate, response codes, request rate
by route, latency percentiles, container restarts/memory/CPU, Node.js event
loop lag and heap, and Loki log panels for errors and Caddy 5xx entries.

Add color coding to Caddy status code panel (green=2xx, blue=3xx, yellow=4xx,
red=5xx). Add log-based error rate panel to the overview dashboard.

New alerts: container restart loop, PostgreSQL connections > 80, application
crash log detection (Loki), and Caddy 502 rate.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 apps/planner/app/lib/brouter.ts               |   7 +-
 .../grafana/dashboards/journal.json           | 312 ++++++++++++++++++
 .../grafana/dashboards/overview.json          |  41 ++-
 .../grafana/dashboards/service-health.json    |  24 +-
 .../grafana/provisioning/alerting/alerts.yml  | 102 +++++-
 5 files changed, 478 insertions(+), 8 deletions(-)
 create mode 100644 infrastructure/grafana/dashboards/journal.json

diff --git a/apps/planner/app/lib/brouter.ts b/apps/planner/app/lib/brouter.ts
index d4a3974..f1ab53e 100644
--- a/apps/planner/app/lib/brouter.ts
+++ b/apps/planner/app/lib/brouter.ts
@@ -68,12 +68,11 @@ export async function computeRoute(request: RouteRequest): Promise<EnrichedRoute
 }
 
 export class BRouterError extends Error {
-  constructor(
-    message: string,
-    public readonly statusCode: number,
-  ) {
+  readonly statusCode: number;
+  constructor(message: string, statusCode: number) {
     super(message);
     this.name = "BRouterError";
+    this.statusCode = statusCode;
   }
 }
 
diff --git a/infrastructure/grafana/dashboards/journal.json b/infrastructure/grafana/dashboards/journal.json
new file mode 100644
index 0000000..6e53e93
--- /dev/null
+++ b/infrastructure/grafana/dashboards/journal.json
@@ -0,0 +1,312 @@
+{
+  "title": "Journal",
+  "uid": "trails-journal",
+  "annotations": {
+    "list": [
+      {
+        "name": "Deploys",
+        "enable": true,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [
+            "deploy"
+          ],
+          "type": "tags"
+        }
+      }
+    ]
+  },
+  "timezone": "browser",
+  "refresh": "30s",
+  "panels": [
+    {
+      "title": "Caddy 502 Rate (Journal)",
+      "description": "502 errors returned by Caddy for the journal upstream. These indicate Caddy could not reach the journal container.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code=\"502\"}[5m])) or vector(0)",
+          "legendFormat": "502/s"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "fillOpacity": 20,
+            "lineWidth": 2
+          },
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "red"
+          }
+        }
+      }
+    },
+    {
+      "title": "Caddy Response Codes (Journal)",
+      "description": "All HTTP response codes from Caddy for the journal host, including Caddy-generated errors like 502.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code!~\"(101|2..|3..)\"}[5m])) by (code)",
+          "legendFormat": "{{code}}"
+        }
+      ]
+    },
+    {
+      "title": "Journal Request Rate by Status",
+      "description": "Request rate from the journal app's own metrics, grouped by HTTP status code.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (status)",
+          "legendFormat": "{{status}}"
+        }
+      ]
+    },
+    {
+      "title": "Journal Latency p50 / p95 / p99",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
+          "legendFormat": "p50"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
+          "legendFormat": "p95"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
+          "legendFormat": "p99"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      }
+    },
+    {
+      "title": "Request Rate by Route",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (route)",
+          "legendFormat": "{{route}}"
+        }
+      ]
+    },
+    {
+      "title": "Container Restarts",
+      "description": "Journal container restart count. Spikes here correlate with 502 errors — Caddy returns 502 while the container is restarting.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 24
+      },
+      "targets": [
+        {
+          "expr": "changes(container_start_time_seconds{name=\"trails-cool-journal-1\"}[5m])",
+          "legendFormat": "journal"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "fillOpacity": 30,
+            "drawStyle": "bars"
+          },
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "orange"
+          }
+        }
+      }
+    },
+    {
+      "title": "Container Memory",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 24
+      },
+      "targets": [
+        {
+          "expr": "container_memory_usage_bytes{name=\"trails-cool-journal-1\"}",
+          "legendFormat": "used"
+        },
+        {
+          "expr": "container_memory_working_set_bytes{name=\"trails-cool-journal-1\"}",
+          "legendFormat": "working set"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        }
+      }
+    },
+    {
+      "title": "Container CPU",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 24
+      },
+      "targets": [
+        {
+          "expr": "rate(container_cpu_usage_seconds_total{name=\"trails-cool-journal-1\"}[5m]) * 100",
+          "legendFormat": "CPU %"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent"
+        }
+      }
+    },
+    {
+      "title": "Node.js Event Loop Lag",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "targets": [
+        {
+          "expr": "nodejs_eventloop_lag_seconds{job=\"journal\"}",
+          "legendFormat": "lag"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      }
+    },
+    {
+      "title": "Node.js Heap Used",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "targets": [
+        {
+          "expr": "nodejs_heap_size_used_bytes{job=\"journal\"}",
+          "legendFormat": "used"
+        },
+        {
+          "expr": "nodejs_heap_size_total_bytes{job=\"journal\"}",
+          "legendFormat": "total"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        }
+      }
+    },
+    {
+      "title": "Journal Logs (Errors & Warnings)",
+      "description": "Recent error and warning logs from the journal container. Requires Promtail docker_sd_configs with Pino JSON parsing.",
+      "type": "logs",
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 40
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "{service=\"journal\"} |~ \"(error|warn|ERR|WARN|level.*(40|50|60))\"",
+          "legendFormat": ""
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "sortOrder": "Descending",
+        "enableLogDetails": true,
+        "dedupStrategy": "none"
+      }
+    },
+    {
+      "title": "Caddy Logs (5xx Responses)",
+      "description": "Caddy access log entries with 5xx status codes. Shows the raw request that triggered the error.",
+      "type": "logs",
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 50
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "{service=\"caddy\"} |~ \"\\\"status\\\":\\s*5\\d\\d\"",
+          "legendFormat": ""
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "sortOrder": "Descending",
+        "enableLogDetails": true,
+        "dedupStrategy": "none"
+      }
+    }
+  ]
+}
diff --git a/infrastructure/grafana/dashboards/overview.json b/infrastructure/grafana/dashboards/overview.json
index 250833c..1d9b9cc 100644
--- a/infrastructure/grafana/dashboards/overview.json
+++ b/infrastructure/grafana/dashboards/overview.json
@@ -52,7 +52,7 @@
       },
       "targets": [
         {
-          "expr": "sum(rate(caddy_http_response_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_response_duration_seconds_count[5m])) * 100",
+          "expr": "sum(rate(caddy_http_request_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_request_duration_seconds_count[5m])) * 100",
           "legendFormat": "Error %"
         }
       ],
@@ -138,5 +138,44 @@
         }
       }
     }
+    {
+      "title": "App Error Rate (from logs)",
+      "description": "Error-level log entries per second from Pino (level 50=error, 60=fatal). Captures crashes and unhandled errors that metrics may miss.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "sum(rate({service=\"journal\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
+          "legendFormat": "journal"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "sum(rate({service=\"planner\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
+          "legendFormat": "planner"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "fillOpacity": 15
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        }
+      }
+    }
   ]
 }
diff --git a/infrastructure/grafana/dashboards/service-health.json b/infrastructure/grafana/dashboards/service-health.json
index 69291e9..3d53131 100644
--- a/infrastructure/grafana/dashboards/service-health.json
+++ b/infrastructure/grafana/dashboards/service-health.json
@@ -418,10 +418,30 @@
       },
       "targets": [
         {
-          "expr": "sum(rate(caddy_http_response_duration_seconds_count[5m])) by (code)",
+          "expr": "sum(rate(caddy_http_request_duration_seconds_count[5m])) by (code)",
           "legendFormat": "{{code}}"
         }
-      ]
+      ],
+      "fieldConfig": {
+        "overrides": [
+          {
+            "matcher": { "id": "byRegexp", "options": "/^2\\d\\d$/" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }]
+          },
+          {
+            "matcher": { "id": "byRegexp", "options": "/^3\\d\\d$/" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }]
+          },
+          {
+            "matcher": { "id": "byRegexp", "options": "/^4\\d\\d$/" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }]
+          },
+          {
+            "matcher": { "id": "byRegexp", "options": "/^5\\d\\d$/" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }]
+          }
+        ]
+      }
     },
     {
       "title": "Caddy \u2014 Request Duration by Host",
diff --git a/infrastructure/grafana/provisioning/alerting/alerts.yml b/infrastructure/grafana/provisioning/alerting/alerts.yml
index e758b25..ac3ce8a 100644
--- a/infrastructure/grafana/provisioning/alerting/alerts.yml
+++ b/infrastructure/grafana/provisioning/alerting/alerts.yml
@@ -61,7 +61,7 @@ groups:
             relativeTimeRange: { from: 600, to: 0 }
             datasourceUid: prometheus
             model:
-              expr: (sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(http_request_duration_seconds_count[5m])), 0.001) * 100
+              expr: (sum(rate(caddy_http_request_duration_seconds_count{code=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(caddy_http_request_duration_seconds_count[5m])), 0.001) * 100
               instant: true
           - refId: B
             datasourceUid: __expr__
@@ -76,6 +76,106 @@ groups:
         annotations:
           summary: "Error rate is above 5%"
 
+      - uid: container-restart-loop
+        title: Container restart loop
+        condition: B
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              expr: changes(container_start_time_seconds{name=~"trails-cool.*"}[5m])
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { params: [2], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 1m
+        annotations:
+          summary: "{{ $labels.name }} has restarted more than 2 times in 5 minutes"
+
+      - uid: db-connections-high
+        title: PostgreSQL connections > 80
+        condition: B
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              expr: sum(pg_stat_activity_count{datname="trails"})
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { params: [80], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 5m
+        annotations:
+          summary: "PostgreSQL active connections above 80 — approaching default max_connections limit"
+
+      - uid: app-crash-log
+        title: Application crash detected
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: loki
+            model:
+              expr: sum(count_over_time({service=~"journal|planner"} |~ "uncaughtException|unhandledRejection|ERR_|FATAL|segfault|OOMKilled"[5m]))
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: reduce
+              expression: A
+              reducer: last
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator: { params: [0], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 0s
+        noDataState: OK
+        annotations:
+          summary: "Crash signature detected in application logs"
+
+      - uid: caddy-502-rate
+        title: Caddy 502 errors detected
+        condition: B
+        noDataState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              expr: sum(rate(caddy_http_request_duration_seconds_count{code="502"}[5m])) or vector(0)
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { params: [0], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 2m
+        annotations:
+          summary: "Caddy is returning 502 errors — journal or planner upstream unreachable"
+
 contactPoints:
   - orgId: 1
     name: email