Merge pull request #222 from trails-cool/fix/observability-and-crash-fix

Fix journal crash loop and improve observability
2026-04-13 20:43:21 +02:00 · 2026-04-13 20:43:21 +02:00 · fc80e66ad4
commit fc80e66ad4
parent fda376267c 37fb5fa1c9
9 changed files with 509 additions and 22 deletions
--- a/apps/journal/app/lib/oauth.server.ts
+++ b/apps/journal/app/lib/oauth.server.ts
@ -264,10 +264,9 @@ export async function getAuthenticatedUser(request: Request) {
 // --- Error type ---

 export class OAuthError extends Error {
-  constructor(
-    public code: string,
-    message: string,
-  ) {
+  code: string;
+  constructor(code: string, message: string) {
    super(message);
+    this.code = code;
  }
 }
--- a/apps/planner/app/lib/brouter.ts
+++ b/apps/planner/app/lib/brouter.ts
@ -68,12 +68,11 @@ export async function computeRoute(request: RouteRequest): Promise<EnrichedRoute
 }

 export class BRouterError extends Error {
-  constructor(
-    message: string,
-    public readonly statusCode: number,
-  ) {
+  readonly statusCode: number;
+  constructor(message: string, statusCode: number) {
    super(message);
    this.name = "BRouterError";
+    this.statusCode = statusCode;
  }
 }

--- a/infrastructure/docker-compose.yml
+++ b/infrastructure/docker-compose.yml
@ -35,7 +35,7 @@ services:
      WAHOO_CLIENT_SECRET: ${WAHOO_CLIENT_SECRET:-}
      WAHOO_WEBHOOK_TOKEN: ${WAHOO_WEBHOOK_TOKEN:-}
    healthcheck:
-      test: ["CMD-SHELL", "node -e \"fetch('http://localhost:3000/api/health').then(r=>{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))\""]
+      test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"]
      interval: 15s
      timeout: 5s
      retries: 3
@ -135,7 +135,7 @@ services:
    image: grafana/promtail:latest
    restart: unless-stopped
    volumes:
-      - /var/log:/var/log:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - ./promtail/promtail-config.yml:/etc/promtail/config.yml:ro
    command: ["-config.file=/etc/promtail/config.yml"]
--- a/infrastructure/grafana/dashboards/journal.json
+++ b/infrastructure/grafana/dashboards/journal.json
@ -0,0 +1,312 @@
+{
+  "title": "Journal",
+  "uid": "trails-journal",
+  "annotations": {
+    "list": [
+      {
+        "name": "Deploys",
+        "enable": true,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [
+            "deploy"
+          ],
+          "type": "tags"
+        }
+      }
+    ]
+  },
+  "timezone": "browser",
+  "refresh": "30s",
+  "panels": [
+    {
+      "title": "Caddy 502 Rate (Journal)",
+      "description": "502 errors returned by Caddy for the journal upstream. These indicate Caddy could not reach the journal container.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code=\"502\"}[5m])) or vector(0)",
+          "legendFormat": "502/s"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "fillOpacity": 20,
+            "lineWidth": 2
+          },
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "red"
+          }
+        }
+      }
+    },
+    {
+      "title": "Caddy Response Codes (Journal)",
+      "description": "All HTTP response codes from Caddy for the journal host, including Caddy-generated errors like 502.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code!~\"(101|2..|3..)\"}[5m])) by (code)",
+          "legendFormat": "{{code}}"
+        }
+      ]
+    },
+    {
+      "title": "Journal Request Rate by Status",
+      "description": "Request rate from the journal app's own metrics, grouped by HTTP status code.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (status)",
+          "legendFormat": "{{status}}"
+        }
+      ]
+    },
+    {
+      "title": "Journal Latency p50 / p95 / p99",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
+          "legendFormat": "p50"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
+          "legendFormat": "p95"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
+          "legendFormat": "p99"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      }
+    },
+    {
+      "title": "Request Rate by Route",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (route)",
+          "legendFormat": "{{route}}"
+        }
+      ]
+    },
+    {
+      "title": "Container Restarts",
+      "description": "Journal container restart count. Spikes here correlate with 502 errors — Caddy returns 502 while the container is restarting.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 24
+      },
+      "targets": [
+        {
+          "expr": "changes(container_start_time_seconds{name=\"trails-cool-journal-1\"}[5m])",
+          "legendFormat": "journal"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "fillOpacity": 30,
+            "drawStyle": "bars"
+          },
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "orange"
+          }
+        }
+      }
+    },
+    {
+      "title": "Container Memory",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 24
+      },
+      "targets": [
+        {
+          "expr": "container_memory_usage_bytes{name=\"trails-cool-journal-1\"}",
+          "legendFormat": "used"
+        },
+        {
+          "expr": "container_memory_working_set_bytes{name=\"trails-cool-journal-1\"}",
+          "legendFormat": "working set"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        }
+      }
+    },
+    {
+      "title": "Container CPU",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 24
+      },
+      "targets": [
+        {
+          "expr": "rate(container_cpu_usage_seconds_total{name=\"trails-cool-journal-1\"}[5m]) * 100",
+          "legendFormat": "CPU %"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent"
+        }
+      }
+    },
+    {
+      "title": "Node.js Event Loop Lag",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "targets": [
+        {
+          "expr": "nodejs_eventloop_lag_seconds{job=\"journal\"}",
+          "legendFormat": "lag"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      }
+    },
+    {
+      "title": "Node.js Heap Used",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "targets": [
+        {
+          "expr": "nodejs_heap_size_used_bytes{job=\"journal\"}",
+          "legendFormat": "used"
+        },
+        {
+          "expr": "nodejs_heap_size_total_bytes{job=\"journal\"}",
+          "legendFormat": "total"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        }
+      }
+    },
+    {
+      "title": "Journal Logs (Errors & Warnings)",
+      "description": "Recent error and warning logs from the journal container. Requires Promtail docker_sd_configs with Pino JSON parsing.",
+      "type": "logs",
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 40
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "{service=\"journal\"} |~ \"(error|warn|ERR|WARN|level.*(40|50|60))\"",
+          "legendFormat": ""
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "sortOrder": "Descending",
+        "enableLogDetails": true,
+        "dedupStrategy": "none"
+      }
+    },
+    {
+      "title": "Caddy Logs (5xx Responses)",
+      "description": "Caddy access log entries with 5xx status codes. Shows the raw request that triggered the error.",
+      "type": "logs",
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 50
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "{service=\"caddy\"} |~ \"\\\"status\\\":\\s*5\\d\\d\"",
+          "legendFormat": ""
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "sortOrder": "Descending",
+        "enableLogDetails": true,
+        "dedupStrategy": "none"
+      }
+    }
+  ]
+}
--- a/infrastructure/grafana/dashboards/overview.json
+++ b/infrastructure/grafana/dashboards/overview.json
@ -52,7 +52,7 @@
      },
      "targets": [
        {
-          "expr": "sum(rate(caddy_http_response_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_response_duration_seconds_count[5m])) * 100",
+          "expr": "sum(rate(caddy_http_request_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_request_duration_seconds_count[5m])) * 100",
          "legendFormat": "Error %"
        }
      ],
@ -138,5 +138,44 @@
        }
      }
    }
+    {
+      "title": "App Error Rate (from logs)",
+      "description": "Error-level log entries per second from Pino (level 50=error, 60=fatal). Captures crashes and unhandled errors that metrics may miss.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "sum(rate({service=\"journal\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
+          "legendFormat": "journal"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "sum(rate({service=\"planner\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
+          "legendFormat": "planner"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "fillOpacity": 15
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        }
+      }
+    }
  ]
 }
--- a/infrastructure/grafana/dashboards/service-health.json
+++ b/infrastructure/grafana/dashboards/service-health.json
@ -418,10 +418,30 @@
      },
      "targets": [
        {
-          "expr": "sum(rate(caddy_http_response_duration_seconds_count[5m])) by (code)",
+          "expr": "sum(rate(caddy_http_request_duration_seconds_count[5m])) by (code)",
          "legendFormat": "{{code}}"
        }
-      ]
+      ],
+      "fieldConfig": {
+        "overrides": [
+          {
+            "matcher": { "id": "byRegexp", "options": "/^2\\d\\d$/" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }]
+          },
+          {
+            "matcher": { "id": "byRegexp", "options": "/^3\\d\\d$/" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }]
+          },
+          {
+            "matcher": { "id": "byRegexp", "options": "/^4\\d\\d$/" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }]
+          },
+          {
+            "matcher": { "id": "byRegexp", "options": "/^5\\d\\d$/" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }]
+          }
+        ]
+      }
    },
    {
      "title": "Caddy \u2014 Request Duration by Host",
--- a/infrastructure/grafana/provisioning/alerting/alerts.yml
+++ b/infrastructure/grafana/provisioning/alerting/alerts.yml
@ -61,7 +61,7 @@ groups:
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: prometheus
            model:
-              expr: (sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(http_request_duration_seconds_count[5m])), 0.001) * 100
+              expr: (sum(rate(caddy_http_request_duration_seconds_count{code=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(caddy_http_request_duration_seconds_count[5m])), 0.001) * 100
              instant: true
          - refId: B
            datasourceUid: __expr__
@ -76,6 +76,106 @@ groups:
        annotations:
          summary: "Error rate is above 5%"

+      - uid: container-restart-loop
+        title: Container restart loop
+        condition: B
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              expr: changes(container_start_time_seconds{name=~"trails-cool.*"}[5m])
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { params: [2], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 1m
+        annotations:
+          summary: "{{ $labels.name }} has restarted more than 2 times in 5 minutes"
+
+      - uid: db-connections-high
+        title: PostgreSQL connections > 80
+        condition: B
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              expr: sum(pg_stat_activity_count{datname="trails"})
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { params: [80], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 5m
+        annotations:
+          summary: "PostgreSQL active connections above 80 — approaching default max_connections limit"
+
+      - uid: app-crash-log
+        title: Application crash detected
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: loki
+            model:
+              expr: sum(count_over_time({service=~"journal|planner"} |~ "uncaughtException|unhandledRejection|ERR_|FATAL|segfault|OOMKilled"[5m]))
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: reduce
+              expression: A
+              reducer: last
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator: { params: [0], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 0s
+        noDataState: OK
+        annotations:
+          summary: "Crash signature detected in application logs"
+
+      - uid: caddy-502-rate
+        title: Caddy 502 errors detected
+        condition: B
+        noDataState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              expr: sum(rate(caddy_http_request_duration_seconds_count{code="502"}[5m])) or vector(0)
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { params: [0], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 2m
+        annotations:
+          summary: "Caddy is returning 502 errors — journal or planner upstream unreachable"
+
 contactPoints:
  - orgId: 1
    name: email
--- a/infrastructure/promtail/promtail-config.yml
+++ b/infrastructure/promtail/promtail-config.yml
@ -10,13 +10,30 @@ clients:

 scrape_configs:
  - job_name: docker
-    static_configs:
-      - targets:
-          - localhost
-        labels:
-          job: docker
-          __path__: /var/lib/docker/containers/*/*.log
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 10s
+    relabel_configs:
+      # Use the container name as a label
+      - source_labels: ["__meta_docker_container_name"]
+        regex: "/?(.*)"
+        target_label: container
+      # Use the compose service name if available
+      - source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
+        target_label: service
+      # Drop containers we don't need logs from (exporters, promtail itself)
+      - source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
+        regex: "promtail|node-exporter|cadvisor|postgres-exporter"
+        action: drop
    pipeline_stages:
      - docker: {}
-      - labeldrop:
-          - filename
+      # Parse Pino JSON logs (journal, planner) to extract level
+      - match:
+          selector: '{service=~"journal|planner"}'
+          stages:
+            - json:
+                expressions:
+                  level: level
+                  msg: msg
+            - labels:
+                level:
--- a/tsconfig.base.json
+++ b/tsconfig.base.json
@ -17,6 +17,7 @@
    "noUncheckedIndexedAccess": true,
    "noUncheckedSideEffectImports": false,
    "noEmit": true,
+    "erasableSyntaxOnly": true,
    "allowImportingTsExtensions": true
  },
  "exclude": ["node_modules", "build", "dist"]