From e69f93800bf2fccabe8d299b964cf3bb7fd3d1f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Sun, 12 Apr 2026 14:00:00 +0200 Subject: [PATCH 1/3] Fix TypeScript parameter property crash in Node strip-only mode oauth.server.ts used a TypeScript parameter property (`public code: string` in constructor) which is not supported by Node.js v25's strip-only TypeScript mode. This caused the journal to crash-loop on startup after seeding OAuth clients. Also enable `erasableSyntaxOnly` in tsconfig.base.json so `pnpm typecheck` catches any future use of non-erasable TypeScript syntax. Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/journal/app/lib/oauth.server.ts | 7 +++---- tsconfig.base.json | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/journal/app/lib/oauth.server.ts b/apps/journal/app/lib/oauth.server.ts index 539c6a3..a4c8c30 100644 --- a/apps/journal/app/lib/oauth.server.ts +++ b/apps/journal/app/lib/oauth.server.ts @@ -264,10 +264,9 @@ export async function getAuthenticatedUser(request: Request) { // --- Error type --- export class OAuthError extends Error { - constructor( - public code: string, - message: string, - ) { + code: string; + constructor(code: string, message: string) { super(message); + this.code = code; } } diff --git a/tsconfig.base.json b/tsconfig.base.json index 9166196..3257189 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -17,6 +17,7 @@ "noUncheckedIndexedAccess": true, "noUncheckedSideEffectImports": false, "noEmit": true, + "erasableSyntaxOnly": true, "allowImportingTsExtensions": true }, "exclude": ["node_modules", "build", "dist"] From 1433997d5bb1e4d33f5c85ba1af82d5151e6c464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Sun, 12 Apr 2026 15:00:00 +0200 Subject: [PATCH 2/3] Fix journal health check and improve log collection Switch journal Docker health check from `node -e "fetch(...)"` to `curl -sf`, matching the planner. The Node.js process spawn was heavy and prone to timeout under memory pressure, causing false health check failures and 502s. Upgrade Promtail from static file scraping to Docker service discovery so logs get `service` and `container` labels. Parse Pino JSON logs to extract the `level` label, enabling Loki queries filtered by log level. Co-Authored-By: Claude Opus 4.6 (1M context) --- infrastructure/docker-compose.yml | 4 +-- infrastructure/promtail/promtail-config.yml | 33 ++++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index 3f63dc5..b9e8055 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -35,7 +35,7 @@ services: WAHOO_CLIENT_SECRET: ${WAHOO_CLIENT_SECRET:-} WAHOO_WEBHOOK_TOKEN: ${WAHOO_WEBHOOK_TOKEN:-} healthcheck: - test: ["CMD-SHELL", "node -e \"fetch('http://localhost:3000/api/health').then(r=>{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))\""] + test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"] interval: 15s timeout: 5s retries: 3 @@ -135,7 +135,7 @@ services: image: grafana/promtail:latest restart: unless-stopped volumes: - - /var/log:/var/log:ro + - /var/run/docker.sock:/var/run/docker.sock:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro - ./promtail/promtail-config.yml:/etc/promtail/config.yml:ro command: ["-config.file=/etc/promtail/config.yml"] diff --git a/infrastructure/promtail/promtail-config.yml b/infrastructure/promtail/promtail-config.yml index f7414fe..248f2a5 100644 --- a/infrastructure/promtail/promtail-config.yml +++ b/infrastructure/promtail/promtail-config.yml @@ -10,13 +10,30 @@ clients: scrape_configs: - job_name: docker - static_configs: - - targets: - - localhost - labels: - job: docker - __path__: /var/lib/docker/containers/*/*.log + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 10s + relabel_configs: + # Use the container name as a label + - source_labels: ["__meta_docker_container_name"] + regex: "/?(.*)" + target_label: container + # Use the compose service name if available + - source_labels: ["__meta_docker_container_label_com_docker_compose_service"] + target_label: service + # Drop containers we don't need logs from (exporters, promtail itself) + - source_labels: ["__meta_docker_container_label_com_docker_compose_service"] + regex: "promtail|node-exporter|cadvisor|postgres-exporter" + action: drop pipeline_stages: - docker: {} - - labeldrop: - - filename + # Parse Pino JSON logs (journal, planner) to extract level + - match: + selector: '{service=~"journal|planner"}' + stages: + - json: + expressions: + level: level + msg: msg + - labels: + level: From 37fb5fa1c942fde89e5325bf29bc645b4b31c93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Sun, 12 Apr 2026 16:00:00 +0200 Subject: [PATCH 3/3] Add Journal dashboard, fix Caddy metrics, and improve alerts Fix blind spot where Caddy 502 errors were invisible: the dashboards and alerts queried caddy_http_response_duration_seconds (upstream responses only), missing 502s that Caddy generates itself. Switch to caddy_http_request_duration_seconds (server-level, all responses). Add Journal Grafana dashboard with: 502 rate, response codes, request rate by route, latency percentiles, container restarts/memory/CPU, Node.js event loop lag and heap, and Loki log panels for errors and Caddy 5xx entries. Add color coding to Caddy status code panel (green=2xx, blue=3xx, yellow=4xx, red=5xx). Add log-based error rate panel to the overview dashboard. New alerts: container restart loop, PostgreSQL connections > 80, application crash log detection (Loki), and Caddy 502 rate. Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/planner/app/lib/brouter.ts | 7 +- .../grafana/dashboards/journal.json | 312 ++++++++++++++++++ .../grafana/dashboards/overview.json | 41 ++- .../grafana/dashboards/service-health.json | 24 +- .../grafana/provisioning/alerting/alerts.yml | 102 +++++- 5 files changed, 478 insertions(+), 8 deletions(-) create mode 100644 infrastructure/grafana/dashboards/journal.json diff --git a/apps/planner/app/lib/brouter.ts b/apps/planner/app/lib/brouter.ts index d4a3974..f1ab53e 100644 --- a/apps/planner/app/lib/brouter.ts +++ b/apps/planner/app/lib/brouter.ts @@ -68,12 +68,11 @@ export async function computeRoute(request: RouteRequest): Promise 80 + condition: B + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + expr: sum(pg_stat_activity_count{datname="trails"}) + instant: true + - refId: B + datasourceUid: __expr__ + model: + type: threshold + expression: A + conditions: + - evaluator: { params: [80], type: gt } + operator: { type: and } + reducer: { type: last } + for: 5m + annotations: + summary: "PostgreSQL active connections above 80 — approaching default max_connections limit" + + - uid: app-crash-log + title: Application crash detected + condition: C + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: loki + model: + expr: sum(count_over_time({service=~"journal|planner"} |~ "uncaughtException|unhandledRejection|ERR_|FATAL|segfault|OOMKilled"[5m])) + instant: true + - refId: B + datasourceUid: __expr__ + model: + type: reduce + expression: A + reducer: last + - refId: C + datasourceUid: __expr__ + model: + type: threshold + expression: B + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + reducer: { type: last } + for: 0s + noDataState: OK + annotations: + summary: "Crash signature detected in application logs" + + - uid: caddy-502-rate + title: Caddy 502 errors detected + condition: B + noDataState: OK + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + expr: sum(rate(caddy_http_request_duration_seconds_count{code="502"}[5m])) or vector(0) + instant: true + - refId: B + datasourceUid: __expr__ + model: + type: threshold + expression: A + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + reducer: { type: last } + for: 2m + annotations: + summary: "Caddy is returning 502 errors — journal or planner upstream unreachable" + contactPoints: - orgId: 1 name: email