diff --git a/apps/journal/app/lib/oauth.server.ts b/apps/journal/app/lib/oauth.server.ts index 539c6a3..a4c8c30 100644 --- a/apps/journal/app/lib/oauth.server.ts +++ b/apps/journal/app/lib/oauth.server.ts @@ -264,10 +264,9 @@ export async function getAuthenticatedUser(request: Request) { // --- Error type --- export class OAuthError extends Error { - constructor( - public code: string, - message: string, - ) { + code: string; + constructor(code: string, message: string) { super(message); + this.code = code; } } diff --git a/apps/planner/app/lib/brouter.ts b/apps/planner/app/lib/brouter.ts index d4a3974..f1ab53e 100644 --- a/apps/planner/app/lib/brouter.ts +++ b/apps/planner/app/lib/brouter.ts @@ -68,12 +68,11 @@ export async function computeRoute(request: RouteRequest): Promise{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))\""] + test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"] interval: 15s timeout: 5s retries: 3 @@ -135,7 +135,7 @@ services: image: grafana/promtail:latest restart: unless-stopped volumes: - - /var/log:/var/log:ro + - /var/run/docker.sock:/var/run/docker.sock:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro - ./promtail/promtail-config.yml:/etc/promtail/config.yml:ro command: ["-config.file=/etc/promtail/config.yml"] diff --git a/infrastructure/grafana/dashboards/journal.json b/infrastructure/grafana/dashboards/journal.json new file mode 100644 index 0000000..6e53e93 --- /dev/null +++ b/infrastructure/grafana/dashboards/journal.json @@ -0,0 +1,312 @@ +{ + "title": "Journal", + "uid": "trails-journal", + "annotations": { + "list": [ + { + "name": "Deploys", + "enable": true, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "iconColor": "rgba(0, 211, 255, 1)", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + "deploy" + ], + "type": "tags" + } + } + ] + }, + "timezone": "browser", + "refresh": "30s", + "panels": [ + { + "title": "Caddy 502 Rate (Journal)", + "description": "502 errors returned by Caddy for the journal upstream. These indicate Caddy could not reach the journal container.", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code=\"502\"}[5m])) or vector(0)", + "legendFormat": "502/s" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 20, + "lineWidth": 2 + }, + "color": { + "mode": "fixed", + "fixedColor": "red" + } + } + } + }, + { + "title": "Caddy Response Codes (Journal)", + "description": "All HTTP response codes from Caddy for the journal host, including Caddy-generated errors like 502.", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code!~\"(101|2..|3..)\"}[5m])) by (code)", + "legendFormat": "{{code}}" + } + ] + }, + { + "title": "Journal Request Rate by Status", + "description": "Request rate from the journal app's own metrics, grouped by HTTP status code.", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (status)", + "legendFormat": "{{status}}" + } + ] + }, + { + "title": "Journal Latency p50 / p95 / p99", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))", + "legendFormat": "p50" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))", + "legendFormat": "p95" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + } + }, + { + "title": "Request Rate by Route", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "targets": [ + { + "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (route)", + "legendFormat": "{{route}}" + } + ] + }, + { + "title": "Container Restarts", + "description": "Journal container restart count. Spikes here correlate with 502 errors — Caddy returns 502 while the container is restarting.", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "targets": [ + { + "expr": "changes(container_start_time_seconds{name=\"trails-cool-journal-1\"}[5m])", + "legendFormat": "journal" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "drawStyle": "bars" + }, + "color": { + "mode": "fixed", + "fixedColor": "orange" + } + } + } + }, + { + "title": "Container Memory", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 24 + }, + "targets": [ + { + "expr": "container_memory_usage_bytes{name=\"trails-cool-journal-1\"}", + "legendFormat": "used" + }, + { + "expr": "container_memory_working_set_bytes{name=\"trails-cool-journal-1\"}", + "legendFormat": "working set" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + } + }, + { + "title": "Container CPU", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 24 + }, + "targets": [ + { + "expr": "rate(container_cpu_usage_seconds_total{name=\"trails-cool-journal-1\"}[5m]) * 100", + "legendFormat": "CPU %" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + } + } + }, + { + "title": "Node.js Event Loop Lag", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "targets": [ + { + "expr": "nodejs_eventloop_lag_seconds{job=\"journal\"}", + "legendFormat": "lag" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + } + }, + { + "title": "Node.js Heap Used", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "targets": [ + { + "expr": "nodejs_heap_size_used_bytes{job=\"journal\"}", + "legendFormat": "used" + }, + { + "expr": "nodejs_heap_size_total_bytes{job=\"journal\"}", + "legendFormat": "total" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + } + }, + { + "title": "Journal Logs (Errors & Warnings)", + "description": "Recent error and warning logs from the journal container. Requires Promtail docker_sd_configs with Pino JSON parsing.", + "type": "logs", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 40 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service=\"journal\"} |~ \"(error|warn|ERR|WARN|level.*(40|50|60))\"", + "legendFormat": "" + } + ], + "options": { + "showTime": true, + "sortOrder": "Descending", + "enableLogDetails": true, + "dedupStrategy": "none" + } + }, + { + "title": "Caddy Logs (5xx Responses)", + "description": "Caddy access log entries with 5xx status codes. Shows the raw request that triggered the error.", + "type": "logs", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 50 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service=\"caddy\"} |~ \"\\\"status\\\":\\s*5\\d\\d\"", + "legendFormat": "" + } + ], + "options": { + "showTime": true, + "sortOrder": "Descending", + "enableLogDetails": true, + "dedupStrategy": "none" + } + } + ] +} diff --git a/infrastructure/grafana/dashboards/overview.json b/infrastructure/grafana/dashboards/overview.json index 250833c..1d9b9cc 100644 --- a/infrastructure/grafana/dashboards/overview.json +++ b/infrastructure/grafana/dashboards/overview.json @@ -52,7 +52,7 @@ }, "targets": [ { - "expr": "sum(rate(caddy_http_response_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_response_duration_seconds_count[5m])) * 100", + "expr": "sum(rate(caddy_http_request_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_request_duration_seconds_count[5m])) * 100", "legendFormat": "Error %" } ], @@ -138,5 +138,44 @@ } } } + { + "title": "App Error Rate (from logs)", + "description": "Error-level log entries per second from Pino (level 50=error, 60=fatal). Captures crashes and unhandled errors that metrics may miss.", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(rate({service=\"journal\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)", + "legendFormat": "journal" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(rate({service=\"planner\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)", + "legendFormat": "planner" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 15 + }, + "color": { + "mode": "palette-classic" + } + } + } + } ] } diff --git a/infrastructure/grafana/dashboards/service-health.json b/infrastructure/grafana/dashboards/service-health.json index 69291e9..3d53131 100644 --- a/infrastructure/grafana/dashboards/service-health.json +++ b/infrastructure/grafana/dashboards/service-health.json @@ -418,10 +418,30 @@ }, "targets": [ { - "expr": "sum(rate(caddy_http_response_duration_seconds_count[5m])) by (code)", + "expr": "sum(rate(caddy_http_request_duration_seconds_count[5m])) by (code)", "legendFormat": "{{code}}" } - ] + ], + "fieldConfig": { + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "/^2\\d\\d$/" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] + }, + { + "matcher": { "id": "byRegexp", "options": "/^3\\d\\d$/" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }] + }, + { + "matcher": { "id": "byRegexp", "options": "/^4\\d\\d$/" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }] + }, + { + "matcher": { "id": "byRegexp", "options": "/^5\\d\\d$/" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] + } + ] + } }, { "title": "Caddy \u2014 Request Duration by Host", diff --git a/infrastructure/grafana/provisioning/alerting/alerts.yml b/infrastructure/grafana/provisioning/alerting/alerts.yml index e758b25..ac3ce8a 100644 --- a/infrastructure/grafana/provisioning/alerting/alerts.yml +++ b/infrastructure/grafana/provisioning/alerting/alerts.yml @@ -61,7 +61,7 @@ groups: relativeTimeRange: { from: 600, to: 0 } datasourceUid: prometheus model: - expr: (sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(http_request_duration_seconds_count[5m])), 0.001) * 100 + expr: (sum(rate(caddy_http_request_duration_seconds_count{code=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(caddy_http_request_duration_seconds_count[5m])), 0.001) * 100 instant: true - refId: B datasourceUid: __expr__ @@ -76,6 +76,106 @@ groups: annotations: summary: "Error rate is above 5%" + - uid: container-restart-loop + title: Container restart loop + condition: B + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + expr: changes(container_start_time_seconds{name=~"trails-cool.*"}[5m]) + instant: true + - refId: B + datasourceUid: __expr__ + model: + type: threshold + expression: A + conditions: + - evaluator: { params: [2], type: gt } + operator: { type: and } + reducer: { type: last } + for: 1m + annotations: + summary: "{{ $labels.name }} has restarted more than 2 times in 5 minutes" + + - uid: db-connections-high + title: PostgreSQL connections > 80 + condition: B + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + expr: sum(pg_stat_activity_count{datname="trails"}) + instant: true + - refId: B + datasourceUid: __expr__ + model: + type: threshold + expression: A + conditions: + - evaluator: { params: [80], type: gt } + operator: { type: and } + reducer: { type: last } + for: 5m + annotations: + summary: "PostgreSQL active connections above 80 — approaching default max_connections limit" + + - uid: app-crash-log + title: Application crash detected + condition: C + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: loki + model: + expr: sum(count_over_time({service=~"journal|planner"} |~ "uncaughtException|unhandledRejection|ERR_|FATAL|segfault|OOMKilled"[5m])) + instant: true + - refId: B + datasourceUid: __expr__ + model: + type: reduce + expression: A + reducer: last + - refId: C + datasourceUid: __expr__ + model: + type: threshold + expression: B + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + reducer: { type: last } + for: 0s + noDataState: OK + annotations: + summary: "Crash signature detected in application logs" + + - uid: caddy-502-rate + title: Caddy 502 errors detected + condition: B + noDataState: OK + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + expr: sum(rate(caddy_http_request_duration_seconds_count{code="502"}[5m])) or vector(0) + instant: true + - refId: B + datasourceUid: __expr__ + model: + type: threshold + expression: A + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + reducer: { type: last } + for: 2m + annotations: + summary: "Caddy is returning 502 errors — journal or planner upstream unreachable" + contactPoints: - orgId: 1 name: email diff --git a/infrastructure/promtail/promtail-config.yml b/infrastructure/promtail/promtail-config.yml index f7414fe..248f2a5 100644 --- a/infrastructure/promtail/promtail-config.yml +++ b/infrastructure/promtail/promtail-config.yml @@ -10,13 +10,30 @@ clients: scrape_configs: - job_name: docker - static_configs: - - targets: - - localhost - labels: - job: docker - __path__: /var/lib/docker/containers/*/*.log + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 10s + relabel_configs: + # Use the container name as a label + - source_labels: ["__meta_docker_container_name"] + regex: "/?(.*)" + target_label: container + # Use the compose service name if available + - source_labels: ["__meta_docker_container_label_com_docker_compose_service"] + target_label: service + # Drop containers we don't need logs from (exporters, promtail itself) + - source_labels: ["__meta_docker_container_label_com_docker_compose_service"] + regex: "promtail|node-exporter|cadvisor|postgres-exporter" + action: drop pipeline_stages: - docker: {} - - labeldrop: - - filename + # Parse Pino JSON logs (journal, planner) to extract level + - match: + selector: '{service=~"journal|planner"}' + stages: + - json: + expressions: + level: level + msg: msg + - labels: + level: diff --git a/tsconfig.base.json b/tsconfig.base.json index 9166196..3257189 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -17,6 +17,7 @@ "noUncheckedIndexedAccess": true, "noUncheckedSideEffectImports": false, "noEmit": true, + "erasableSyntaxOnly": true, "allowImportingTsExtensions": true }, "exclude": ["node_modules", "build", "dist"]