trails-cool/trails
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add observability: health endpoints, structured logging, metrics, Grafana stack

Browse source 
Health endpoints:
- /api/health (Journal) and /health (Planner) with DB connectivity check
- Docker healthchecks updated to use app health endpoints

Structured logging:
- Pino with JSON output in production, pretty-print in dev
- Request logging middleware in Planner (method, path, status, duration)
- Replaced console.log/error with structured logger in email and auth flows

Prometheus metrics:
- prom-client with default Node.js metrics + custom histograms/gauges
- /metrics endpoints on both apps
- http_request_duration, planner_active_sessions, brouter_request_duration

Monitoring stack:
- Prometheus, Loki, Grafana containers in docker-compose
- Grafana provisioned with datasources, dashboards, and alert rules
- Caddy access logging (JSON to stdout for Loki)
- grafana.trails.cool with basic auth via Caddy

Dashboards and alerting:
- Overview: request rate, error rate, latency p50/p95/p99
- Planner: active sessions, connected clients, BRouter latency
- Infrastructure: memory, CPU, event loop lag
- Alerts: disk >80%, app down 2min, error rate >5%

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ullrich Schäfer 2026-03-26 22:59:44 +01:00
parent ce964cae96
commit 49aadd04a9
No known key found for this signature in database
GPG key ID: A32FF691A0F752D9
24 changed files with 770 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

15
infrastructure/Caddyfile
View file

@ -17,6 +17,10 @@
{$DOMAIN:trails.cool} {
import security_headers
import block_scanners
log {
output stdout
format json
}
reverse_proxy journal:3000
}
@ -24,8 +28,19 @@ www.{$DOMAIN:trails.cool} {
redir https://{$DOMAIN:trails.cool}{uri} permanent
}
grafana.{$DOMAIN:trails.cool} {
basicauth {
{$GRAFANA_USER:admin} {$GRAFANA_PASSWORD_HASH}
}
reverse_proxy grafana:3000
}
planner.{$DOMAIN:trails.cool} {
import security_headers
import block_scanners
log {
output stdout
format json
}
reverse_proxy planner:3001
}

47
infrastructure/docker-compose.yml
View file

@ -29,6 +29,11 @@ services:
SENTRY_RELEASE: ${SENTRY_RELEASE:-}
SMTP_URL: ${SMTP_URL:-}
SMTP_FROM: ${SMTP_FROM:-trails.cool <noreply@trails.cool>}
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"]
interval: 15s
timeout: 5s
retries: 3
depends_on:
postgres:
condition: service_healthy
@ -42,6 +47,11 @@ services:
NODE_ENV: production
PORT: 3001
SENTRY_RELEASE: ${SENTRY_RELEASE:-}
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:3001/health || exit 1"]
interval: 15s
timeout: 5s
retries: 3
depends_on:
postgres:
condition: service_healthy
@ -72,6 +82,40 @@ services:
timeout: 5s
retries: 5
prometheus:
image: prom/prometheus:latest
restart: unless-stopped
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.retention.time=15d"
- "--storage.tsdb.retention.size=1GB"
loki:
image: grafana/loki:latest
restart: unless-stopped
volumes:
- ./loki/loki-config.yml:/etc/loki/local-config.yaml:ro
- loki_data:/loki
command: ["-config.file=/etc/loki/local-config.yaml"]
grafana:
image: grafana/grafana:latest
restart: unless-stopped
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_USER:-admin}
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin}
GF_SERVER_ROOT_URL: https://grafana.${DOMAIN:-trails.cool}
volumes:
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
depends_on:
- prometheus
- loki
# garage:
# image: dxflrs/garage:v1.0
# restart: unless-stopped
@ -83,3 +127,6 @@ volumes:
caddy_data:
caddy_config:
pgdata:
prometheus_data:
loki_data:
grafana_data:

45
infrastructure/grafana/dashboards/infrastructure.json Normal file
View file

@ -0,0 +1,45 @@
{
"dashboard": {
"title": "Infrastructure",
"uid": "trails-infra",
"timezone": "browser",
"refresh": "30s",
"panels": [
{
"title": "Process Memory (RSS)",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
"targets": [
{ "expr": "process_resident_memory_bytes", "legendFormat": "{{job}}" }
],
"fieldConfig": { "defaults": { "unit": "bytes" } }
},
{
"title": "CPU Usage",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
"targets": [
{ "expr": "rate(process_cpu_seconds_total[5m])", "legendFormat": "{{job}}" }
],
"fieldConfig": { "defaults": { "unit": "percentunit" } }
},
{
"title": "Event Loop Lag",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
"targets": [
{ "expr": "nodejs_eventloop_lag_seconds", "legendFormat": "{{job}}" }
],
"fieldConfig": { "defaults": { "unit": "s" } }
},
{
"title": "Open File Descriptors",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
"targets": [
{ "expr": "process_open_fds", "legendFormat": "{{job}}" }
]
}
]
}
}

57
infrastructure/grafana/dashboards/overview.json Normal file
View file

@ -0,0 +1,57 @@
{
"dashboard": {
"title": "trails.cool Overview",
"uid": "trails-overview",
"timezone": "browser",
"refresh": "30s",
"panels": [
{
"title": "Request Rate",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
"targets": [
{
"expr": "sum(rate(http_request_duration_seconds_count[5m])) by (job)",
"legendFormat": "{{job}}"
}
]
},
{
"title": "Error Rate",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
"targets": [
{
"expr": "sum(rate(http_request_duration_seconds_count{status=~\"5..\"}[5m])) / sum(rate(http_request_duration_seconds_count[5m])) * 100",
"legendFormat": "Error %"
}
],
"fieldConfig": {
"defaults": { "unit": "percent", "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 5 }] } }
}
},
{
"title": "Latency p50 / p95 / p99",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
"targets": [
{ "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
{ "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
{ "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
],
"fieldConfig": { "defaults": { "unit": "s" } }
},
{
"title": "Health Status",
"type": "stat",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
"targets": [
{ "expr": "up", "legendFormat": "{{job}}" }
],
"fieldConfig": {
"defaults": { "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }] }
}
}
]
}
}

40
infrastructure/grafana/dashboards/planner.json Normal file
View file

@ -0,0 +1,40 @@
{
"dashboard": {
"title": "Planner",
"uid": "trails-planner",
"timezone": "browser",
"refresh": "30s",
"panels": [
{
"title": "Active Sessions",
"type": "stat",
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 },
"targets": [{ "expr": "planner_active_sessions", "legendFormat": "Sessions" }]
},
{
"title": "Connected Clients",
"type": "stat",
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 },
"targets": [{ "expr": "planner_connected_clients", "legendFormat": "Clients" }]
},
{
"title": "BRouter Latency",
"type": "timeseries",
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
"targets": [
{ "expr": "histogram_quantile(0.50, rate(brouter_request_duration_seconds_bucket[5m]))", "legendFormat": "p50" },
{ "expr": "histogram_quantile(0.95, rate(brouter_request_duration_seconds_bucket[5m]))", "legendFormat": "p95" }
],
"fieldConfig": { "defaults": { "unit": "s" } }
},
{
"title": "Request Rate by Route",
"type": "timeseries",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
"targets": [
{ "expr": "sum(rate(http_request_duration_seconds_count{job=\"planner\"}[5m])) by (route)", "legendFormat": "{{route}}" }
]
}
]
}
}

89
infrastructure/grafana/provisioning/alerting/alerts.yml Normal file
View file

@ -0,0 +1,89 @@
apiVersion: 1
groups:
- orgId: 1
name: trails.cool alerts
folder: trails.cool
interval: 1m
rules:
- uid: disk-usage-high
title: Disk usage > 80%
condition: C
data:
- refId: A
relativeTimeRange: { from: 600, to: 0 }
datasourceUid: prometheus
model:
expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100
intervalMs: 60000
maxDataPoints: 43200
- refId: C
relativeTimeRange: { from: 600, to: 0 }
datasourceUid: __expr__
model:
type: threshold
conditions:
- evaluator: { params: [80], type: gt }
reducer: { type: last }
for: 15m
annotations:
summary: "Disk usage is above 80%"
- uid: app-health-failing
title: App health check failing
condition: C
data:
- refId: A
relativeTimeRange: { from: 300, to: 0 }
datasourceUid: prometheus
model:
expr: up{job=~"journal|planner"} == 0
intervalMs: 15000
maxDataPoints: 43200
- refId: C
relativeTimeRange: { from: 300, to: 0 }
datasourceUid: __expr__
model:
type: threshold
conditions:
- evaluator: { params: [0], type: gt }
reducer: { type: last }
for: 2m
annotations:
summary: "{{ $labels.job }} is down"
- uid: error-rate-high
title: Error rate > 5%
condition: C
data:
- refId: A
relativeTimeRange: { from: 600, to: 0 }
datasourceUid: prometheus
model:
expr: sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m])) / sum(rate(http_request_duration_seconds_count[5m])) * 100
intervalMs: 60000
maxDataPoints: 43200
- refId: C
relativeTimeRange: { from: 600, to: 0 }
datasourceUid: __expr__
model:
type: threshold
conditions:
- evaluator: { params: [5], type: gt }
reducer: { type: last }
for: 5m
annotations:
summary: "Error rate is above 5%"
contactPoints:
- orgId: 1
name: email
receivers:
- uid: email-default
type: email
settings:
addresses: ${ALERT_EMAIL:-admin@trails.cool}
policies:
- orgId: 1
receiver: email

12
infrastructure/grafana/provisioning/dashboards/dashboards.yml Normal file
View file

@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: trails.cool
orgId: 1
folder: ""
type: file
disableDeletion: false
editable: true
options:
path: /var/lib/grafana/dashboards
foldersFromFilesStructure: false

15
infrastructure/grafana/provisioning/datasources/datasources.yml Normal file
View file

@ -0,0 +1,15 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
- name: Loki
type: loki
access: proxy
url: http://loki:3100
editable: false

32
infrastructure/loki/loki-config.yml Normal file
View file

@ -0,0 +1,32 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
retention_period: 168h # 7 days
compactor:
working_directory: /loki/compactor
retention_enabled: true

18
infrastructure/prometheus/prometheus.yml Normal file
View file

@ -0,0 +1,18 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: "journal"
metrics_path: "/api/metrics"
static_configs:
- targets: ["journal:3000"]
- job_name: "planner"
metrics_path: "/metrics"
static_configs:
- targets: ["planner:3001"]
- job_name: "caddy"
static_configs:
- targets: ["caddy:2019"]