Add observability: health endpoints, structured logging, metrics, Grafana stack

Health endpoints:
- /api/health (Journal) and /health (Planner) with DB connectivity check
- Docker healthchecks updated to use app health endpoints

Structured logging:
- Pino with JSON output in production, pretty-print in dev
- Request logging middleware in Planner (method, path, status, duration)
- Replaced console.log/error with structured logger in email and auth flows

Prometheus metrics:
- prom-client with default Node.js metrics + custom histograms/gauges
- /metrics endpoints on both apps
- http_request_duration, planner_active_sessions, brouter_request_duration

Monitoring stack:
- Prometheus, Loki, Grafana containers in docker-compose
- Grafana provisioned with datasources, dashboards, and alert rules
- Caddy access logging (JSON to stdout for Loki)
- grafana.trails.cool with basic auth via Caddy

Dashboards and alerting:
- Overview: request rate, error rate, latency p50/p95/p99
- Planner: active sessions, connected clients, BRouter latency
- Infrastructure: memory, CPU, event loop lag
- Alerts: disk >80%, app down 2min, error rate >5%

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ullrich Schäfer 2026-03-26 22:59:44 +01:00
parent ce964cae96
commit 49aadd04a9
No known key found for this signature in database
GPG key ID: A32FF691A0F752D9
24 changed files with 770 additions and 48 deletions

View file

@ -1,4 +1,6 @@
import * as Sentry from "@sentry/node";
import { logger } from "./app/lib/logger.server.ts";
import { httpRequestDuration } from "./app/lib/metrics.server.ts";
import { createRequestListener } from "@react-router/node";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { createReadStream, statSync } from "node:fs";
@ -62,7 +64,44 @@ const listener = createRequestListener({
build: () => import("./build/server/index.js") as never,
});
async function handleMetrics(_req: IncomingMessage, res: ServerResponse): Promise<void> {
const { registry } = await import("./app/lib/metrics.server.ts");
const metrics = await registry.metrics();
res.writeHead(200, { "Content-Type": registry.contentType });
res.end(metrics);
}
async function handleHealth(_req: IncomingMessage, res: ServerResponse): Promise<void> {
try {
const { withDb, db } = await import("@trails-cool/db");
const { sql } = await import("drizzle-orm");
await withDb(async () => { await db.execute(sql`SELECT 1`); });
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({ status: "ok", db: "connected" }));
} catch {
res.writeHead(503, { "Content-Type": "application/json" });
res.end(JSON.stringify({ status: "degraded", db: "unreachable" }));
}
}
const server = createServer((req, res) => {
const url = req.url ?? "/";
const start = Date.now();
// Log and track request on finish (skip static assets and health/metrics)
if (!url.startsWith("/assets/") && url !== "/health" && url !== "/metrics") {
res.on("finish", () => {
const duration = Date.now() - start;
logger.info({ method: req.method, path: url, status: res.statusCode, duration }, "request");
httpRequestDuration.observe(
{ method: req.method ?? "GET", route: url.split("?")[0]!, status: String(res.statusCode) },
duration / 1000,
);
});
}
if (url === "/health") { handleHealth(req, res); return; }
if (url === "/metrics") { handleMetrics(req, res); return; }
if (!serveStatic(req, res)) {
listener(req, res);
}
@ -71,6 +110,6 @@ const server = createServer((req, res) => {
setupYjsWebSocket(server);
server.listen(port, () => {
console.log(`Planner server listening on http://localhost:${port}`);
console.log(`Yjs WebSocket available at ws://localhost:${port}/sync/:sessionId`);
logger.info({ port }, "Planner server listening");
logger.info({ port, path: "/sync/:sessionId" }, "Yjs WebSocket available");
});