Add observability: health endpoints, structured logging, metrics, Grafana stack

Health endpoints:
- /api/health (Journal) and /health (Planner) with DB connectivity check
- Docker healthchecks updated to use app health endpoints

Structured logging:
- Pino with JSON output in production, pretty-print in dev
- Request logging middleware in Planner (method, path, status, duration)
- Replaced console.log/error with structured logger in email and auth flows

Prometheus metrics:
- prom-client with default Node.js metrics + custom histograms/gauges
- /metrics endpoints on both apps
- http_request_duration, planner_active_sessions, brouter_request_duration

Monitoring stack:
- Prometheus, Loki, Grafana containers in docker-compose
- Grafana provisioned with datasources, dashboards, and alert rules
- Caddy access logging (JSON to stdout for Loki)
- grafana.trails.cool with basic auth via Caddy

Dashboards and alerting:
- Overview: request rate, error rate, latency p50/p95/p99
- Planner: active sessions, connected clients, BRouter latency
- Infrastructure: memory, CPU, event loop lag
- Alerts: disk >80%, app down 2min, error rate >5%

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

apps/journal/app/lib/email.server.test.ts

@@ -7,6 +7,11 @@ vi.mock("nodemailer", () => ({
   }),
 }));
 
+// Mock logger
+vi.mock("./logger.server", () => ({
+  logger: { info: vi.fn(), debug: vi.fn(), error: vi.fn(), warn: vi.fn() },
+}));
+
 describe("email.server", () => {
   const originalEnv = process.env.NODE_ENV;
 
@@ -19,29 +24,27 @@ describe("email.server", () => {
     delete process.env.SMTP_URL;
   });
 
-  it("logs to console in dev mode", async () => {
+  it("uses logger in dev mode instead of sending email", async () => {
     process.env.NODE_ENV = "development";
     const { sendEmail } = await import("./email.server");
-    const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {});
+    const { logger } = await import("./logger.server");
 
     await sendEmail("test@example.com", "Test Subject", "<p>Hello</p>", "Hello");
 
-    expect(consoleSpy).toHaveBeenCalledWith(
-      expect.stringContaining("test@example.com"),
+    expect(logger.info).toHaveBeenCalledWith(
+      expect.objectContaining({ to: "test@example.com" }),
+      expect.any(String),
     );
-    consoleSpy.mockRestore();
   });
 
   it("does not call SMTP in dev mode", async () => {
     process.env.NODE_ENV = "development";
     const nodemailer = await import("nodemailer");
     const { sendEmail } = await import("./email.server");
-    const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {});
 
     await sendEmail("test@example.com", "Test", "<p>Hi</p>", "Hi");
 
     expect(nodemailer.createTransport).not.toHaveBeenCalled();
-    consoleSpy.mockRestore();
   });
 
   it("magicLinkTemplate includes link and expiry note", async () => {

apps/journal/app/lib/email.server.ts

@@ -1,4 +1,5 @@
 import { createTransport, type Transporter } from "nodemailer";
+import { logger } from "./logger.server";
 
 const FROM = process.env.SMTP_FROM ?? "trails.cool <noreply@trails.cool>";
 
@@ -19,8 +20,8 @@ export async function sendEmail(
   text: string,
 ): Promise<void> {
   if (process.env.NODE_ENV !== "production") {
-    console.log(`[Email] To: ${to} | Subject: ${subject}`);
-    console.log(`[Email] Text:\n${text}`);
+    logger.info({ to, subject }, "Email sent (dev mode — logged, not delivered)");
+    logger.debug({ text }, "Email text content");
     return;
   }
 

apps/journal/app/lib/logger.server.ts

@@ -0,0 +1,8 @@
+import pino from "pino";
+
+export const logger = pino({
+  level: process.env.LOG_LEVEL ?? "info",
+  ...(process.env.NODE_ENV !== "production"
+    ? { transport: { target: "pino-pretty" } }
+    : {}),
+});

apps/journal/app/lib/metrics.server.ts

@@ -0,0 +1,13 @@
+import client from "prom-client";
+
+// Collect default Node.js metrics (event loop, heap, GC)
+client.collectDefaultMetrics();
+
+export const httpRequestDuration = new client.Histogram({
+  name: "http_request_duration_seconds",
+  help: "Duration of HTTP requests in seconds",
+  labelNames: ["method", "route", "status"] as const,
+  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
+});
+
+export const registry = client.register;

apps/journal/app/routes/api.auth.register.ts

@@ -2,6 +2,7 @@ import { data } from "react-router";
 import type { Route } from "./+types/api.auth.register";
 import { startRegistration, finishRegistration, createSession, addPasskeyStart, addPasskeyFinish } from "~/lib/auth.server";
 import { sendWelcome } from "~/lib/email.server";
+import { logger } from "~/lib/logger.server";
 
 export async function action({ request }: Route.ActionArgs) {
   const body = await request.json();
@@ -18,7 +19,7 @@ export async function action({ request }: Route.ActionArgs) {
       const cookie = await createSession(newUserId, request);
       // Send welcome email (fire-and-forget — don't block registration on email)
       sendWelcome(email, username).catch((err) =>
-        console.error("[Email] Failed to send welcome email:", err),
+        logger.error({ err }, "Failed to send welcome email"),
       );
       return data({ step: "done" }, { headers: { "Set-Cookie": cookie } });
     }

apps/journal/app/routes/api.health.ts

@@ -0,0 +1,13 @@
+import { data } from "react-router";
+import { withDb } from "@trails-cool/db";
+
+export async function loader() {
+  try {
+    await withDb(async () => {
+      // withDb creates a connection — if it succeeds, DB is reachable
+    });
+    return data({ status: "ok", db: "connected" });
+  } catch {
+    return data({ status: "degraded", db: "unreachable" }, { status: 503 });
+  }
+}

apps/journal/app/routes/api.metrics.ts

@@ -0,0 +1,8 @@
+import { registry } from "~/lib/metrics.server";
+
+export async function loader() {
+  const metrics = await registry.metrics();
+  return new Response(metrics, {
+    headers: { "Content-Type": registry.contentType },
+  });
+}

apps/journal/package.json

@@ -27,6 +27,8 @@
     "isbot": "^5.1.0",
     "jose": "^6.2.2",
     "nodemailer": "^8.0.4",
+    "pino": "^10.3.1",
+    "prom-client": "^15.1.3",
     "react": "catalog:",
     "react-dom": "catalog:",
     "react-router": "catalog:"
@@ -38,6 +40,7 @@
     "@types/nodemailer": "^7.0.11",
     "@types/react": "catalog:",
     "@types/react-dom": "catalog:",
+    "pino-pretty": "^13.1.3",
     "tailwindcss": "catalog:",
     "typescript": "catalog:",
     "vite": "catalog:"

apps/planner/app/lib/logger.server.ts

@@ -0,0 +1,8 @@
+import pino from "pino";
+
+export const logger = pino({
+  level: process.env.LOG_LEVEL ?? "info",
+  ...(process.env.NODE_ENV !== "production"
+    ? { transport: { target: "pino-pretty" } }
+    : {}),
+});

apps/planner/app/lib/metrics.server.ts

@@ -0,0 +1,29 @@
+import client from "prom-client";
+
+// Collect default Node.js metrics (event loop, heap, GC)
+client.collectDefaultMetrics();
+
+export const httpRequestDuration = new client.Histogram({
+  name: "http_request_duration_seconds",
+  help: "Duration of HTTP requests in seconds",
+  labelNames: ["method", "route", "status"] as const,
+  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
+});
+
+export const plannerActiveSessions = new client.Gauge({
+  name: "planner_active_sessions",
+  help: "Number of active planner sessions",
+});
+
+export const plannerConnectedClients = new client.Gauge({
+  name: "planner_connected_clients",
+  help: "Number of connected WebSocket clients",
+});
+
+export const brouterRequestDuration = new client.Histogram({
+  name: "brouter_request_duration_seconds",
+  help: "Duration of BRouter API requests in seconds",
+  buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
+});
+
+export const registry = client.register;

apps/planner/package.json

@@ -25,6 +25,8 @@
     "drizzle-orm": "catalog:",
     "isbot": "^5.1.0",
     "lib0": "^0.2.117",
+    "pino": "^10.3.1",
+    "prom-client": "^15.1.3",
     "react": "catalog:",
     "react-dom": "catalog:",
     "react-router": "catalog:",
@@ -39,6 +41,7 @@
     "@types/react": "catalog:",
     "@types/react-dom": "catalog:",
     "@types/ws": "^8.18.1",
+    "pino-pretty": "^13.1.3",
     "tailwindcss": "catalog:",
     "typescript": "catalog:",
     "vite": "catalog:"

apps/planner/server.ts

@@ -1,4 +1,6 @@
 import * as Sentry from "@sentry/node";
+import { logger } from "./app/lib/logger.server.ts";
+import { httpRequestDuration } from "./app/lib/metrics.server.ts";
 import { createRequestListener } from "@react-router/node";
 import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
 import { createReadStream, statSync } from "node:fs";
@@ -62,7 +64,44 @@ const listener = createRequestListener({
   build: () => import("./build/server/index.js") as never,
 });
 
+async function handleMetrics(_req: IncomingMessage, res: ServerResponse): Promise<void> {
+  const { registry } = await import("./app/lib/metrics.server.ts");
+  const metrics = await registry.metrics();
+  res.writeHead(200, { "Content-Type": registry.contentType });
+  res.end(metrics);
+}
+
+async function handleHealth(_req: IncomingMessage, res: ServerResponse): Promise<void> {
+  try {
+    const { withDb, db } = await import("@trails-cool/db");
+    const { sql } = await import("drizzle-orm");
+    await withDb(async () => { await db.execute(sql`SELECT 1`); });
+    res.writeHead(200, { "Content-Type": "application/json" });
+    res.end(JSON.stringify({ status: "ok", db: "connected" }));
+  } catch {
+    res.writeHead(503, { "Content-Type": "application/json" });
+    res.end(JSON.stringify({ status: "degraded", db: "unreachable" }));
+  }
+}
+
 const server = createServer((req, res) => {
+  const url = req.url ?? "/";
+  const start = Date.now();
+
+  // Log and track request on finish (skip static assets and health/metrics)
+  if (!url.startsWith("/assets/") && url !== "/health" && url !== "/metrics") {
+    res.on("finish", () => {
+      const duration = Date.now() - start;
+      logger.info({ method: req.method, path: url, status: res.statusCode, duration }, "request");
+      httpRequestDuration.observe(
+        { method: req.method ?? "GET", route: url.split("?")[0]!, status: String(res.statusCode) },
+        duration / 1000,
+      );
+    });
+  }
+
+  if (url === "/health") { handleHealth(req, res); return; }
+  if (url === "/metrics") { handleMetrics(req, res); return; }
   if (!serveStatic(req, res)) {
     listener(req, res);
   }
@@ -71,6 +110,6 @@ const server = createServer((req, res) => {
 setupYjsWebSocket(server);
 
 server.listen(port, () => {
-  console.log(`Planner server listening on http://localhost:${port}`);
-  console.log(`Yjs WebSocket available at ws://localhost:${port}/sync/:sessionId`);
+  logger.info({ port }, "Planner server listening");
+  logger.info({ port, path: "/sync/:sessionId" }, "Yjs WebSocket available");
 });