881991ca18
Task group 4 of federation-hardening.
4.1 — FEDERATION.md at the repo root: actor discovery (WebFinger, actor,
NodeInfo), object/activity types with real JSON examples (Note, Create,
Delete, the narrow follow-graph inbox), addressing, HTTP-Signature
expectations, the two-layer dedup contract, durable delivery/retry
policy, and blocklist moderation semantics — precise enough for another
implementation to interoperate. Linked from README and docs/architecture.
4.2 — three prom-client metrics + a journal dashboard row:
- `federation_delivery_total{outcome}` — incremented in deliver-activity
(delivered/skipped/failed).
- `federation_inbox_dropped_total{reason}` — incremented at every inbox
drop (duplicate | blocked); this is the counter deferred from task 3.2.
- `federation_queue_depth` — gauge sampled at scrape time in
/api/metrics from PgBossMessageQueue.getDepth(); the restart-loss
regression detector.
Grafana journal.json gains a Federation row (delivery rate, queue depth,
inbox drops); the logs panels shift down to make room.
Verified: dashboard JSON valid; journal typecheck + lint clean; unit
suite 357 passing (route-template guard unaffected).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
391 lines
8.3 KiB
JSON
391 lines
8.3 KiB
JSON
{
|
|
"title": "Journal",
|
|
"uid": "trails-journal",
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"name": "Deploys",
|
|
"enable": true,
|
|
"datasource": {
|
|
"type": "grafana",
|
|
"uid": "-- Grafana --"
|
|
},
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"target": {
|
|
"limit": 100,
|
|
"matchAny": false,
|
|
"tags": [
|
|
"deploy"
|
|
],
|
|
"type": "tags"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"timezone": "browser",
|
|
"refresh": "30s",
|
|
"panels": [
|
|
{
|
|
"title": "Caddy 502 Rate (Journal)",
|
|
"description": "502 errors returned by Caddy for the journal upstream. These indicate Caddy could not reach the journal container.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code=\"502\"}[5m])) or vector(0)",
|
|
"legendFormat": "502/s"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2
|
|
},
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Caddy Response Codes (Journal)",
|
|
"description": "All HTTP response codes from Caddy for the journal host, including Caddy-generated errors like 502.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code!~\"(101|2..|3..)\"}[5m])) by (code)",
|
|
"legendFormat": "{{code}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"title": "Journal Request Rate by Status",
|
|
"description": "Request rate from the journal app's own metrics, grouped by HTTP status code.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (status)",
|
|
"legendFormat": "{{status}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"title": "Journal Latency p50 / p95 / p99",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Request Rate by Route",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (route)",
|
|
"legendFormat": "{{route}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"title": "Container Restarts",
|
|
"description": "Journal container restart count. Spikes here correlate with 502 errors \u2014 Caddy returns 502 while the container is restarting.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "changes(container_start_time_seconds{name=\"trails-cool-journal-1\"}[5m])",
|
|
"legendFormat": "journal"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 30,
|
|
"drawStyle": "bars"
|
|
},
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Container Memory",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "container_memory_usage_bytes{name=\"trails-cool-journal-1\"}",
|
|
"legendFormat": "used"
|
|
},
|
|
{
|
|
"expr": "container_memory_working_set_bytes{name=\"trails-cool-journal-1\"}",
|
|
"legendFormat": "working set"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Container CPU",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(container_cpu_usage_seconds_total{name=\"trails-cool-journal-1\"}[5m]) * 100",
|
|
"legendFormat": "CPU %"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Node.js Event Loop Lag",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "nodejs_eventloop_lag_seconds{job=\"journal\"}",
|
|
"legendFormat": "lag"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Node.js Heap Used",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 32
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "nodejs_heap_size_used_bytes{job=\"journal\"}",
|
|
"legendFormat": "used"
|
|
},
|
|
{
|
|
"expr": "nodejs_heap_size_total_bytes{job=\"journal\"}",
|
|
"legendFormat": "total"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Federation Delivery Rate by Outcome",
|
|
"description": "Outbound federation delivery attempts/sec by outcome (delivered/skipped/failed). Sustained 'failed' means an unreachable or misconfigured remote; retries continue until the budget is exhausted.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 40
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (outcome) (rate(federation_delivery_total[5m]))",
|
|
"legendFormat": "{{outcome}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Federation Queue Depth",
|
|
"description": "Messages waiting in the durable Fedify (pg-boss) queue. A depth that climbs and never drains is the restart-loss / stuck-consumer regression detector \u2014 before the pg-boss backing (PR #570) a restart silently zeroed this by dropping the work.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 40
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "federation_queue_depth",
|
|
"legendFormat": "queued"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 2
|
|
},
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Federation Inbox Drops by Reason",
|
|
"description": "Inbound activities dropped before side effects, by reason: 'duplicate' (replay defense) and 'blocked' (instance blocklist). A spike in 'blocked' after a block confirms it is taking effect; sustained 'duplicate' can indicate a misbehaving remote redelivering.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 48
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (reason) (rate(federation_inbox_dropped_total[5m]))",
|
|
"legendFormat": "{{reason}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Journal Logs (Errors & Warnings)",
|
|
"description": "Recent error and warning logs from the journal container. Requires Promtail docker_sd_configs with Pino JSON parsing.",
|
|
"type": "logs",
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 56
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{service=\"journal\"} |~ \"(error|warn|ERR|WARN|level.*(40|50|60))\"",
|
|
"legendFormat": ""
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"enableLogDetails": true,
|
|
"dedupStrategy": "none"
|
|
}
|
|
},
|
|
{
|
|
"title": "Caddy Logs (5xx Responses)",
|
|
"description": "Caddy access log entries with 5xx status codes. Shows the raw request that triggered the error.",
|
|
"type": "logs",
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 66
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{service=\"caddy\"} |~ \"\\\"status\\\":\\s*5\\d\\d\"",
|
|
"legendFormat": ""
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"enableLogDetails": true,
|
|
"dedupStrategy": "none"
|
|
}
|
|
}
|
|
]
|
|
}
|