{
 "title": "Journal",
 "uid": "trails-journal",
 "annotations": {
  "list": [
   {
    "name": "Deploys",
    "enable": true,
    "datasource": {
     "type": "grafana",
     "uid": "-- Grafana --"
    },
    "iconColor": "rgba(0, 211, 255, 1)",
    "target": {
     "limit": 100,
     "matchAny": false,
     "tags": [
      "deploy"
     ],
     "type": "tags"
    }
   }
  ]
 },
 "timezone": "browser",
 "refresh": "30s",
 "panels": [
  {
   "title": "Caddy 502 Rate (Journal)",
   "description": "502 errors returned by Caddy for the journal upstream. These indicate Caddy could not reach the journal container.",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 0,
    "y": 0
   },
   "targets": [
    {
     "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code=\"502\"}[5m])) or vector(0)",
     "legendFormat": "502/s"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "custom": {
      "fillOpacity": 20,
      "lineWidth": 2
     },
     "color": {
      "mode": "fixed",
      "fixedColor": "red"
     }
    }
   }
  },
  {
   "title": "Caddy Response Codes (Journal)",
   "description": "All HTTP response codes from Caddy for the journal host, including Caddy-generated errors like 502.",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 12,
    "y": 0
   },
   "targets": [
    {
     "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code!~\"(101|2..|3..)\"}[5m])) by (code)",
     "legendFormat": "{{code}}"
    }
   ]
  },
  {
   "title": "Journal Request Rate by Status",
   "description": "Request rate from the journal app's own metrics, grouped by HTTP status code.",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 0,
    "y": 8
   },
   "targets": [
    {
     "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (status)",
     "legendFormat": "{{status}}"
    }
   ]
  },
  {
   "title": "Journal Latency p50 / p95 / p99",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 12,
    "y": 8
   },
   "targets": [
    {
     "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
     "legendFormat": "p50"
    },
    {
     "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
     "legendFormat": "p95"
    },
    {
     "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
     "legendFormat": "p99"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "unit": "s"
    }
   }
  },
  {
   "title": "Request Rate by Route",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 24,
    "x": 0,
    "y": 16
   },
   "targets": [
    {
     "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (route)",
     "legendFormat": "{{route}}"
    }
   ]
  },
  {
   "title": "Container Restarts",
   "description": "Journal container restart count. Spikes here correlate with 502 errors \u2014 Caddy returns 502 while the container is restarting.",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 8,
    "x": 0,
    "y": 24
   },
   "targets": [
    {
     "expr": "changes(container_start_time_seconds{name=\"trails-cool-journal-1\"}[5m])",
     "legendFormat": "journal"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "custom": {
      "fillOpacity": 30,
      "drawStyle": "bars"
     },
     "color": {
      "mode": "fixed",
      "fixedColor": "orange"
     }
    }
   }
  },
  {
   "title": "Container Memory",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 8,
    "x": 8,
    "y": 24
   },
   "targets": [
    {
     "expr": "container_memory_usage_bytes{name=\"trails-cool-journal-1\"}",
     "legendFormat": "used"
    },
    {
     "expr": "container_memory_working_set_bytes{name=\"trails-cool-journal-1\"}",
     "legendFormat": "working set"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "unit": "bytes"
    }
   }
  },
  {
   "title": "Container CPU",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 8,
    "x": 16,
    "y": 24
   },
   "targets": [
    {
     "expr": "rate(container_cpu_usage_seconds_total{name=\"trails-cool-journal-1\"}[5m]) * 100",
     "legendFormat": "CPU %"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "unit": "percent"
    }
   }
  },
  {
   "title": "Node.js Event Loop Lag",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 0,
    "y": 32
   },
   "targets": [
    {
     "expr": "nodejs_eventloop_lag_seconds{job=\"journal\"}",
     "legendFormat": "lag"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "unit": "s"
    }
   }
  },
  {
   "title": "Node.js Heap Used",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 12,
    "y": 32
   },
   "targets": [
    {
     "expr": "nodejs_heap_size_used_bytes{job=\"journal\"}",
     "legendFormat": "used"
    },
    {
     "expr": "nodejs_heap_size_total_bytes{job=\"journal\"}",
     "legendFormat": "total"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "unit": "bytes"
    }
   }
  },
  {
   "title": "Federation Delivery Rate by Outcome",
   "description": "Outbound federation delivery attempts/sec by outcome (delivered/skipped/failed). Sustained 'failed' means an unreachable or misconfigured remote; retries continue until the budget is exhausted.",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 0,
    "y": 40
   },
   "targets": [
    {
     "expr": "sum by (outcome) (rate(federation_delivery_total[5m]))",
     "legendFormat": "{{outcome}}"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "custom": {
      "fillOpacity": 20,
      "lineWidth": 2
     }
    }
   }
  },
  {
   "title": "Federation Queue Depth",
   "description": "Messages waiting in the durable Fedify (pg-boss) queue. A depth that climbs and never drains is the restart-loss / stuck-consumer regression detector \u2014 before the pg-boss backing (PR #570) a restart silently zeroed this by dropping the work.",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 12,
    "y": 40
   },
   "targets": [
    {
     "expr": "federation_queue_depth",
     "legendFormat": "queued"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "custom": {
      "fillOpacity": 10,
      "lineWidth": 2
     },
     "color": {
      "mode": "fixed",
      "fixedColor": "blue"
     }
    }
   }
  },
  {
   "title": "Federation Inbox Drops by Reason",
   "description": "Inbound activities dropped before side effects, by reason: 'duplicate' (replay defense) and 'blocked' (instance blocklist). A spike in 'blocked' after a block confirms it is taking effect; sustained 'duplicate' can indicate a misbehaving remote redelivering.",
   "type": "timeseries",
   "gridPos": {
    "h": 8,
    "w": 24,
    "x": 0,
    "y": 48
   },
   "targets": [
    {
     "expr": "sum by (reason) (rate(federation_inbox_dropped_total[5m]))",
     "legendFormat": "{{reason}}"
    }
   ],
   "fieldConfig": {
    "defaults": {
     "custom": {
      "fillOpacity": 20,
      "lineWidth": 2
     }
    }
   }
  },
  {
   "title": "Journal Logs (Errors & Warnings)",
   "description": "Recent error and warning logs from the journal container. Requires Promtail docker_sd_configs with Pino JSON parsing.",
   "type": "logs",
   "gridPos": {
    "h": 10,
    "w": 24,
    "x": 0,
    "y": 56
   },
   "targets": [
    {
     "datasource": {
      "type": "loki",
      "uid": "loki"
     },
     "expr": "{service=\"journal\"} |~ \"(error|warn|ERR|WARN|level.*(40|50|60))\"",
     "legendFormat": ""
    }
   ],
   "options": {
    "showTime": true,
    "sortOrder": "Descending",
    "enableLogDetails": true,
    "dedupStrategy": "none"
   }
  },
  {
   "title": "Caddy Logs (5xx Responses)",
   "description": "Caddy access log entries with 5xx status codes. Shows the raw request that triggered the error.",
   "type": "logs",
   "gridPos": {
    "h": 10,
    "w": 24,
    "x": 0,
    "y": 66
   },
   "targets": [
    {
     "datasource": {
      "type": "loki",
      "uid": "loki"
     },
     "expr": "{service=\"caddy\"} |~ \"\\\"status\\\":\\s*5\\d\\d\"",
     "legendFormat": ""
    }
   ],
   "options": {
    "showTime": true,
    "sortOrder": "Descending",
    "enableLogDetails": true,
    "dedupStrategy": "none"
   }
  }
 ]
}
