{
  "title": "Journal",
  "uid": "trails-journal",
  "annotations": {
    "list": [
      {
        "name": "Deploys",
        "enable": true,
        "datasource": {
          "type": "grafana",
          "uid": "-- Grafana --"
        },
        "iconColor": "rgba(0, 211, 255, 1)",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [
            "deploy"
          ],
          "type": "tags"
        }
      }
    ]
  },
  "timezone": "browser",
  "refresh": "30s",
  "panels": [
    {
      "title": "Caddy 502 Rate (Journal)",
      "description": "502 errors returned by Caddy for the journal upstream. These indicate Caddy could not reach the journal container.",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "targets": [
        {
          "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code=\"502\"}[5m])) or vector(0)",
          "legendFormat": "502/s"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 20,
            "lineWidth": 2
          },
          "color": {
            "mode": "fixed",
            "fixedColor": "red"
          }
        }
      }
    },
    {
      "title": "Caddy Response Codes (Journal)",
      "description": "All HTTP response codes from Caddy for the journal host, including Caddy-generated errors like 502.",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "targets": [
        {
          "expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code!~\"(101|2..|3..)\"}[5m])) by (code)",
          "legendFormat": "{{code}}"
        }
      ]
    },
    {
      "title": "Journal Request Rate by Status",
      "description": "Request rate from the journal app's own metrics, grouped by HTTP status code.",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "targets": [
        {
          "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (status)",
          "legendFormat": "{{status}}"
        }
      ]
    },
    {
      "title": "Journal Latency p50 / p95 / p99",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
          "legendFormat": "p50"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
          "legendFormat": "p95"
        },
        {
          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
          "legendFormat": "p99"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "title": "Request Rate by Route",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 16
      },
      "targets": [
        {
          "expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (route)",
          "legendFormat": "{{route}}"
        }
      ]
    },
    {
      "title": "Container Restarts",
      "description": "Journal container restart count. Spikes here correlate with 502 errors — Caddy returns 502 while the container is restarting.",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 24
      },
      "targets": [
        {
          "expr": "changes(container_start_time_seconds{name=\"trails-cool-journal-1\"}[5m])",
          "legendFormat": "journal"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 30,
            "drawStyle": "bars"
          },
          "color": {
            "mode": "fixed",
            "fixedColor": "orange"
          }
        }
      }
    },
    {
      "title": "Container Memory",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 24
      },
      "targets": [
        {
          "expr": "container_memory_usage_bytes{name=\"trails-cool-journal-1\"}",
          "legendFormat": "used"
        },
        {
          "expr": "container_memory_working_set_bytes{name=\"trails-cool-journal-1\"}",
          "legendFormat": "working set"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        }
      }
    },
    {
      "title": "Container CPU",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 24
      },
      "targets": [
        {
          "expr": "rate(container_cpu_usage_seconds_total{name=\"trails-cool-journal-1\"}[5m]) * 100",
          "legendFormat": "CPU %"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        }
      }
    },
    {
      "title": "Node.js Event Loop Lag",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 32
      },
      "targets": [
        {
          "expr": "nodejs_eventloop_lag_seconds{job=\"journal\"}",
          "legendFormat": "lag"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "title": "Node.js Heap Used",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 32
      },
      "targets": [
        {
          "expr": "nodejs_heap_size_used_bytes{job=\"journal\"}",
          "legendFormat": "used"
        },
        {
          "expr": "nodejs_heap_size_total_bytes{job=\"journal\"}",
          "legendFormat": "total"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        }
      }
    },
    {
      "title": "Journal Logs (Errors & Warnings)",
      "description": "Recent error and warning logs from the journal container. Requires Promtail docker_sd_configs with Pino JSON parsing.",
      "type": "logs",
      "gridPos": {
        "h": 10,
        "w": 24,
        "x": 0,
        "y": 40
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "loki"
          },
          "expr": "{service=\"journal\"} |~ \"(error|warn|ERR|WARN|level.*(40|50|60))\"",
          "legendFormat": ""
        }
      ],
      "options": {
        "showTime": true,
        "sortOrder": "Descending",
        "enableLogDetails": true,
        "dedupStrategy": "none"
      }
    },
    {
      "title": "Caddy Logs (5xx Responses)",
      "description": "Caddy access log entries with 5xx status codes. Shows the raw request that triggered the error.",
      "type": "logs",
      "gridPos": {
        "h": 10,
        "w": 24,
        "x": 0,
        "y": 50
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "loki"
          },
          "expr": "{service=\"caddy\"} |~ \"\\\"status\\\":\\s*5\\d\\d\"",
          "legendFormat": ""
        }
      ],
      "options": {
        "showTime": true,
        "sortOrder": "Descending",
        "enableLogDetails": true,
        "dedupStrategy": "none"
      }
    }
  ]
}