Fix empty Caddy and pg_stat_statements panels in service-health dashboard

- Caddy Response Status Codes: caddy_http_responses_total doesn't exist,
  use caddy_http_response_duration_seconds_count which has the code label
- pg_stat_statements: enable extension in init script, add custom queries
  config for postgres-exporter, remove invalid datname filter from query
- Add postgres/queries.yml to cd-infra SCP sources

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.github/workflows/cd-infra.yml

@@ -39,7 +39,7 @@ jobs:
           host: ${{ secrets.DEPLOY_HOST }}
           username: root
           key: ${{ secrets.DEPLOY_SSH_KEY }}
-          source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards"
+          source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards"
           target: /opt/trails-cool
           strip_components: 1
 

infrastructure/docker-compose.yml

@@ -99,6 +99,9 @@ services:
     restart: unless-stopped
     environment:
       DATA_SOURCE_NAME: postgresql://trails:${POSTGRES_PASSWORD:-trails}@postgres:5432/trails?sslmode=disable
+      PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres-exporter/queries.yml
+    volumes:
+      - ./postgres/queries.yml:/etc/postgres-exporter/queries.yml:ro
     depends_on:
       postgres:
         condition: service_healthy

infrastructure/grafana/dashboards/service-health.json

@@ -93,7 +93,7 @@
       },
       "targets": [
         {
-          "expr": "topk(10, pg_stat_statements_mean_time_seconds{datname=\"trails\"})",
+          "expr": "topk(10, pg_stat_statements_mean_time_seconds)",
           "legendFormat": "{{query}}",
           "format": "table",
           "instant": true
@@ -418,7 +418,7 @@
       },
       "targets": [
         {
-          "expr": "sum(rate(caddy_http_responses_total[5m])) by (code)",
+          "expr": "sum(rate(caddy_http_response_duration_seconds_count[5m])) by (code)",
           "legendFormat": "{{code}}"
         }
       ]

infrastructure/postgres/init-grafana-user.sql

@@ -1,3 +1,6 @@
+-- Enable pg_stat_statements for query performance monitoring
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+
 -- Read-only user for Grafana dashboards
 -- Password is set via: ALTER ROLE grafana_reader PASSWORD '<from SOPS>'
 -- Run by cd-infra deploy script after container start

infrastructure/postgres/queries.yml

@@ -0,0 +1,28 @@
+pg_stat_statements:
+  query: |
+    SELECT
+      queryid,
+      LEFT(query, 80) AS query,
+      calls,
+      mean_exec_time / 1000.0 AS mean_time_seconds,
+      total_exec_time / 1000.0 AS total_time_seconds
+    FROM pg_stat_statements
+    WHERE dbid = (SELECT oid FROM pg_database WHERE datname = 'trails')
+    ORDER BY mean_exec_time DESC
+    LIMIT 20
+  metrics:
+    - queryid:
+        usage: "LABEL"
+        description: "Query ID"
+    - query:
+        usage: "LABEL"
+        description: "Query text (truncated)"
+    - calls:
+        usage: "GAUGE"
+        description: "Number of calls"
+    - mean_time_seconds:
+        usage: "GAUGE"
+        description: "Mean execution time in seconds"
+    - total_time_seconds:
+        usage: "GAUGE"
+        description: "Total execution time in seconds"