From 3adf6fbfc49d9a779c70629687f5d45cf2e7d540 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= <hi+github@ullrich.is>
Date: Sun, 5 Apr 2026 16:02:23 +0200
Subject: [PATCH] Fix empty Caddy and pg_stat_statements panels in
 service-health dashboard

- Caddy Response Status Codes: caddy_http_responses_total doesn't exist,
  use caddy_http_response_duration_seconds_count which has the code label
- pg_stat_statements: enable extension in init script, add custom queries
  config for postgres-exporter, remove invalid datname filter from query
- Add postgres/queries.yml to cd-infra SCP sources

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .github/workflows/cd-infra.yml                |  2 +-
 infrastructure/docker-compose.yml             |  3 ++
 .../grafana/dashboards/service-health.json    |  4 +--
 infrastructure/postgres/init-grafana-user.sql |  3 ++
 infrastructure/postgres/queries.yml           | 28 +++++++++++++++++++
 5 files changed, 37 insertions(+), 3 deletions(-)
 create mode 100644 infrastructure/postgres/queries.yml

diff --git a/.github/workflows/cd-infra.yml b/.github/workflows/cd-infra.yml
index ba11467..b1839c4 100644
--- a/.github/workflows/cd-infra.yml
+++ b/.github/workflows/cd-infra.yml
@@ -39,7 +39,7 @@ jobs:
           host: ${{ secrets.DEPLOY_HOST }}
           username: root
           key: ${{ secrets.DEPLOY_SSH_KEY }}
-          source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards"
+          source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards"
           target: /opt/trails-cool
           strip_components: 1
 
diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml
index d8b61b7..64858da 100644
--- a/infrastructure/docker-compose.yml
+++ b/infrastructure/docker-compose.yml
@@ -99,6 +99,9 @@ services:
     restart: unless-stopped
     environment:
       DATA_SOURCE_NAME: postgresql://trails:${POSTGRES_PASSWORD:-trails}@postgres:5432/trails?sslmode=disable
+      PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres-exporter/queries.yml
+    volumes:
+      - ./postgres/queries.yml:/etc/postgres-exporter/queries.yml:ro
     depends_on:
       postgres:
         condition: service_healthy
diff --git a/infrastructure/grafana/dashboards/service-health.json b/infrastructure/grafana/dashboards/service-health.json
index 811bde3..69291e9 100644
--- a/infrastructure/grafana/dashboards/service-health.json
+++ b/infrastructure/grafana/dashboards/service-health.json
@@ -93,7 +93,7 @@
       },
       "targets": [
         {
-          "expr": "topk(10, pg_stat_statements_mean_time_seconds{datname=\"trails\"})",
+          "expr": "topk(10, pg_stat_statements_mean_time_seconds)",
           "legendFormat": "{{query}}",
           "format": "table",
           "instant": true
@@ -418,7 +418,7 @@
       },
       "targets": [
         {
-          "expr": "sum(rate(caddy_http_responses_total[5m])) by (code)",
+          "expr": "sum(rate(caddy_http_response_duration_seconds_count[5m])) by (code)",
           "legendFormat": "{{code}}"
         }
       ]
diff --git a/infrastructure/postgres/init-grafana-user.sql b/infrastructure/postgres/init-grafana-user.sql
index 099f5dc..98a56e3 100644
--- a/infrastructure/postgres/init-grafana-user.sql
+++ b/infrastructure/postgres/init-grafana-user.sql
@@ -1,3 +1,6 @@
+-- Enable pg_stat_statements for query performance monitoring
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+
 -- Read-only user for Grafana dashboards
 -- Password is set via: ALTER ROLE grafana_reader PASSWORD '<from SOPS>'
 -- Run by cd-infra deploy script after container start
diff --git a/infrastructure/postgres/queries.yml b/infrastructure/postgres/queries.yml
new file mode 100644
index 0000000..74b3b5b
--- /dev/null
+++ b/infrastructure/postgres/queries.yml
@@ -0,0 +1,28 @@
+pg_stat_statements:
+  query: |
+    SELECT
+      queryid,
+      LEFT(query, 80) AS query,
+      calls,
+      mean_exec_time / 1000.0 AS mean_time_seconds,
+      total_exec_time / 1000.0 AS total_time_seconds
+    FROM pg_stat_statements
+    WHERE dbid = (SELECT oid FROM pg_database WHERE datname = 'trails')
+    ORDER BY mean_exec_time DESC
+    LIMIT 20
+  metrics:
+    - queryid:
+        usage: "LABEL"
+        description: "Query ID"
+    - query:
+        usage: "LABEL"
+        description: "Query text (truncated)"
+    - calls:
+        usage: "GAUGE"
+        description: "Number of calls"
+    - mean_time_seconds:
+        usage: "GAUGE"
+        description: "Mean execution time in seconds"
+    - total_time_seconds:
+        usage: "GAUGE"
+        description: "Total execution time in seconds"