Merge pull request #498 from trails-cool/fix/prometheus-retention-size

fix(infra): right-size Prometheus storage (retention 6GB + 20s scrape + cAdvisor drops)

infrastructure/docker-compose.yml

@@ -201,7 +201,7 @@ services:
     command:
       - "--config.file=/etc/prometheus/prometheus.yml"
       - "--storage.tsdb.retention.time=15d"
-      - "--storage.tsdb.retention.size=1GB"
+      - "--storage.tsdb.retention.size=2GB"
 
   loki:
     image: grafana/loki:latest

infrastructure/prometheus/prometheus.yml

@@ -1,6 +1,10 @@
 global:
-  scrape_interval: 5s
+  # 20s is plenty of resolution for a single-box self-hosted deployment.
-  evaluation_interval: 5s
+  # The previous 5s tripled (vs the 15s default) the sample volume and
+  # WAL growth for no practical benefit; all alert/recording rules use
+  # [5m] windows and `for:` >= 1m, so 20s evaluation is fine.
+  scrape_interval: 20s
+  evaluation_interval: 20s
 
 scrape_configs:
   - job_name: "journal"
@@ -24,6 +28,16 @@ scrape_configs:
   - job_name: "cadvisor"
     static_configs:
       - targets: ["cadvisor:8080"]
+    # cAdvisor emits ~100 per-container metric families; our dashboards
+    # and alerts only use CPU, memory, and container_start_time. Drop the
+    # high-cardinality families we never query (per-filesystem, block-IO,
+    # per-task state, memory-failure counters, socket tables). These
+    # dominated the ~12k active series — dropping them is the main
+    # cardinality cut. Add a name back here if you start charting it.
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: 'container_(fs_.*|blkio_.*|tasks_state|memory_failures_total|memory_numa_.*|network_tcp.*|network_udp.*)'
+        action: drop
 
   - job_name: "caddy"
     static_configs:
@@ -38,3 +52,8 @@ scrape_configs:
       - targets: ["10.0.1.10:8080"]
         labels:
           host: "brouter"
+    # Same high-cardinality drop as the local cadvisor job above.
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: 'container_(fs_.*|blkio_.*|tasks_state|memory_failures_total|memory_numa_.*|network_tcp.*|network_udp.*)'
+        action: drop