From 114f6c2ce7cd3904065d043ba95b5f94df4e0847 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Tue, 9 Jun 2026 11:52:10 +0200 Subject: [PATCH 1/3] fix(infra): raise Prometheus retention.size 1GB -> 6GB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prometheus size-based retention counts the WAL toward its budget. With retention.size=1GB and a stuck/corrupt WAL that had grown to 2.1GB, the total was permanently over budget, so Prometheus deleted every freshly-compacted block the moment it was written. Net effect: only the in-memory ~2h head block was ever queryable — metrics appeared to "start this morning" no matter when you looked, and last night's Caddy 502 spike (planner /api/overpass upstream stall, 18:33-18:34 UTC) was unobservable in Prometheus (logs survived in Loki). The corrupt WAL was cleared out-of-band on the flagship. This raises the size backstop to 6GB so compacted blocks survive to the intended 15d time-based retention. Disk is 38G (43% used), so 6GB is safe headroom. Co-Authored-By: Claude Opus 4.8 (1M context) --- infrastructure/docker-compose.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index 1a7c623..9cd36dc 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -201,7 +201,13 @@ services: command: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.retention.time=15d" - - "--storage.tsdb.retention.size=1GB" + # Size-based retention is a disk-safety backstop, not the primary + # policy (time=15d is). It counts the WAL toward the budget, so a + # too-small cap makes Prometheus delete freshly-compacted blocks on + # creation — which it did at 1GB, leaving only the ~2h head block + # queryable (a stuck/corrupt WAL had bloated to 2.1GB). 6GB holds + # ~15d of metrics at current cardinality with headroom; disk is 38G. + - "--storage.tsdb.retention.size=6GB" loki: image: grafana/loki:latest From f38a224376005178cfd521797280cdb6766b3a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Tue, 9 Jun 2026 11:59:27 +0200 Subject: [PATCH 2/3] perf(infra): scrape every 20s + drop unused cAdvisor metric families Two cardinality/volume cuts to keep Prometheus storage sustainable within the size budget: - scrape_interval 5s -> 20s (evaluation_interval too). 5s tripled the sample volume vs the 15s default for no benefit on a single-box deployment; all rules use [5m] windows and for: >= 1m. - Drop high-cardinality cAdvisor families we never chart or alert on (container_fs_*, blkio_*, tasks_state, memory_failures_total, memory_numa_*, network_tcp*/udp*) on both cadvisor jobs. Dashboards only use container CPU, memory, and start_time. This was the bulk of the ~12.4k active series. Validated with `promtool check config`. Co-Authored-By: Claude Opus 4.8 (1M context) --- infrastructure/prometheus/prometheus.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/infrastructure/prometheus/prometheus.yml b/infrastructure/prometheus/prometheus.yml index 0c692f4..6785c59 100644 --- a/infrastructure/prometheus/prometheus.yml +++ b/infrastructure/prometheus/prometheus.yml @@ -1,6 +1,10 @@ global: - scrape_interval: 5s - evaluation_interval: 5s + # 20s is plenty of resolution for a single-box self-hosted deployment. + # The previous 5s tripled (vs the 15s default) the sample volume and + # WAL growth for no practical benefit; all alert/recording rules use + # [5m] windows and `for:` >= 1m, so 20s evaluation is fine. + scrape_interval: 20s + evaluation_interval: 20s scrape_configs: - job_name: "journal" @@ -24,6 +28,16 @@ scrape_configs: - job_name: "cadvisor" static_configs: - targets: ["cadvisor:8080"] + # cAdvisor emits ~100 per-container metric families; our dashboards + # and alerts only use CPU, memory, and container_start_time. Drop the + # high-cardinality families we never query (per-filesystem, block-IO, + # per-task state, memory-failure counters, socket tables). These + # dominated the ~12k active series — dropping them is the main + # cardinality cut. Add a name back here if you start charting it. + metric_relabel_configs: + - source_labels: [__name__] + regex: 'container_(fs_.*|blkio_.*|tasks_state|memory_failures_total|memory_numa_.*|network_tcp.*|network_udp.*)' + action: drop - job_name: "caddy" static_configs: @@ -38,3 +52,8 @@ scrape_configs: - targets: ["10.0.1.10:8080"] labels: host: "brouter" + # Same high-cardinality drop as the local cadvisor job above. + metric_relabel_configs: + - source_labels: [__name__] + regex: 'container_(fs_.*|blkio_.*|tasks_state|memory_failures_total|memory_numa_.*|network_tcp.*|network_udp.*)' + action: drop From 715cd3dce8948a263e908b120dee39b1a9525e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Tue, 9 Jun 2026 12:17:54 +0200 Subject: [PATCH 3/3] Apply suggestion from @stigi --- infrastructure/docker-compose.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index 9cd36dc..57a8057 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -201,13 +201,7 @@ services: command: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.retention.time=15d" - # Size-based retention is a disk-safety backstop, not the primary - # policy (time=15d is). It counts the WAL toward the budget, so a - # too-small cap makes Prometheus delete freshly-compacted blocks on - # creation — which it did at 1GB, leaving only the ~2h head block - # queryable (a stuck/corrupt WAL had bloated to 2.1GB). 6GB holds - # ~15d of metrics at current cardinality with headroom; disk is 38G. - - "--storage.tsdb.retention.size=6GB" + - "--storage.tsdb.retention.size=2GB" loki: image: grafana/loki:latest