diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index 1a7c623..57a8057 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -201,7 +201,7 @@ services: command: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.retention.time=15d" - - "--storage.tsdb.retention.size=1GB" + - "--storage.tsdb.retention.size=2GB" loki: image: grafana/loki:latest diff --git a/infrastructure/prometheus/prometheus.yml b/infrastructure/prometheus/prometheus.yml index 0c692f4..6785c59 100644 --- a/infrastructure/prometheus/prometheus.yml +++ b/infrastructure/prometheus/prometheus.yml @@ -1,6 +1,10 @@ global: - scrape_interval: 5s - evaluation_interval: 5s + # 20s is plenty of resolution for a single-box self-hosted deployment. + # The previous 5s tripled (vs the 15s default) the sample volume and + # WAL growth for no practical benefit; all alert/recording rules use + # [5m] windows and `for:` >= 1m, so 20s evaluation is fine. + scrape_interval: 20s + evaluation_interval: 20s scrape_configs: - job_name: "journal" @@ -24,6 +28,16 @@ scrape_configs: - job_name: "cadvisor" static_configs: - targets: ["cadvisor:8080"] + # cAdvisor emits ~100 per-container metric families; our dashboards + # and alerts only use CPU, memory, and container_start_time. Drop the + # high-cardinality families we never query (per-filesystem, block-IO, + # per-task state, memory-failure counters, socket tables). These + # dominated the ~12k active series — dropping them is the main + # cardinality cut. Add a name back here if you start charting it. + metric_relabel_configs: + - source_labels: [__name__] + regex: 'container_(fs_.*|blkio_.*|tasks_state|memory_failures_total|memory_numa_.*|network_tcp.*|network_udp.*)' + action: drop - job_name: "caddy" static_configs: @@ -38,3 +52,8 @@ scrape_configs: - targets: ["10.0.1.10:8080"] labels: host: "brouter" + # Same high-cardinality drop as the local cadvisor job above. + metric_relabel_configs: + - source_labels: [__name__] + regex: 'container_(fs_.*|blkio_.*|tasks_state|memory_failures_total|memory_numa_.*|network_tcp.*|network_udp.*)' + action: drop