bddc3d7620
The postgres init script grants read permissions to `grafana_reader` on `public`, `planner`, `journal`, and `pgboss` schemas. It's mounted into the postgres container via bind mount from `/opt/trails-cool/postgres/init-grafana-user.sql`, and cd-infra re-runs it with `psql -f` after postgres comes up. But `cd-infra.yml` only scp's `infrastructure/postgres/queries.yml` to the flagship — not `init-grafana-user.sql`. The file on the server was the pre-pg-boss-upgrade March version with no pgboss grants. After pg-boss v12 landed and (re)created the `pgboss` schema, the grafana_reader lost schema-level USAGE, the "Background job failures" alert query started erroring with `permission denied for schema pgboss`, and the alert fired as "Alerting (Error)" for hours. Fix: add the init SQL to the scp source list so changes to it actually reach the flagship. I also manually re-granted on the live postgres to clear the firing alert; the next cd-infra deploy will now pick up future edits to the init file too. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
100 lines
4.3 KiB
YAML
100 lines
4.3 KiB
YAML
name: CD Infra
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- "infrastructure/**"
|
|
workflow_dispatch:
|
|
inputs:
|
|
restart_all:
|
|
description: "Restart all containers (not just infra)"
|
|
type: boolean
|
|
default: false
|
|
|
|
concurrency:
|
|
group: deploy-infra
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
deploy:
|
|
name: Deploy Infrastructure
|
|
runs-on: ubuntu-latest
|
|
environment: infra
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Decrypt secrets
|
|
run: |
|
|
curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
|
|
chmod +x sops-v3.9.4.linux.amd64
|
|
# Merge app + infra secrets into one .env for docker-compose
|
|
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/.env
|
|
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.infra.env >> infrastructure/.env
|
|
echo "DOMAIN=trails.cool" >> infrastructure/.env
|
|
|
|
- name: Copy configs to server
|
|
uses: appleboy/scp-action@v1
|
|
with:
|
|
host: ${{ secrets.DEPLOY_HOST }}
|
|
username: root
|
|
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/promtail/promtail-config.yml,infrastructure/postgres/queries.yml,infrastructure/postgres/init-grafana-user.sql,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards"
|
|
target: /opt/trails-cool
|
|
strip_components: 1
|
|
|
|
- name: Copy secrets to server
|
|
uses: appleboy/scp-action@v1
|
|
with:
|
|
host: ${{ secrets.DEPLOY_HOST }}
|
|
username: root
|
|
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
source: "infrastructure/.env"
|
|
target: /opt/trails-cool
|
|
strip_components: 1
|
|
|
|
- name: Deploy via SSH
|
|
uses: appleboy/ssh-action@v1
|
|
with:
|
|
host: ${{ secrets.DEPLOY_HOST }}
|
|
username: root
|
|
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
script: |
|
|
cd /opt/trails-cool
|
|
|
|
# .env was placed by the SCP step (decrypted app + infra secrets)
|
|
|
|
# Login to ghcr.io (for pulling monitoring images)
|
|
GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN .env | cut -d= -f2-)
|
|
echo "$GHCR_TOKEN" | docker login ghcr.io -u stigi --password-stdin 2>/dev/null || true
|
|
|
|
# Setup Grafana read-only DB user (idempotent)
|
|
docker compose exec -T postgres psql -U trails -d trails -f /docker-entrypoint-initdb.d/init-grafana-user.sql 2>/dev/null || true
|
|
GRAFANA_DB_PW=$(grep GRAFANA_DB_PASSWORD .env | cut -d= -f2-)
|
|
if [ -n "$GRAFANA_DB_PW" ]; then
|
|
docker compose exec -T postgres psql -U trails -d trails -c "ALTER ROLE grafana_reader PASSWORD '$GRAFANA_DB_PW'" 2>/dev/null || true
|
|
fi
|
|
|
|
# Full restart: gh workflow run cd-infra.yml -f restart_all=true
|
|
if [ "${{ github.event.inputs.restart_all }}" = "true" ]; then
|
|
docker compose --env-file .env up -d --remove-orphans
|
|
else
|
|
# Restart infra services (except Caddy — just reload its config).
|
|
# --remove-orphans cleans up containers whose service was deleted
|
|
# from the compose file (e.g., the flagship `brouter` removal in
|
|
# PR #297 left an orphan that had to be removed by hand).
|
|
docker compose --env-file .env up -d --remove-orphans postgres prometheus loki promtail grafana postgres-exporter node-exporter cadvisor
|
|
docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile
|
|
fi
|
|
|
|
docker compose ps
|
|
|
|
# Annotate deploy in Grafana
|
|
GRAFANA_TOKEN=$(grep GRAFANA_SERVICE_TOKEN .env | cut -d= -f2-)
|
|
if [ -n "$GRAFANA_TOKEN" ]; then
|
|
docker compose exec -T grafana curl -sf -X POST \
|
|
-H "Authorization: Bearer $GRAFANA_TOKEN" \
|
|
-H "Content-Type: application/json" \
|
|
-d '{"text":"Deploy infra ${{ github.sha }}","tags":["deploy","infra"]}' \
|
|
http://localhost:3000/api/annotations || true
|
|
fi
|