From 57094323d2fa6bcc6c8457fd7b9dcb224aef84f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Fri, 27 Mar 2026 16:56:01 +0100 Subject: [PATCH] SOPS+age secrets, split CD workflows, GitHub OAuth for Grafana MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Secrets: - Add .sops.yaml with age encryption config - Add encrypted secrets.app.env (app secrets) and secrets.infra.env (Grafana OAuth) - CD decrypts at deploy time with AGE_SECRET_KEY — all other secrets move out of GitHub Actions into version-controlled encrypted files Split CD: - cd-apps.yml: triggered by apps/packages changes, builds Docker images, deploys apps - cd-infra.yml: triggered by infrastructure/ changes, copies configs, restarts services - Remove monolithic cd.yml Grafana auth: - GitHub OAuth (trails-cool org), disable login form - Remove Caddy basic_auth block and all GRAFANA_* env vars/secrets Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/cd-apps.yml | 109 +++++++++++++++ .github/workflows/cd-brouter.yml | 66 +++++++++ .github/workflows/cd-infra.yml | 81 +++++++++++ .github/workflows/cd.yml | 131 ------------------ .sops.yaml | 3 + infrastructure/Caddyfile | 3 - infrastructure/docker-compose.yml | 10 +- infrastructure/secrets.app.env | 15 ++ infrastructure/secrets.infra.env | 10 ++ .../changes/sops-age-split-cd/.openspec.yaml | 2 + openspec/changes/sops-age-split-cd/design.md | 126 +++++++++++++++++ .../changes/sops-age-split-cd/proposal.md | 44 ++++++ .../specs/infrastructure/spec.md | 27 ++++ .../specs/secret-management/spec.md | 16 +++ openspec/changes/sops-age-split-cd/tasks.md | 36 +++++ 15 files changed, 541 insertions(+), 138 deletions(-) create mode 100644 .github/workflows/cd-apps.yml create mode 100644 .github/workflows/cd-brouter.yml create mode 100644 .github/workflows/cd-infra.yml delete mode 100644 .github/workflows/cd.yml create mode 100644 .sops.yaml create mode 100644 infrastructure/secrets.app.env create mode 100644 infrastructure/secrets.infra.env create mode 100644 openspec/changes/sops-age-split-cd/.openspec.yaml create mode 100644 openspec/changes/sops-age-split-cd/design.md create mode 100644 openspec/changes/sops-age-split-cd/proposal.md create mode 100644 openspec/changes/sops-age-split-cd/specs/infrastructure/spec.md create mode 100644 openspec/changes/sops-age-split-cd/specs/secret-management/spec.md create mode 100644 openspec/changes/sops-age-split-cd/tasks.md diff --git a/.github/workflows/cd-apps.yml b/.github/workflows/cd-apps.yml new file mode 100644 index 0000000..0fa9bfe --- /dev/null +++ b/.github/workflows/cd-apps.yml @@ -0,0 +1,109 @@ +name: CD Apps + +on: + push: + branches: [main] + paths: + - "apps/**" + - "packages/**" + - "pnpm-lock.yaml" + workflow_dispatch: {} + +concurrency: + group: deploy-apps + cancel-in-progress: true + +jobs: + build-images: + name: Build & Push Docker Images + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + matrix: + app: [journal, planner] + steps: + - uses: actions/checkout@v6 + + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Decrypt app secrets + run: | + curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 + chmod +x sops-v3.9.4.linux.amd64 + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > /tmp/secrets.env + source /tmp/secrets.env + + - uses: docker/build-push-action@v7 + with: + context: . + file: apps/${{ matrix.app }}/Dockerfile + push: true + tags: | + ghcr.io/trails-cool/${{ matrix.app }}:latest + ghcr.io/trails-cool/${{ matrix.app }}:${{ github.sha }} + build-args: | + SENTRY_AUTH_TOKEN=${{ env.SENTRY_AUTH_TOKEN }} + SENTRY_RELEASE=${{ github.sha }} + + deploy: + name: Deploy Apps + needs: [build-images] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Decrypt secrets + run: | + curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 + chmod +x sops-v3.9.4.linux.amd64 + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > /tmp/app.env + echo "SENTRY_RELEASE=${{ github.sha }}" >> /tmp/app.env + echo "DOMAIN=trails.cool" >> /tmp/app.env + + - name: Copy files to server + uses: appleboy/scp-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile" + target: /opt/trails-cool + strip_components: 1 + + - name: Copy secrets to server + uses: appleboy/scp-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + source: "/tmp/app.env" + target: /opt/trails-cool + strip_components: 2 + + - name: Deploy via SSH + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + cd /opt/trails-cool + + # Login to ghcr.io + GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN app.env | cut -d= -f2-) + echo "$GHCR_TOKEN" | docker login ghcr.io -u stigi --password-stdin + + # Pull and deploy app containers + docker compose --env-file app.env pull journal planner + docker compose --env-file app.env run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force + docker compose --env-file app.env up -d journal planner + + # Clean up + docker image prune -af + docker compose ps diff --git a/.github/workflows/cd-brouter.yml b/.github/workflows/cd-brouter.yml new file mode 100644 index 0000000..ba369df --- /dev/null +++ b/.github/workflows/cd-brouter.yml @@ -0,0 +1,66 @@ +name: CD BRouter + +on: + push: + branches: [main] + paths: + - "docker/brouter/**" + workflow_dispatch: {} + +concurrency: + group: deploy-brouter + cancel-in-progress: true + +jobs: + build: + name: Build & Push BRouter Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v6 + + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@v7 + with: + context: docker/brouter + push: true + tags: | + ghcr.io/trails-cool/brouter:latest + ghcr.io/trails-cool/brouter:${{ github.sha }} + + deploy: + name: Deploy BRouter + needs: [build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Deploy via SSH + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + cd /opt/trails-cool + + # Download BRouter segments if missing + if [ ! -d /opt/trails-cool/segments ] || [ -z "$(ls /opt/trails-cool/segments/*.rd5 2>/dev/null)" ]; then + mkdir -p /opt/trails-cool/segments + echo "Downloading Germany BRouter segments..." + for tile in E5_N45 E5_N50 E10_N45 E10_N50; do + [ -f "/opt/trails-cool/segments/${tile}.rd5" ] || \ + wget -q "https://brouter.de/brouter/segments4/${tile}.rd5" -O "/opt/trails-cool/segments/${tile}.rd5" + done + fi + + docker pull ghcr.io/trails-cool/brouter:latest + docker compose up -d brouter + docker image prune -af diff --git a/.github/workflows/cd-infra.yml b/.github/workflows/cd-infra.yml new file mode 100644 index 0000000..b85c5b7 --- /dev/null +++ b/.github/workflows/cd-infra.yml @@ -0,0 +1,81 @@ +name: CD Infra + +on: + push: + branches: [main] + paths: + - "infrastructure/**" + workflow_dispatch: + inputs: + restart_all: + description: "Restart all containers (not just infra)" + type: boolean + default: false + +concurrency: + group: deploy-infra + cancel-in-progress: true + +jobs: + deploy: + name: Deploy Infrastructure + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Decrypt secrets + run: | + curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 + chmod +x sops-v3.9.4.linux.amd64 + # Merge app + infra secrets into one .env for docker-compose + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > /tmp/all.env + SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.infra.env >> /tmp/all.env + echo "DOMAIN=trails.cool" >> /tmp/all.env + + - name: Copy configs to server + uses: appleboy/scp-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards" + target: /opt/trails-cool + strip_components: 1 + + - name: Copy secrets to server + uses: appleboy/scp-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + source: "/tmp/all.env" + target: /opt/trails-cool + strip_components: 2 + + - name: Deploy via SSH + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: root + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + cd /opt/trails-cool + + # Merge env files: app.env (from cd-apps) + all.env (from cd-infra) + # all.env has both app + infra secrets + cp all.env .env + + # Login to ghcr.io (for pulling monitoring images) + GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN .env | cut -d= -f2-) + echo "$GHCR_TOKEN" | docker login ghcr.io -u stigi --password-stdin 2>/dev/null || true + + # Restart services + if [ "${{ github.event.inputs.restart_all }}" = "true" ]; then + docker compose --env-file .env up -d --remove-orphans + else + # Restart infra services (except Caddy — just reload its config) + docker compose --env-file .env up -d postgres prometheus loki grafana + docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile + fi + + docker compose ps diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml deleted file mode 100644 index 987f602..0000000 --- a/.github/workflows/cd.yml +++ /dev/null @@ -1,131 +0,0 @@ -name: CD - -# Triggers: -# - push: merges via merge queue or manual gh pr merge -# - workflow_dispatch: manual trigger via gh workflow run cd.yml -on: - push: - branches: [main] - workflow_dispatch: {} - -concurrency: - group: deploy - cancel-in-progress: false - -jobs: - build-images: - name: Build & Push Docker Images - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - matrix: - app: [journal, planner] - steps: - - uses: actions/checkout@v6 - - - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - uses: docker/build-push-action@v7 - with: - context: . - file: apps/${{ matrix.app }}/Dockerfile - push: true - tags: | - ghcr.io/trails-cool/${{ matrix.app }}:latest - ghcr.io/trails-cool/${{ matrix.app }}:${{ github.sha }} - build-args: | - SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_RELEASE=${{ github.sha }} - - build-brouter: - name: Build & Push BRouter Image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@v6 - - - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - uses: docker/build-push-action@v7 - with: - context: docker/brouter - push: true - tags: | - ghcr.io/trails-cool/brouter:latest - ghcr.io/trails-cool/brouter:${{ github.sha }} - - deploy: - name: Deploy to Hetzner - needs: [build-images, build-brouter] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - name: Copy files to server - uses: appleboy/scp-action@v1 - with: - host: ${{ secrets.DEPLOY_HOST }} - username: root - key: ${{ secrets.DEPLOY_SSH_KEY }} - source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile,infrastructure/prometheus/prometheus.yml,infrastructure/loki/loki-config.yml,infrastructure/grafana/provisioning,infrastructure/grafana/dashboards" - target: /opt/trails-cool - strip_components: 1 - - - name: Deploy via SSH - uses: appleboy/ssh-action@v1 - with: - host: ${{ secrets.DEPLOY_HOST }} - username: root - key: ${{ secrets.DEPLOY_SSH_KEY }} - script: | - cd /opt/trails-cool - - # Login to ghcr.io to pull private images - echo "${{ secrets.DEPLOY_GHCR_TOKEN }}" | docker login ghcr.io -u stigi --password-stdin - - # Download BRouter segments if missing - if [ ! -d /opt/trails-cool/segments ] || [ -z "$(ls /opt/trails-cool/segments/*.rd5 2>/dev/null)" ]; then - mkdir -p /opt/trails-cool/segments - echo "Downloading Germany BRouter segments..." - for tile in E5_N45 E5_N50 E10_N45 E10_N50; do - [ -f "/opt/trails-cool/segments/${tile}.rd5" ] || \ - wget -q "https://brouter.de/brouter/segments4/${tile}.rd5" -O "/opt/trails-cool/segments/${tile}.rd5" - done - fi - - # Pull latest images - export SENTRY_RELEASE="${{ github.sha }}" - export SMTP_URL="${{ secrets.SMTP_URL }}" - export SMTP_FROM="${{ secrets.SMTP_FROM }}" - export GRAFANA_USER="${{ secrets.GRAFANA_USER }}" - export GRAFANA_PASSWORD="${{ secrets.GRAFANA_PASSWORD }}" - export GRAFANA_PASSWORD_HASH='${{ secrets.GRAFANA_PASSWORD_HASH }}' - docker compose pull - - # Push database schema (starts postgres, waits for healthy) - docker compose run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force - - # Start all services - docker compose up -d --remove-orphans - - # Clean up old images to prevent disk full - docker image prune -af - - # Sync Grafana admin credentials (env vars only work on first boot) - sleep 10 - docker compose exec -T grafana grafana cli admin reset-admin-password "$GRAFANA_PASSWORD" || true - - # Verify services are running - docker compose ps diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 0000000..759e1c2 --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,3 @@ +creation_rules: + - path_regex: secrets\..*\.env$ + age: age1vukt8py0s2mm47fx6qh6vykckfkdqslf9w68ekytfzvc8xp7e3rqn6p64n diff --git a/infrastructure/Caddyfile b/infrastructure/Caddyfile index ed3fdba..70513bd 100644 --- a/infrastructure/Caddyfile +++ b/infrastructure/Caddyfile @@ -29,9 +29,6 @@ www.{$DOMAIN:trails.cool} { } grafana.internal.{$DOMAIN:trails.cool} { - basic_auth { - {$GRAFANA_USER:admin} {$GRAFANA_PASSWORD_HASH} - } reverse_proxy grafana:3000 } diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index e32352d..9c20fcf 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -8,8 +8,6 @@ services: - "443:443/udp" environment: DOMAIN: ${DOMAIN:-trails.cool} - GRAFANA_USER: ${GRAFANA_USER:-admin} - GRAFANA_PASSWORD_HASH: ${GRAFANA_PASSWORD_HASH:-} volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro - caddy_data:/data @@ -109,9 +107,13 @@ services: image: grafana/grafana:latest restart: unless-stopped environment: - GF_SECURITY_ADMIN_USER: ${GRAFANA_USER:-admin} - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin} GF_SERVER_ROOT_URL: https://grafana.internal.${DOMAIN:-trails.cool} + GF_AUTH_GITHUB_ENABLED: "true" + GF_AUTH_GITHUB_CLIENT_ID: ${GF_AUTH_GITHUB_CLIENT_ID:-} + GF_AUTH_GITHUB_CLIENT_SECRET: ${GF_AUTH_GITHUB_CLIENT_SECRET:-} + GF_AUTH_GITHUB_ALLOWED_ORGANIZATIONS: trails-cool + GF_AUTH_GITHUB_SCOPES: user:email,read:org + GF_AUTH_DISABLE_LOGIN_FORM: "true" volumes: - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro diff --git a/infrastructure/secrets.app.env b/infrastructure/secrets.app.env new file mode 100644 index 0000000..9713047 --- /dev/null +++ b/infrastructure/secrets.app.env @@ -0,0 +1,15 @@ +#ENC[AES256_GCM,data:4AVDH0K+AknlSfn7+uBxY1bQrJaLgu7e4H9a2r0DJTw4cxYMNfDJ4qtcnOAJXQLmf9tdQMQ=,iv:eJ4Cck3LvSi3W58QhZjkmL2+Q+TYYO1w8ucSM9Z2tF0=,tag:OAiQY06xOXDRQqdtTgyVqQ==,type:comment] +#ENC[AES256_GCM,data:+hScU/d5XG/4YJfrbf7wvQgQxnivPnZiRb43u8WePdmopML3CY0J5A03qpxEJdKYNFcALB/nOd1rMjhjGLexTNjdy+pjPfs+82xlEUU=,iv:azEAINahXBDJeglTu7FSBbd0OvUy5vyiER5aYqDsMfE=,tag:fEwUHGMq5SS5J8NN3auCyw==,type:comment] +POSTGRES_PASSWORD=ENC[AES256_GCM,data:8GlpWQ8J,iv:P2ZXmY+PIzuRt7IplHIR2Fgqv3v86Js9kadlVLloyv4=,tag:Hnp2UDtrZ6J3zbLO5lydkw==,type:str] +JWT_SECRET=ENC[AES256_GCM,data:VMN5O225nhWNcjTMXySpGzejCa4Kanat0qN1FjYYGljWkHrgRSOmFEl+lDg=,iv:p+/uFO9tjfuy6dcRkijqvhPDVb9ST90WCSRiFfDJ3aU=,tag:R4dPEwVAe1u9wilvr0e67Q==,type:str] +SESSION_SECRET=ENC[AES256_GCM,data:sbyyoq+gsVZZbf/YfSAzRN8K1ny+6b4bDoWj5yjJr6qpk7dYoeYKoMGTOek=,iv:Jk81deYxkdyg6MNGMB/DiQRR27cr87CD8K7g70YDzko=,tag:7hAAxEd0DIPZcQ4IgoHrzA==,type:str] +SMTP_URL=ENC[AES256_GCM,data:o4kgOpqNnbtoAfZMM3+cnk0pHIQ/xfQ7Z2wg003qkx+7aU0JdNW8FXlxPc7QAOHT/jreIirLe9VV6K5tLMA6ICuQeA==,iv:6o06J4DR5IWdNSY9fCBZGRhE6qWnDQR/Dq9HhO4viqA=,tag:gFQs4no1VGTZSzYDfgZeCw==,type:str] +SMTP_FROM=ENC[AES256_GCM,data:pS4B7UxNc9DIyhveRV0LP3wchAFtCY1TN7NhTK9YsM92,iv:jnz4zEjU++PVC1OOyQTvr4mnDNe1hmx4oqZMLJ/7jcQ=,tag:ifclwnVpmcD8x0SLMK+ZPw==,type:str] +SENTRY_AUTH_TOKEN=ENC[AES256_GCM,data:Bz+XkGYAJDhCN5iSSPdTkd+21BO1zc7S2rU2NvxaBzmskYve6gHt1KR+OzQ9YLuy7VkhpFlBeYKS592HrpdkCA7Sp3DZvl8=,iv:E3cyWEhDX8WCpvROJdhGhdgkKjV+mhmrj4kMpBz9hWM=,tag:BNb7rsdPQQ4pBS3UX2G4EQ==,type:str] +DEPLOY_GHCR_TOKEN=ENC[AES256_GCM,data:cmyUDf24Mt5iS8rRwjpCMPfGBjKDXW4vXQjEqlO1KWNNxO0fW5SFlg==,iv:rhKAgWZ48JGfq2vS6FJ58E9pU41s6zHYe0xo7+YapoM=,tag:LKl86akqu7E32Lp0ScDWYg==,type:str] +sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWblo1Rm9rUlFCQkVVRUVR\nSCtPR1kwMlFvRXplNmdxWVVSOEJzUVB3VlZBCk9Hb1FJSCs1R2llMkk4eGVydnlh\nUmJmU0Joa1pneEZSWFJ5L1VZRnNiZ0UKLS0tIG9aaUkzT0o5dERSekM0cjNxOXBM\nRVhnU1prNjFOa0RQdEVsSlB5QnZzMGcKFFPBeSgQozLR2zohwr/lYBRFzeAK8Hzd\nV635q8YQrXBt9ZUSYvy/b7Gse61+ynaqI7ukLfbc/cRNszQFSuMJyg==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_0__map_recipient=age1vukt8py0s2mm47fx6qh6vykckfkdqslf9w68ekytfzvc8xp7e3rqn6p64n +sops_lastmodified=2026-03-27T15:54:10Z +sops_mac=ENC[AES256_GCM,data:ESmrI1rbshwEKB3ttElBFD0vpbRQjydPBi5zlP5DNhc9ldBPFiIIFCA/v5MBquPsA4VG0C3RyKJBXrryPiADcvj/bhOWWllVlJ0OPr9GSoNFRpYRsWSTJfu7NG3wP9B24Cm/QpW2p2wYi+Z4nm0KHdHWTshxgUBBFOAOiVJPWIU=,iv:/IzX+/MP+3eTeePcmgfwRmVMAHe/WmRYBfrEFp7JOoA=,tag:nhNbuk8GlbJJMghvx8CnKw==,type:str] +sops_unencrypted_suffix=_unencrypted +sops_version=3.9.4 diff --git a/infrastructure/secrets.infra.env b/infrastructure/secrets.infra.env new file mode 100644 index 0000000..e41e191 --- /dev/null +++ b/infrastructure/secrets.infra.env @@ -0,0 +1,10 @@ +#ENC[AES256_GCM,data:IbWmlFr0kUJNYnRkD9Rt6tPAgtE+8TrECPY3MstykO8EPIZOvkg0T1hjGsDLca5NA8cdWqWWEmSF4o7ewvk=,iv:UD1YxHEH9PxnSrkVI/Nm9WVazGk9Kh+B8uzfNqjmf3I=,tag:+mWHdGvo+OeQFa4Ak/S3Pg==,type:comment] +#ENC[AES256_GCM,data:BDOEQHz+QsQcQCzrD8XWcxREuNWELctcjsZahTmwEAYolct7EaW4G9bGI1K8BL8V5OejHvqtxgd34LHBAzTrTL66ATW/pWGY2lF/G69J4Q==,iv:4RyNDeZUZOgibkI7+aIyCJDxd/7jZMMaBIepQCbbHFA=,tag:v/mPe5ILLVS2bLUKFpOoJg==,type:comment] +GF_AUTH_GITHUB_CLIENT_ID=ENC[AES256_GCM,data:4a1+g2U6a+XlfwSLLoJFWruvOLg=,iv:h1yDpE21XPxqOGBsE+E1Xunr0eAC2ALLP9x11XMPd6Q=,tag:zKWMUrCi1b2Ir8g1BcsAcA==,type:str] +GF_AUTH_GITHUB_CLIENT_SECRET=ENC[AES256_GCM,data:LGbZHhQeLcS01UJXtZefsGUi0P8lscZRrho0HGq+hKRuoy5RO1HssA==,iv:NATDvm3fKe4zjJ36PWb/iJLVA2lJsognRLg0TZY5yw4=,tag:JYatVnyvrc0dYue2aTh55Q==,type:str] +sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4UiszMUJMYzFlWE5Qc3Zk\nUDF2V1BIYmIwbjVydHI5OTJoRVJXdFhDZDEwCmlEQTFKRUZwYVlQbVRJMEpHL2Vj\nd21pS3BwZHc3QklCMVl4ZHFzZUJYRWMKLS0tIEtQeFQvNGxRUGhqNnBzQ0ZvaXRY\nMFZjQlhobXBKT08yc0dLVGNwd0pHbmcKEAS/YRu2NhGCZLX+nkVjo7PEu7ZV4iCW\nyO9GIsaIQcN6FWmp5IkjdeM6jkHxSk+qHCL5bKx78EgPMgzKobjgzQ==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_0__map_recipient=age1vukt8py0s2mm47fx6qh6vykckfkdqslf9w68ekytfzvc8xp7e3rqn6p64n +sops_lastmodified=2026-03-27T15:40:55Z +sops_mac=ENC[AES256_GCM,data:pkYMdGTyjc6/ig/hdBD4GGeyiOkJBp/z9+w3vlT/bfHeje4FBZxjdmgykfCmNFJqov1MoysEsP0z8VZpdcSa+cjcJXVIhXcSpl4Uo/NXZuYRIuPkQztSGna+tStrpus4Zxk2SLe8IF5AFgPtuCwZvS7R7JE4KKbkKG/kfcTMrR8=,iv:SXzctdK5QEV6RZuSSxDor7MYPgEJGvSih4l3G2+lwYI=,tag:umAgJ94DM7DKVPPniCdRvw==,type:str] +sops_unencrypted_suffix=_unencrypted +sops_version=3.9.4 diff --git a/openspec/changes/sops-age-split-cd/.openspec.yaml b/openspec/changes/sops-age-split-cd/.openspec.yaml new file mode 100644 index 0000000..a61e7c1 --- /dev/null +++ b/openspec/changes/sops-age-split-cd/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-03-27 diff --git a/openspec/changes/sops-age-split-cd/design.md b/openspec/changes/sops-age-split-cd/design.md new file mode 100644 index 0000000..2bfc2be --- /dev/null +++ b/openspec/changes/sops-age-split-cd/design.md @@ -0,0 +1,126 @@ +## Context + +The current CD workflow has ~10 GitHub Actions secrets passed via `export` +statements in a deploy script. Some secrets contain `$` characters that get +mangled. Grafana authentication required managing a bcrypt hash, a plaintext +password, and a username across three different systems (Caddy, Grafana, CD). +The single CD workflow rebuilds Docker images even for config-only changes. + +## Goals / Non-Goals + +**Goals:** +- All secrets in one encrypted file in the repo (SOPS + age) +- Only one GitHub secret needed for decryption (AGE_SECRET_KEY) +- App deploys (Docker build + push) separate from infra deploys (config copy) +- Grafana login via GitHub OAuth (no passwords) +- Remove Caddy basic auth for Grafana + +**Non-Goals:** +- Secret rotation automation (manual for now) +- Per-environment secret files (only production) +- Grafana RBAC / team-based access (just org membership check) +- Moving DEPLOY_SSH_KEY or GITHUB_TOKEN into SOPS (these are GitHub-native) + +## Decisions + +### D1: SOPS + age file-based encryption + +Create `.sops.yaml` at the repo root defining age as the encryption method. +Two encrypted secret files, split by deployment scope: + +- **`infrastructure/secrets.app.env`** — app secrets needed by journal/planner. + Used by both cd-apps (production) and future staging deploys. +- **`infrastructure/secrets.infra.env`** — monitoring/Grafana secrets. + Used only by cd-infra (production). Staging doesn't run Grafana. + +```yaml +# .sops.yaml +creation_rules: + - path_regex: secrets\..*\.env$ + age: +``` + +Workflow: +- **Edit secrets**: `sops infrastructure/secrets.app.env` (decrypts in editor, + re-encrypts on save) +- **CD decrypts**: Install sops + age, decrypt the relevant file(s) to a temp + `.env`, pass to `docker compose --env-file` +- **Only one GitHub secret**: `AGE_SECRET_KEY` (the private key) + +**secrets.app.env** contents (used by cd-apps + cd-infra): +- POSTGRES_PASSWORD, JWT_SECRET, SESSION_SECRET +- SMTP_URL, SMTP_FROM +- SENTRY_AUTH_TOKEN +- DEPLOY_GHCR_TOKEN + +**secrets.infra.env** contents (used by cd-infra only): +- GF_AUTH_GITHUB_CLIENT_ID, GF_AUTH_GITHUB_CLIENT_SECRET + +Secrets that stay as GitHub Actions secrets: +- DEPLOY_SSH_KEY (used by SCP/SSH actions, not by docker-compose) +- AGE_SECRET_KEY (chicken-and-egg: can't encrypt the decryption key) +- DEPLOY_HOST (not really secret, but convenient) + +### D2: Split CD into two workflows + +**cd-apps.yml** — triggered by changes to `apps/`, `packages/`, `pnpm-lock.yaml`: +1. Build and push Docker images (journal, planner, brouter) +2. SSH to server, pull images, run migrations, restart app containers + +**cd-infra.yml** — triggered by changes to `infrastructure/`: +1. Copy config files (docker-compose, Caddyfile, Prometheus, Loki, Grafana) +2. SSH to server, decrypt secrets, `docker compose up -d` + +Both workflows also trigger on `workflow_dispatch` for manual runs. +Both share the decrypt-secrets step. + +### D3: GitHub OAuth for Grafana + +Register a GitHub OAuth app at github.com/settings/applications: +- Callback URL: `https://grafana.internal.trails.cool/login/github` +- Homepage: `https://grafana.internal.trails.cool` + +Grafana config via environment variables: +```yaml +GF_AUTH_GITHUB_ENABLED: "true" +GF_AUTH_GITHUB_CLIENT_ID: +GF_AUTH_GITHUB_CLIENT_SECRET: +GF_AUTH_GITHUB_ALLOWED_ORGANIZATIONS: trails-cool +GF_AUTH_GITHUB_SCOPES: user:email,read:org +GF_AUTH_DISABLE_LOGIN_FORM: "true" +``` + +Remove from Caddyfile: +- `basic_auth` block for grafana.internal +- GRAFANA_USER, GRAFANA_PASSWORD, GRAFANA_PASSWORD_HASH env vars + +Remove from docker-compose.yml: +- GF_SECURITY_ADMIN_USER, GF_SECURITY_ADMIN_PASSWORD +- Caddy GRAFANA_USER, GRAFANA_PASSWORD_HASH env vars + +Remove from CD: +- grafana cli password reset step +- All GRAFANA_* secret exports + +### D4: Server-side secret decryption + +The CD deploy step: +1. Install sops + age on the runner +2. Decrypt: `SOPS_AGE_KEY=${{ secrets.AGE_SECRET_KEY }} sops -d infrastructure/secrets.env > /tmp/secrets.env` +3. SCP the decrypted `.env` to server as `/opt/trails-cool/.env` +4. `docker compose --env-file .env up -d` +5. Clean up `/tmp/secrets.env` from runner + +The server never has the age private key — secrets arrive as a plain `.env` +file via SCP (same security as current `export` approach, but now +version-controlled and auditable). + +## Risks / Trade-offs + +- **age key loss** → If the AGE_SECRET_KEY is lost, secrets can't be decrypted. + Mitigate: store a backup of the age key outside GitHub (e.g., password + manager). +- **Encrypted file merge conflicts** → SOPS encrypted files don't merge well. + Mitigate: only one person edits secrets at a time (fine for solo/small team). +- **GitHub OAuth requires internet** → If GitHub is down, Grafana login fails. + Acceptable for a monitoring dashboard. diff --git a/openspec/changes/sops-age-split-cd/proposal.md b/openspec/changes/sops-age-split-cd/proposal.md new file mode 100644 index 0000000..5ec5276 --- /dev/null +++ b/openspec/changes/sops-age-split-cd/proposal.md @@ -0,0 +1,44 @@ +## Why + +Secrets are scattered across GitHub Actions secrets with no version control, +no audit trail, and painful manual management (the Grafana password hash saga). +The CD pipeline is monolithic — changing a Grafana dashboard rebuilds both app +Docker images. And Grafana authentication requires managing bcrypt hashes and +basic auth layers. + +## What Changes + +- **SOPS + age for secrets**: Encrypt a `.env.production` file in the repo. + CD decrypts at deploy time with a single age private key stored as one + GitHub secret. All other secrets move from GitHub Actions secrets into the + encrypted file — version-controlled, diffable, auditable. +- **Split CD into apps vs infra**: Two workflows triggered by path filters. + App changes (apps/, packages/) build Docker images and deploy. Infra changes + (infrastructure/) copy configs and restart services. No unnecessary rebuilds. +- **GitHub OAuth for Grafana**: Replace Caddy basic auth + Grafana login with + GitHub OAuth. One login, restricted to the trails-cool GitHub org. Remove + GRAFANA_PASSWORD_HASH, GRAFANA_USER, GRAFANA_PASSWORD secrets entirely. + +## Capabilities + +### New Capabilities + +- `secret-management`: SOPS + age encrypted secrets in the repository with + CD decryption + +### Modified Capabilities + +- `infrastructure`: Split CD workflows, GitHub OAuth for Grafana, remove + Caddy basic auth for Grafana + +## Impact + +- **Files**: New `.env.production.enc` (encrypted), `.sops.yaml` config, + split `cd-apps.yml` and `cd-infra.yml` workflows, updated docker-compose.yml + and Caddyfile +- **Dependencies**: `sops` and `age` CLI tools in CD runner (install step) +- **GitHub secrets**: Reduced from ~10 secrets to 2 (AGE_SECRET_KEY + + DEPLOY_SSH_KEY). Everything else moves into the encrypted env file. +- **Grafana**: GitHub OAuth app registration needed (Client ID + Secret go + into the SOPS-encrypted file) +- **Caddy**: Remove basic auth block for grafana.internal, just proxy through diff --git a/openspec/changes/sops-age-split-cd/specs/infrastructure/spec.md b/openspec/changes/sops-age-split-cd/specs/infrastructure/spec.md new file mode 100644 index 0000000..5f69261 --- /dev/null +++ b/openspec/changes/sops-age-split-cd/specs/infrastructure/spec.md @@ -0,0 +1,27 @@ +## MODIFIED Requirements + +### Requirement: CI/CD pipeline +GitHub Actions SHALL use separate workflows for app deployment and infrastructure deployment, with secrets decrypted from a SOPS-encrypted file. + +#### Scenario: App deployment +- **WHEN** code changes are pushed to main in apps/ or packages/ +- **THEN** the cd-apps workflow builds Docker images, pushes to ghcr.io, and deploys app containers + +#### Scenario: Infrastructure deployment +- **WHEN** changes are pushed to main in infrastructure/ +- **THEN** the cd-infra workflow copies configs and restarts infrastructure services without rebuilding app images + +#### Scenario: Secret decryption at deploy time +- **WHEN** either CD workflow runs +- **THEN** the SOPS-encrypted secrets file is decrypted and provided to docker-compose as an env file + +### Requirement: Grafana authentication +Grafana SHALL authenticate users via GitHub OAuth, restricted to the trails-cool GitHub organization. + +#### Scenario: GitHub OAuth login +- **WHEN** a user navigates to grafana.internal.trails.cool +- **THEN** they are redirected to GitHub for authentication and granted access if they are a member of the trails-cool organization + +#### Scenario: No password-based login +- **WHEN** Grafana is deployed +- **THEN** the login form is disabled and only GitHub OAuth is available diff --git a/openspec/changes/sops-age-split-cd/specs/secret-management/spec.md b/openspec/changes/sops-age-split-cd/specs/secret-management/spec.md new file mode 100644 index 0000000..b1ade56 --- /dev/null +++ b/openspec/changes/sops-age-split-cd/specs/secret-management/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: Encrypted secrets in repository +Production secrets SHALL be stored as a SOPS-encrypted file in the repository, decryptable only with a single age private key. + +#### Scenario: Edit secrets +- **WHEN** a developer runs `sops infrastructure/secrets.env` +- **THEN** the file is decrypted in a temporary editor, and re-encrypted on save + +#### Scenario: CD decryption +- **WHEN** the CD workflow runs +- **THEN** the encrypted secrets file is decrypted using the AGE_SECRET_KEY GitHub secret and provided to docker-compose as an env file + +#### Scenario: Secret audit trail +- **WHEN** a secret is changed +- **THEN** the change appears in git history as a diff of the encrypted file with a commit message describing what changed diff --git a/openspec/changes/sops-age-split-cd/tasks.md b/openspec/changes/sops-age-split-cd/tasks.md new file mode 100644 index 0000000..13c05f3 --- /dev/null +++ b/openspec/changes/sops-age-split-cd/tasks.md @@ -0,0 +1,36 @@ +## 1. SOPS + age Setup + +- [x] 1.1 Generate age key pair (`age-keygen`), store public key in `.sops.yaml`, private key as `AGE_SECRET_KEY` GitHub secret +- [x] 1.2 Create `.sops.yaml` at repo root with age encryption rule for `secrets.*.env` +- [x] 1.3 Create `infrastructure/secrets.app.env` with app secrets (POSTGRES_PASSWORD, JWT_SECRET, SESSION_SECRET, SMTP_URL, SMTP_FROM, SENTRY_AUTH_TOKEN, DEPLOY_GHCR_TOKEN), encrypt with `sops -e` +- [x] 1.4 Create `infrastructure/secrets.infra.env` with infra secrets (GF_AUTH_GITHUB_CLIENT_ID, GF_AUTH_GITHUB_CLIENT_SECRET), encrypt with `sops -e` +- [ ] 1.5 Remove migrated secrets from GitHub Actions (keep only AGE_SECRET_KEY, DEPLOY_SSH_KEY, DEPLOY_HOST) + +## 2. GitHub OAuth for Grafana + +- [x] 2.1 Register GitHub OAuth app (callback: grafana.internal.trails.cool/login/github) +- [x] 2.2 Add GF_AUTH_GITHUB_CLIENT_ID and GF_AUTH_GITHUB_CLIENT_SECRET to secrets.env +- [x] 2.3 Update docker-compose.yml: add GitHub OAuth env vars to Grafana, remove GF_SECURITY_ADMIN_USER/PASSWORD +- [x] 2.4 Update Caddyfile: remove basic_auth block for grafana.internal, just reverse_proxy +- [x] 2.5 Remove Caddy GRAFANA_USER/GRAFANA_PASSWORD_HASH env vars from docker-compose.yml + +## 3. Split CD Workflows + +- [x] 3.1 Create `.github/workflows/cd-apps.yml` — triggered by apps/, packages/, pnpm-lock.yaml changes +- [x] 3.2 Create `.github/workflows/cd-infra.yml` — triggered by infrastructure/ changes +- [x] 3.3 Add sops + age install step to both workflows +- [x] 3.4 cd-apps decrypt step: `sops -d secrets.app.env > .env`, SCP to server +- [x] 3.5 cd-infra decrypt step: merge `secrets.app.env` + `secrets.infra.env` into `.env`, SCP to server +- [x] 3.6 cd-apps: build images, push, SSH deploy (pull, migrate, restart apps) +- [x] 3.7 cd-infra: copy configs, SSH deploy (restart infra services) +- [x] 3.8 Remove old `cd.yml` workflow +- [x] 3.9 Remove GRAFANA_* exports and password reset from deploy script +- [x] 3.10 Both workflows use `docker compose --env-file .env up -d` instead of inline exports + +## 4. Verify + +- [x] 4.1 Test sops encrypt/decrypt cycle locally +- [ ] 4.2 Test cd-apps deploys only on app changes (not infra) +- [ ] 4.3 Test cd-infra deploys only on infra changes (not apps) +- [ ] 4.4 Test Grafana GitHub OAuth login +- [ ] 4.5 Verify old GitHub secrets can be removed after migration