From 01506ef837f70f7d8d8d015f1b601c1a814b9ab3 Mon Sep 17 00:00:00 2001 From: Wanja Krah Date: Thu, 5 Nov 2026 11:20:49 +0100 Subject: [PATCH] Fix pipeline deployment with postgres --- ci/ReleaseShellTests.java | 37 ++++++++++++++++++++++++++++++++++- deploy/remote.sh | 5 ++--- docs/persistent-storage.md | 23 ++++++++++++++++++++-- docs/production-deployment.md | 6 +++--- 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/ci/ReleaseShellTests.java b/ci/ReleaseShellTests.java index 25cfbe4..bfb5be9 100644 --- a/ci/ReleaseShellTests.java +++ b/ci/ReleaseShellTests.java @@ -22,6 +22,30 @@ class ReleaseShellTests { CiTests.equal(false, calls.contains("prune")); } }); + ReleaseTests.test("actual NAS script rejects a missing database password before pulling images", () -> { + try (var f = new Fixture()) { + Path config = f.directory.resolve("production.env"); + Files.writeString( + config, Files.readString(config).replace("DATABASE_PASSWORD=fixture-only-password\n", "")); + f.call("lock", null, "one").checked(); + Commands.Result result = f.call("preflight", null, "one"); + CiTests.equal(1, result.code()); + CiTests.equal(true, result.output().contains("Missing DATABASE_PASSWORD")); + CiTests.equal(false, Files.exists(f.directory.resolve("live.env"))); + } + }); + ReleaseTests.test("actual NAS script passes database configuration through sudo", () -> { + try (var f = new Fixture()) { + f.executable("sudo", "#!/bin/sh\n[ \"$1\" != -n ] || shift\nunset DATABASE_PASSWORD\nexec \"$@\"\n"); + Path config = f.directory.resolve("production.env"); + Files.writeString(config, Files.readString(config).replace("DOCKER_SUDO=0", "DOCKER_SUDO=1")); + f.prepare(); + CiTests.equal( + false, + Files.readString(f.directory.resolve("releases/one/release.env")) + .contains("PASSWORD")); + } + }); ReleaseTests.test("actual NAS script rejects wrong image revision before replacement", () -> { try (var f = new Fixture()) { f.env.put("WRONG_REVISION", "1"); @@ -102,7 +126,11 @@ class ReleaseShellTests { Files.writeString(directory.resolve("docker-auth/config.json"), "{}"); Files.writeString( directory.resolve("production.env"), - "BIND_IP=127.0.0.1\nFRONTEND_PORT=8088\nDOCKER_SUDO=0\nDOCKER_BIN=" + "DATABASE_PASSWORD=fixture-only-password\n" + + "BIND_IP=127.0.0.1\n" + + "FRONTEND_PORT=8088\n" + + "DOCKER_SUDO=0\n" + + "DOCKER_BIN=" + directory.resolve("bin/docker") + "\nDOCKER_CONFIG=" + directory.resolve("docker-auth") + "\n"); } @@ -171,8 +199,15 @@ class ReleaseShellTests { [ "$1" != version ] || exit 0 shift 4 [ "$1" = --env-file ] || exit 1 + production=$2 + shift 2 + [ "$1" = --env-file ] || exit 1 file=$2 shift 4 + # Compose must read the secret from disk, even after sudo clears the environment. + test -f "$production" + . "$production" + [ "$DATABASE_PASSWORD" = fixture-only-password ] || exit 15 case "$1" in config) test -f "$file";; up) cp "$file" "$FIXTURE/live.env";; diff --git a/deploy/remote.sh b/deploy/remote.sh index 8c8be3a..8aeabab 100644 --- a/deploy/remote.sh +++ b/deploy/remote.sh @@ -65,8 +65,6 @@ test -f production.env || die 'Missing production.env; follow provisioning runbo . "$root/production.env" : "${BIND_IP:?}" "${FRONTEND_PORT:?}" "${DOCKER_BIN:?}" "${DOCKER_CONFIG:?}" export DOCKER_CONFIG -# Compose expands database credentials from trusted production.env, never release metadata. -if [ -n "${DATABASE_PASSWORD:-}" ]; then export DATABASE_PASSWORD; fi docker_cmd() { if [ "${DOCKER_SUDO:-1}" = 1 ]; then sudo -n "$DOCKER_BIN" --config "$DOCKER_CONFIG" "$@" @@ -76,7 +74,7 @@ docker_cmd() { } compose() { docker_cmd compose --project-name agiledancers-production --project-directory "$root" \ - --env-file "$release/release.env" -f "$release/compose.yaml" "$@" + --env-file "$root/production.env" --env-file "$release/release.env" -f "$release/compose.yaml" "$@" } identity() { image=$1 @@ -111,6 +109,7 @@ health() { case "$action" in preflight) + test -n "${DATABASE_PASSWORD:-}" || die 'Missing DATABASE_PASSWORD in production.env; follow docs/persistent-storage.md before deploying S16' command -v timeout >/dev/null || die 'timeout is required for bounded remote operations' command -v curl >/dev/null || die 'curl is required' test -x "$DOCKER_BIN" || die 'Docker executable missing' diff --git a/docs/persistent-storage.md b/docs/persistent-storage.md index ab27c57..bce4e1a 100644 --- a/docs/persistent-storage.md +++ b/docs/persistent-storage.md @@ -12,8 +12,10 @@ school ownership of every relationship. Production uses `deploy/compose.yaml`: PostgreSQL has no published port and keeps its data in the named volume `agiledancers-production-database`. Set a long random `DATABASE_PASSWORD` in the mode-600 `production.env` on the NAS before the first -S16 deployment. The release helper exports it for Compose without writing it to -release metadata. The backend receives: +S16 deployment. Compose reads it directly from this file, including when Docker runs through +`sudo`; exporting it in the SSH shell does not survive the default sudo environment +filter. The password is never written to release metadata. Deployment preflight +rejects a missing password before pulling images or replacing containers. The backend receives: - `DATABASE_URL=jdbc:postgresql://database:5432/agiledancers` - `DATABASE_USER=agiledancers` @@ -26,6 +28,23 @@ WAL, fsync and synchronous-commit defaults must remain enabled. Application writ are acknowledged only after commit. A network loss during acknowledgement can leave the client uncertain; reload current data before deciding to retry. +## Deployment failure before the first S16 release + +A missing `DATABASE_PASSWORD` can make `compose config --quiet` exit with status +15. Configure the password in the existing NAS `production.env` (trusted shell +configuration, mode `600`), for example `DATABASE_PASSWORD=''`. +Keep the same password after PostgreSQL initializes its volume: changing only this +file does not change the database account password. Use Compose v2.17 or newer; +the deployment reads `production.env` and then the retained `release.env` through +two `--env-file` options, so sudo cannot discard the credential. Do not print +expanded Compose configuration because it contains the password; use `config --quiet`. + +If the attempt was rejected before container replacement, no pending journal or +lock remains and the old release continues running. After merging the deployment +fix and completing the export below, run a new deployment. Use retry only if the +failed attempt already created a version tag. Do not remove a lock or pending +journal if one does remain; follow the production recovery runbook. + ## Transition from the previous release 1. Export **every school** through JSON while the old backend is still running. diff --git a/docs/production-deployment.md b/docs/production-deployment.md index 2eb68ab..8c68c41 100644 --- a/docs/production-deployment.md +++ b/docs/production-deployment.md @@ -30,7 +30,7 @@ Gebruik je bestaande beheeraccount voor deze stappen: 5. Schakel **Terminal en SNMP → Terminal → SSH-service inschakelen** in. De maintainer heeft SSH-poort **22** bevestigd. Beperk toegang tot mini-pc/LAN/VPN; configureer geen internet-portforwarding. -6. Controleer dat Container Manager/Docker draait, dat Docker Compose **v2**, +6. Controleer dat Container Manager/Docker draait, dat Docker Compose **v2.17 of nieuwer**, `curl`, `timeout` met `-k` en `sh` beschikbaar zijn. Op de NAS is geen Java, Python of Git nodig. Controleer het Docker-pad, doorgaans `/usr/local/bin/docker`. @@ -287,8 +287,8 @@ nieuwe release door te drukken. ## Na S16 en lokale verificatie De databaseservice en het volume `agiledancers-production-database` blijven buiten -de vervanging van backend en frontend. `production.env` levert het databasewachtwoord -ook bij retry en rollback; het staat niet in releasebestanden. De procedure vereist +de vervanging van backend en frontend. Compose leest `production.env` rechtstreeks via `--env-file`, zodat het +databasewachtwoord ook via sudo en bij retry en rollback beschikbaar is; het staat niet in releasebestanden. De procedure vereist databasecompatibiliteit met de vorige applicatieversie. Plan incompatibele migraties apart: imageherstel maakt geen migratie ongedaan. Rollback naar een oude release van vóór S16 kan de database niet lezen. Backup/restore blijven S17-werk.