Merge pull request 'Fix pipeline deployment with postgres' (#16) from S16_deploy-fix into main

Reviewed-on: #16

ci/ReleaseShellTests.java

@@ -22,6 +22,30 @@ class ReleaseShellTests {
                 CiTests.equal(false, calls.contains("prune"));
             }
         });
+        ReleaseTests.test("actual NAS script rejects a missing database password before pulling images", () -> {
+            try (var f = new Fixture()) {
+                Path config = f.directory.resolve("production.env");
+                Files.writeString(
+                        config, Files.readString(config).replace("DATABASE_PASSWORD=fixture-only-password\n", ""));
+                f.call("lock", null, "one").checked();
+                Commands.Result result = f.call("preflight", null, "one");
+                CiTests.equal(1, result.code());
+                CiTests.equal(true, result.output().contains("Missing DATABASE_PASSWORD"));
+                CiTests.equal(false, Files.exists(f.directory.resolve("live.env")));
+            }
+        });
+        ReleaseTests.test("actual NAS script passes database configuration through sudo", () -> {
+            try (var f = new Fixture()) {
+                f.executable("sudo", "#!/bin/sh\n[ \"$1\" != -n ] || shift\nunset DATABASE_PASSWORD\nexec \"$@\"\n");
+                Path config = f.directory.resolve("production.env");
+                Files.writeString(config, Files.readString(config).replace("DOCKER_SUDO=0", "DOCKER_SUDO=1"));
+                f.prepare();
+                CiTests.equal(
+                        false,
+                        Files.readString(f.directory.resolve("releases/one/release.env"))
+                                .contains("PASSWORD"));
+            }
+        });
         ReleaseTests.test("actual NAS script rejects wrong image revision before replacement", () -> {
             try (var f = new Fixture()) {
                 f.env.put("WRONG_REVISION", "1");
@@ -102,7 +126,11 @@ class ReleaseShellTests {
             Files.writeString(directory.resolve("docker-auth/config.json"), "{}");
             Files.writeString(
                     directory.resolve("production.env"),
-                    "BIND_IP=127.0.0.1\nFRONTEND_PORT=8088\nDOCKER_SUDO=0\nDOCKER_BIN="
+                    "DATABASE_PASSWORD=fixture-only-password\n"
+                            + "BIND_IP=127.0.0.1\n"
+                            + "FRONTEND_PORT=8088\n"
+                            + "DOCKER_SUDO=0\n"
+                            + "DOCKER_BIN="
                             + directory.resolve("bin/docker") + "\nDOCKER_CONFIG=" + directory.resolve("docker-auth")
                             + "\n");
         }
@@ -171,8 +199,15 @@ class ReleaseShellTests {
             [ "$1" != version ] || exit 0
             shift 4
             [ "$1" = --env-file ] || exit 1
+            production=$2
+            shift 2
+            [ "$1" = --env-file ] || exit 1
             file=$2
             shift 4
+            # Compose must read the secret from disk, even after sudo clears the environment.
+            test -f "$production"
+            . "$production"
+            [ "$DATABASE_PASSWORD" = fixture-only-password ] || exit 15
             case "$1" in
               config) test -f "$file";;
               up) cp "$file" "$FIXTURE/live.env";;

deploy/remote.sh

@@ -65,8 +65,6 @@ test -f production.env || die 'Missing production.env; follow provisioning runbo
 . "$root/production.env"
 : "${BIND_IP:?}" "${FRONTEND_PORT:?}" "${DOCKER_BIN:?}" "${DOCKER_CONFIG:?}"
 export DOCKER_CONFIG
-# Compose expands database credentials from trusted production.env, never release metadata.
-if [ -n "${DATABASE_PASSWORD:-}" ]; then export DATABASE_PASSWORD; fi
 docker_cmd() {
     if [ "${DOCKER_SUDO:-1}" = 1 ]; then
         sudo -n "$DOCKER_BIN" --config "$DOCKER_CONFIG" "$@"
@@ -76,7 +74,7 @@ docker_cmd() {
 }
 compose() {
     docker_cmd compose --project-name agiledancers-production --project-directory "$root" \
-        --env-file "$release/release.env" -f "$release/compose.yaml" "$@"
+        --env-file "$root/production.env" --env-file "$release/release.env" -f "$release/compose.yaml" "$@"
 }
 identity() {
     image=$1
@@ -111,6 +109,7 @@ health() {
 
 case "$action" in
     preflight)
+        test -n "${DATABASE_PASSWORD:-}" || die 'Missing DATABASE_PASSWORD in production.env; follow docs/persistent-storage.md before deploying S16'
         command -v timeout >/dev/null || die 'timeout is required for bounded remote operations'
         command -v curl >/dev/null || die 'curl is required'
         test -x "$DOCKER_BIN" || die 'Docker executable missing'

docs/persistent-storage.md

@@ -12,8 +12,10 @@ school ownership of every relationship.
 Production uses `deploy/compose.yaml`: PostgreSQL has no published port and keeps
 its data in the named volume `agiledancers-production-database`. Set a long random
 `DATABASE_PASSWORD` in the mode-600 `production.env` on the NAS before the first
-S16 deployment. The release helper exports it for Compose without writing it to
-release metadata. The backend receives:
+S16 deployment. Compose reads it directly from this file, including when Docker runs through
+`sudo`; exporting it in the SSH shell does not survive the default sudo environment
+filter. The password is never written to release metadata. Deployment preflight
+rejects a missing password before pulling images or replacing containers. The backend receives:
 
 - `DATABASE_URL=jdbc:postgresql://database:5432/agiledancers`
 - `DATABASE_USER=agiledancers`
@@ -26,6 +28,23 @@ WAL, fsync and synchronous-commit defaults must remain enabled. Application writ
 are acknowledged only after commit. A network loss during acknowledgement can
 leave the client uncertain; reload current data before deciding to retry.
 
+## Deployment failure before the first S16 release
+
+A missing `DATABASE_PASSWORD` can make `compose config --quiet` exit with status
+15. Configure the password in the existing NAS `production.env` (trusted shell
+configuration, mode `600`), for example `DATABASE_PASSWORD='<long random value>'`.
+Keep the same password after PostgreSQL initializes its volume: changing only this
+file does not change the database account password. Use Compose v2.17 or newer;
+the deployment reads `production.env` and then the retained `release.env` through
+two `--env-file` options, so sudo cannot discard the credential. Do not print
+expanded Compose configuration because it contains the password; use `config --quiet`.
+
+If the attempt was rejected before container replacement, no pending journal or
+lock remains and the old release continues running. After merging the deployment
+fix and completing the export below, run a new deployment. Use retry only if the
+failed attempt already created a version tag. Do not remove a lock or pending
+journal if one does remain; follow the production recovery runbook.
+
 ## Transition from the previous release
 
 1. Export **every school** through JSON while the old backend is still running.

docs/production-deployment.md

@@ -30,7 +30,7 @@ Gebruik je bestaande beheeraccount voor deze stappen:
 5. Schakel **Terminal en SNMP → Terminal → SSH-service inschakelen** in. De
    maintainer heeft SSH-poort **22** bevestigd. Beperk toegang tot mini-pc/LAN/VPN;
    configureer geen internet-portforwarding.
-6. Controleer dat Container Manager/Docker draait, dat Docker Compose **v2**,
+6. Controleer dat Container Manager/Docker draait, dat Docker Compose **v2.17 of nieuwer**,
    `curl`, `timeout` met `-k` en `sh` beschikbaar zijn. Op de NAS is geen Java,
    Python of Git nodig. Controleer het Docker-pad, doorgaans `/usr/local/bin/docker`.
 
@@ -287,8 +287,8 @@ nieuwe release door te drukken.
 ## Na S16 en lokale verificatie
 
 De databaseservice en het volume `agiledancers-production-database` blijven buiten
-de vervanging van backend en frontend. `production.env` levert het databasewachtwoord
-ook bij retry en rollback; het staat niet in releasebestanden. De procedure vereist
+de vervanging van backend en frontend. Compose leest `production.env` rechtstreeks via `--env-file`, zodat het
+databasewachtwoord ook via sudo en bij retry en rollback beschikbaar is; het staat niet in releasebestanden. De procedure vereist
 databasecompatibiliteit met de vorige applicatieversie. Plan incompatibele migraties
 apart: imageherstel maakt geen migratie ongedaan. Rollback naar een oude release
 van vóór S16 kan de database niet lezen. Backup/restore blijven S17-werk.