Implement manual versioned Synology deployments for S6

ci/Ci.java

@@ -37,6 +37,9 @@ public class Ci {
                 case "install-runner" ->
                     Setup.installRunner(root, env, commands, args.length == 2 && args[1].equals("--prepare-only"));
                 case "test" -> CiTests.run();
+                case "deploy", "rollback" -> Release.execute(root, env, args[0]);
+                case "release-test" -> ReleaseTests.run();
+                case "configure-deployment" -> ReleaseSetup.configure(root, env);
                 default -> throw new IllegalArgumentException(usage());
             }
         } catch (Commands.Failure e) {
@@ -54,6 +57,6 @@ public class Ci {
 
     static String usage() {
         return "Usage: java ci/Ci.java verify|publish|cleanup|attempt|test|configure-gitea|configure-docker"
-                + " <path>|install-runner [--prepare-only]";
+                + " <path>|install-runner [--prepare-only]|deploy|rollback|release-test|configure-deployment";
     }
 }

ci/Pipeline.java

@@ -101,6 +101,7 @@ class Pipeline {
             run("node", "--version");
             run("mvn", "-B", "-ntp", "-Dformat.mode=check", "clean", "verify");
             CiTests.run();
+            ReleaseTests.run();
             Path frontend = root.resolve("agiledancers-fe");
             verifyFrontend();
             String platform = env.get("TARGET_PLATFORM");

ci/Release.java

@@ -0,0 +1,234 @@
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/** Manual release coordinator. All mutation decisions are made under the NAS lock. */
+class Release {
+    final Path root, artifacts;
+    final Map<String, String> env;
+    final ReleaseApi api;
+    final ReleaseRemote remote;
+    final String mode, increment, retry, pinned, attempt;
+    ReleaseState.Deployment selected;
+    boolean locked, pending;
+
+    static void execute(Path root, Map<String, String> env, String operation) throws Exception {
+        validate(env, operation);
+        String attempt = "release-" + env.get("CI_RUN_ID") + "-" + UUID.randomUUID();
+        Path artifacts = root.resolve("ci/artifacts/deployment");
+        Files.createDirectories(artifacts);
+        var commands = new Commands(Duration.ofMinutes(15));
+        commands.log = artifacts.resolve(attempt + ".log");
+        Files.writeString(commands.log, "");
+        try (var remote = new ReleaseRemote(root, env, commands, attempt)) {
+            new Release(root, env, new ReleaseApi(env), remote, operation, attempt).run();
+        }
+    }
+
+    static void validate(Map<String, String> env, String operation) {
+        Pipeline.check("refs/heads/main".equals(env.get("CI_REF")), "Production workflows must run from main");
+        Pipeline.check(
+                "workflow_dispatch".equals(env.get("CI_EVENT")), "Production requires a manual workflow dispatch");
+        Pipeline.check(Setup.required(env, "CI_RUN_ID").matches("[0-9]+"), "Invalid Gitea run ID");
+        ReleaseApi.sha(env.get("CI_COMMIT"));
+        Pipeline.check(List.of("deploy", "rollback").contains(operation), "Unknown release operation");
+        if (operation.equals("rollback")) return;
+        String mode = Setup.required(env, "RELEASE_MODE");
+        String increment = env.getOrDefault("RELEASE_INCREMENT", "");
+        String retry = env.getOrDefault("RELEASE_VERSION", "");
+        if (mode.equals("new")) {
+            Pipeline.check(List.of("patch", "minor", "major").contains(increment), "Choose patch, minor or major");
+            Pipeline.check(retry.isBlank(), "New release cannot specify an existing version");
+        } else {
+            Pipeline.check(mode.equals("retry"), "Choose new or retry release mode");
+            ReleaseVersion.parse(retry);
+            // Gitea shows all inputs; the new-release increment is ignored for retries.
+        }
+    }
+
+    Release(Path root, Map<String, String> env, ReleaseApi api, ReleaseRemote remote, String operation, String attempt)
+            throws Exception {
+        this.root = root;
+        this.env = env;
+        this.api = api;
+        this.remote = remote;
+        this.attempt = attempt;
+        artifacts = root.resolve("ci/artifacts/deployment");
+        Files.createDirectories(artifacts);
+        mode = operation.equals("rollback") ? "rollback" : env.get("RELEASE_MODE");
+        increment = env.getOrDefault("RELEASE_INCREMENT", "");
+        retry = env.getOrDefault("RELEASE_VERSION", "");
+        pinned = ReleaseApi.sha(env.get("CI_COMMIT"));
+    }
+
+    void run() throws Exception {
+        String outcome = "rejected";
+        try {
+            remote.acquire();
+            locked = true;
+            remote.call("preflight", null);
+            ReleaseState before = ReleaseState.parse(remote.call("state", null));
+            selected = select(before);
+            System.out.println("Selected " + selected.version() + " at " + selected.sha());
+            // Validate the recovery pair before any replacement too.
+            if (before.current() != null) ensure(before.current());
+            if (mode.equals("new")) {
+                api.create(selected.version(), selected.sha(), attempt);
+                Pipeline.check(
+                        selected.sha().equals(api.head()),
+                        "Main advanced; release tagged but production unchanged. Retry explicitly.");
+            }
+            var journal = new LinkedHashMap<String, Object>();
+            journal.put("attempt", attempt);
+            journal.put("mode", mode);
+            journal.put("candidate", selected.json());
+            journal.put("before", Json.parse(before.current() == null ? "{}" : before.json()));
+            remote.call("pending", Json.write(journal));
+            pending = true;
+            outcome = "replacement-started";
+            try {
+                remote.call("apply", null, selected.directory());
+            } catch (Exception failure) {
+                diagnose(selected);
+                outcome = recover(before);
+                throw new IllegalArgumentException(
+                        "Deployment failed: " + outcome + ". See deployment diagnostics and recovery runbook.");
+            }
+            // A lost reply here is ambiguous: retain journal and lock, never guess
+            // whether a state commit happened and launch an overlapping recovery.
+            outcome = "healthy-state-commit-pending";
+            remote.call("commit", before.success(selected).json());
+            outcome = "success";
+            record(outcome);
+            remote.call("finish", null);
+            pending = false;
+            System.out.println("Production " + selected.version() + " is healthy (" + selected.sha() + ")");
+        } finally {
+            try {
+                record(outcome);
+            } finally {
+                if (locked && !pending) remote.release();
+                else if (locked)
+                    System.err.println("NAS lock and pending journal retained. Follow docs/production-deployment.md for"
+                            + " recovery.");
+            }
+        }
+    }
+
+    ReleaseState.Deployment select(ReleaseState before) throws Exception {
+        if (mode.equals("rollback")) {
+            Pipeline.check(before.previous() != null, "No previous successful production release is available");
+            ensure(before.previous());
+            return before.previous();
+        }
+        List<ReleaseApi.Tag> tags = api.tags();
+        ReleaseVersion version;
+        String commit;
+        if (mode.equals("new")) {
+            Pipeline.check(
+                    pinned.equals(api.head()), "Main advanced since this dispatch; start a new release workflow");
+            Pipeline.check(
+                    tags.stream().noneMatch(t -> t.sha().equals(pinned)),
+                    "This commit already has a release tag; use retry or rollback");
+            version = ReleaseVersion.next(
+                    tags.stream().map(ReleaseApi.Tag::version).toList(), increment);
+            commit = pinned;
+        } else {
+            version = ReleaseVersion.parse(retry);
+            var tag = tags.stream()
+                    .filter(t -> t.version().equals(version))
+                    .findFirst()
+                    .orElseThrow(() -> new IllegalArgumentException("Retry release tag does not exist"));
+            api.annotated(tag);
+            commit = tag.sha();
+        }
+        Pipeline.check(
+                before.highest() == null || version.compareTo(before.highest()) >= (mode.equals("new") ? 1 : 0),
+                "A newer release has already succeeded; use explicit rollback for older versions");
+        String build = api.verified(commit);
+        remote.call("stage", Files.readString(root.resolve("deploy/compose.yaml")));
+        String prepared = remote.call("prepare", null, version.toString(), commit, prefix());
+        return new ReleaseState.Deployment(
+                version, commit, digest(prepared, "BACKEND"), digest(prepared, "FRONTEND"), attempt, build);
+    }
+
+    String prefix() {
+        var uri = URI.create(api.server);
+        String authority = uri.getRawAuthority();
+        String registry = env.getOrDefault("DEPLOY_REGISTRY", "").strip();
+        if (registry.isEmpty()) registry = authority;
+        int port = uri.getPort() >= 0 ? uri.getPort() : uri.getScheme().equals("https") ? 443 : 80;
+        Pipeline.check(
+                registry.equals(authority) || registry.equals("127.0.0.1:" + port),
+                "Deployment registry must be the existing Gitea registry or its NAS loopback alias");
+        String owner = Setup.required(env, "CI_REPOSITORY").split("/")[0].toLowerCase(java.util.Locale.ROOT);
+        return registry + "/" + owner + "/agiledancers-";
+    }
+
+    void ensure(ReleaseState.Deployment release) throws Exception {
+        remote.call(
+                "ensure",
+                null,
+                release.directory(),
+                release.version().toString(),
+                release.sha(),
+                release.backend(),
+                release.frontend());
+    }
+
+    String digest(String response, String role) {
+        String value = response.lines()
+                .filter(line -> line.startsWith("S6_" + role + "="))
+                .map(line -> line.substring(("S6_" + role + "=").length()))
+                .reduce((a, b) -> {
+                    throw new IllegalArgumentException("Ambiguous prepared image digest");
+                })
+                .orElseThrow(() -> new IllegalArgumentException("Missing prepared image digest"));
+        Pipeline.check(
+                value.startsWith(prefix() + role.toLowerCase(java.util.Locale.ROOT) + "@sha256:"),
+                "Unexpected image repository");
+        return value;
+    }
+
+    String recover(ReleaseState before) throws Exception {
+        if (before.current() == null) return "first-release-failed-manual-recovery-required";
+        try {
+            remote.call("apply", null, before.current().directory());
+        } catch (Exception failure) {
+            diagnose(before.current());
+            return "recovery-failed-manual-recovery-required";
+        }
+        record("failed-previous-release-restored");
+        remote.call("finish", null);
+        pending = false;
+        return "previous-release-restored";
+    }
+
+    void diagnose(ReleaseState.Deployment release) {
+        try {
+            Files.writeString(
+                    artifacts.resolve(attempt + "-" + release.directory() + "-containers.log"),
+                    remote.call("diagnose", null, release.directory()));
+        } catch (Exception ignored) {
+            System.err.println("Container diagnostics unavailable; use the NAS recovery runbook");
+        }
+    }
+
+    void record(String outcome) throws Exception {
+        var data = new LinkedHashMap<String, Object>();
+        data.put("attempt", attempt);
+        data.put("mode", mode);
+        data.put("outcome", outcome);
+        data.put("time", Instant.now().toString());
+        data.put("release", selected == null ? null : selected.json());
+        String json = Json.write(data) + "\n";
+        Files.writeString(artifacts.resolve(attempt + ".json"), json);
+        if (locked) remote.call("event", json);
+    }
+}

ci/ReleaseApi.java

@@ -0,0 +1,124 @@
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/** Gitea 1.25 API adapter. Never include response bodies or tokens in errors. */
+class ReleaseApi {
+    record Tag(ReleaseVersion version, String sha, String object) {}
+
+    final String server, token, base;
+
+    ReleaseApi(Map<String, String> env) {
+        server = Setup.required(env, "GITEA_SERVER_URL");
+        var uri = URI.create(server);
+        Pipeline.check(
+                List.of("http", "https").contains(uri.getScheme())
+                        && uri.getUserInfo() == null
+                        && uri.getQuery() == null
+                        && uri.getFragment() == null
+                        && (uri.getPath().isEmpty() || uri.getPath().equals("/")),
+                "Gitea must use a server-root URL");
+        String repository = Setup.required(env, "CI_REPOSITORY");
+        Pipeline.check(repository.matches("[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+"), "Invalid repository name");
+        base = "/api/v1/repos/" + repository;
+        token = Setup.required(env, "RELEASE_TOKEN");
+    }
+
+    Object request(String method, String path, Object body) throws Exception {
+        return Setup.api(server, token, method, base + path, body);
+    }
+
+    String head() throws Exception {
+        return sha(
+                Json.object(Json.object(request("GET", "/branches/main", null)).get("commit"))
+                        .get("id"));
+    }
+
+    static String sha(Object value) {
+        String result = Objects.toString(value, "");
+        Pipeline.check(result.matches("[0-9a-f]{40}"), "Invalid release commit SHA");
+        return result;
+    }
+
+    List<Tag> tags() throws Exception {
+        List<Tag> tags = new ArrayList<>();
+        var seen = new java.util.HashSet<String>();
+        for (int page = 1; ; page++) {
+            var entries = (List<?>) request("GET", "/tags?limit=50&page=" + page, null);
+            if (entries.isEmpty()) return tags;
+            for (Object entry : entries) {
+                var tag = Json.object(entry);
+                String name = Objects.toString(tag.get("name"), "");
+                Pipeline.check(
+                        seen.add(name),
+                        "Remote tag pagination repeated a tag; cannot establish a safe version baseline");
+                // Unknown version-looking tags require explicit reconciliation, not a second baseline.
+                if (!name.matches("v?[0-9].*")) continue;
+                ReleaseVersion version = ReleaseVersion.parse(name);
+                String commit = sha(Json.object(tag.get("commit")).get("sha"));
+                String object = sha(tag.get("id"));
+                tags.add(new Tag(version, commit, object));
+            }
+        }
+    }
+
+    void annotated(Tag tag) throws Exception {
+        var data = Json.object(request("GET", "/git/tags/" + tag.object(), null));
+        Pipeline.check(
+                tag.version().toString().equals(data.get("tag"))
+                        && tag.sha().equals(Json.object(data.get("object")).get("sha")),
+                "Release tag must be annotated and point directly to its recorded commit");
+    }
+
+    void create(ReleaseVersion version, String commit, String run) throws Exception {
+        request(
+                "POST",
+                "/tags",
+                Map.of(
+                        "tag_name",
+                        version.toString(),
+                        "target",
+                        commit,
+                        "message",
+                        "Agiledancers " + version + "\nCommit: " + commit + "\nDeployment attempt: " + run));
+        Tag tag = tags().stream()
+                .filter(t -> t.version().equals(version))
+                .findFirst()
+                .orElseThrow();
+        Pipeline.check(tag.sha().equals(commit), "Created release tag does not match selected commit");
+        annotated(tag);
+    }
+
+    String verified(String commit) throws Exception {
+        // A successful build.yaml main-push run includes verification AND successful image publication.
+        // Do not accept another workflow, a branch build, or merely an existing image/tag.
+        var seen = new java.util.HashSet<String>();
+        for (int page = 1; ; page++) {
+            var response = Json.object(request(
+                    "GET",
+                    "/actions/runs?head_sha=" + commit + "&event=push&branch=main&status=success&limit=50&page=" + page,
+                    null));
+            var runs = (List<?>) response.get("workflow_runs");
+            if (runs.isEmpty()) break;
+            boolean fresh = false;
+            for (Object entry : runs) {
+                var run = Json.object(entry);
+                fresh |= seen.add(Objects.toString(run.get("id")));
+                if (commit.equals(run.get("head_sha"))
+                        && "push".equals(run.get("event"))
+                        && List.of("main", "refs/heads/main").contains(Objects.toString(run.get("head_branch")))
+                        && "completed".equals(run.get("status"))
+                        && "success".equals(run.get("conclusion"))
+                        && "build.yaml@refs/heads/main".equals(run.get("path"))) {
+                    return Objects.toString(run.get("id"));
+                }
+            }
+            // Older Gitea installations may ignore some list filters/pagination.
+            // Validate each entry locally and stop if a page makes no progress.
+            if (!fresh) break;
+        }
+        throw new IllegalArgumentException("No successful build.yaml main-push run verifies and publishes this SHA");
+    }
+}

ci/ReleaseRemote.java

@@ -0,0 +1,119 @@
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/** Argument-safe SSH adapter; server-side mkdir lock is shared by every runner. */
+class ReleaseRemote implements AutoCloseable {
+    final Path root, credentials;
+    final Map<String, String> env;
+    final Commands commands;
+    final String owner, directory, host, user, port, script;
+
+    ReleaseRemote(Path root, Map<String, String> env, Commands commands, String owner) throws Exception {
+        this.root = root;
+        this.env = env;
+        this.commands = commands;
+        this.owner = owner;
+        directory = Setup.required(env, "DEPLOY_DIRECTORY");
+        host = Setup.required(env, "DEPLOY_HOST");
+        user = Setup.required(env, "DEPLOY_USER");
+        port = env.getOrDefault("DEPLOY_SSH_PORT", "22");
+        Pipeline.check(
+                directory.matches("/[a-zA-Z0-9_./-]+") && !directory.contains("..") && !directory.equals("/"),
+                "Use an absolute dedicated deployment directory without spaces or parent traversal");
+        Pipeline.check(
+                host.matches("[a-zA-Z0-9][a-zA-Z0-9.-]*")
+                        && user.matches("[a-zA-Z0-9_][a-zA-Z0-9_-]*")
+                        && port.matches("[0-9]{1,5}")
+                        && Integer.parseInt(port) > 0
+                        && Integer.parseInt(port) <= 65535,
+                "Invalid deployment SSH host, user or port");
+        script = Files.readString(root.resolve("deploy/remote.sh"));
+        credentials = Files.createTempDirectory("agiledancers-deploy-ssh-");
+        try {
+            writePrivate("identity", Setup.required(env, "DEPLOY_SSH_KEY") + "\n");
+            writePrivate("known_hosts", Setup.required(env, "DEPLOY_KNOWN_HOSTS") + "\n");
+        } catch (Exception e) {
+            close();
+            throw e;
+        }
+    }
+
+    void writePrivate(String name, String value) throws Exception {
+        Path file = credentials.resolve(name);
+        Files.createFile(file, PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------")));
+        Files.writeString(file, value);
+    }
+
+    static String quote(String value) {
+        return "'" + value.replace("'", "'\"'\"'") + "'";
+    }
+
+    String call(String action, String input, String... arguments) throws Exception {
+        int seconds = switch (action) {
+            case "apply" -> 300;
+            case "prepare", "ensure" -> 600;
+            default -> 60;
+        };
+        var parameters = new ArrayList<>(List.of(action, directory, owner));
+        parameters.addAll(List.of(arguments));
+        return executeScript(script, input, seconds, parameters);
+    }
+
+    String executeScript(String source, String input, int seconds, List<String> parameters) throws Exception {
+        String remote = "timeout -k 5 " + seconds + " sh -c " + quote(source) + " --";
+        for (String argument : parameters) remote += " " + quote(argument);
+        var args = new ArrayList<>(List.of(
+                "ssh",
+                "-F",
+                "/dev/null",
+                "-T",
+                "-o",
+                "BatchMode=yes",
+                "-o",
+                "StrictHostKeyChecking=yes",
+                "-o",
+                "IdentitiesOnly=yes",
+                "-o",
+                "IdentityAgent=none",
+                "-o",
+                "ForwardAgent=no",
+                "-o",
+                "ConnectTimeout=10",
+                "-o",
+                "ServerAliveInterval=10",
+                "-o",
+                "ServerAliveCountMax=3",
+                "-o",
+                "LogLevel=ERROR",
+                "-o",
+                "UserKnownHostsFile=" + credentials.resolve("known_hosts"),
+                "-o",
+                "GlobalKnownHostsFile=/dev/null",
+                "-i",
+                credentials.resolve("identity").toString(),
+                "-p",
+                port,
+                user + "@" + host,
+                remote));
+        commands.renew(Duration.ofSeconds(seconds + 30));
+        return commands.execute(args, root, env, input, false).checked();
+    }
+
+    void acquire() throws Exception {
+        call("lock", null);
+    }
+
+    void release() throws Exception {
+        call("unlock", null);
+    }
+
+    @Override
+    public void close() throws Exception {
+        Pipeline.deleteTree(credentials);
+    }
+}

ci/ReleaseSetup.java

@@ -0,0 +1,107 @@
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** One-time provisioning; secrets stay in owner-only files or Gitea secret storage. */
+class ReleaseSetup {
+    static void configure(Path root, Map<String, String> input) throws Exception {
+        var env = new HashMap<>(input);
+        env.putIfAbsent("GITEA_SERVER_URL", "http://192.168.2.28:3000");
+        env.putIfAbsent("CI_REPOSITORY", "wanjakrah/agiledancers");
+        env.putIfAbsent("DEPLOY_HOST", "192.168.2.28");
+        env.putIfAbsent("DEPLOY_USER", "agiledancers-deploy");
+        env.putIfAbsent("DEPLOY_SSH_PORT", "22");
+        env.putIfAbsent("DEPLOY_DIRECTORY", "/volume1/docker/agiledancers-production");
+        env.putIfAbsent("DEPLOY_BIND_IP", "192.168.2.28");
+        env.putIfAbsent("DEPLOY_FRONTEND_PORT", "8088");
+        env.putIfAbsent("DEPLOY_REGISTRY", "127.0.0.1:3000");
+        Path config = Setup.home().resolve(".config/agiledancers-runner");
+        env.putIfAbsent("GITEA_ADMIN_TOKEN_FILE", config.resolve("admin-token").toString());
+        env.putIfAbsent("RELEASE_TOKEN_FILE", config.resolve("s6-release-token").toString());
+        env.putIfAbsent("PULL_TOKEN_FILE", config.resolve("s6-pull-token").toString());
+        env.putIfAbsent("DEPLOY_SSH_KEY_FILE", config.resolve("s6-deploy-key").toString());
+        String admin = Setup.secret(env, "GITEA_ADMIN_TOKEN_FILE");
+        String pull = Setup.secret(env, "PULL_TOKEN_FILE");
+        env.put("RELEASE_TOKEN", Setup.secret(env, "RELEASE_TOKEN_FILE"));
+        env.put("DEPLOY_SSH_KEY", Setup.secret(env, "DEPLOY_SSH_KEY_FILE"));
+        String bind = env.get("DEPLOY_BIND_IP"), port = env.get("DEPLOY_FRONTEND_PORT");
+        Pipeline.check(
+                bind.matches("[0-9.]+")
+                        && port.matches("[0-9]{4,5}")
+                        && Integer.parseInt(port) >= 1024
+                        && Integer.parseInt(port) <= 65535,
+                "Invalid private bind address or port");
+        var commands = new Commands(Duration.ofMinutes(15));
+        Path diagnostics = root.resolve("ci/artifacts/deployment");
+        Files.createDirectories(diagnostics);
+        commands.log = diagnostics.resolve("setup.log");
+        Files.writeString(commands.log, "");
+        String host = env.get("DEPLOY_HOST");
+        String lookup = env.get("DEPLOY_SSH_PORT").equals("22") ? host : "[" + host + "]:" + env.get("DEPLOY_SSH_PORT");
+        env.put(
+                "DEPLOY_KNOWN_HOSTS",
+                commands.execute(
+                                List.of(
+                                        "ssh-keygen",
+                                        "-F",
+                                        lookup,
+                                        "-f",
+                                        Setup.home().resolve(".ssh/known_hosts").toString()),
+                                root,
+                                env,
+                                null,
+                                false)
+                        .checked());
+        Pipeline.check(!env.get("DEPLOY_KNOWN_HOSTS").isBlank(), "A previously trusted NAS hostkey is required");
+        var api = new ReleaseApi(env);
+        api.verified(api.head());
+        var uri = URI.create(api.server);
+        String registry = env.get("DEPLOY_REGISTRY");
+        int registryPort = uri.getPort() >= 0 ? uri.getPort() : uri.getScheme().equals("https") ? 443 : 80;
+        Pipeline.check(
+                registry.equals(uri.getRawAuthority()) || registry.equals("127.0.0.1:" + registryPort),
+                "Deployment registry must be the Gitea registry or its NAS loopback alias");
+        String registryUser =
+                env.getOrDefault("PULL_USER", env.get("CI_REPOSITORY").split("/")[0]);
+        Pipeline.check(registryUser.matches("[a-zA-Z0-9_.-]+"), "Invalid registry user");
+        try (var remote = new ReleaseRemote(root, env, commands, "setup")) {
+            System.out.println(remote.executeScript(
+                    Files.readString(root.resolve("deploy/provision.sh")),
+                    pull + "\n",
+                    300,
+                    List.of(remote.directory, registryUser, registry, bind, port)));
+            remote.acquire();
+            try {
+                remote.call("preflight", null);
+            } finally {
+                remote.release();
+            }
+        }
+        // Provision credentials only after successful NAS login and release-read verification.
+        for (String secret : List.of("RELEASE_TOKEN", "DEPLOY_SSH_KEY", "DEPLOY_KNOWN_HOSTS"))
+            Setup.api(
+                    api.server, admin, "PUT", api.base + "/actions/secrets/" + secret, Map.of("data", env.get(secret)));
+        for (String variable :
+                List.of("DEPLOY_HOST", "DEPLOY_USER", "DEPLOY_SSH_PORT", "DEPLOY_DIRECTORY", "DEPLOY_REGISTRY"))
+            variable(api, admin, variable, env.get(variable));
+        System.out.println(
+                "Production configuration and Gitea deployment secrets/variables installed; no release was created.");
+    }
+
+    static void variable(ReleaseApi api, String admin, String name, String value) throws Exception {
+        String path = api.base + "/actions/variables/" + name;
+        boolean exists;
+        try {
+            Setup.api(api.server, admin, "GET", path, null);
+            exists = true;
+        } catch (Setup.ApiFailure error) {
+            if (error.status != 404) throw error;
+            exists = false;
+        }
+        Setup.api(api.server, admin, exists ? "PUT" : "POST", path, Map.of("value", value));
+    }
+}

ci/ReleaseShellTests.java

@@ -0,0 +1,197 @@
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Runs the actual NAS script with disposable command adapters, no Docker daemon. */
+class ReleaseShellTests {
+    static void run() throws Exception {
+        ReleaseTests.test("actual NAS script prepares digest images and checks the published endpoint", () -> {
+            try (var f = new Fixture()) {
+                f.prepare();
+                f.call("pending", "{}", "one").checked();
+                CiTests.equal("S6_HEALTHY", f.call("apply", null, "one", "one").checked());
+                String calls = Files.readString(f.directory.resolve("calls"));
+                CiTests.equal(true, calls.contains("http://backend:8080/q/health/ready"));
+                CiTests.equal(true, calls.contains("http://127.0.0.1:8088/api/dance-schools"));
+                CiTests.equal(false, calls.contains(" down"));
+                CiTests.equal(false, calls.contains("prune"));
+            }
+        });
+        ReleaseTests.test("actual NAS script rejects wrong image revision before replacement", () -> {
+            try (var f = new Fixture()) {
+                f.env.put("WRONG_REVISION", "1");
+                CiTests.failure(f::prepare);
+                CiTests.equal(false, Files.exists(f.directory.resolve("live.env")));
+            }
+        });
+        ReleaseTests.test("actual NAS script rejects unavailable image before replacement", () -> {
+            try (var f = new Fixture()) {
+                f.env.put("FAIL_PULL", "1");
+                CiTests.failure(f::prepare);
+                CiTests.equal(false, Files.exists(f.directory.resolve("live.env")));
+            }
+        });
+        ReleaseTests.test("actual NAS script rejects a changed retained release record", () -> {
+            try (var f = new Fixture()) {
+                f.prepare();
+                CiTests.failure(() -> f.call(
+                                "ensure", null, "one", "one", "v9.0.0", ReleaseTests.SHA, f.backend(), f.frontend())
+                        .checked());
+                f.call("ensure", null, "one", "one", "v1.0.0", ReleaseTests.SHA, f.backend(), f.frontend())
+                        .checked();
+            }
+        });
+        ReleaseTests.test("actual NAS replacement timeout is bounded and leaves recovery journal", () -> {
+            try (var f = new Fixture()) {
+                f.prepare();
+                f.call("pending", "{}", "one").checked();
+                f.env.put("FAIL_HEALTH", "1");
+                CiTests.equal(1, f.call("apply", null, "one", "one", "1").code());
+                CiTests.equal(true, Files.exists(f.directory.resolve("pending.json")));
+                // Termination releases only the operation fence, not the deployment lock.
+                CiTests.equal(false, Files.exists(f.directory.resolve("deploy.lock/operation")));
+                CiTests.equal(true, Files.exists(f.directory.resolve("deploy.lock/owner")));
+                f.env.remove("FAIL_HEALTH");
+                f.call("apply", null, "one", "one").checked();
+            }
+        });
+        ReleaseTests.test("actual NAS script fences overlapping remote replacements", () -> {
+            try (var f = new Fixture()) {
+                f.prepare();
+                Files.createDirectory(f.directory.resolve("deploy.lock/operation"));
+                CiTests.failure(() -> f.call("apply", null, "one", "one").checked());
+                CiTests.equal(false, Files.exists(f.directory.resolve("live.env")));
+            }
+        });
+        ReleaseTests.test("actual NAS script stores state atomically and archives resolved journal", () -> {
+            try (var f = new Fixture()) {
+                f.call("lock", null, "one").checked();
+                f.call("pending", "{\"attempt\":\"one\"}", "one").checked();
+                f.call("commit", "{\"current\":\"one\"}", "one").checked();
+                CiTests.equal(
+                        "{\"current\":\"one\"}", f.call("state", null, "one").checked());
+                f.call("event", "{\"outcome\":\"success\"}", "one").checked();
+                f.call("finish", null, "one").checked();
+                f.call("unlock", null, "one").checked();
+                CiTests.equal(true, Files.exists(f.directory.resolve("history/one.json")));
+                CiTests.equal(false, Files.exists(f.directory.resolve("pending.json")));
+            }
+        });
+    }
+
+    static class Fixture implements AutoCloseable {
+        final Path directory = Files.createTempDirectory("nas-script-test-");
+        final Map<String, String> env = new HashMap<>(System.getenv());
+        final String script = Files.readString(Path.of("deploy/remote.sh"));
+        final Commands commands = new Commands(Duration.ofMinutes(1));
+
+        Fixture() throws Exception {
+            Files.createDirectory(directory.resolve("bin"));
+            executable("docker", DOCKER);
+            executable(
+                    "curl",
+                    "#!/bin/sh\nprintf 'curl %s\\n' \"$*\" >> \"$FIXTURE/calls\"\n[ \"${FAIL_HEALTH:-0}\" = 0 ]\n");
+            env.put("PATH", directory.resolve("bin") + ":" + env.get("PATH"));
+            env.put("FIXTURE", directory.toString());
+            Files.createDirectory(directory.resolve("docker-auth"));
+            Files.writeString(directory.resolve("docker-auth/config.json"), "{}");
+            Files.writeString(
+                    directory.resolve("production.env"),
+                    "BIND_IP=127.0.0.1\nFRONTEND_PORT=8088\nDOCKER_SUDO=0\nDOCKER_BIN="
+                            + directory.resolve("bin/docker") + "\nDOCKER_CONFIG=" + directory.resolve("docker-auth")
+                            + "\n");
+        }
+
+        void executable(String name, String body) throws Exception {
+            Path path = directory.resolve("bin").resolve(name);
+            Files.writeString(path, body);
+            Files.setPosixFilePermissions(path, PosixFilePermissions.fromString("rwx------"));
+        }
+
+        Commands.Result call(String action, String input, String owner, String... args) throws Exception {
+            var command = new ArrayList<>(
+                    List.of("timeout", "-k", "1", "10", "sh", "-c", script, "--", action, directory.toString(), owner));
+            command.addAll(List.of(args));
+            commands.renew(Duration.ofSeconds(15));
+            return commands.execute(command, directory, env, input, false);
+        }
+
+        String backend() {
+            return "nas:3000/owner/agiledancers-backend@sha256:" + "1".repeat(64);
+        }
+
+        String frontend() {
+            return "nas:3000/owner/agiledancers-frontend@sha256:" + "2".repeat(64);
+        }
+
+        void prepare() throws Exception {
+            call("lock", null, "one").checked();
+            call("preflight", null, "one").checked();
+            call("stage", Files.readString(Path.of("deploy/compose.yaml")), "one")
+                    .checked();
+            String result = call("prepare", null, "one", "v1.0.0", ReleaseTests.SHA, "nas:3000/owner/agiledancers-")
+                    .checked();
+            CiTests.equal(true, result.contains("S6_BACKEND=" + backend()));
+            CiTests.equal(true, result.contains("S6_FRONTEND=" + frontend()));
+        }
+
+        @Override
+        public void close() throws Exception {
+            Pipeline.deleteTree(directory);
+        }
+    }
+
+    static final String DOCKER = """
+        #!/bin/sh
+        set -eu
+        printf 'docker %s\n' "$*" >> "$FIXTURE/calls"
+        [ "$1" != --config ] || shift 2
+        case "$1" in
+          info) echo linux/x86_64;;
+          pull) [ "${FAIL_PULL:-0}" = 0 ];;
+          image)
+            format=$4; image=$5
+            case "$image" in *backend*) role=backend; digit=1;; *) role=frontend; digit=2;; esac
+            case "$format" in
+              *revision*)
+                if [ "${WRONG_REVISION:-0}" = 1 ]; then echo wrong; else printf '%040d\n' 0 | tr 0 a; fi;;
+              *Architecture*) echo linux/amd64;;
+              *RepoDigests*)
+                printf '%s@sha256:' "${image%:*}"; printf '%064d\n' 0 | tr 0 "$digit";;
+              *Id*) echo "sha256:$role";;
+              *) exit 1;;
+            esac;;
+          compose)
+            shift
+            [ "$1" != version ] || exit 0
+            shift 4
+            [ "$1" = --env-file ] || exit 1
+            file=$2
+            shift 4
+            case "$1" in
+              config) test -f "$file";;
+              up) cp "$file" "$FIXTURE/live.env";;
+              ps) eval 'role=${'"$#"'}'; echo "$role";;
+              logs) echo 'sample container diagnostics';;
+              *) exit 1;;
+            esac;;
+          inspect)
+            . "$FIXTURE/live.env"
+            format=$3; id=$4
+            case "$format" in
+              *State.Running*) echo true;;
+              *Image*) echo "sha256:$id";;
+              *revision*) echo "$RELEASE_SHA";;
+              *release*) echo "$RELEASE_VERSION";;
+              *) exit 1;;
+            esac;;
+          exec) [ "${FAIL_HEALTH:-0}" = 0 ];;
+          *) exit 1;;
+        esac
+        """;
+}

ci/ReleaseState.java

@@ -0,0 +1,80 @@
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/** Successful deployment state is separate from immutable allocated version tags. */
+record ReleaseState(ReleaseState.Deployment current, ReleaseState.Deployment previous, ReleaseVersion highest) {
+    record Deployment(
+            ReleaseVersion version, String sha, String backend, String frontend, String directory, String build) {
+        Deployment {
+            ReleaseApi.sha(sha);
+            Pipeline.check(directory.matches("[a-zA-Z0-9-]+"), "Invalid retained release directory");
+            for (String image : new String[] {backend, frontend})
+                Pipeline.check(image.matches("[a-zA-Z0-9.:/_-]+@sha256:[0-9a-f]{64}"), "Invalid retained image digest");
+        }
+
+        Map<String, Object> json() {
+            return Map.of(
+                    "version",
+                    version.toString(),
+                    "sha",
+                    sha,
+                    "backend",
+                    backend,
+                    "frontend",
+                    frontend,
+                    "directory",
+                    directory,
+                    "build",
+                    build);
+        }
+
+        static Deployment parse(Object value) {
+            if (value == null) return null;
+            var data = Json.object(value);
+            return new Deployment(
+                    ReleaseVersion.parse(text(data, "version")),
+                    text(data, "sha"),
+                    text(data, "backend"),
+                    text(data, "frontend"),
+                    text(data, "directory"),
+                    text(data, "build"));
+        }
+    }
+
+    static String text(Map<String, Object> data, String key) {
+        return Objects.toString(data.get(key), "");
+    }
+
+    static ReleaseState parse(String value) {
+        var data = Json.object(Json.parse(value));
+        if (data.isEmpty()) return new ReleaseState(null, null, null);
+        Pipeline.check("1".equals(text(data, "schema")), "Unsupported deployment state schema");
+        Deployment current = Deployment.parse(data.get("current"));
+        Deployment previous = Deployment.parse(data.get("previous"));
+        ReleaseVersion highest = ReleaseVersion.parse(text(data, "highest"));
+        Pipeline.check(
+                current != null
+                        && highest.compareTo(current.version()) >= 0
+                        && (previous == null || highest.compareTo(previous.version()) >= 0),
+                "Inconsistent successful release state");
+        return new ReleaseState(current, previous, highest);
+    }
+
+    String json() {
+        var data = new LinkedHashMap<String, Object>();
+        data.put("schema", "1");
+        data.put("current", current == null ? null : current.json());
+        data.put("previous", previous == null ? null : previous.json());
+        data.put("highest", highest == null ? null : highest.toString());
+        return Json.write(data) + "\n";
+    }
+
+    ReleaseState success(Deployment selected) {
+        ReleaseVersion watermark =
+                highest == null || selected.version().compareTo(highest) > 0 ? selected.version() : highest;
+        // A redeploy of the same release must not discard the distinct rollback target.
+        Deployment fallback = current != null && current.version().equals(selected.version()) ? previous : current;
+        return new ReleaseState(selected, fallback, watermark);
+    }
+}

ci/ReleaseTests.java

@@ -0,0 +1,549 @@
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Release state-machine and real POSIX remote-script tests, without server writes. */
+class ReleaseTests {
+    static final String SHA = "a".repeat(40), OLD = "b".repeat(40);
+    static int count;
+
+    static void test(String name, CiTests.Test test) throws Exception {
+        test.run();
+        count++;
+        System.out.println("PASS release: " + name);
+    }
+
+    static Map<String, String> environment() {
+        var env = new HashMap<>(System.getenv());
+        env.putAll(Map.of(
+                "CI_REF",
+                "refs/heads/main",
+                "CI_EVENT",
+                "workflow_dispatch",
+                "CI_RUN_ID",
+                "123",
+                "CI_COMMIT",
+                SHA,
+                "RELEASE_MODE",
+                "new",
+                "RELEASE_INCREMENT",
+                "patch",
+                "RELEASE_VERSION",
+                "",
+                "GITEA_SERVER_URL",
+                "http://nas:3000",
+                "CI_REPOSITORY",
+                "owner/agiledancers",
+                "RELEASE_TOKEN",
+                "test"));
+        env.putAll(Map.of(
+                "DEPLOY_HOST",
+                "nas",
+                "DEPLOY_USER",
+                "deploy",
+                "DEPLOY_DIRECTORY",
+                "/tmp/production",
+                "DEPLOY_SSH_KEY",
+                "test-private-key",
+                "DEPLOY_KNOWN_HOSTS",
+                "test-pinned-host"));
+        return env;
+    }
+
+    static ReleaseState.Deployment deployment(String version, String sha, String directory) {
+        return new ReleaseState.Deployment(
+                ReleaseVersion.parse(version),
+                sha,
+                "nas:3000/owner/agiledancers-backend@sha256:" + "1".repeat(64),
+                "nas:3000/owner/agiledancers-frontend@sha256:" + "2".repeat(64),
+                directory,
+                "10");
+    }
+
+    static ReleaseState prior() {
+        return new ReleaseState(
+                deployment("v1.0.1", OLD, "previous-current"),
+                deployment("v1.0.0", OLD, "previous-old"),
+                ReleaseVersion.parse("v1.0.1"));
+    }
+
+    static void run() throws Exception {
+        count = 0;
+        test("first release and all increments reset the correct components", () -> {
+            CiTests.equal("v1.0.0", ReleaseVersion.next(List.of(), "major").toString());
+            var versions = List.of(ReleaseVersion.parse("v1.2.3"), ReleaseVersion.parse("v1.1.9"));
+            CiTests.equal("v1.2.4", ReleaseVersion.next(versions, "patch").toString());
+            CiTests.equal("v1.3.0", ReleaseVersion.next(versions, "minor").toString());
+            CiTests.equal("v2.0.0", ReleaseVersion.next(versions, "major").toString());
+            CiTests.equal(
+                    "v1.0.100000000000000000000",
+                    ReleaseVersion.next(List.of(ReleaseVersion.parse("v1.0.99999999999999999999")), "patch")
+                            .toString());
+            for (String invalid : List.of("1.0.0", "v01.0.0", "v1.0.0-rc1", "v1.0.0+build", "v1.0", "v-1.0.0"))
+                CiTests.failure(() -> ReleaseVersion.parse(invalid));
+        });
+        test("branch, nonmanual, malformed input and conflicting new-release inputs are rejected", () -> {
+            var env = environment();
+            Release.validate(env, "deploy");
+            env.put("CI_REF", "refs/heads/feature/S6");
+            CiTests.failure(() -> Release.validate(env, "deploy"));
+            env.put("CI_REF", "refs/heads/main");
+            env.put("CI_EVENT", "push");
+            CiTests.failure(() -> Release.validate(env, "rollback"));
+            env.put("CI_EVENT", "workflow_dispatch");
+            env.put("RELEASE_VERSION", "v1.0.0");
+            CiTests.failure(() -> Release.validate(env, "deploy"));
+            env.put("RELEASE_MODE", "retry");
+            Release.validate(env, "deploy");
+            env.put("RELEASE_VERSION", "bad");
+            CiTests.failure(() -> Release.validate(env, "deploy"));
+        });
+        test("successful release stages images before tag, checks main again and commits state", () -> {
+            try (var f = new Fixture()) {
+                f.release.run();
+                CiTests.equal("v1.0.0", f.remote.state.current().version().toString());
+                CiTests.equal(false, f.remote.locked);
+                CiTests.equal(false, f.remote.pending);
+                CiTests.equal(true, f.events.indexOf("prepare") < f.events.indexOf("tag"));
+                CiTests.equal(true, f.events.indexOf("tag") < f.events.lastIndexOf("head"));
+                CiTests.equal(true, f.events.lastIndexOf("head") < f.events.indexOf("apply:attempt"));
+            }
+        });
+        test("unverified commit never creates a tag or replaces production", () -> {
+            try (var f = new Fixture()) {
+                f.api.verified = false;
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(false, f.events.contains("apply:attempt"));
+                CiTests.equal(false, f.remote.locked);
+            }
+        });
+        test("missing images prevent tag allocation and replacement", () -> {
+            try (var f = new Fixture()) {
+                f.remote.failAction = "prepare";
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(false, f.remote.pending);
+            }
+        });
+        test("a tagged commit cannot receive another new version", () -> {
+            try (var f = new Fixture()) {
+                f.api.tags.add(new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.0"), SHA, OLD));
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("prepare"));
+            }
+        });
+        test("stale dispatch is rejected before pulling or tagging", () -> {
+            try (var f = new Fixture()) {
+                f.api.head = OLD;
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("prepare"));
+            }
+        });
+        test("main advancing after tag creation preserves tag without changing production", () -> {
+            try (var f = new Fixture()) {
+                f.api.advanceAfterTag = true;
+                CiTests.failure(f.release::run);
+                CiTests.equal(1, f.api.tags.size());
+                CiTests.equal(false, f.remote.pending);
+                CiTests.equal(false, f.events.contains("apply:attempt"));
+            }
+        });
+        test("unavailable recovery images block replacement", () -> {
+            try (var f = new Fixture()) {
+                f.withPrior();
+                f.remote.failAction = "ensure";
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(false, f.remote.pending);
+            }
+        });
+        test("failed candidate restores prior version and still fails the workflow", () -> {
+            try (var f = new Fixture()) {
+                f.withPrior();
+                f.remote.failedDirectories.add("attempt");
+                CiTests.failure(f.release::run);
+                CiTests.equal(
+                        List.of("apply:attempt", "apply:previous-current"),
+                        f.events.stream().filter(e -> e.startsWith("apply:")).toList());
+                CiTests.equal(prior(), f.remote.state);
+                CiTests.equal(false, f.remote.pending);
+                CiTests.equal(false, f.remote.locked);
+                CiTests.equal(2, f.api.tags.size());
+            }
+        });
+        test("failed first release retains journal and lock for manual recovery", () -> {
+            try (var f = new Fixture()) {
+                f.remote.failedDirectories.add("attempt");
+                CiTests.failure(f.release::run);
+                CiTests.equal(true, f.remote.pending);
+                CiTests.equal(true, f.remote.locked);
+                CiTests.equal(null, f.remote.state.current());
+            }
+        });
+        test("failed automatic recovery retains journal and lock without retry loop", () -> {
+            try (var f = new Fixture()) {
+                f.withPrior();
+                f.remote.failedDirectories.addAll(List.of("attempt", "previous-current"));
+                CiTests.failure(f.release::run);
+                CiTests.equal(
+                        2L,
+                        f.events.stream().filter(e -> e.startsWith("apply:")).count());
+                CiTests.equal(true, f.remote.pending);
+                CiTests.equal(true, f.remote.locked);
+            }
+        });
+        test("ambiguous state commit never launches speculative rollback", () -> {
+            try (var f = new Fixture()) {
+                f.withPrior();
+                f.remote.failAction = "commit";
+                CiTests.failure(f.release::run);
+                CiTests.equal(
+                        1L,
+                        f.events.stream().filter(e -> e.startsWith("apply:")).count());
+                CiTests.equal(true, f.remote.pending);
+                CiTests.equal(true, f.remote.locked);
+            }
+        });
+        test("retry keeps existing version and SHA after main advances", () -> {
+            try (var f = new Fixture("retry")) {
+                f.api.head = OLD;
+                f.release.run();
+                CiTests.equal(SHA, f.remote.state.current().sha());
+                CiTests.equal("v1.0.0", f.remote.state.current().version().toString());
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(true, f.events.contains("annotated"));
+            }
+        });
+        test("retry cannot cross the successful-version high watermark after rollback", () -> {
+            try (var f = new Fixture("retry")) {
+                f.remote.state = prior().success(prior().previous());
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("apply:attempt"));
+            }
+        });
+        test("manual rollback preserves version baseline and creates no tag", () -> {
+            try (var f = new Fixture("rollback")) {
+                f.remote.state = prior();
+                f.release.run();
+                CiTests.equal("v1.0.0", f.remote.state.current().version().toString());
+                CiTests.equal("v1.0.1", f.remote.state.previous().version().toString());
+                CiTests.equal("v1.0.1", f.remote.state.highest().toString());
+                CiTests.equal(false, f.events.contains("tag"));
+            }
+        });
+        test("redeploying current version preserves a distinct rollback target", () -> {
+            var state = prior();
+            CiTests.equal(state.previous(), state.success(state.current()).previous());
+            CiTests.equal(state, ReleaseState.parse(state.json()));
+        });
+        test("NAS lock contention cannot allocate tags or replace production", () -> {
+            try (var f = new Fixture()) {
+                f.remote.locked = true;
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(true, f.remote.locked);
+            }
+        });
+        apiTests();
+        shellTests();
+        ReleaseShellTests.run();
+        System.out.println(count + " release behavior tests passed");
+    }
+
+    static class FakeApi extends ReleaseApi {
+        final List<String> events;
+        final List<Tag> tags = new ArrayList<>();
+        String head = SHA;
+        boolean verified = true, advanceAfterTag;
+
+        FakeApi(Map<String, String> env, List<String> events) {
+            super(env);
+            this.events = events;
+        }
+
+        @Override
+        String head() {
+            events.add("head");
+            return head;
+        }
+
+        @Override
+        List<Tag> tags() {
+            return List.copyOf(tags);
+        }
+
+        @Override
+        String verified(String sha) {
+            Pipeline.check(verified, "Unverified test commit");
+            events.add("verified");
+            return "10";
+        }
+
+        @Override
+        void annotated(Tag tag) {
+            events.add("annotated");
+        }
+
+        @Override
+        void create(ReleaseVersion version, String sha, String attempt) {
+            events.add("tag");
+            tags.add(new Tag(version, sha, OLD));
+            if (advanceAfterTag) head = OLD;
+        }
+    }
+
+    static class FakeRemote extends ReleaseRemote {
+        final List<String> events;
+        final List<String> failedDirectories = new ArrayList<>();
+        ReleaseState state = new ReleaseState(null, null, null);
+        boolean locked, pending;
+        String failAction = "";
+
+        FakeRemote(Path root, Map<String, String> env, List<String> events) throws Exception {
+            super(root, env, new Commands(Duration.ofMinutes(1)), "attempt");
+            this.events = events;
+        }
+
+        @Override
+        String call(String action, String input, String... args) {
+            events.add(action.equals("apply") ? action + ":" + args[0] : action);
+            Pipeline.check(!action.equals(failAction), "Injected remote failure");
+            switch (action) {
+                case "lock" -> {
+                    Pipeline.check(!locked && !pending, "Locked");
+                    locked = true;
+                }
+                case "unlock" -> {
+                    Pipeline.check(!pending, "Pending");
+                    locked = false;
+                }
+                case "state" -> {
+                    return state.current() == null ? "{}" : state.json();
+                }
+                case "pending" -> pending = true;
+                case "finish" -> pending = false;
+                case "commit" -> state = ReleaseState.parse(input);
+                case "prepare" -> {
+                    return "S6_BACKEND=" + deployment("v1.0.0", SHA, "attempt").backend() + "\nS6_FRONTEND="
+                            + deployment("v1.0.0", SHA, "attempt").frontend();
+                }
+                case "apply" -> Pipeline.check(!failedDirectories.contains(args[0]), "Injected unhealthy release");
+                default -> {}
+            }
+            return "";
+        }
+    }
+
+    static class Fixture implements AutoCloseable {
+        final Path root = Files.createTempDirectory("release-test-");
+        final Map<String, String> env = environment();
+        final List<String> events = new ArrayList<>();
+        final FakeApi api;
+        final FakeRemote remote;
+        final Release release;
+
+        Fixture() throws Exception {
+            this("new");
+        }
+
+        Fixture(String mode) throws Exception {
+            Files.createDirectories(root.resolve("deploy"));
+            Files.writeString(root.resolve("deploy/remote.sh"), "");
+            Files.writeString(root.resolve("deploy/compose.yaml"), "services: {}\n");
+            env.put("RELEASE_MODE", mode);
+            if (mode.equals("retry")) env.put("RELEASE_VERSION", "v1.0.0");
+            api = new FakeApi(env, events);
+            if (mode.equals("retry")) api.tags.add(new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.0"), SHA, OLD));
+            remote = new FakeRemote(root, env, events);
+            release = new Release(root, env, api, remote, mode.equals("rollback") ? "rollback" : "deploy", "attempt");
+        }
+
+        void withPrior() {
+            remote.state = prior();
+            api.tags.add(new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.1"), OLD, OLD));
+        }
+
+        @Override
+        public void close() throws Exception {
+            remote.close();
+            Pipeline.deleteTree(root);
+        }
+    }
+
+    static void apiTests() throws Exception {
+        test("NAS registry alias keeps the same repository and rejects unrelated registries", () -> {
+            try (var f = new Fixture()) {
+                f.env.put("DEPLOY_REGISTRY", "127.0.0.1:3000");
+                CiTests.equal("127.0.0.1:3000/owner/agiledancers-", f.release.prefix());
+                f.env.put("DEPLOY_REGISTRY", "other-registry:3000");
+                CiTests.failure(f.release::prefix);
+            }
+        });
+        test("repeating remote tag pages fail closed instead of allocating from incomplete history", () -> {
+            var api = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return List.of(Map.of("name", "v1.0.0", "id", OLD, "commit", Map.of("sha", SHA)));
+                }
+            };
+            CiTests.failure(api::tags);
+        });
+        test("annotated release tag must identify the recorded version and exact SHA", () -> {
+            var tag = new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.0"), SHA, OLD);
+            var api = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return Map.of("tag", "v1.0.0", "object", Map.of("sha", SHA));
+                }
+            };
+            api.annotated(tag);
+            var wrong = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return Map.of("tag", "v1.0.0", "object", Map.of("sha", OLD));
+                }
+            };
+            CiTests.failure(() -> wrong.annotated(tag));
+        });
+        test("Gitea API eligibility validates SHA, event, branch, workflow path and successful conclusion", () -> {
+            var valid = Map.<String, Object>of(
+                    "id",
+                    1,
+                    "head_sha",
+                    SHA,
+                    "event",
+                    "push",
+                    "head_branch",
+                    "main",
+                    "path",
+                    "build.yaml@refs/heads/main",
+                    "status",
+                    "completed",
+                    "conclusion",
+                    "success");
+            for (String field : List.of("head_sha", "event", "head_branch", "path", "status", "conclusion")) {
+                var invalid = new HashMap<>(valid);
+                invalid.put(field, "wrong");
+                var api = new ReleaseApi(environment()) {
+                    @Override
+                    Object request(String method, String path, Object body) {
+                        return Map.of("workflow_runs", List.of(invalid));
+                    }
+                };
+                CiTests.failure(() -> api.verified(SHA));
+            }
+            var api = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return Map.of("workflow_runs", List.of(valid));
+                }
+            };
+            CiTests.equal("1", api.verified(SHA));
+        });
+        test("remote tag pagination includes failed-release baseline and rejects ambiguous tag formats", () -> {
+            var api = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return path.endsWith("page=1")
+                            ? List.of(Map.of("name", "v1.2.3", "id", OLD, "commit", Map.of("sha", SHA)))
+                            : List.of();
+                }
+            };
+            CiTests.equal(
+                    "v1.2.4",
+                    ReleaseVersion.next(
+                                    api.tags().stream()
+                                            .map(ReleaseApi.Tag::version)
+                                            .toList(),
+                                    "patch")
+                            .toString());
+            var invalid = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return List.of(Map.of("name", "1.0", "id", OLD, "commit", Map.of("sha", SHA)));
+                }
+            };
+            CiTests.failure(invalid::tags);
+        });
+    }
+
+    static void shellTests() throws Exception {
+        test("real remote shell atomically excludes a second owner and retains unresolved journal", () -> {
+            Path directory = Files.createTempDirectory("release-shell-test-");
+            try {
+                String script = Files.readString(Path.of("deploy/remote.sh"));
+                var commands = new Commands(Duration.ofSeconds(30));
+                var env = environment();
+                var first = commands.execute(
+                        List.of("sh", "-c", script, "--", "lock", directory.toString(), "one"),
+                        directory,
+                        env,
+                        null,
+                        false);
+                first.checked();
+                CiTests.equal(
+                        1,
+                        commands.execute(
+                                        List.of("sh", "-c", script, "--", "lock", directory.toString(), "two"),
+                                        directory,
+                                        env,
+                                        null,
+                                        false)
+                                .code());
+                commands.execute(
+                                List.of("sh", "-c", script, "--", "pending", directory.toString(), "one"),
+                                directory,
+                                env,
+                                "{\"candidate\":\"test\"}",
+                                false)
+                        .checked();
+                CiTests.equal(
+                        1,
+                        commands.execute(
+                                        List.of("sh", "-c", script, "--", "unlock", directory.toString(), "one"),
+                                        directory,
+                                        env,
+                                        null,
+                                        false)
+                                .code());
+                CiTests.equal(true, Files.exists(directory.resolve("pending.json")));
+                commands.execute(
+                                List.of("sh", "-c", script, "--", "finish", directory.toString(), "one"),
+                                directory,
+                                env,
+                                null,
+                                false)
+                        .checked();
+                commands.execute(
+                                List.of("sh", "-c", script, "--", "unlock", directory.toString(), "one"),
+                                directory,
+                                env,
+                                null,
+                                false)
+                        .checked();
+                CiTests.equal(false, Files.exists(directory.resolve("deploy.lock")));
+                CiTests.equal(true, Files.exists(directory.resolve("history/one.json")));
+            } finally {
+                Pipeline.deleteTree(directory);
+            }
+        });
+        test("SSH shell quoting preserves metacharacters as data", () -> {
+            String value = "quote' dollar$ backtick` newline\n";
+            var commands = new Commands(Duration.ofSeconds(5));
+            CiTests.equal(
+                    value,
+                    commands.execute(
+                                    List.of("sh", "-c", "printf '%s' " + ReleaseRemote.quote(value)),
+                                    Path.of("."),
+                                    environment(),
+                                    null,
+                                    false)
+                            .output());
+        });
+    }
+}

ci/ReleaseVersion.java

@@ -0,0 +1,39 @@
+import java.math.BigInteger;
+import java.util.List;
+import java.util.regex.Pattern;
+
+/** Strict stable release versions; arbitrary-size integers avoid overflow/reuse. */
+record ReleaseVersion(BigInteger major, BigInteger minor, BigInteger patch) implements Comparable<ReleaseVersion> {
+    private static final Pattern FORMAT = Pattern.compile("v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)");
+
+    static ReleaseVersion parse(String value) {
+        var match = FORMAT.matcher(value);
+        Pipeline.check(match.matches(), "Expected release version vMAJOR.MINOR.PATCH");
+        return new ReleaseVersion(
+                new BigInteger(match.group(1)), new BigInteger(match.group(2)), new BigInteger(match.group(3)));
+    }
+
+    static ReleaseVersion next(List<ReleaseVersion> versions, String increment) {
+        Pipeline.check(List.of("patch", "minor", "major").contains(increment), "Choose patch, minor or major");
+        var highest = versions.stream().max(ReleaseVersion::compareTo);
+        if (highest.isEmpty()) return parse("v1.0.0");
+        var current = highest.get();
+        return switch (increment) {
+            case "major" -> new ReleaseVersion(current.major.add(BigInteger.ONE), BigInteger.ZERO, BigInteger.ZERO);
+            case "minor" -> new ReleaseVersion(current.major, current.minor.add(BigInteger.ONE), BigInteger.ZERO);
+            default -> new ReleaseVersion(current.major, current.minor, current.patch.add(BigInteger.ONE));
+        };
+    }
+
+    @Override
+    public int compareTo(ReleaseVersion other) {
+        int value = major.compareTo(other.major);
+        if (value == 0) value = minor.compareTo(other.minor);
+        return value == 0 ? patch.compareTo(other.patch) : value;
+    }
+
+    @Override
+    public String toString() {
+        return "v" + major + "." + minor + "." + patch;
+    }
+}