Implement manual versioned Synology deployments for S6

ci/ReleaseTests.java

@@ -0,0 +1,549 @@
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Release state-machine and real POSIX remote-script tests, without server writes. */
+class ReleaseTests {
+    static final String SHA = "a".repeat(40), OLD = "b".repeat(40);
+    static int count;
+
+    static void test(String name, CiTests.Test test) throws Exception {
+        test.run();
+        count++;
+        System.out.println("PASS release: " + name);
+    }
+
+    static Map<String, String> environment() {
+        var env = new HashMap<>(System.getenv());
+        env.putAll(Map.of(
+                "CI_REF",
+                "refs/heads/main",
+                "CI_EVENT",
+                "workflow_dispatch",
+                "CI_RUN_ID",
+                "123",
+                "CI_COMMIT",
+                SHA,
+                "RELEASE_MODE",
+                "new",
+                "RELEASE_INCREMENT",
+                "patch",
+                "RELEASE_VERSION",
+                "",
+                "GITEA_SERVER_URL",
+                "http://nas:3000",
+                "CI_REPOSITORY",
+                "owner/agiledancers",
+                "RELEASE_TOKEN",
+                "test"));
+        env.putAll(Map.of(
+                "DEPLOY_HOST",
+                "nas",
+                "DEPLOY_USER",
+                "deploy",
+                "DEPLOY_DIRECTORY",
+                "/tmp/production",
+                "DEPLOY_SSH_KEY",
+                "test-private-key",
+                "DEPLOY_KNOWN_HOSTS",
+                "test-pinned-host"));
+        return env;
+    }
+
+    static ReleaseState.Deployment deployment(String version, String sha, String directory) {
+        return new ReleaseState.Deployment(
+                ReleaseVersion.parse(version),
+                sha,
+                "nas:3000/owner/agiledancers-backend@sha256:" + "1".repeat(64),
+                "nas:3000/owner/agiledancers-frontend@sha256:" + "2".repeat(64),
+                directory,
+                "10");
+    }
+
+    static ReleaseState prior() {
+        return new ReleaseState(
+                deployment("v1.0.1", OLD, "previous-current"),
+                deployment("v1.0.0", OLD, "previous-old"),
+                ReleaseVersion.parse("v1.0.1"));
+    }
+
+    static void run() throws Exception {
+        count = 0;
+        test("first release and all increments reset the correct components", () -> {
+            CiTests.equal("v1.0.0", ReleaseVersion.next(List.of(), "major").toString());
+            var versions = List.of(ReleaseVersion.parse("v1.2.3"), ReleaseVersion.parse("v1.1.9"));
+            CiTests.equal("v1.2.4", ReleaseVersion.next(versions, "patch").toString());
+            CiTests.equal("v1.3.0", ReleaseVersion.next(versions, "minor").toString());
+            CiTests.equal("v2.0.0", ReleaseVersion.next(versions, "major").toString());
+            CiTests.equal(
+                    "v1.0.100000000000000000000",
+                    ReleaseVersion.next(List.of(ReleaseVersion.parse("v1.0.99999999999999999999")), "patch")
+                            .toString());
+            for (String invalid : List.of("1.0.0", "v01.0.0", "v1.0.0-rc1", "v1.0.0+build", "v1.0", "v-1.0.0"))
+                CiTests.failure(() -> ReleaseVersion.parse(invalid));
+        });
+        test("branch, nonmanual, malformed input and conflicting new-release inputs are rejected", () -> {
+            var env = environment();
+            Release.validate(env, "deploy");
+            env.put("CI_REF", "refs/heads/feature/S6");
+            CiTests.failure(() -> Release.validate(env, "deploy"));
+            env.put("CI_REF", "refs/heads/main");
+            env.put("CI_EVENT", "push");
+            CiTests.failure(() -> Release.validate(env, "rollback"));
+            env.put("CI_EVENT", "workflow_dispatch");
+            env.put("RELEASE_VERSION", "v1.0.0");
+            CiTests.failure(() -> Release.validate(env, "deploy"));
+            env.put("RELEASE_MODE", "retry");
+            Release.validate(env, "deploy");
+            env.put("RELEASE_VERSION", "bad");
+            CiTests.failure(() -> Release.validate(env, "deploy"));
+        });
+        test("successful release stages images before tag, checks main again and commits state", () -> {
+            try (var f = new Fixture()) {
+                f.release.run();
+                CiTests.equal("v1.0.0", f.remote.state.current().version().toString());
+                CiTests.equal(false, f.remote.locked);
+                CiTests.equal(false, f.remote.pending);
+                CiTests.equal(true, f.events.indexOf("prepare") < f.events.indexOf("tag"));
+                CiTests.equal(true, f.events.indexOf("tag") < f.events.lastIndexOf("head"));
+                CiTests.equal(true, f.events.lastIndexOf("head") < f.events.indexOf("apply:attempt"));
+            }
+        });
+        test("unverified commit never creates a tag or replaces production", () -> {
+            try (var f = new Fixture()) {
+                f.api.verified = false;
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(false, f.events.contains("apply:attempt"));
+                CiTests.equal(false, f.remote.locked);
+            }
+        });
+        test("missing images prevent tag allocation and replacement", () -> {
+            try (var f = new Fixture()) {
+                f.remote.failAction = "prepare";
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(false, f.remote.pending);
+            }
+        });
+        test("a tagged commit cannot receive another new version", () -> {
+            try (var f = new Fixture()) {
+                f.api.tags.add(new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.0"), SHA, OLD));
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("prepare"));
+            }
+        });
+        test("stale dispatch is rejected before pulling or tagging", () -> {
+            try (var f = new Fixture()) {
+                f.api.head = OLD;
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("prepare"));
+            }
+        });
+        test("main advancing after tag creation preserves tag without changing production", () -> {
+            try (var f = new Fixture()) {
+                f.api.advanceAfterTag = true;
+                CiTests.failure(f.release::run);
+                CiTests.equal(1, f.api.tags.size());
+                CiTests.equal(false, f.remote.pending);
+                CiTests.equal(false, f.events.contains("apply:attempt"));
+            }
+        });
+        test("unavailable recovery images block replacement", () -> {
+            try (var f = new Fixture()) {
+                f.withPrior();
+                f.remote.failAction = "ensure";
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(false, f.remote.pending);
+            }
+        });
+        test("failed candidate restores prior version and still fails the workflow", () -> {
+            try (var f = new Fixture()) {
+                f.withPrior();
+                f.remote.failedDirectories.add("attempt");
+                CiTests.failure(f.release::run);
+                CiTests.equal(
+                        List.of("apply:attempt", "apply:previous-current"),
+                        f.events.stream().filter(e -> e.startsWith("apply:")).toList());
+                CiTests.equal(prior(), f.remote.state);
+                CiTests.equal(false, f.remote.pending);
+                CiTests.equal(false, f.remote.locked);
+                CiTests.equal(2, f.api.tags.size());
+            }
+        });
+        test("failed first release retains journal and lock for manual recovery", () -> {
+            try (var f = new Fixture()) {
+                f.remote.failedDirectories.add("attempt");
+                CiTests.failure(f.release::run);
+                CiTests.equal(true, f.remote.pending);
+                CiTests.equal(true, f.remote.locked);
+                CiTests.equal(null, f.remote.state.current());
+            }
+        });
+        test("failed automatic recovery retains journal and lock without retry loop", () -> {
+            try (var f = new Fixture()) {
+                f.withPrior();
+                f.remote.failedDirectories.addAll(List.of("attempt", "previous-current"));
+                CiTests.failure(f.release::run);
+                CiTests.equal(
+                        2L,
+                        f.events.stream().filter(e -> e.startsWith("apply:")).count());
+                CiTests.equal(true, f.remote.pending);
+                CiTests.equal(true, f.remote.locked);
+            }
+        });
+        test("ambiguous state commit never launches speculative rollback", () -> {
+            try (var f = new Fixture()) {
+                f.withPrior();
+                f.remote.failAction = "commit";
+                CiTests.failure(f.release::run);
+                CiTests.equal(
+                        1L,
+                        f.events.stream().filter(e -> e.startsWith("apply:")).count());
+                CiTests.equal(true, f.remote.pending);
+                CiTests.equal(true, f.remote.locked);
+            }
+        });
+        test("retry keeps existing version and SHA after main advances", () -> {
+            try (var f = new Fixture("retry")) {
+                f.api.head = OLD;
+                f.release.run();
+                CiTests.equal(SHA, f.remote.state.current().sha());
+                CiTests.equal("v1.0.0", f.remote.state.current().version().toString());
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(true, f.events.contains("annotated"));
+            }
+        });
+        test("retry cannot cross the successful-version high watermark after rollback", () -> {
+            try (var f = new Fixture("retry")) {
+                f.remote.state = prior().success(prior().previous());
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("apply:attempt"));
+            }
+        });
+        test("manual rollback preserves version baseline and creates no tag", () -> {
+            try (var f = new Fixture("rollback")) {
+                f.remote.state = prior();
+                f.release.run();
+                CiTests.equal("v1.0.0", f.remote.state.current().version().toString());
+                CiTests.equal("v1.0.1", f.remote.state.previous().version().toString());
+                CiTests.equal("v1.0.1", f.remote.state.highest().toString());
+                CiTests.equal(false, f.events.contains("tag"));
+            }
+        });
+        test("redeploying current version preserves a distinct rollback target", () -> {
+            var state = prior();
+            CiTests.equal(state.previous(), state.success(state.current()).previous());
+            CiTests.equal(state, ReleaseState.parse(state.json()));
+        });
+        test("NAS lock contention cannot allocate tags or replace production", () -> {
+            try (var f = new Fixture()) {
+                f.remote.locked = true;
+                CiTests.failure(f.release::run);
+                CiTests.equal(false, f.events.contains("tag"));
+                CiTests.equal(true, f.remote.locked);
+            }
+        });
+        apiTests();
+        shellTests();
+        ReleaseShellTests.run();
+        System.out.println(count + " release behavior tests passed");
+    }
+
+    static class FakeApi extends ReleaseApi {
+        final List<String> events;
+        final List<Tag> tags = new ArrayList<>();
+        String head = SHA;
+        boolean verified = true, advanceAfterTag;
+
+        FakeApi(Map<String, String> env, List<String> events) {
+            super(env);
+            this.events = events;
+        }
+
+        @Override
+        String head() {
+            events.add("head");
+            return head;
+        }
+
+        @Override
+        List<Tag> tags() {
+            return List.copyOf(tags);
+        }
+
+        @Override
+        String verified(String sha) {
+            Pipeline.check(verified, "Unverified test commit");
+            events.add("verified");
+            return "10";
+        }
+
+        @Override
+        void annotated(Tag tag) {
+            events.add("annotated");
+        }
+
+        @Override
+        void create(ReleaseVersion version, String sha, String attempt) {
+            events.add("tag");
+            tags.add(new Tag(version, sha, OLD));
+            if (advanceAfterTag) head = OLD;
+        }
+    }
+
+    static class FakeRemote extends ReleaseRemote {
+        final List<String> events;
+        final List<String> failedDirectories = new ArrayList<>();
+        ReleaseState state = new ReleaseState(null, null, null);
+        boolean locked, pending;
+        String failAction = "";
+
+        FakeRemote(Path root, Map<String, String> env, List<String> events) throws Exception {
+            super(root, env, new Commands(Duration.ofMinutes(1)), "attempt");
+            this.events = events;
+        }
+
+        @Override
+        String call(String action, String input, String... args) {
+            events.add(action.equals("apply") ? action + ":" + args[0] : action);
+            Pipeline.check(!action.equals(failAction), "Injected remote failure");
+            switch (action) {
+                case "lock" -> {
+                    Pipeline.check(!locked && !pending, "Locked");
+                    locked = true;
+                }
+                case "unlock" -> {
+                    Pipeline.check(!pending, "Pending");
+                    locked = false;
+                }
+                case "state" -> {
+                    return state.current() == null ? "{}" : state.json();
+                }
+                case "pending" -> pending = true;
+                case "finish" -> pending = false;
+                case "commit" -> state = ReleaseState.parse(input);
+                case "prepare" -> {
+                    return "S6_BACKEND=" + deployment("v1.0.0", SHA, "attempt").backend() + "\nS6_FRONTEND="
+                            + deployment("v1.0.0", SHA, "attempt").frontend();
+                }
+                case "apply" -> Pipeline.check(!failedDirectories.contains(args[0]), "Injected unhealthy release");
+                default -> {}
+            }
+            return "";
+        }
+    }
+
+    static class Fixture implements AutoCloseable {
+        final Path root = Files.createTempDirectory("release-test-");
+        final Map<String, String> env = environment();
+        final List<String> events = new ArrayList<>();
+        final FakeApi api;
+        final FakeRemote remote;
+        final Release release;
+
+        Fixture() throws Exception {
+            this("new");
+        }
+
+        Fixture(String mode) throws Exception {
+            Files.createDirectories(root.resolve("deploy"));
+            Files.writeString(root.resolve("deploy/remote.sh"), "");
+            Files.writeString(root.resolve("deploy/compose.yaml"), "services: {}\n");
+            env.put("RELEASE_MODE", mode);
+            if (mode.equals("retry")) env.put("RELEASE_VERSION", "v1.0.0");
+            api = new FakeApi(env, events);
+            if (mode.equals("retry")) api.tags.add(new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.0"), SHA, OLD));
+            remote = new FakeRemote(root, env, events);
+            release = new Release(root, env, api, remote, mode.equals("rollback") ? "rollback" : "deploy", "attempt");
+        }
+
+        void withPrior() {
+            remote.state = prior();
+            api.tags.add(new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.1"), OLD, OLD));
+        }
+
+        @Override
+        public void close() throws Exception {
+            remote.close();
+            Pipeline.deleteTree(root);
+        }
+    }
+
+    static void apiTests() throws Exception {
+        test("NAS registry alias keeps the same repository and rejects unrelated registries", () -> {
+            try (var f = new Fixture()) {
+                f.env.put("DEPLOY_REGISTRY", "127.0.0.1:3000");
+                CiTests.equal("127.0.0.1:3000/owner/agiledancers-", f.release.prefix());
+                f.env.put("DEPLOY_REGISTRY", "other-registry:3000");
+                CiTests.failure(f.release::prefix);
+            }
+        });
+        test("repeating remote tag pages fail closed instead of allocating from incomplete history", () -> {
+            var api = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return List.of(Map.of("name", "v1.0.0", "id", OLD, "commit", Map.of("sha", SHA)));
+                }
+            };
+            CiTests.failure(api::tags);
+        });
+        test("annotated release tag must identify the recorded version and exact SHA", () -> {
+            var tag = new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.0"), SHA, OLD);
+            var api = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return Map.of("tag", "v1.0.0", "object", Map.of("sha", SHA));
+                }
+            };
+            api.annotated(tag);
+            var wrong = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return Map.of("tag", "v1.0.0", "object", Map.of("sha", OLD));
+                }
+            };
+            CiTests.failure(() -> wrong.annotated(tag));
+        });
+        test("Gitea API eligibility validates SHA, event, branch, workflow path and successful conclusion", () -> {
+            var valid = Map.<String, Object>of(
+                    "id",
+                    1,
+                    "head_sha",
+                    SHA,
+                    "event",
+                    "push",
+                    "head_branch",
+                    "main",
+                    "path",
+                    "build.yaml@refs/heads/main",
+                    "status",
+                    "completed",
+                    "conclusion",
+                    "success");
+            for (String field : List.of("head_sha", "event", "head_branch", "path", "status", "conclusion")) {
+                var invalid = new HashMap<>(valid);
+                invalid.put(field, "wrong");
+                var api = new ReleaseApi(environment()) {
+                    @Override
+                    Object request(String method, String path, Object body) {
+                        return Map.of("workflow_runs", List.of(invalid));
+                    }
+                };
+                CiTests.failure(() -> api.verified(SHA));
+            }
+            var api = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return Map.of("workflow_runs", List.of(valid));
+                }
+            };
+            CiTests.equal("1", api.verified(SHA));
+        });
+        test("remote tag pagination includes failed-release baseline and rejects ambiguous tag formats", () -> {
+            var api = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return path.endsWith("page=1")
+                            ? List.of(Map.of("name", "v1.2.3", "id", OLD, "commit", Map.of("sha", SHA)))
+                            : List.of();
+                }
+            };
+            CiTests.equal(
+                    "v1.2.4",
+                    ReleaseVersion.next(
+                                    api.tags().stream()
+                                            .map(ReleaseApi.Tag::version)
+                                            .toList(),
+                                    "patch")
+                            .toString());
+            var invalid = new ReleaseApi(environment()) {
+                @Override
+                Object request(String method, String path, Object body) {
+                    return List.of(Map.of("name", "1.0", "id", OLD, "commit", Map.of("sha", SHA)));
+                }
+            };
+            CiTests.failure(invalid::tags);
+        });
+    }
+
+    static void shellTests() throws Exception {
+        test("real remote shell atomically excludes a second owner and retains unresolved journal", () -> {
+            Path directory = Files.createTempDirectory("release-shell-test-");
+            try {
+                String script = Files.readString(Path.of("deploy/remote.sh"));
+                var commands = new Commands(Duration.ofSeconds(30));
+                var env = environment();
+                var first = commands.execute(
+                        List.of("sh", "-c", script, "--", "lock", directory.toString(), "one"),
+                        directory,
+                        env,
+                        null,
+                        false);
+                first.checked();
+                CiTests.equal(
+                        1,
+                        commands.execute(
+                                        List.of("sh", "-c", script, "--", "lock", directory.toString(), "two"),
+                                        directory,
+                                        env,
+                                        null,
+                                        false)
+                                .code());
+                commands.execute(
+                                List.of("sh", "-c", script, "--", "pending", directory.toString(), "one"),
+                                directory,
+                                env,
+                                "{\"candidate\":\"test\"}",
+                                false)
+                        .checked();
+                CiTests.equal(
+                        1,
+                        commands.execute(
+                                        List.of("sh", "-c", script, "--", "unlock", directory.toString(), "one"),
+                                        directory,
+                                        env,
+                                        null,
+                                        false)
+                                .code());
+                CiTests.equal(true, Files.exists(directory.resolve("pending.json")));
+                commands.execute(
+                                List.of("sh", "-c", script, "--", "finish", directory.toString(), "one"),
+                                directory,
+                                env,
+                                null,
+                                false)
+                        .checked();
+                commands.execute(
+                                List.of("sh", "-c", script, "--", "unlock", directory.toString(), "one"),
+                                directory,
+                                env,
+                                null,
+                                false)
+                        .checked();
+                CiTests.equal(false, Files.exists(directory.resolve("deploy.lock")));
+                CiTests.equal(true, Files.exists(directory.resolve("history/one.json")));
+            } finally {
+                Pipeline.deleteTree(directory);
+            }
+        });
+        test("SSH shell quoting preserves metacharacters as data", () -> {
+            String value = "quote' dollar$ backtick` newline\n";
+            var commands = new Commands(Duration.ofSeconds(5));
+            CiTests.equal(
+                    value,
+                    commands.execute(
+                                    List.of("sh", "-c", "printf '%s' " + ReleaseRemote.quote(value)),
+                                    Path.of("."),
+                                    environment(),
+                                    null,
+                                    false)
+                            .output());
+        });
+    }
+}