import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.UUID; /** Manual release coordinator. All mutation decisions are made under the NAS lock. */ class Release { final Path root, artifacts; final Map env; final ReleaseApi api; final ReleaseRemote remote; final String mode, increment, retry, pinned, attempt; ReleaseState.Deployment selected; boolean locked, pending; static void execute(Path root, Map env, String operation) throws Exception { validate(env, operation); String attempt = "release-" + env.get("CI_RUN_ID") + "-" + UUID.randomUUID(); Path artifacts = root.resolve("ci/artifacts/deployment"); Files.createDirectories(artifacts); var commands = new Commands(Duration.ofMinutes(15)); commands.log = artifacts.resolve(attempt + ".log"); Files.writeString(commands.log, ""); try (var remote = new ReleaseRemote(root, env, commands, attempt)) { new Release(root, env, new ReleaseApi(env), remote, operation, attempt).run(); } } static void validate(Map env, String operation) { Pipeline.check("refs/heads/main".equals(env.get("CI_REF")), "Production workflows must run from main"); Pipeline.check( "workflow_dispatch".equals(env.get("CI_EVENT")), "Production requires a manual workflow dispatch"); Pipeline.check(Setup.required(env, "CI_RUN_ID").matches("[0-9]+"), "Invalid Gitea run ID"); ReleaseApi.sha(env.get("CI_COMMIT")); Pipeline.check(List.of("deploy", "rollback").contains(operation), "Unknown release operation"); if (operation.equals("rollback")) return; String mode = Setup.required(env, "RELEASE_MODE"); String increment = env.getOrDefault("RELEASE_INCREMENT", ""); String retry = env.getOrDefault("RELEASE_VERSION", ""); if (mode.equals("new")) { Pipeline.check(List.of("patch", "minor", "major").contains(increment), "Choose patch, minor or major"); Pipeline.check(retry.isBlank(), "New release cannot specify an existing version"); } else { Pipeline.check(mode.equals("retry"), "Choose new or retry release mode"); ReleaseVersion.parse(retry); // Gitea shows all inputs; the new-release increment is ignored for retries. } } Release(Path root, Map env, ReleaseApi api, ReleaseRemote remote, String operation, String attempt) throws Exception { this.root = root; this.env = env; this.api = api; this.remote = remote; this.attempt = attempt; artifacts = root.resolve("ci/artifacts/deployment"); Files.createDirectories(artifacts); mode = operation.equals("rollback") ? "rollback" : env.get("RELEASE_MODE"); increment = env.getOrDefault("RELEASE_INCREMENT", ""); retry = env.getOrDefault("RELEASE_VERSION", ""); pinned = ReleaseApi.sha(env.get("CI_COMMIT")); } void run() throws Exception { String outcome = "rejected"; try { remote.acquire(); locked = true; remote.call("preflight", null); ReleaseState before = ReleaseState.parse(remote.call("state", null)); selected = select(before); System.out.println("Selected " + selected.version() + " at " + selected.sha()); // Validate the recovery pair before any replacement too. if (before.current() != null) ensure(before.current()); if (mode.equals("new")) { api.create(selected.version(), selected.sha(), attempt); Pipeline.check( selected.sha().equals(api.head()), "Main advanced; release tagged but production unchanged. Retry explicitly."); } var journal = new LinkedHashMap(); journal.put("attempt", attempt); journal.put("mode", mode); journal.put("candidate", selected.json()); journal.put("before", Json.parse(before.current() == null ? "{}" : before.json())); remote.call("pending", Json.write(journal)); pending = true; outcome = "replacement-started"; try { remote.call("apply", null, selected.directory()); } catch (Exception failure) { diagnose(selected); outcome = recover(before); throw new IllegalArgumentException( "Deployment failed: " + outcome + ". See deployment diagnostics and recovery runbook."); } // A lost reply here is ambiguous: retain journal and lock, never guess // whether a state commit happened and launch an overlapping recovery. outcome = "healthy-state-commit-pending"; remote.call("commit", before.success(selected).json()); outcome = "success"; record(outcome); remote.call("finish", null); pending = false; System.out.println("Production " + selected.version() + " is healthy (" + selected.sha() + ")"); } finally { try { record(outcome); } finally { if (locked && !pending) remote.release(); else if (locked) System.err.println("NAS lock and pending journal retained. Follow docs/production-deployment.md for" + " recovery."); } } } ReleaseState.Deployment select(ReleaseState before) throws Exception { if (mode.equals("rollback")) { Pipeline.check(before.previous() != null, "No previous successful production release is available"); ensure(before.previous()); return before.previous(); } List tags = api.tags(); ReleaseVersion version; String commit; if (mode.equals("new")) { Pipeline.check( pinned.equals(api.head()), "Main advanced since this dispatch; start a new release workflow"); Pipeline.check( tags.stream().noneMatch(t -> t.sha().equals(pinned)), "This commit already has a release tag; use retry or rollback"); version = ReleaseVersion.next( tags.stream().map(ReleaseApi.Tag::version).toList(), increment); commit = pinned; } else { version = ReleaseVersion.parse(retry); var tag = tags.stream() .filter(t -> t.version().equals(version)) .findFirst() .orElseThrow(() -> new IllegalArgumentException("Retry release tag does not exist")); api.annotated(tag); commit = tag.sha(); } Pipeline.check( before.highest() == null || version.compareTo(before.highest()) >= (mode.equals("new") ? 1 : 0), "A newer release has already succeeded; use explicit rollback for older versions"); String build = api.verified(commit); remote.call("stage", Files.readString(root.resolve("deploy/compose.yaml"))); remote.call("stage-health", Files.readString(root.resolve("deploy/health.sh"))); String prepared = remote.call("prepare", null, version.toString(), commit, prefix()); return new ReleaseState.Deployment( version, commit, digest(prepared, "BACKEND"), digest(prepared, "FRONTEND"), attempt, build); } String prefix() { var uri = URI.create(api.server); String authority = uri.getRawAuthority(); String registry = env.getOrDefault("DEPLOY_REGISTRY", "").strip(); if (registry.isEmpty()) registry = authority; int port = uri.getPort() >= 0 ? uri.getPort() : uri.getScheme().equals("https") ? 443 : 80; Pipeline.check( registry.equals(authority) || registry.equals("127.0.0.1:" + port), "Deployment registry must be the existing Gitea registry or its NAS loopback alias"); String owner = Setup.required(env, "CI_REPOSITORY").split("/")[0].toLowerCase(java.util.Locale.ROOT); return registry + "/" + owner + "/" + Recipe.project(env) + "-"; } void ensure(ReleaseState.Deployment release) throws Exception { remote.call( "ensure", null, release.directory(), release.version().toString(), release.sha(), release.backend(), release.frontend()); } String digest(String response, String role) { if (!Recipe.roles(env).contains(role.toLowerCase(java.util.Locale.ROOT))) return "-"; String value = response.lines() .filter(line -> line.startsWith("S6_" + role + "=")) .map(line -> line.substring(("S6_" + role + "=").length())) .reduce((a, b) -> { throw new IllegalArgumentException("Ambiguous prepared image digest"); }) .orElseThrow(() -> new IllegalArgumentException("Missing prepared image digest")); Pipeline.check( value.startsWith(prefix() + role.toLowerCase(java.util.Locale.ROOT) + "@sha256:"), "Unexpected image repository"); return value; } String recover(ReleaseState before) throws Exception { if (before.current() == null) return "first-release-failed-manual-recovery-required"; try { remote.call("apply", null, before.current().directory()); } catch (Exception failure) { diagnose(before.current()); return "recovery-failed-manual-recovery-required"; } record("failed-previous-release-restored"); remote.call("finish", null); pending = false; return "previous-release-restored"; } void diagnose(ReleaseState.Deployment release) { try { Files.writeString( artifacts.resolve(attempt + "-" + release.directory() + "-containers.log"), remote.call("diagnose", null, release.directory())); } catch (Exception ignored) { System.err.println("Container diagnostics unavailable; use the NAS recovery runbook"); } } void record(String outcome) throws Exception { var data = new LinkedHashMap(); data.put("attempt", attempt); data.put("mode", mode); data.put("outcome", outcome); data.put("time", Instant.now().toString()); data.put("release", selected == null ? null : selected.json()); String json = Json.write(data) + "\n"; Files.writeString(artifacts.resolve(attempt + ".json"), json); if (locked) remote.call("event", json); } }