237 lines
11 KiB
Java
237 lines
11 KiB
Java
import java.net.URI;
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Path;
|
|
import java.time.Duration;
|
|
import java.time.Instant;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.UUID;
|
|
|
|
/** Manual release coordinator. All mutation decisions are made under the NAS lock. */
|
|
class Release {
|
|
final Path root, artifacts;
|
|
final Map<String, String> env;
|
|
final ReleaseApi api;
|
|
final ReleaseRemote remote;
|
|
final String mode, increment, retry, pinned, attempt;
|
|
ReleaseState.Deployment selected;
|
|
boolean locked, pending;
|
|
|
|
static void execute(Path root, Map<String, String> env, String operation) throws Exception {
|
|
validate(env, operation);
|
|
String attempt = "release-" + env.get("CI_RUN_ID") + "-" + UUID.randomUUID();
|
|
Path artifacts = root.resolve("ci/artifacts/deployment");
|
|
Files.createDirectories(artifacts);
|
|
var commands = new Commands(Duration.ofMinutes(15));
|
|
commands.log = artifacts.resolve(attempt + ".log");
|
|
Files.writeString(commands.log, "");
|
|
try (var remote = new ReleaseRemote(root, env, commands, attempt)) {
|
|
new Release(root, env, new ReleaseApi(env), remote, operation, attempt).run();
|
|
}
|
|
}
|
|
|
|
static void validate(Map<String, String> env, String operation) {
|
|
Pipeline.check("refs/heads/main".equals(env.get("CI_REF")), "Production workflows must run from main");
|
|
Pipeline.check(
|
|
"workflow_dispatch".equals(env.get("CI_EVENT")), "Production requires a manual workflow dispatch");
|
|
Pipeline.check(Setup.required(env, "CI_RUN_ID").matches("[0-9]+"), "Invalid Gitea run ID");
|
|
ReleaseApi.sha(env.get("CI_COMMIT"));
|
|
Pipeline.check(List.of("deploy", "rollback").contains(operation), "Unknown release operation");
|
|
if (operation.equals("rollback")) return;
|
|
String mode = Setup.required(env, "RELEASE_MODE");
|
|
String increment = env.getOrDefault("RELEASE_INCREMENT", "");
|
|
String retry = env.getOrDefault("RELEASE_VERSION", "");
|
|
if (mode.equals("new")) {
|
|
Pipeline.check(List.of("patch", "minor", "major").contains(increment), "Choose patch, minor or major");
|
|
Pipeline.check(retry.isBlank(), "New release cannot specify an existing version");
|
|
} else {
|
|
Pipeline.check(mode.equals("retry"), "Choose new or retry release mode");
|
|
ReleaseVersion.parse(retry);
|
|
// Gitea shows all inputs; the new-release increment is ignored for retries.
|
|
}
|
|
}
|
|
|
|
Release(Path root, Map<String, String> env, ReleaseApi api, ReleaseRemote remote, String operation, String attempt)
|
|
throws Exception {
|
|
this.root = root;
|
|
this.env = env;
|
|
this.api = api;
|
|
this.remote = remote;
|
|
this.attempt = attempt;
|
|
artifacts = root.resolve("ci/artifacts/deployment");
|
|
Files.createDirectories(artifacts);
|
|
mode = operation.equals("rollback") ? "rollback" : env.get("RELEASE_MODE");
|
|
increment = env.getOrDefault("RELEASE_INCREMENT", "");
|
|
retry = env.getOrDefault("RELEASE_VERSION", "");
|
|
pinned = ReleaseApi.sha(env.get("CI_COMMIT"));
|
|
}
|
|
|
|
void run() throws Exception {
|
|
String outcome = "rejected";
|
|
try {
|
|
remote.acquire();
|
|
locked = true;
|
|
remote.call("preflight", null);
|
|
ReleaseState before = ReleaseState.parse(remote.call("state", null));
|
|
selected = select(before);
|
|
IO.println("Selected " + selected.version() + " at " + selected.sha());
|
|
// Validate the recovery pair before any replacement too.
|
|
if (before.current() != null) ensure(before.current());
|
|
if (mode.equals("new")) {
|
|
api.create(selected.version(), selected.sha(), attempt);
|
|
Pipeline.check(
|
|
selected.sha().equals(api.head()),
|
|
"Main advanced; release tagged but production unchanged. Retry explicitly.");
|
|
}
|
|
var journal = new LinkedHashMap<String, Object>();
|
|
journal.put("attempt", attempt);
|
|
journal.put("mode", mode);
|
|
journal.put("candidate", selected.json());
|
|
journal.put("before", Json.parse(before.current() == null ? "{}" : before.json()));
|
|
remote.call("pending", Json.write(journal));
|
|
pending = true;
|
|
outcome = "replacement-started";
|
|
try {
|
|
remote.call("apply", null, selected.directory());
|
|
} catch (Exception failure) {
|
|
diagnose(selected);
|
|
outcome = recover(before);
|
|
throw new IllegalArgumentException(
|
|
"Deployment failed: " + outcome + ". See deployment diagnostics and recovery runbook.");
|
|
}
|
|
// A lost reply here is ambiguous: retain journal and lock, never guess
|
|
// whether a state commit happened and launch an overlapping recovery.
|
|
outcome = "healthy-state-commit-pending";
|
|
remote.call("commit", before.success(selected).json());
|
|
outcome = "success";
|
|
record(outcome);
|
|
remote.call("finish", null);
|
|
pending = false;
|
|
IO.println("Production " + selected.version() + " is healthy (" + selected.sha() + ")");
|
|
} finally {
|
|
try {
|
|
record(outcome);
|
|
} finally {
|
|
if (locked && !pending) remote.release();
|
|
else if (locked)
|
|
System.err.println("NAS lock and pending journal retained. Follow docs/production-deployment.md for"
|
|
+ " recovery.");
|
|
}
|
|
}
|
|
}
|
|
|
|
ReleaseState.Deployment select(ReleaseState before) throws Exception {
|
|
if (mode.equals("rollback")) {
|
|
Pipeline.check(before.previous() != null, "No previous successful production release is available");
|
|
ensure(before.previous());
|
|
return before.previous();
|
|
}
|
|
List<ReleaseApi.Tag> tags = api.tags();
|
|
ReleaseVersion version;
|
|
String commit;
|
|
if (mode.equals("new")) {
|
|
Pipeline.check(
|
|
pinned.equals(api.head()), "Main advanced since this dispatch; start a new release workflow");
|
|
Pipeline.check(
|
|
tags.stream().noneMatch(t -> t.sha().equals(pinned)),
|
|
"This commit already has a release tag; use retry or rollback");
|
|
version = ReleaseVersion.next(
|
|
tags.stream().map(ReleaseApi.Tag::version).toList(), increment);
|
|
commit = pinned;
|
|
} else {
|
|
version = ReleaseVersion.parse(retry);
|
|
var tag = tags.stream()
|
|
.filter(t -> t.version().equals(version))
|
|
.findFirst()
|
|
.orElseThrow(() -> new IllegalArgumentException("Retry release tag does not exist"));
|
|
api.annotated(tag);
|
|
commit = tag.sha();
|
|
}
|
|
Pipeline.check(
|
|
before.highest() == null || version.compareTo(before.highest()) >= (mode.equals("new") ? 1 : 0),
|
|
"A newer release has already succeeded; use explicit rollback for older versions");
|
|
String build = api.verified(commit);
|
|
remote.call("stage", Files.readString(root.resolve("deploy/compose.yaml")));
|
|
remote.call("stage-health", Files.readString(root.resolve("deploy/health.sh")));
|
|
String prepared = remote.call("prepare", null, version.toString(), commit, prefix());
|
|
return new ReleaseState.Deployment(
|
|
version, commit, digest(prepared, "BACKEND"), digest(prepared, "FRONTEND"), attempt, build);
|
|
}
|
|
|
|
String prefix() {
|
|
var uri = URI.create(api.server);
|
|
String authority = uri.getRawAuthority();
|
|
String registry = env.getOrDefault("DEPLOY_REGISTRY", "").strip();
|
|
if (registry.isEmpty()) registry = authority;
|
|
int port = uri.getPort() >= 0 ? uri.getPort() : uri.getScheme().equals("https") ? 443 : 80;
|
|
Pipeline.check(
|
|
registry.equals(authority) || registry.equals("127.0.0.1:" + port),
|
|
"Deployment registry must be the existing Gitea registry or its NAS loopback alias");
|
|
String owner = Setup.required(env, "CI_REPOSITORY").split("/")[0].toLowerCase(java.util.Locale.ROOT);
|
|
return registry + "/" + owner + "/" + Recipe.project(env) + "-";
|
|
}
|
|
|
|
void ensure(ReleaseState.Deployment release) throws Exception {
|
|
remote.call(
|
|
"ensure",
|
|
null,
|
|
release.directory(),
|
|
release.version().toString(),
|
|
release.sha(),
|
|
release.backend(),
|
|
release.frontend());
|
|
}
|
|
|
|
String digest(String response, String role) {
|
|
if (!Recipe.roles(env).contains(role.toLowerCase(java.util.Locale.ROOT))) return "-";
|
|
String value = response.lines()
|
|
.filter(line -> line.startsWith("S6_" + role + "="))
|
|
.map(line -> line.substring(("S6_" + role + "=").length()))
|
|
.reduce((a, b) -> {
|
|
throw new IllegalArgumentException("Ambiguous prepared image digest");
|
|
})
|
|
.orElseThrow(() -> new IllegalArgumentException("Missing prepared image digest"));
|
|
Pipeline.check(
|
|
value.startsWith(prefix() + role.toLowerCase(java.util.Locale.ROOT) + "@sha256:"),
|
|
"Unexpected image repository");
|
|
return value;
|
|
}
|
|
|
|
String recover(ReleaseState before) throws Exception {
|
|
if (before.current() == null) return "first-release-failed-manual-recovery-required";
|
|
try {
|
|
remote.call("apply", null, before.current().directory());
|
|
} catch (Exception failure) {
|
|
diagnose(before.current());
|
|
return "recovery-failed-manual-recovery-required";
|
|
}
|
|
record("failed-previous-release-restored");
|
|
remote.call("finish", null);
|
|
pending = false;
|
|
return "previous-release-restored";
|
|
}
|
|
|
|
void diagnose(ReleaseState.Deployment release) {
|
|
try {
|
|
Files.writeString(
|
|
artifacts.resolve(attempt + "-" + release.directory() + "-containers.log"),
|
|
remote.call("diagnose", null, release.directory()));
|
|
} catch (Exception ignored) {
|
|
System.err.println("Container diagnostics unavailable; use the NAS recovery runbook");
|
|
}
|
|
}
|
|
|
|
void record(String outcome) throws Exception {
|
|
var data = new LinkedHashMap<String, Object>();
|
|
data.put("attempt", attempt);
|
|
data.put("mode", mode);
|
|
data.put("outcome", outcome);
|
|
data.put("time", Instant.now().toString());
|
|
data.put("release", selected == null ? null : selected.json());
|
|
String json = Json.write(data) + "\n";
|
|
Files.writeString(artifacts.resolve(attempt + ".json"), json);
|
|
if (locked) remote.call("event", json);
|
|
}
|
|
}
|