556 lines
24 KiB
Java
556 lines
24 KiB
Java
import java.nio.file.Files;
|
|
import java.nio.file.Path;
|
|
import java.time.Duration;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
|
|
/** Release state-machine and real POSIX remote-script tests, without server writes. */
|
|
class ReleaseTests {
|
|
static final String SHA = "a".repeat(40), OLD = "b".repeat(40);
|
|
static int count;
|
|
|
|
static void test(String name, CiTests.Test test) throws Exception {
|
|
test.run();
|
|
count++;
|
|
System.out.println("PASS release: " + name);
|
|
}
|
|
|
|
static Map<String, String> environment() {
|
|
var env = new HashMap<>(System.getenv());
|
|
env.putAll(Map.of(
|
|
"CI_REF",
|
|
"refs/heads/main",
|
|
"CI_EVENT",
|
|
"workflow_dispatch",
|
|
"CI_RUN_ID",
|
|
"123",
|
|
"CI_COMMIT",
|
|
SHA,
|
|
"RELEASE_MODE",
|
|
"new",
|
|
"RELEASE_INCREMENT",
|
|
"patch",
|
|
"RELEASE_VERSION",
|
|
"",
|
|
"GITEA_SERVER_URL",
|
|
"http://nas:3000",
|
|
"CI_REPOSITORY",
|
|
"owner/recipe-demo",
|
|
"RELEASE_TOKEN",
|
|
"test"));
|
|
env.putAll(Map.of(
|
|
"DEPLOY_HOST",
|
|
"nas",
|
|
"DEPLOY_USER",
|
|
"deploy",
|
|
"DEPLOY_DIRECTORY",
|
|
"/tmp/production",
|
|
"DEPLOY_SSH_KEY",
|
|
"test-private-key",
|
|
"DEPLOY_KNOWN_HOSTS",
|
|
"test-pinned-host"));
|
|
return env;
|
|
}
|
|
|
|
static ReleaseState.Deployment deployment(String version, String sha, String directory) {
|
|
return new ReleaseState.Deployment(
|
|
ReleaseVersion.parse(version),
|
|
sha,
|
|
"nas:3000/owner/recipe-demo-backend@sha256:" + "1".repeat(64),
|
|
"nas:3000/owner/recipe-demo-frontend@sha256:" + "2".repeat(64),
|
|
directory,
|
|
"10");
|
|
}
|
|
|
|
static ReleaseState prior() {
|
|
return new ReleaseState(
|
|
deployment("v1.0.1", OLD, "previous-current"),
|
|
deployment("v1.0.0", OLD, "previous-old"),
|
|
ReleaseVersion.parse("v1.0.1"));
|
|
}
|
|
|
|
static void run() throws Exception {
|
|
count = 0;
|
|
test("first release and all increments reset the correct components", () -> {
|
|
CiTests.equal("v1.0.0", ReleaseVersion.next(List.of(), "major").toString());
|
|
var versions = List.of(ReleaseVersion.parse("v1.2.3"), ReleaseVersion.parse("v1.1.9"));
|
|
CiTests.equal("v1.2.4", ReleaseVersion.next(versions, "patch").toString());
|
|
CiTests.equal("v1.3.0", ReleaseVersion.next(versions, "minor").toString());
|
|
CiTests.equal("v2.0.0", ReleaseVersion.next(versions, "major").toString());
|
|
CiTests.equal(
|
|
"v1.0.100000000000000000000",
|
|
ReleaseVersion.next(List.of(ReleaseVersion.parse("v1.0.99999999999999999999")), "patch")
|
|
.toString());
|
|
for (String invalid : List.of("1.0.0", "v01.0.0", "v1.0.0-rc1", "v1.0.0+build", "v1.0", "v-1.0.0"))
|
|
CiTests.failure(() -> ReleaseVersion.parse(invalid));
|
|
});
|
|
test("branch, nonmanual, malformed input and conflicting new-release inputs are rejected", () -> {
|
|
var env = environment();
|
|
Release.validate(env, "deploy");
|
|
env.put("CI_REF", "refs/heads/feature/S6");
|
|
CiTests.failure(() -> Release.validate(env, "deploy"));
|
|
env.put("CI_REF", "refs/heads/main");
|
|
env.put("CI_EVENT", "push");
|
|
CiTests.failure(() -> Release.validate(env, "rollback"));
|
|
env.put("CI_EVENT", "workflow_dispatch");
|
|
env.put("RELEASE_VERSION", "v1.0.0");
|
|
CiTests.failure(() -> Release.validate(env, "deploy"));
|
|
env.put("RELEASE_MODE", "retry");
|
|
Release.validate(env, "deploy");
|
|
env.put("RELEASE_VERSION", "bad");
|
|
CiTests.failure(() -> Release.validate(env, "deploy"));
|
|
});
|
|
test("successful release stages images before tag, checks main again and commits state", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.release.run();
|
|
CiTests.equal("v1.0.0", f.remote.state.current().version().toString());
|
|
CiTests.equal(false, f.remote.locked);
|
|
CiTests.equal(false, f.remote.pending);
|
|
CiTests.equal(true, f.events.indexOf("prepare") < f.events.indexOf("tag"));
|
|
CiTests.equal(true, f.events.indexOf("tag") < f.events.lastIndexOf("head"));
|
|
CiTests.equal(true, f.events.lastIndexOf("head") < f.events.indexOf("apply:attempt"));
|
|
}
|
|
});
|
|
test("unverified commit never creates a tag or replaces production", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.api.verified = false;
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(false, f.events.contains("tag"));
|
|
CiTests.equal(false, f.events.contains("apply:attempt"));
|
|
CiTests.equal(false, f.remote.locked);
|
|
}
|
|
});
|
|
test("missing images prevent tag allocation and replacement", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.remote.failAction = "prepare";
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(false, f.events.contains("tag"));
|
|
CiTests.equal(false, f.remote.pending);
|
|
}
|
|
});
|
|
test("a tagged commit cannot receive another new version", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.api.tags.add(new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.0"), SHA, OLD));
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(false, f.events.contains("prepare"));
|
|
}
|
|
});
|
|
test("stale dispatch is rejected before pulling or tagging", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.api.head = OLD;
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(false, f.events.contains("prepare"));
|
|
}
|
|
});
|
|
test("main advancing after tag creation preserves tag without changing production", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.api.advanceAfterTag = true;
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(1, f.api.tags.size());
|
|
CiTests.equal(false, f.remote.pending);
|
|
CiTests.equal(false, f.events.contains("apply:attempt"));
|
|
}
|
|
});
|
|
test("unavailable recovery images block replacement", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.withPrior();
|
|
f.remote.failAction = "ensure";
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(false, f.events.contains("tag"));
|
|
CiTests.equal(false, f.remote.pending);
|
|
}
|
|
});
|
|
test("failed candidate restores prior version and still fails the workflow", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.withPrior();
|
|
f.remote.failedDirectories.add("attempt");
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(
|
|
List.of("apply:attempt", "apply:previous-current"),
|
|
f.events.stream().filter(e -> e.startsWith("apply:")).toList());
|
|
CiTests.equal(prior(), f.remote.state);
|
|
CiTests.equal(false, f.remote.pending);
|
|
CiTests.equal(false, f.remote.locked);
|
|
CiTests.equal(2, f.api.tags.size());
|
|
}
|
|
});
|
|
test("failed first release retains journal and lock for manual recovery", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.remote.failedDirectories.add("attempt");
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(true, f.remote.pending);
|
|
CiTests.equal(true, f.remote.locked);
|
|
CiTests.equal(null, f.remote.state.current());
|
|
}
|
|
});
|
|
test("failed automatic recovery retains journal and lock without retry loop", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.withPrior();
|
|
f.remote.failedDirectories.addAll(List.of("attempt", "previous-current"));
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(
|
|
2L,
|
|
f.events.stream().filter(e -> e.startsWith("apply:")).count());
|
|
CiTests.equal(true, f.remote.pending);
|
|
CiTests.equal(true, f.remote.locked);
|
|
}
|
|
});
|
|
test("ambiguous state commit never launches speculative rollback", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.withPrior();
|
|
f.remote.failAction = "commit";
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(
|
|
1L,
|
|
f.events.stream().filter(e -> e.startsWith("apply:")).count());
|
|
CiTests.equal(true, f.remote.pending);
|
|
CiTests.equal(true, f.remote.locked);
|
|
}
|
|
});
|
|
test("retry keeps existing version and SHA after main advances", () -> {
|
|
try (var f = new Fixture("retry")) {
|
|
f.api.head = OLD;
|
|
f.release.run();
|
|
CiTests.equal(SHA, f.remote.state.current().sha());
|
|
CiTests.equal("v1.0.0", f.remote.state.current().version().toString());
|
|
CiTests.equal(false, f.events.contains("tag"));
|
|
CiTests.equal(true, f.events.contains("annotated"));
|
|
}
|
|
});
|
|
test("retry cannot cross the successful-version high watermark after rollback", () -> {
|
|
try (var f = new Fixture("retry")) {
|
|
f.remote.state = prior().success(prior().previous());
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(false, f.events.contains("apply:attempt"));
|
|
}
|
|
});
|
|
test("manual rollback preserves version baseline and creates no tag", () -> {
|
|
try (var f = new Fixture("rollback")) {
|
|
f.remote.state = prior();
|
|
f.release.run();
|
|
CiTests.equal("v1.0.0", f.remote.state.current().version().toString());
|
|
CiTests.equal("v1.0.1", f.remote.state.previous().version().toString());
|
|
CiTests.equal("v1.0.1", f.remote.state.highest().toString());
|
|
CiTests.equal(false, f.events.contains("tag"));
|
|
}
|
|
});
|
|
test("redeploying current version preserves a distinct rollback target", () -> {
|
|
var state = prior();
|
|
CiTests.equal(state.previous(), state.success(state.current()).previous());
|
|
CiTests.equal(state, ReleaseState.parse(state.json()));
|
|
});
|
|
test("NAS lock contention cannot allocate tags or replace production", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.remote.locked = true;
|
|
CiTests.failure(f.release::run);
|
|
CiTests.equal(false, f.events.contains("tag"));
|
|
CiTests.equal(true, f.remote.locked);
|
|
}
|
|
});
|
|
apiTests();
|
|
shellTests();
|
|
ReleaseShellTests.run();
|
|
System.out.println(count + " release behavior tests passed");
|
|
}
|
|
|
|
static class FakeApi extends ReleaseApi {
|
|
final List<String> events;
|
|
final List<Tag> tags = new ArrayList<>();
|
|
String head = SHA;
|
|
boolean verified = true, advanceAfterTag;
|
|
|
|
FakeApi(Map<String, String> env, List<String> events) {
|
|
super(env);
|
|
this.events = events;
|
|
}
|
|
|
|
@Override
|
|
String head() {
|
|
events.add("head");
|
|
return head;
|
|
}
|
|
|
|
@Override
|
|
List<Tag> tags() {
|
|
return List.copyOf(tags);
|
|
}
|
|
|
|
@Override
|
|
String verified(String sha) {
|
|
Pipeline.check(verified, "Unverified test commit");
|
|
events.add("verified");
|
|
return "10";
|
|
}
|
|
|
|
@Override
|
|
void annotated(Tag tag) {
|
|
events.add("annotated");
|
|
}
|
|
|
|
@Override
|
|
void create(ReleaseVersion version, String sha, String attempt) {
|
|
events.add("tag");
|
|
tags.add(new Tag(version, sha, OLD));
|
|
if (advanceAfterTag) head = OLD;
|
|
}
|
|
}
|
|
|
|
static class FakeRemote extends ReleaseRemote {
|
|
final List<String> events;
|
|
final List<String> failedDirectories = new ArrayList<>();
|
|
ReleaseState state = new ReleaseState(null, null, null);
|
|
boolean locked, pending;
|
|
String failAction = "";
|
|
|
|
FakeRemote(Path root, Map<String, String> env, List<String> events) throws Exception {
|
|
super(root, env, new Commands(Duration.ofMinutes(1)), "attempt");
|
|
this.events = events;
|
|
}
|
|
|
|
@Override
|
|
String call(String action, String input, String... args) {
|
|
events.add(action.equals("apply") ? action + ":" + args[0] : action);
|
|
Pipeline.check(!action.equals(failAction), "Injected remote failure");
|
|
switch (action) {
|
|
case "lock" -> {
|
|
Pipeline.check(!locked && !pending, "Locked");
|
|
locked = true;
|
|
}
|
|
case "unlock" -> {
|
|
Pipeline.check(!pending, "Pending");
|
|
locked = false;
|
|
}
|
|
case "state" -> {
|
|
return state.current() == null ? "{}" : state.json();
|
|
}
|
|
case "pending" -> pending = true;
|
|
case "finish" -> pending = false;
|
|
case "commit" -> state = ReleaseState.parse(input);
|
|
case "prepare" -> {
|
|
return "S6_BACKEND=" + deployment("v1.0.0", SHA, "attempt").backend() + "\nS6_FRONTEND="
|
|
+ deployment("v1.0.0", SHA, "attempt").frontend();
|
|
}
|
|
case "apply" -> Pipeline.check(!failedDirectories.contains(args[0]), "Injected unhealthy release");
|
|
default -> {}
|
|
}
|
|
return "";
|
|
}
|
|
}
|
|
|
|
static class Fixture implements AutoCloseable {
|
|
final Path root = Files.createTempDirectory("release-test-");
|
|
final Map<String, String> env = environment();
|
|
final List<String> events = new ArrayList<>();
|
|
final FakeApi api;
|
|
final FakeRemote remote;
|
|
final Release release;
|
|
|
|
Fixture() throws Exception {
|
|
this("new");
|
|
}
|
|
|
|
Fixture(String mode) throws Exception {
|
|
Files.createDirectories(root.resolve("deploy"));
|
|
Files.createDirectories(root.resolve("generated/deploy"));
|
|
env.put("PROJECT_NAME", "recipe-demo");
|
|
env.put("BACKEND_MODULE", "backend");
|
|
env.put("FRONTEND_MODULE", "frontend");
|
|
env.put("COMPOSE_PROJECT", "recipe-demo-test");
|
|
Files.writeString(root.resolve("generated/deploy/remote.sh"), "");
|
|
Files.writeString(root.resolve("deploy/compose.yaml"), "services: {}\n");
|
|
Files.writeString(root.resolve("deploy/health.sh"), "application_health() { true; }\n");
|
|
env.put("RELEASE_MODE", mode);
|
|
if (mode.equals("retry")) env.put("RELEASE_VERSION", "v1.0.0");
|
|
api = new FakeApi(env, events);
|
|
if (mode.equals("retry")) api.tags.add(new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.0"), SHA, OLD));
|
|
remote = new FakeRemote(root, env, events);
|
|
release = new Release(root, env, api, remote, mode.equals("rollback") ? "rollback" : "deploy", "attempt");
|
|
}
|
|
|
|
void withPrior() {
|
|
remote.state = prior();
|
|
api.tags.add(new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.1"), OLD, OLD));
|
|
}
|
|
|
|
@Override
|
|
public void close() throws Exception {
|
|
remote.close();
|
|
Pipeline.deleteTree(root);
|
|
}
|
|
}
|
|
|
|
static void apiTests() throws Exception {
|
|
test("NAS registry alias keeps the same repository and rejects unrelated registries", () -> {
|
|
try (var f = new Fixture()) {
|
|
f.env.put("DEPLOY_REGISTRY", "127.0.0.1:3000");
|
|
CiTests.equal("127.0.0.1:3000/owner/recipe-demo-", f.release.prefix());
|
|
f.env.put("DEPLOY_REGISTRY", "other-registry:3000");
|
|
CiTests.failure(f.release::prefix);
|
|
}
|
|
});
|
|
test("repeating remote tag pages fail closed instead of allocating from incomplete history", () -> {
|
|
var api = new ReleaseApi(environment()) {
|
|
@Override
|
|
Object request(String method, String path, Object body) {
|
|
return List.of(Map.of("name", "v1.0.0", "id", OLD, "commit", Map.of("sha", SHA)));
|
|
}
|
|
};
|
|
CiTests.failure(api::tags);
|
|
});
|
|
test("annotated release tag must identify the recorded version and exact SHA", () -> {
|
|
var tag = new ReleaseApi.Tag(ReleaseVersion.parse("v1.0.0"), SHA, OLD);
|
|
var api = new ReleaseApi(environment()) {
|
|
@Override
|
|
Object request(String method, String path, Object body) {
|
|
return Map.of("tag", "v1.0.0", "object", Map.of("sha", SHA));
|
|
}
|
|
};
|
|
api.annotated(tag);
|
|
var wrong = new ReleaseApi(environment()) {
|
|
@Override
|
|
Object request(String method, String path, Object body) {
|
|
return Map.of("tag", "v1.0.0", "object", Map.of("sha", OLD));
|
|
}
|
|
};
|
|
CiTests.failure(() -> wrong.annotated(tag));
|
|
});
|
|
test("Gitea API eligibility validates SHA, event, branch, workflow path and successful conclusion", () -> {
|
|
var valid = Map.<String, Object>of(
|
|
"id",
|
|
1,
|
|
"head_sha",
|
|
SHA,
|
|
"event",
|
|
"push",
|
|
"head_branch",
|
|
"main",
|
|
"path",
|
|
"build.yaml@refs/heads/main",
|
|
"status",
|
|
"completed",
|
|
"conclusion",
|
|
"success");
|
|
for (String field : List.of("head_sha", "event", "head_branch", "path", "status", "conclusion")) {
|
|
var invalid = new HashMap<>(valid);
|
|
invalid.put(field, "wrong");
|
|
var api = new ReleaseApi(environment()) {
|
|
@Override
|
|
Object request(String method, String path, Object body) {
|
|
return Map.of("workflow_runs", List.of(invalid));
|
|
}
|
|
};
|
|
CiTests.failure(() -> api.verified(SHA));
|
|
}
|
|
var api = new ReleaseApi(environment()) {
|
|
@Override
|
|
Object request(String method, String path, Object body) {
|
|
return Map.of("workflow_runs", List.of(valid));
|
|
}
|
|
};
|
|
CiTests.equal("1", api.verified(SHA));
|
|
});
|
|
test("remote tag pagination includes failed-release baseline and rejects ambiguous tag formats", () -> {
|
|
var api = new ReleaseApi(environment()) {
|
|
@Override
|
|
Object request(String method, String path, Object body) {
|
|
return path.endsWith("page=1")
|
|
? List.of(Map.of("name", "v1.2.3", "id", OLD, "commit", Map.of("sha", SHA)))
|
|
: List.of();
|
|
}
|
|
};
|
|
CiTests.equal(
|
|
"v1.2.4",
|
|
ReleaseVersion.next(
|
|
api.tags().stream()
|
|
.map(ReleaseApi.Tag::version)
|
|
.toList(),
|
|
"patch")
|
|
.toString());
|
|
var invalid = new ReleaseApi(environment()) {
|
|
@Override
|
|
Object request(String method, String path, Object body) {
|
|
return List.of(Map.of("name", "1.0", "id", OLD, "commit", Map.of("sha", SHA)));
|
|
}
|
|
};
|
|
CiTests.failure(invalid::tags);
|
|
});
|
|
}
|
|
|
|
static void shellTests() throws Exception {
|
|
test("real remote shell atomically excludes a second owner and retains unresolved journal", () -> {
|
|
Path directory = Files.createTempDirectory("release-shell-test-");
|
|
try {
|
|
String script = Files.readString(Path.of("generated/deploy/remote.sh"));
|
|
var commands = new Commands(Duration.ofSeconds(30));
|
|
var env = environment();
|
|
var first = commands.execute(
|
|
List.of("sh", "-c", script, "--", "lock", directory.toString(), "one"),
|
|
directory,
|
|
env,
|
|
null,
|
|
false);
|
|
first.checked();
|
|
CiTests.equal(
|
|
1,
|
|
commands.execute(
|
|
List.of("sh", "-c", script, "--", "lock", directory.toString(), "two"),
|
|
directory,
|
|
env,
|
|
null,
|
|
false)
|
|
.code());
|
|
commands.execute(
|
|
List.of("sh", "-c", script, "--", "pending", directory.toString(), "one"),
|
|
directory,
|
|
env,
|
|
"{\"candidate\":\"test\"}",
|
|
false)
|
|
.checked();
|
|
CiTests.equal(
|
|
1,
|
|
commands.execute(
|
|
List.of("sh", "-c", script, "--", "unlock", directory.toString(), "one"),
|
|
directory,
|
|
env,
|
|
null,
|
|
false)
|
|
.code());
|
|
CiTests.equal(true, Files.exists(directory.resolve("pending.json")));
|
|
commands.execute(
|
|
List.of("sh", "-c", script, "--", "finish", directory.toString(), "one"),
|
|
directory,
|
|
env,
|
|
null,
|
|
false)
|
|
.checked();
|
|
commands.execute(
|
|
List.of("sh", "-c", script, "--", "unlock", directory.toString(), "one"),
|
|
directory,
|
|
env,
|
|
null,
|
|
false)
|
|
.checked();
|
|
CiTests.equal(false, Files.exists(directory.resolve("deploy.lock")));
|
|
CiTests.equal(true, Files.exists(directory.resolve("history/one.json")));
|
|
} finally {
|
|
Pipeline.deleteTree(directory);
|
|
}
|
|
});
|
|
test("SSH shell quoting preserves metacharacters as data", () -> {
|
|
String value = "quote' dollar$ backtick` newline\n";
|
|
var commands = new Commands(Duration.ofSeconds(5));
|
|
CiTests.equal(
|
|
value,
|
|
commands.execute(
|
|
List.of("sh", "-c", "printf '%s' " + ReleaseRemote.quote(value)),
|
|
Path.of("."),
|
|
environment(),
|
|
null,
|
|
false)
|
|
.output());
|
|
});
|
|
}
|
|
}
|