wanjakrah/engineering
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

S19: establish engineering repository

Browse Source
This commit is contained in:
Wanja Krah
2026-11-08 15:15:50 +01:00
commit c25ce11a1b
84 changed files with 6599 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
tests/acceptance.py Normal file
View File

@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Deploy two example revisions and roll back using the real shared remote script."""
import argparse
import os
from pathlib import Path
import shutil
import socket
import subprocess
import tempfile
import uuid
def run(args, *, cwd=None, env=None, input=None):
result = subprocess.run(args, cwd=cwd, env=env, input=input, text=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=600)
if result.returncode:
raise RuntimeError('Command failed: ' + args[0] + '\n' + result.stdout)
return result.stdout.strip()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('project', type=Path)
parser.add_argument('--registry', required=True)
parser.add_argument('--namespace', required=True)
parser.add_argument('--user', required=True)
parser.add_argument('--token-file', type=Path, required=True)
args = parser.parse_args()
project = args.project.resolve()
name = 'recipe-acceptance-' + uuid.uuid4().hex[:10]
env = dict(os.environ)
env['DOCKER_HOST'] = run(['docker', 'context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'])
env.pop('DOCKER_CONTEXT', None)
with tempfile.TemporaryDirectory(prefix='recipe-deploy-') as temporary:
root = Path(temporary)
auth = root / 'docker-auth'
auth.mkdir()
env['DOCKER_CONFIG'] = str(auth)
run(['docker', 'login', args.registry, '--username', args.user, '--password-stdin'],
env=env, input=args.token_file.read_text().strip() + '\n')
with socket.socket() as listener:
listener.bind(('127.0.0.1', 0))
port = listener.getsockname()[1]
(root / 'production.env').write_text(f'BIND_IP=127.0.0.1\nFRONTEND_PORT={port}\nDOCKER_BIN={shutil.which("docker")}\nDOCKER_SUDO=0\nDOCKER_CONFIG={auth}\n')
os.chmod(root / 'production.env', 0o600)
source = (project / 'generated/deploy/remote.sh').read_text()
prefix = f"COMPOSE_PROJECT={name}\nDEPLOY_ROLES='backend frontend'\nDEPENDENCY_SERVICES=''\nTARGET_PLATFORM=linux/amd64\nREQUIRED_SECRETS=''\n"
script = prefix + (project / 'deploy/health.sh').read_text() + '\n' + source
image_prefix = args.registry + '/' + args.namespace + '/' + name + '-'
def remote(action, owner, *values, input=None):
return run(['sh', '-c', script, '--', action, str(root), owner, *values],
env=env, input=input)
def apply(owner, version, revision):
for role in ('backend', 'frontend'):
image = image_prefix + role + ':' + revision
context = project if role == 'backend' else project / 'frontend'
run(['docker', 'build', '--target', 'prebuilt', '--label',
'org.opencontainers.image.revision=' + revision, '-t', image, str(context)], env=env)
run(['docker', 'push', image], env=env)
remote('lock', owner)
remote('preflight', owner)
remote('stage', owner, input=(project / 'deploy/compose.yaml').read_text())
remote('stage-health', owner, input=(project / 'deploy/health.sh').read_text())
prepared = remote('prepare', owner, version, revision, image_prefix)
assert prepared.count('@sha256:') == 2
remote('pending', owner, input='{"isolated_acceptance":true}\n')
assert remote('apply', owner, owner, '60').endswith('S6_HEALTHY')
remote('commit', owner, input='{"isolated_acceptance":true}\n')
remote('finish', owner)
remote('unlock', owner)
try:
apply('first', 'v1.0.0', 'a' * 40)
apply('second', 'v1.0.1', 'b' * 40)
remote('lock', 'rollback')
remote('pending', 'rollback', input='{"rollback":true}\n')
assert remote('apply', 'rollback', 'first', '60').endswith('S6_HEALTHY')
revisions = run(['docker', 'compose', '--project-name', name, '--env-file',
str(root / 'production.env'), '--env-file', str(root / 'releases/first/release.env'),
'-f', str(root / 'releases/first/compose.yaml'), 'ps', '-q'], env=env).splitlines()
assert len(revisions) == 2
for container in revisions:
assert run(['docker', 'inspect', '--format', '{{index .Config.Labels "nl.agilecc.revision"}}', container], env=env) == 'a' * 40
remote('finish', 'rollback')
remote('unlock', 'rollback')
print('PASS: actual digest preparation, two isolated deployments, readiness and rollback to the first revision')
finally:
config = root / 'releases/first/compose.yaml'
if config.exists():
run(['docker', 'compose', '--project-name', name, '--env-file', str(root / 'production.env'),
'--env-file', str(root / 'releases/first/release.env'), '-f', str(config), 'down', '--volumes'], env=env)
if __name__ == '__main__':
main()

51
tests/format_integration.py Normal file
View File

@@ -0,0 +1,51 @@
"""Integration checks against an isolated checkout; run python3 tooling/format/test.py."""
from pathlib import Path
import shutil
import subprocess
import tempfile
root = Path(__file__).resolve().parents[1]
with tempfile.TemporaryDirectory(prefix="palantir-test-") as temporary:
fixture = Path(temporary)
pom = (root / "templates/common/pom.xml").read_text().replace("@PROJECT@", "format-fixture").replace("@MODULES@", "")
(fixture / "pom.xml").write_text(pom)
(fixture / "generated/tooling").mkdir(parents=True)
shutil.copytree(root / "resources/tooling/format", fixture / "generated/tooling/format")
(fixture / "engineering.properties").write_text("FORMAT_DIRECTORIES=backend/src,ci\n")
for name in ("backend/src/main/java", "backend/src/test/java", "ci/runner"):
(fixture / name).mkdir(parents=True)
source = fixture / "ci/runner/Example.java"
original = "import java.util.List;\nclass Example{String text=\"héllo\";int value(){return 1;}}\n"
source.write_text(original)
def run(mode, success=True):
result = subprocess.run(
["mvn", "-N", "-B", "-ntp", "exec:exec@format", "-Dformat.mode=" + mode],
cwd=fixture, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
assert (result.returncode == 0) == success, result.stdout
return result.stdout
output = run("check", False)
assert "ci/runner/Example.java" in output and "mvn -N exec:exec@format" in output
assert source.read_text() == original
run("apply")
formatted = source.read_text()
assert "import java.util.List" not in formatted and "héllo" in formatted
run("check")
run("apply")
assert source.read_text() == formatted
# Force a full (uncached) second pass to prove engine idempotence.
(fixture / "target/palantir-cache.properties").unlink()
run("check")
source.write_text(formatted + "class Broken { ??? }\n")
invalid = source.read_bytes()
output = run("apply", False)
assert "ci/runner/Example.java" in output
assert source.read_bytes() == invalid
# A new file is discovered even after a successful cached check.
source.write_text(formatted)
extra = fixture / "ci/New.java"
extra.write_text("class New{}")
run("check", False)
assert extra.read_text() == "class New{}"
print("PASS: check is read-only, apply/check agree, imports, UTF-8, idempotence, parser errors, new files")

95
tests/test_bootstrap.py Normal file
View File

@@ -0,0 +1,95 @@
"""Exercise release pinning, drift detection, replacement and recovery without network."""
import importlib.util
import json
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
HOME = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location('bootstrap', HOME / 'bootstrap.py')
bootstrap = importlib.util.module_from_spec(spec)
spec.loader.exec_module(bootstrap)
class BootstrapTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name)
self.source = self.root / 'engineering'
self.source.mkdir()
self.git('init', '-q')
self.git('config', 'user.email', 'test@example.invalid')
self.git('config', 'user.name', 'Engineering test')
(self.source / 'resources').mkdir()
(self.source / 'resources/tool').write_text('version one\n')
self.commit = self.release('v1.0.0')
self.project = self.root / 'consumer'
self.project.mkdir()
self.lock = {'repository': str(self.source), 'tag': 'v1.0.0', 'commit': self.commit}
self.save_lock()
def git(self, *args):
return subprocess.check_output(['git', '-C', str(self.source), *args], text=True, stderr=subprocess.DEVNULL).strip()
def release(self, tag):
self.git('add', '.')
self.git('commit', '-qm', tag)
self.git('tag', tag)
return self.git('rev-parse', 'HEAD')
def save_lock(self):
(self.project / 'engineering.lock.json').write_text(json.dumps(self.lock))
def test_absent_generated_and_repeat_are_deterministic(self):
bootstrap.materialize(self.project)
first = bootstrap.files(self.project / 'generated')
bootstrap.materialize(self.project)
self.assertEqual(first, bootstrap.files(self.project / 'generated'))
bootstrap.materialize(self.project, True)
def test_drift_includes_modified_added_and_missing_files(self):
bootstrap.materialize(self.project)
target = self.project / 'generated/tool'
for operation in ('modify', 'remove', 'add'):
with self.subTest(operation=operation):
bootstrap.materialize(self.project)
if operation == 'modify': target.write_text('hand edit')
elif operation == 'remove': target.unlink()
else: (target.parent / 'extra').write_text('untracked')
before = bootstrap.files(target.parent)
with self.assertRaisesRegex(ValueError, 'differ'):
bootstrap.materialize(self.project, True)
self.assertEqual(before, bootstrap.files(target.parent))
def test_update_removes_obsolete_files_and_preserves_project_files(self):
bootstrap.materialize(self.project)
(self.project / 'pom.xml').write_text('owned by project')
(self.source / 'resources/tool').unlink()
(self.source / 'resources/new').write_text('new release')
self.lock.update(tag='v1.1.0', commit=self.release('v1.1.0'))
self.save_lock()
bootstrap.materialize(self.project)
self.assertFalse((self.project / 'generated/tool').exists())
self.assertEqual('owned by project', (self.project / 'pom.xml').read_text())
def test_moved_tag_fails_before_replacing_existing_content(self):
bootstrap.materialize(self.project)
before = bootstrap.files(self.project / 'generated')
(self.source / 'resources/tool').write_text('different commit')
self.release('v1.1.0')
self.git('tag', '-f', 'v1.0.0')
with self.assertRaisesRegex(ValueError, 'tag moved'):
bootstrap.materialize(self.project)
self.assertEqual(before, bootstrap.files(self.project / 'generated'))
def test_symlink_output_is_rejected(self):
(self.project / 'generated').symlink_to(self.source / 'resources', target_is_directory=True)
with self.assertRaisesRegex(ValueError, 'symlink'):
bootstrap.materialize(self.project)
if __name__ == '__main__':
unittest.main()

59
tests/test_generator.py Normal file
View File

@@ -0,0 +1,59 @@
"""Check component selection and real Maven bootstrapping with generated/ absent."""
import importlib.util
import json
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
HOME = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location('generator', HOME / 'create-project.py')
generator = importlib.util.module_from_spec(spec)
spec.loader.exec_module(generator)
class GeneratorTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.temporary = tempfile.TemporaryDirectory()
cls.addClassCleanup(cls.temporary.cleanup)
cls.root = Path(cls.temporary.name)
cls.source = cls.root / 'engineering'
shutil.copytree(HOME, cls.source, ignore=shutil.ignore_patterns('.git','target','__pycache__'))
for args in (['init', '-q'], ['config','user.name','Recipe test'],
['config','user.email','test@example.invalid'], ['add','.'],
['commit','-qm','Test release'], ['tag','v1.0.0']):
subprocess.run(['git','-C',str(cls.source),*args], check=True, stdout=subprocess.DEVNULL)
generator.HOME = cls.source
def test_component_choices_and_bootstrap_without_generated_files(self):
for components in (['backend'], ['frontend'], ['backend','frontend']):
with self.subTest(components=components):
project = self.root / ('consumer-' + '-'.join(components))
generator.create(project, 'sample', components, str(self.source), 'v1.0.0', 'test-runner', 'https://landscape.invalid')
self.assertEqual('backend' in components, (project / 'backend/pom.xml').exists())
self.assertEqual('frontend' in components, (project / 'frontend/package.json').exists())
self.assertIn('test-runner', (project / '.gitea/workflows/build.yaml').read_text())
shutil.rmtree(project / 'generated')
result = subprocess.run(['mvn','-N','-B','-ntp','-Pregenerate-engineering','validate'], cwd=project,
text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=180)
self.assertEqual(0, result.returncode, result.stdout)
self.assertTrue((project / 'generated/ci/Ci.java').exists())
subprocess.run(['python3','scripts/engineering.py','--check'], cwd=project, check=True, stdout=subprocess.DEVNULL)
if components == ['frontend']:
for args in (['npm','ci'], ['npm','run','lint'], ['npm','test'], ['npm','run','build']):
subprocess.run(args, cwd=project / 'frontend', check=True, stdout=subprocess.DEVNULL)
self.assertTrue((project / 'frontend/dist/index.html').exists())
def test_generator_preserves_existing_destination(self):
project = self.root / 'already-exists'
project.mkdir()
(project / 'important').write_text('preserve')
with self.assertRaisesRegex(ValueError, 'must not exist'):
generator.create(project, 'sample', ['backend'], str(self.source), 'v1.0.0', 'runner', 'landscape')
self.assertEqual('preserve', (project / 'important').read_text())
if __name__ == '__main__':
unittest.main()