chore: preserve current workspace changes
Save pending configuration, deployment, and documentation work.
This commit is contained in:
@@ -0,0 +1,446 @@
|
||||
import argparse
|
||||
import base64
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
PACKAGE = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_module(name, filename):
|
||||
spec = importlib.util.spec_from_file_location(name, PACKAGE / filename)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
ns = load_module("notification", "notification.py")
|
||||
storage = load_module("prepare_storage", "prepare-storage.py")
|
||||
|
||||
|
||||
def fixture():
|
||||
config = ns.read_json(PACKAGE / "config.example.json")
|
||||
artifacts = ns.read_json(PACKAGE / "artifacts.example.json")
|
||||
artifacts["sourceCommit"] = "a" * 40
|
||||
artifacts["sourceArchiveSha256"] = "b" * 64
|
||||
for key, value in artifacts["images"].items():
|
||||
value["image"] = "registry.hrynco.com/test/" + key + "@sha256:" + "c" * 64
|
||||
if key in ("web", "worker", "migrator"):
|
||||
value["sourceArchiveSha256"] = artifacts["sourceArchiveSha256"]
|
||||
return config, artifacts
|
||||
|
||||
|
||||
class FakeCluster:
|
||||
def __init__(self, history=None, worker=0, worker_error=False, consumer_count=1,
|
||||
postgres_ready=True, postgres_exists=True, storage_error=False):
|
||||
self.schema = list(ns.MIGRATIONS if history is None else history)
|
||||
self.replicas = {"web": 1, "worker": worker, "postgres": 1}
|
||||
if not postgres_exists:
|
||||
self.replicas.pop("postgres")
|
||||
elif not postgres_ready:
|
||||
self.replicas["postgres"] = 0
|
||||
self.events = []
|
||||
self.worker_error = worker_error
|
||||
self.consumer_counts = list(consumer_count) if isinstance(consumer_count, list) else [consumer_count]
|
||||
self.postgres_ready = postgres_ready
|
||||
self.storage_error = storage_error
|
||||
self.applications = []
|
||||
|
||||
def get(self, kind, name=None, namespace=None):
|
||||
if kind in ("namespace", "secret"):
|
||||
return {"metadata": {"labels": ns.labels()}}
|
||||
if kind == "deployment" and name in self.replicas:
|
||||
value = {"metadata": {"generation": 1}, "spec": {"replicas": self.replicas[name]}}
|
||||
if name == "postgres":
|
||||
value["status"] = {"readyReplicas": int(self.postgres_ready), "updatedReplicas": int(self.postgres_ready), "observedGeneration": 1}
|
||||
return value
|
||||
return None
|
||||
|
||||
def history(self):
|
||||
if not self.postgres_ready or self.replicas.get("postgres", 0) != 1:
|
||||
raise RuntimeError("cannot query stopped database")
|
||||
self.events.append(("history",))
|
||||
return list(self.schema)
|
||||
|
||||
def scale(self, component, replicas):
|
||||
self.replicas[component] = replicas
|
||||
self.events.append(("scale", component, replicas))
|
||||
|
||||
def apply(self, resources):
|
||||
self.events.append(("apply",))
|
||||
self.applications.append(copy.deepcopy(resources))
|
||||
for item in resources:
|
||||
if item["kind"] == "Deployment":
|
||||
self.replicas[item["metadata"]["name"]] = item["spec"]["replicas"]
|
||||
|
||||
def rollout(self, component):
|
||||
self.events.append(("rollout", component))
|
||||
if component == "postgres":
|
||||
if self.storage_error:
|
||||
raise RuntimeError("postgres rollout failed")
|
||||
self.postgres_ready = True
|
||||
if component == "worker" and self.worker_error:
|
||||
raise RuntimeError("worker rollout failed")
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
if "psql" in args:
|
||||
return b"t\n"
|
||||
if "messages_ready" in args:
|
||||
return b"notification.send-email\t0\t0\n"
|
||||
self.events.append(("broker-consumer-check",))
|
||||
count = self.consumer_counts.pop(0) if len(self.consumer_counts) > 1 else self.consumer_counts[0]
|
||||
return ("notification.send-email\t" + str(count) + "\n").encode()
|
||||
|
||||
|
||||
class ContractTests(unittest.TestCase):
|
||||
def test_example_requires_real_pinned_artifacts(self):
|
||||
with self.assertRaises(ValueError):
|
||||
ns.manifest(ns.read_json(PACKAGE / "config.example.json"), ns.read_json(PACKAGE / "artifacts.example.json"))
|
||||
|
||||
def test_rejects_mutable_image_and_different_migrator_source(self):
|
||||
config, artifacts = fixture()
|
||||
artifacts["images"]["worker"]["image"] = "registry.hrynco.com/test:latest"
|
||||
with self.assertRaises(ValueError):
|
||||
ns.validate(config, artifacts)
|
||||
config, artifacts = fixture()
|
||||
artifacts["images"]["migrator"]["sourceArchiveSha256"] = "d" * 64
|
||||
with self.assertRaises(ValueError):
|
||||
ns.validate(config, artifacts)
|
||||
|
||||
def test_retained_prebound_storage_and_nonroot_exact_guard(self):
|
||||
config, artifacts = fixture()
|
||||
resources = ns.manifest(config, artifacts)
|
||||
pvs = [x for x in resources if x["kind"] == "PersistentVolume"]
|
||||
self.assertEqual(len(pvs), 2)
|
||||
for pv in pvs:
|
||||
self.assertEqual(pv["spec"]["persistentVolumeReclaimPolicy"], "Retain")
|
||||
self.assertEqual(pv["spec"]["claimRef"]["namespace"], ns.NAMESPACE)
|
||||
self.assertTrue(pv["spec"]["local"]["path"].startswith("/mnt/shared/notification-service/"))
|
||||
for resource in resources:
|
||||
if resource["kind"] != "Deployment":
|
||||
continue
|
||||
pod = resource["spec"]["template"]["spec"]
|
||||
self.assertFalse(pod["automountServiceAccountToken"])
|
||||
self.assertTrue(pod["containers"][0]["securityContext"]["runAsNonRoot"])
|
||||
if resource["metadata"]["name"] in ("postgres", "rabbitmq"):
|
||||
self.assertNotIn("fsGroup", pod.get("securityContext", {}))
|
||||
guard = pod["initContainers"][0]
|
||||
values = {x["name"]: x["value"] for x in guard["env"]}
|
||||
self.assertEqual(values["EXPECTED_UUID"], config["storage"]["uuid"])
|
||||
self.assertEqual(values["EXPECTED_ROLE"], ns.NAMESPACE + "/" + resource["metadata"]["name"])
|
||||
if resource["metadata"]["name"] == "postgres":
|
||||
self.assertFalse(any(m["mountPath"] == "/var/run/postgresql" for m in pod["containers"][0]["volumeMounts"]))
|
||||
|
||||
def test_rabbit_probes_allow_observed_slow_diagnostics_without_rapid_restarts(self):
|
||||
config, artifacts = fixture()
|
||||
broker = next(x for x in ns.manifest(config, artifacts)
|
||||
if x["kind"] == "Deployment" and x["metadata"]["name"] == "rabbitmq")
|
||||
container = broker["spec"]["template"]["spec"]["containers"][0]
|
||||
for name in ("startupProbe", "readinessProbe", "livenessProbe"):
|
||||
probe = container[name]
|
||||
# The constrained live node measured seven seconds for a healthy CLI call.
|
||||
self.assertGreaterEqual(probe["timeoutSeconds"], 15)
|
||||
self.assertGreater(probe["periodSeconds"], probe["timeoutSeconds"])
|
||||
self.assertEqual(probe["exec"]["command"][:2], ["rabbitmq-diagnostics", "-q"])
|
||||
startup = container["startupProbe"]
|
||||
self.assertEqual(startup["periodSeconds"] * startup["failureThreshold"], 180)
|
||||
self.assertEqual(container["readinessProbe"]["exec"]["command"][-1], "check_running")
|
||||
self.assertGreaterEqual(container["livenessProbe"]["periodSeconds"], 30)
|
||||
|
||||
def test_no_public_ports_or_internet_allowance(self):
|
||||
config, artifacts = fixture()
|
||||
resources = ns.manifest(config, artifacts)
|
||||
for item in resources:
|
||||
self.assertNotEqual(item["kind"], "Ingress")
|
||||
if item["kind"] == "Service":
|
||||
self.assertNotIn("type", item["spec"])
|
||||
if item["kind"] == "NetworkPolicy":
|
||||
for rule in item["spec"].get("egress", []):
|
||||
for target in rule.get("to", []):
|
||||
self.assertNotIn("ipBlock", target)
|
||||
self.assertIn("namespaceSelector", target)
|
||||
self.assertIn("podSelector", target)
|
||||
worker = next(x for x in resources if x["kind"] == "Deployment" and x["metadata"]["name"] == "worker")
|
||||
self.assertEqual(worker["spec"]["replicas"], 0)
|
||||
|
||||
def test_broker_hashes_and_scope_are_deterministic(self):
|
||||
config, _ = fixture()
|
||||
credentials = {}
|
||||
for key in ("rabbitAdminPassword", "rabbitWorkerPassword", "rabbitPublisherPassword"):
|
||||
credentials[key] = "secret-test-value"
|
||||
credentials[key + "Salt"] = base64.b64encode(b"salt").decode()
|
||||
first = ns.rabbit_definitions(config, credentials)
|
||||
self.assertEqual(first, ns.rabbit_definitions(config, credentials))
|
||||
self.assertNotIn("secret-test-value", json.dumps(first))
|
||||
worker = next(x for x in first["permissions"] if x["user"] == "notification_worker")
|
||||
self.assertEqual(worker["read"], r"^notification\.send\-email$")
|
||||
self.assertNotIn(".*", worker["write"])
|
||||
for user in ("notification_worker", "invemory_staging"):
|
||||
permissions = next(x for x in first["permissions"] if x["user"] == user)
|
||||
self.assertIsNotNone(re.fullmatch(permissions["write"], "amq.default"))
|
||||
self.assertIsNone(re.fullmatch(permissions["write"], "unreviewed.exchange"))
|
||||
|
||||
def test_secrets_not_present_in_rendered_resources(self):
|
||||
config, artifacts = fixture()
|
||||
self.assertFalse(any(x["kind"] == "Secret" for x in ns.manifest(config, artifacts)))
|
||||
|
||||
def test_storage_provision_rejects_wrong_disk_before_creating(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
target = Path(folder) / "notification-service"
|
||||
result = argparse.Namespace(stdout=json.dumps({"filesystems": [{"target": "/mnt/files", "fstype": "ext4", "uuid": "wrong"}]}).encode())
|
||||
with patch.object(storage.sys, "platform", "linux"), patch.object(storage.os, "geteuid", return_value=0, create=True), patch.object(storage, "BASE", target), patch.object(storage.Path, "is_dir", return_value=True), patch.object(storage.Path, "is_symlink", return_value=False), patch.object(storage.subprocess, "run", return_value=result):
|
||||
with self.assertRaises(RuntimeError):
|
||||
storage.main()
|
||||
self.assertFalse(target.exists())
|
||||
|
||||
|
||||
class ClusterQueries(unittest.TestCase):
|
||||
def cluster(self):
|
||||
return ns.Cluster(argparse.Namespace(kubectl="kubectl", kubeconfig="test-config"))
|
||||
|
||||
def test_empty_pod_listing_uses_json_list_without_ignore_not_found(self):
|
||||
cluster = self.cluster()
|
||||
pod_list = {"apiVersion": "v1", "kind": "PodList", "items": []}
|
||||
with patch.object(cluster, "run", return_value=json.dumps(pod_list).encode()) as run:
|
||||
self.assertEqual(cluster.get("pods"), pod_list)
|
||||
self.assertEqual(run.call_args.args, ("get", "pods", "-n", ns.NAMESPACE, "-o", "json"))
|
||||
|
||||
def test_missing_named_resource_retains_ignore_not_found_semantics(self):
|
||||
cluster = self.cluster()
|
||||
with patch.object(cluster, "run", return_value=b"") as run:
|
||||
self.assertIsNone(cluster.get("deployment", "worker"))
|
||||
self.assertIn("--ignore-not-found", run.call_args.args)
|
||||
|
||||
def test_empty_non_json_listing_fails_with_explicit_gate(self):
|
||||
cluster = self.cluster()
|
||||
with patch.object(cluster, "run", return_value=b""):
|
||||
with self.assertRaisesRegex(RuntimeError, "no JSON for resource listing"):
|
||||
cluster.get("pods")
|
||||
|
||||
def test_stopping_worker_with_no_pods_succeeds_without_wait(self):
|
||||
cluster = self.cluster()
|
||||
calls = []
|
||||
|
||||
def run(*args, **kwargs):
|
||||
calls.append(args)
|
||||
if args[:3] == ("get", "deployment", "worker"):
|
||||
return json.dumps({"spec": {"replicas": 0}}).encode()
|
||||
if args[:2] == ("get", "pods"):
|
||||
return json.dumps({"apiVersion": "v1", "kind": "PodList", "items": []}).encode()
|
||||
if args[:2] == ("scale", "deployment/worker"):
|
||||
return b"deployment.apps/worker scaled\n"
|
||||
raise AssertionError("unexpected operation during empty-pod scale")
|
||||
|
||||
with patch.object(cluster, "run", side_effect=run):
|
||||
cluster.scale("worker", 0)
|
||||
self.assertTrue(any(call[0] == "scale" for call in calls))
|
||||
self.assertFalse(any(call[0] == "wait" for call in calls))
|
||||
self.assertNotIn("--ignore-not-found", next(call for call in calls if call[:2] == ("get", "pods")))
|
||||
|
||||
|
||||
class DeploymentGates(unittest.TestCase):
|
||||
def execute(self, cluster, folder, migrate=False, enable=False, verified=False, backup_error=False, job_error=False):
|
||||
config, artifacts = fixture()
|
||||
args = argparse.Namespace(private_dir=folder, migrate=migrate, enable_worker=enable, sandbox_verified=verified)
|
||||
|
||||
def save_backup(client, private_dir):
|
||||
client.events.append(("backup",))
|
||||
if backup_error:
|
||||
raise RuntimeError("backup failed")
|
||||
|
||||
def run_job(client, value):
|
||||
client.events.append(("job", value["metadata"]["name"]))
|
||||
if job_error:
|
||||
raise RuntimeError("migration failed")
|
||||
client.schema = list(ns.MIGRATIONS)
|
||||
|
||||
elapsed = [0]
|
||||
|
||||
def sleep(seconds):
|
||||
elapsed[0] += seconds
|
||||
|
||||
with patch.object(ns, "preflight"), patch.object(ns, "host_storage_check"), patch.object(ns, "backup", side_effect=save_backup), patch.object(ns, "run_job", side_effect=run_job), patch.object(ns.time, "monotonic", side_effect=lambda: elapsed[0]), patch.object(ns.time, "sleep", side_effect=sleep):
|
||||
ns.deploy(cluster, config, artifacts, args)
|
||||
|
||||
def test_ordinary_repeat_preserves_stopped_worker(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=0)
|
||||
self.execute(cluster, folder)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertNotIn(("backup",), cluster.events)
|
||||
self.assertNotIn(("rollout", "worker"), cluster.events)
|
||||
|
||||
def test_ordinary_repeat_waits_for_running_worker_and_checks_consumer(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1)
|
||||
self.execute(cluster, folder)
|
||||
self.assertIn(("rollout", "worker"), cluster.events)
|
||||
self.assertIn(("broker-consumer-check",), cluster.events)
|
||||
|
||||
def test_ordinary_repeat_rejects_failed_worker_rollout(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1, worker_error=True)
|
||||
with self.assertRaisesRegex(RuntimeError, "worker rollout failed"):
|
||||
self.execute(cluster, folder)
|
||||
self.assertNotIn(("broker-consumer-check",), cluster.events)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
|
||||
def test_ordinary_repeat_stops_worker_when_consumer_is_missing(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1, consumer_count=0)
|
||||
with self.assertRaisesRegex(RuntimeError, "exactly one"):
|
||||
self.execute(cluster, folder)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertEqual(cluster.events.count(("broker-consumer-check",)), 30)
|
||||
|
||||
def test_enabling_worker_waits_for_delayed_consumer(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(consumer_count=[0, 0, 1])
|
||||
self.execute(cluster, folder, enable=True, verified=True)
|
||||
self.assertEqual(cluster.replicas["worker"], 1)
|
||||
self.assertEqual(cluster.events.count(("broker-consumer-check",)), 3)
|
||||
self.assertLess(cluster.events.index(("rollout", "worker")), cluster.events.index(("broker-consumer-check",)))
|
||||
|
||||
def test_ordinary_repeat_waits_for_delayed_consumer(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1, consumer_count=[0, 1])
|
||||
self.execute(cluster, folder)
|
||||
self.assertEqual(cluster.replicas["worker"], 1)
|
||||
self.assertEqual(cluster.events.count(("broker-consumer-check",)), 2)
|
||||
|
||||
def test_extra_consumer_stops_worker_immediately(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(consumer_count=[2, 1])
|
||||
with self.assertRaisesRegex(RuntimeError, "unexpected broker count"):
|
||||
self.execute(cluster, folder, enable=True, verified=True)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertEqual(cluster.events.count(("broker-consumer-check",)), 1)
|
||||
|
||||
def test_consumer_gate_zero_timeout_performs_no_broker_query(self):
|
||||
config, _ = fixture()
|
||||
cluster = FakeCluster()
|
||||
with self.assertRaisesRegex(RuntimeError, "before timeout"):
|
||||
ns.wait_for_worker_consumer(cluster, config, timeout=0)
|
||||
self.assertEqual(cluster.events, [])
|
||||
|
||||
def test_consumer_query_has_bounded_timeout_and_redacts_failure(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1)
|
||||
with patch.object(cluster, "run", side_effect=RuntimeError("private broker detail")) as run:
|
||||
with self.assertRaisesRegex(RuntimeError, "could not verify") as error:
|
||||
self.execute(cluster, folder)
|
||||
self.assertNotIn("private broker detail", str(error.exception))
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertEqual(run.call_args.kwargs, {"timeout": 20, "sensitive": True})
|
||||
|
||||
def test_consumer_query_cannot_succeed_after_deadline(self):
|
||||
config, _ = fixture()
|
||||
with patch.object(ns.time, "monotonic", side_effect=[0, 0, 61]):
|
||||
with self.assertRaisesRegex(RuntimeError, "before timeout"):
|
||||
ns.wait_for_worker_consumer(FakeCluster(), config)
|
||||
|
||||
def test_backup_failure_leaves_both_writers_stopped(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1)
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.execute(cluster, folder, migrate=True, backup_error=True)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertEqual(cluster.replicas["web"], 0)
|
||||
self.assertNotIn(("apply",), cluster.events)
|
||||
|
||||
def test_migration_failure_leaves_both_writers_stopped(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1)
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.execute(cluster, folder, migrate=True, job_error=True)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertEqual(cluster.replicas["web"], 0)
|
||||
self.assertLess(cluster.events.index(("backup",)), cluster.events.index(("job", "notification-migrator")))
|
||||
|
||||
def test_noop_migration_resumes_web_after_grants_and_keeps_worker_stopped(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1)
|
||||
self.execute(cluster, folder, migrate=True)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertEqual(cluster.replicas["web"], 1)
|
||||
self.assertLess(cluster.events.index(("job", "notification-grants")), cluster.events.index(("scale", "web", 1)))
|
||||
self.assertLess(cluster.events.index(("job", "notification-sandbox")), cluster.events.index(("scale", "web", 1)))
|
||||
|
||||
def test_unknown_history_stops_before_mutation(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(history=["unknown"])
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.execute(cluster, folder, migrate=True)
|
||||
self.assertEqual(cluster.events, [("history",)])
|
||||
|
||||
def test_interrupted_first_install_recovers_storage_before_history_then_backs_up(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(history=[], worker=0, postgres_ready=False)
|
||||
self.execute(cluster, folder, migrate=True)
|
||||
self.assertLess(cluster.events.index(("rollout", "postgres")), cluster.events.index(("history",)))
|
||||
self.assertLess(cluster.events.index(("history",)), cluster.events.index(("backup",)))
|
||||
self.assertLess(cluster.events.index(("backup",)), cluster.events.index(("job", "notification-migrator")))
|
||||
first_deployments = {v["metadata"]["name"] for v in cluster.applications[0] if v["kind"] == "Deployment"}
|
||||
self.assertEqual(first_deployments, {"postgres", "rabbitmq", "mailpit"})
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertEqual(cluster.replicas["web"], 1)
|
||||
|
||||
def test_missing_deployment_does_not_skip_backup_of_retained_database(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(postgres_exists=False, postgres_ready=False)
|
||||
self.execute(cluster, folder, migrate=True)
|
||||
self.assertIn(("backup",), cluster.events)
|
||||
self.assertLess(cluster.events.index(("backup",)), cluster.events.index(("job", "notification-migrator")))
|
||||
|
||||
def test_recovery_backup_failure_keeps_writers_stopped_and_never_migrates(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1, postgres_ready=False)
|
||||
with self.assertRaisesRegex(RuntimeError, "backup failed"):
|
||||
self.execute(cluster, folder, migrate=True, backup_error=True)
|
||||
self.assertEqual(cluster.replicas["web"], 0)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertEqual(len(cluster.applications), 1) # Foundation only.
|
||||
self.assertFalse(any(event[0] == "job" for event in cluster.events))
|
||||
|
||||
def test_recovery_storage_failure_does_not_query_history_or_start_writers(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(worker=1, postgres_ready=False, storage_error=True)
|
||||
with self.assertRaisesRegex(RuntimeError, "postgres rollout failed"):
|
||||
self.execute(cluster, folder, migrate=True)
|
||||
self.assertNotIn(("history",), cluster.events)
|
||||
self.assertNotIn(("backup",), cluster.events)
|
||||
self.assertEqual(cluster.replicas["web"], 0)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
|
||||
def test_recovery_unknown_history_keeps_writers_stopped_and_never_migrates(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(history=["unknown"], worker=1, postgres_ready=False)
|
||||
with self.assertRaisesRegex(RuntimeError, "unknown or non-prefix"):
|
||||
self.execute(cluster, folder, migrate=True)
|
||||
self.assertEqual(cluster.replicas["web"], 0)
|
||||
self.assertEqual(cluster.replicas["worker"], 0)
|
||||
self.assertFalse(any(event[0] == "job" for event in cluster.events))
|
||||
|
||||
def test_ordinary_deploy_of_stopped_database_requires_recovery_without_mutation(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster(postgres_ready=False)
|
||||
with self.assertRaisesRegex(RuntimeError, "recovery requires --migrate"):
|
||||
self.execute(cluster, folder)
|
||||
self.assertEqual(cluster.events, [])
|
||||
|
||||
def test_enabling_worker_without_inspection_fails_before_mutation(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
cluster = FakeCluster()
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.execute(cluster, folder, enable=True)
|
||||
self.assertEqual(cluster.events, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,317 @@
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
PACKAGE = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location("prepare_seq_key", PACKAGE / "prepare-seq-key.py")
|
||||
seq = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(seq)
|
||||
|
||||
|
||||
def key(token="a" * 64):
|
||||
return {"Id": "apikey-notification", "Title": seq.TITLE, "Token": token,
|
||||
"AssignedPermissions": ["Ingest"], "InputSettings": {"AppliedProperties": copy.deepcopy(seq.PROPERTIES)}}
|
||||
|
||||
|
||||
class FakeSeq:
|
||||
def __init__(self, keys=None, token_file=None, fail_after_create=False, fail_before_put=False, fail_after_put=False):
|
||||
self.keys = copy.deepcopy(keys or [])
|
||||
self.calls = []
|
||||
self.token_file = token_file
|
||||
self.fail_after_create = fail_after_create
|
||||
self.events = []
|
||||
self.fail_before_put = fail_before_put
|
||||
self.fail_after_put = fail_after_put
|
||||
|
||||
def request(self, method, path, body=None, key=None):
|
||||
self.calls.append((method, path))
|
||||
if method == "GET" and path in ("api/apikeys", seq.KEY_LIST):
|
||||
# Actual API may omit Token: probe must verify it independently.
|
||||
visible = self.keys if path == seq.KEY_LIST else [v for v in self.keys if v.get("OwnerId") is not None]
|
||||
return [{k: copy.deepcopy(v) for k, v in value.items() if k != "Token"} for value in visible]
|
||||
if path == "api/apikeys/template":
|
||||
return {"InputSettings": {"AppliedProperties": []}}
|
||||
if path == "api/apikeys/":
|
||||
self.assert_saved_before_post(body["Token"])
|
||||
value = copy.deepcopy(body)
|
||||
value["Id"] = "apikey-notification"
|
||||
self.keys.append(value)
|
||||
if self.fail_after_create:
|
||||
raise RuntimeError("trusted Seq endpoint unavailable")
|
||||
return copy.deepcopy(value)
|
||||
if method == "PUT" and path.startswith("api/apikeys/"):
|
||||
if self.fail_before_put:
|
||||
raise RuntimeError("trusted Seq endpoint unavailable")
|
||||
index = next(i for i, value in enumerate(self.keys) if path == "api/apikeys/" + value["Id"])
|
||||
original_token = self.keys[index].get("Token")
|
||||
self.keys[index] = copy.deepcopy(body)
|
||||
self.keys[index]["Token"] = original_token
|
||||
if self.fail_after_put:
|
||||
raise RuntimeError("trusted Seq endpoint unavailable")
|
||||
return copy.deepcopy(self.keys[index])
|
||||
if path == "api/events/raw":
|
||||
origin = next((value for value in self.keys if value.get("Token") == key), None)
|
||||
if origin is None:
|
||||
raise RuntimeError("Seq request denied (HTTP 401)")
|
||||
canary = json.loads(body)
|
||||
if set(canary) != {"@t", "@mt", "Application", "ProvisioningCheckId"}:
|
||||
raise AssertionError("unexpected canary scope")
|
||||
properties = {k: v for k, v in canary.items() if not k.startswith("@")}
|
||||
properties.update({p["Name"]: p["Value"] for p in origin["InputSettings"]["AppliedProperties"]})
|
||||
self.events.append({"ApiKeyId": origin["Id"], "Properties": [{"Name": k, "Value": v} for k, v in properties.items()]})
|
||||
return None
|
||||
if method == "GET" and path.startswith("api/events?count=2&filter="):
|
||||
query = seq.urllib.parse.unquote(path.split("filter=", 1)[1])
|
||||
check_id = query.split("'", 2)[1]
|
||||
return [event for event in self.events if any(p == {"Name": "ProvisioningCheckId", "Value": check_id} for p in event["Properties"])]
|
||||
raise AssertionError("unexpected API operation")
|
||||
|
||||
def assert_saved_before_post(self, token):
|
||||
if self.token_file is None or self.token_file.read_text().strip() != token:
|
||||
raise AssertionError("token was not persisted before create")
|
||||
|
||||
|
||||
class SeqKeyTests(unittest.TestCase):
|
||||
def test_create_preserves_token_before_post_and_does_not_touch_staging_key(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
staging = {"Id": "apikey-staging", "Title": "invemory-staging-ingest", "AssignedPermissions": ["Ingest"]}
|
||||
client = FakeSeq([staging], path)
|
||||
self.assertEqual(seq.provision(client, path), "apikey-notification")
|
||||
self.assertEqual(client.keys[0], staging)
|
||||
self.assertEqual(client.keys[1]["AssignedPermissions"], ["Ingest"])
|
||||
self.assertEqual(client.keys[1]["InputSettings"]["AppliedProperties"], seq.PROPERTIES + [{"Name": seq.MARKER, "Value": "apikey-notification"}])
|
||||
self.assertEqual(path.read_text().strip(), client.keys[1]["Token"])
|
||||
|
||||
def test_repeat_verifies_existing_key_without_rotation(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
client = FakeSeq(token_file=path)
|
||||
seq.provision(client, path)
|
||||
original = path.read_bytes()
|
||||
client.calls.clear()
|
||||
seq.provision(client, path)
|
||||
self.assertEqual(path.read_bytes(), original)
|
||||
self.assertEqual(len(client.keys), 1)
|
||||
self.assertNotIn(("POST", "api/apikeys/"), client.calls)
|
||||
self.assertFalse(any(method == "PUT" for method, _ in client.calls))
|
||||
self.assertTrue(all(seq.allowed_path(method, path) for method, path in client.calls))
|
||||
|
||||
def test_ambiguous_keys_fail_before_token_generation(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
with self.assertRaisesRegex(RuntimeError, "ambiguous"):
|
||||
seq.provision(FakeSeq([key(), key()]), path)
|
||||
self.assertFalse(path.exists())
|
||||
|
||||
def test_existing_key_with_extra_permissions_is_not_modified(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
existing = key()
|
||||
existing["AssignedPermissions"].append("Read")
|
||||
client = FakeSeq([existing])
|
||||
with self.assertRaisesRegex(RuntimeError, "only Ingest"):
|
||||
seq.provision(client, path)
|
||||
self.assertFalse(path.exists())
|
||||
self.assertEqual(client.keys, [existing])
|
||||
|
||||
def test_existing_key_with_wrong_stamp_is_not_modified(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
existing = key()
|
||||
existing["InputSettings"]["AppliedProperties"][0]["Value"] = "Staging"
|
||||
client = FakeSeq([existing])
|
||||
with self.assertRaisesRegex(RuntimeError, "properties mismatch"):
|
||||
seq.provision(client, path)
|
||||
self.assertFalse(path.exists())
|
||||
self.assertEqual(client.keys, [existing])
|
||||
|
||||
def test_missing_local_token_does_not_rotate_remote_key(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
client = FakeSeq([key()])
|
||||
with self.assertRaisesRegex(RuntimeError, "local token is missing"):
|
||||
seq.provision(client, path)
|
||||
self.assertFalse(path.exists())
|
||||
self.assertEqual(client.calls, [("GET", seq.KEY_LIST)])
|
||||
|
||||
def test_unknown_post_outcome_is_recovered_with_original_token(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
client = FakeSeq(token_file=path, fail_after_create=True)
|
||||
with self.assertRaises(RuntimeError):
|
||||
seq.provision(client, path)
|
||||
original = path.read_bytes()
|
||||
client.fail_after_create = False
|
||||
seq.provision(client, path)
|
||||
self.assertEqual(path.read_bytes(), original)
|
||||
self.assertEqual(len(client.keys), 1)
|
||||
|
||||
def test_wrong_preserved_token_fails_probe_without_rotation(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
seq.private_directory(path.parent)
|
||||
path.write_text("b" * 64)
|
||||
path.chmod(0o600)
|
||||
client = FakeSeq([key()])
|
||||
with self.assertRaisesRegex(RuntimeError, "HTTP 401"):
|
||||
seq.provision(client, path)
|
||||
self.assertEqual(path.read_text(), "b" * 64)
|
||||
self.assertNotIn(("POST", "api/apikeys/"), client.calls)
|
||||
|
||||
def test_valid_foreign_staging_token_is_rejected_by_canary_identity(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
seq.private_directory(path.parent)
|
||||
path.write_text("b" * 64)
|
||||
path.chmod(0o600)
|
||||
staging = key("b" * 64)
|
||||
staging.update(Id="apikey-staging", Title="invemory-staging-ingest")
|
||||
staging["InputSettings"]["AppliedProperties"] = [{"Name": "Environment", "Value": "Staging"}]
|
||||
client = FakeSeq([key(), staging])
|
||||
with self.assertRaisesRegex(RuntimeError, "different or unverified key"):
|
||||
seq.provision(client, path)
|
||||
self.assertIn(("POST", "api/events/raw"), client.calls) # Valid ingestion alone is insufficient.
|
||||
self.assertNotIn(("POST", "api/apikeys/"), client.calls)
|
||||
self.assertEqual(path.read_text(), "b" * 64)
|
||||
self.assertEqual(client.keys[1], staging)
|
||||
self.assertEqual(client.keys[0]["Token"], key()["Token"])
|
||||
|
||||
def test_foreign_key_with_matching_properties_is_still_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
seq.private_directory(path.parent)
|
||||
path.write_text("b" * 64)
|
||||
path.chmod(0o600)
|
||||
foreign = key("b" * 64)
|
||||
foreign.update(Id="apikey-foreign", Title="Other Service")
|
||||
with self.assertRaisesRegex(RuntimeError, "different or unverified key"):
|
||||
seq.provision(FakeSeq([key(), foreign]), path)
|
||||
|
||||
def test_server_stamped_marker_proves_identity_without_event_key_id_or_token(self):
|
||||
client = FakeSeq()
|
||||
check_id = "c" * 32
|
||||
client.events = [{"Properties": [{"Name": "ProvisioningCheckId", "Value": check_id},
|
||||
{"Name": seq.MARKER, "Value": "apikey-notification"}] + copy.deepcopy(seq.PROPERTIES)}]
|
||||
redacted = key()
|
||||
redacted["Token"] = None
|
||||
seq.verify_canary(client, check_id, redacted)
|
||||
|
||||
def test_missing_server_stamped_marker_fails_closed(self):
|
||||
client = FakeSeq()
|
||||
check_id = "c" * 32
|
||||
client.events = [{"Properties": [{"Name": "ProvisioningCheckId", "Value": check_id}] + copy.deepcopy(seq.PROPERTIES)}]
|
||||
redacted = key()
|
||||
redacted.pop("Token")
|
||||
with self.assertRaisesRegex(RuntimeError, "unverified key"):
|
||||
seq.verify_canary(client, check_id, redacted)
|
||||
|
||||
def test_correct_key_id_still_requires_actual_shared_stamps(self):
|
||||
client = FakeSeq()
|
||||
check_id = "c" * 32
|
||||
client.events = [{"ApiKeyId": "apikey-notification", "Properties": [
|
||||
{"Name": "ProvisioningCheckId", "Value": check_id}, {"Name": seq.MARKER, "Value": "apikey-notification"},
|
||||
{"Name": "Environment", "Value": "Staging"}]}]
|
||||
with self.assertRaisesRegex(RuntimeError, "stamped identity mismatch"):
|
||||
seq.verify_canary(client, check_id, key())
|
||||
|
||||
def test_marker_collision_fails_before_put_or_ingestion(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
seq.private_directory(path.parent)
|
||||
path.write_text("a" * 64)
|
||||
path.chmod(0o600)
|
||||
foreign = key("b" * 64)
|
||||
foreign.update(Id="apikey-foreign", Title="Other Service")
|
||||
foreign["InputSettings"]["AppliedProperties"].append({"Name": seq.MARKER, "Value": "apikey-notification"})
|
||||
client = FakeSeq([key(), foreign])
|
||||
with self.assertRaisesRegex(RuntimeError, "marker collision"):
|
||||
seq.provision(client, path)
|
||||
self.assertFalse(any(method == "PUT" for method, _ in client.calls))
|
||||
self.assertNotIn(("POST", "api/events/raw"), client.calls)
|
||||
self.assertEqual(client.keys, [key(), foreign])
|
||||
|
||||
def test_shared_foreign_marker_collision_is_visible_in_full_listing(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
seq.private_directory(path.parent)
|
||||
path.write_text("a" * 64)
|
||||
path.chmod(0o600)
|
||||
dedicated = key()
|
||||
dedicated["OwnerId"] = "user-operator"
|
||||
shared = key("b" * 64)
|
||||
shared.update(Id="apikey-shared-foreign", Title="Shared foreign service", OwnerId=None)
|
||||
shared["InputSettings"]["AppliedProperties"].append({"Name": seq.MARKER, "Value": dedicated["Id"]})
|
||||
client = FakeSeq([dedicated, shared])
|
||||
# Demonstrate the narrower default endpoint would conceal the collision.
|
||||
self.assertEqual(len(client.request("GET", "api/apikeys")), 1)
|
||||
client.calls.clear()
|
||||
with self.assertRaisesRegex(RuntimeError, "marker collision"):
|
||||
seq.provision(client, path)
|
||||
self.assertTrue(all(call == ("GET", seq.KEY_LIST) for call in client.calls))
|
||||
self.assertEqual(client.keys, [dedicated, shared])
|
||||
self.assertEqual(path.read_text(), "a" * 64)
|
||||
|
||||
def test_allowlist_requires_full_shared_key_listing(self):
|
||||
self.assertTrue(seq.allowed_path("GET", "api/apikeys?shared=true"))
|
||||
self.assertFalse(seq.allowed_path("GET", "api/apikeys"))
|
||||
self.assertFalse(seq.allowed_path("GET", "api/apikeys?shared=false"))
|
||||
|
||||
def test_guarded_put_preserves_token_permissions_and_all_other_settings(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
seq.private_directory(path.parent)
|
||||
path.write_text("a" * 64)
|
||||
path.chmod(0o600)
|
||||
original = key()
|
||||
original.update(OwnerId=None, IsDefault=False, TokenPrefix="aaaa")
|
||||
original["InputSettings"].update(MinimumLevel="Information", Filter=None, CustomSettings={"retained": True})
|
||||
expected = copy.deepcopy(original)
|
||||
expected["InputSettings"]["AppliedProperties"].append({"Name": seq.MARKER, "Value": "apikey-notification"})
|
||||
client = FakeSeq([original])
|
||||
seq.provision(client, path)
|
||||
self.assertEqual(client.keys, [expected])
|
||||
self.assertEqual(path.read_text(), "a" * 64)
|
||||
self.assertEqual([call for call in client.calls if call[0] == "PUT"], [("PUT", "api/apikeys/apikey-notification")])
|
||||
|
||||
def test_put_failure_preserves_original_key_and_local_token(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
seq.private_directory(path.parent)
|
||||
path.write_text("a" * 64)
|
||||
path.chmod(0o600)
|
||||
client = FakeSeq([key()], fail_before_put=True)
|
||||
with self.assertRaisesRegex(RuntimeError, "unavailable"):
|
||||
seq.provision(client, path)
|
||||
self.assertEqual(client.keys, [key()])
|
||||
self.assertEqual(path.read_text(), "a" * 64)
|
||||
self.assertNotIn(("POST", "api/events/raw"), client.calls)
|
||||
|
||||
def test_unknown_put_outcome_is_recovered_without_another_put_or_rotation(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "private" / "seq-key.txt"
|
||||
seq.private_directory(path.parent)
|
||||
path.write_text("a" * 64)
|
||||
path.chmod(0o600)
|
||||
client = FakeSeq([key()], fail_after_put=True)
|
||||
with self.assertRaisesRegex(RuntimeError, "unavailable"):
|
||||
seq.provision(client, path)
|
||||
client.calls.clear()
|
||||
client.fail_after_put = False
|
||||
seq.provision(client, path)
|
||||
self.assertFalse(any(method == "PUT" for method, _ in client.calls))
|
||||
self.assertNotIn(("POST", "api/apikeys/"), client.calls)
|
||||
self.assertEqual(path.read_text(), "a" * 64)
|
||||
self.assertEqual(len(client.keys), 1)
|
||||
|
||||
def test_redirect_is_rejected_before_credentials_can_be_forwarded(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "redirect denied"):
|
||||
seq.NoRedirects().redirect_request(None, None, 302, "Found", {}, "https://untrusted.example/")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user