feat(notification): enable production publisher

Preserve production broker definitions and allow bounded SMTP delivery.

Ref: IT-1508

docker/kubernetes/shared-notification/notification.py

@@ -6,6 +6,7 @@ import base64
 import contextlib
 import datetime
 import hashlib
+import hmac
 import json
 import os
 from pathlib import Path, PurePosixPath
@@ -20,6 +21,7 @@ MANAGED = "hrynco-shared-notification"
 MIGRATIONS = ["20260501214629_InitialCreate", "20260502154249_PendingChanges"]
 IMAGE = re.compile(r"registry\.hrynco\.com/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$")
 SHA = re.compile(r"[a-f0-9]{64}$")
+PRODUCTION_RESULT_QUEUE = "invemory.production.notification-results"
 
 # Marker must be provisioned only after findmnt proves the backing filesystem UUID.
 # XFS/ext4 statfs plus exact marker prevents a missing-mount fallback onto another FS.
@@ -313,7 +315,7 @@ def network_policies(config):
     return output
 
 
-def rabbit_definitions(config, credentials):
+def rabbit_definitions(config, credentials, production_password_hash=None):
     def hashed(password, salt):
         return base64.b64encode(salt + hashlib.sha256(salt + password.encode()).digest()).decode()
 
@@ -321,27 +323,58 @@ def rabbit_definitions(config, credentials):
     for name, key, tag in (("notification_admin", "rabbitAdminPassword", "administrator"), ("notification_worker", "rabbitWorkerPassword", ""), ("invemory_staging", "rabbitPublisherPassword", "")):
         salt = base64.b64decode(credentials[key + "Salt"])
         users.append({"name": name, "password_hash": hashed(credentials[key], salt), "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": tag})
+    if production_password_hash:
+        users.append({"name": "invemory_production", "password_hash": production_password_hash,
+                      "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": ""})
     incoming = re.escape("notification.send-email")
     reply = re.escape(config["replyQueue"])
+    production_reply = re.escape(PRODUCTION_RESULT_QUEUE)
+    worker_replies = reply + ("|" + production_reply if production_password_hash else "")
     vhost = config["rabbitVirtualHost"]
-    return {"users": users, "vhosts": [{"name": vhost}], "permissions": [
+    permissions = [
         {"user": "notification_admin", "vhost": vhost, "configure": ".*", "write": ".*", "read": ".*"},
-        {"user": "notification_worker", "vhost": vhost, "configure": "^(" + incoming + "|" + reply + ")$", "write": "^(amq\\.default|" + reply + ")$", "read": "^" + incoming + "$"},
-        {"user": "invemory_staging", "vhost": vhost, "configure": "^(" + incoming + "|" + reply + ")$", "write": "^(amq\\.default|" + incoming + ")$", "read": "^" + reply + "$"}],
-        "queues": [{"name": name, "vhost": vhost, "durable": True, "auto_delete": False, "arguments": {}} for name in ("notification.send-email", config["replyQueue"])], "exchanges": [], "bindings": []}
+        {"user": "notification_worker", "vhost": vhost, "configure": "^(" + incoming + "|" + worker_replies + ")$", "write": "^(amq\\.default|" + worker_replies + ")$", "read": "^" + incoming + "$"},
+        {"user": "invemory_staging", "vhost": vhost, "configure": "^(" + incoming + "|" + reply + ")$", "write": "^(amq\\.default|" + incoming + ")$", "read": "^" + reply + "$"}]
+    if production_password_hash:
+        permissions.append({"user": "invemory_production", "vhost": vhost,
+                            "configure": "^(" + incoming + "|" + production_reply + ")$",
+                            "write": "^(amq\\.default|" + incoming + ")$", "read": "^" + production_reply + "$"})
+    queues = ["notification.send-email", config["replyQueue"]]
+    if production_password_hash:
+        queues.append(PRODUCTION_RESULT_QUEUE)
+    return {"users": users, "vhosts": [{"name": vhost}], "permissions": permissions,
+            "queues": [{"name": name, "vhost": vhost, "durable": True, "auto_delete": False, "arguments": {}} for name in queues],
+            "exchanges": [], "bindings": []}
 
 
-def secret_manifests(config, credentials, registry):
+def secret_manifests(config, credentials, registry, production_password_hash=None):
     def secret(name, data, kind="Opaque"):
         return obj("Secret", name, type=kind, data={key: base64.b64encode(value.encode()).decode() for key, value in data.items()})
 
     data = {key: credentials[key] for key in ("postgresPassword", "runtimePassword", "rabbitAdminPassword", "rabbitWorkerPassword", "rabbitPublisherPassword", "seqApiKey")}
     data["ownerConnectionString"] = "Host=postgres;Database=notification_service;Username=notification_owner;Password=" + credentials["postgresPassword"]
     data["runtimeConnectionString"] = "Host=postgres;Database=notification_service;Username=notification_runtime;Password=" + credentials["runtimePassword"]
-    return [secret("notification-private", data), secret("rabbit-definitions", {"definitions.json": json.dumps(rabbit_definitions(config, credentials))}),
+    return [secret("notification-private", data), secret("rabbit-definitions", {"definitions.json": json.dumps(rabbit_definitions(config, credentials, production_password_hash))}),
             secret("registry-pull", {".dockerconfigjson": json.dumps(registry)}, "kubernetes.io/dockerconfigjson")]
 
 
+def preserved_production_password_hash(private_dir):
+    password_path = Path(private_dir) / "invemory-production-rabbitmq-password.txt"
+    hash_path = Path(private_dir) / "invemory-production-rabbitmq-password-hash.txt"
+    if password_path.exists() != hash_path.exists():
+        raise RuntimeError("production RabbitMQ password and boot hash must be preserved together")
+    if not password_path.exists():
+        return None
+    if any(path.stat().st_mode & 0o077 for path in (password_path, hash_path)):
+        raise RuntimeError("production RabbitMQ private file permissions are too broad")
+    password = password_path.read_text(encoding="utf-8").strip()
+    password_hash = hash_path.read_text(encoding="utf-8").strip()
+    decoded = base64.b64decode(password_hash, validate=True)
+    if len(decoded) != 36 or not hmac.compare_digest(decoded[4:], hashlib.sha256(decoded[:4] + password.encode()).digest()):
+        raise RuntimeError("production RabbitMQ boot hash does not match preserved password")
+    return password_hash
+
+
 class Cluster:
     def __init__(self, args):
         self.command = [args.kubectl, "--kubeconfig=" + args.kubeconfig, "--request-timeout=20s"]
@@ -662,7 +695,7 @@ def main(argv=None):
         if namespace and namespace["metadata"].get("labels", {}).get("app.kubernetes.io/managed-by") != MANAGED:
             raise RuntimeError("refusing to adopt existing namespace")
         cluster.apply([resources[0]])
-        values = secret_manifests(config, credentials, registry)
+        values = secret_manifests(config, credentials, registry, preserved_production_password_hash(args.private_dir))
         for value in values:
             current = cluster.get("secret", value["metadata"]["name"])
             if current and current["metadata"].get("labels", {}).get("app.kubernetes.io/managed-by") != MANAGED: