From 9e268b8ec3d21c1ee0b927b790ab5beb86fb4934 Mon Sep 17 00:00:00 2001 From: Anatolii Hrynchuk Date: Sat, 24 Oct 2026 13:14:16 +0300 Subject: [PATCH] fix(ops): allow google api egress and document recovery Add scoped google api network policy, its refresh command, and the verified production operations runbook. Ref: IT-1074 --- README.md | 1 + .../kubernetes/shared-notification/README.md | 7 + .../google-api-network.json | 1534 +++++++++++++++++ .../refresh-google-api-network.py | 62 + docs/production-operations.md | 564 ++++++ 5 files changed, 2168 insertions(+) create mode 100644 docker/kubernetes/shared-notification/google-api-network.json create mode 100644 docker/kubernetes/shared-notification/refresh-google-api-network.py create mode 100644 docs/production-operations.md diff --git a/README.md b/README.md index 9ed94a2..00cf6fd 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Production image references now use `registry.hrynco.com`, preserving the existi - [Service and whole-template API for AI clients](docs/email-template-definition-api.md) - [ItemTracker outbox email consumer](docs/itemtracker-outbox-email-consumer.md) - [Shared Notification Service Kubernetes package](docker/kubernetes/shared-notification/README.md) +- [Production operations and recovery runbook](docs/production-operations.md) — actual Kubernetes environment, backup/restore, grants, Google egress and owner access. ## Windows host development diff --git a/docker/kubernetes/shared-notification/README.md b/docker/kubernetes/shared-notification/README.md index 62200cb..5e0a743 100644 --- a/docker/kubernetes/shared-notification/README.md +++ b/docker/kubernetes/shared-notification/README.md @@ -1,5 +1,12 @@ # Shared Notification Service on MicroK8s +For the current six-migration production release, use the +[production operations and recovery runbook](../../../docs/production-operations.md). +The original bootstrap/deploy schema below predates push channels; do not run its +full deploy against the current database. The current additive Google HTTPS policy +is `google-api-network.json`; refresh it with `refresh-google-api-network.py` and +review/server-dry-run it before applying. Existing SMTP and broker policies remain. + This package deploys one shared Web/Worker, PostgreSQL 17, RabbitMQ 4 and a temporary Mailpit sandbox in `shared-notification`, on `mini-pc-2`. It does not expose public Ingress, NodePort or unauthenticated administration. Application administration, diff --git a/docker/kubernetes/shared-notification/google-api-network.json b/docker/kubernetes/shared-notification/google-api-network.json new file mode 100644 index 0000000..baa9d33 --- /dev/null +++ b/docker/kubernetes/shared-notification/google-api-network.json @@ -0,0 +1,1534 @@ +{ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": "web-worker-to-google-apis", + "namespace": "shared-notification", + "labels": { + "app.kubernetes.io/name": "hrynco-shared-notification", + "app.kubernetes.io/managed-by": "hrynco-shared-notification" + }, + "annotations": { + "hrynco.com/google-ranges-date": "2026-10-24T01:06:28.876297", + "hrynco.com/cloud-ranges-date": "2026-10-24T01:06:28.876297" + } + }, + "spec": { + "podSelector": { + "matchLabels": { + "app.kubernetes.io/name": "hrynco-shared-notification", + "app.kubernetes.io/managed-by": "hrynco-shared-notification" + }, + "matchExpressions": [ + { + "key": "app.kubernetes.io/component", + "operator": "In", + "values": [ + "web", + "worker" + ] + } + ] + }, + "policyTypes": [ + "Egress" + ], + "egress": [ + { + "to": [ + { + "ipBlock": { + "cidr": "8.8.4.0/24" + } + }, + { + "ipBlock": { + "cidr": "8.8.8.0/24" + } + }, + { + "ipBlock": { + "cidr": "8.35.200.0/21" + } + }, + { + "ipBlock": { + "cidr": "8.230.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "8.231.0.0/19" + } + }, + { + "ipBlock": { + "cidr": "8.234.0.0/23" + } + }, + { + "ipBlock": { + "cidr": "8.234.18.0/23" + } + }, + { + "ipBlock": { + "cidr": "8.236.0.0/17" + } + }, + { + "ipBlock": { + "cidr": "8.236.128.0/20" + } + }, + { + "ipBlock": { + "cidr": "8.236.160.0/19" + } + }, + { + "ipBlock": { + "cidr": "8.237.128.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.0.228.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.0.232.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.1.64.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.2.64.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.2.72.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.2.78.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.2.80.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.2.112.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.3.0.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.3.3.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.3.4.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.3.8.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.3.16.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.3.48.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.3.64.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.3.72.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.3.112.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.4.0.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.4.20.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.4.101.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.4.112.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.4.193.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.4.194.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.4.196.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.4.200.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.4.224.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.5.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "34.13.64.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.13.80.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.13.96.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.34.192.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.34.208.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.34.224.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.43.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "34.50.128.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.52.0.0/17" + } + }, + { + "ipBlock": { + "cidr": "34.64.0.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.96.0.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.98.0.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.98.136.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.98.144.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.98.160.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.98.192.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.99.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "34.100.0.0/17" + } + }, + { + "ipBlock": { + "cidr": "34.101.0.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.101.16.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.101.19.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.101.28.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.103.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "34.104.0.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.104.16.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.104.24.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.104.26.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.104.28.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.104.32.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.104.48.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.104.53.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.104.54.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.109.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "34.110.0.0/17" + } + }, + { + "ipBlock": { + "cidr": "34.114.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "34.116.8.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.116.16.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.116.32.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.118.208.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.118.224.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.119.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "34.124.64.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.124.96.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.126.0.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.126.224.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.127.128.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.127.144.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.127.152.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.127.176.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.127.181.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.127.182.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.127.192.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.128.0.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.128.8.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.128.16.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.128.38.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.128.40.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.128.50.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.128.56.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.128.192.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.128.240.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.143.0.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.143.72.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.143.80.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.143.96.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.144.0.0/17" + } + }, + { + "ipBlock": { + "cidr": "34.144.128.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.144.160.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.144.168.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.152.70.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.152.82.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.152.88.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.152.99.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.152.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "34.153.0.0/20" + } + }, + { + "ipBlock": { + "cidr": "34.153.34.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.153.36.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.153.39.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.153.56.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.153.60.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.153.64.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.153.226.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.153.228.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.153.231.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.153.248.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.157.10.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.157.86.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.157.120.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.157.123.128/25" + } + }, + { + "ipBlock": { + "cidr": "34.157.138.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.157.214.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.157.248.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.158.0.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.158.248.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.167.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "34.177.0.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.177.38.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.177.56.0/21" + } + }, + { + "ipBlock": { + "cidr": "34.177.67.0/24" + } + }, + { + "ipBlock": { + "cidr": "34.177.78.128/25" + } + }, + { + "ipBlock": { + "cidr": "34.177.124.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.177.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "34.180.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "34.183.10.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.20.0/25" + } + }, + { + "ipBlock": { + "cidr": "34.183.22.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.30.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.38.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.48.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.62.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.64.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.126.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.134.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.142.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.183.160.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.183.192.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.184.10.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.184.20.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.184.28.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.184.36.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.184.60.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.184.124.0/23" + } + }, + { + "ipBlock": { + "cidr": "34.184.140.0/22" + } + }, + { + "ipBlock": { + "cidr": "34.184.160.0/19" + } + }, + { + "ipBlock": { + "cidr": "34.184.192.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.185.0.0/18" + } + }, + { + "ipBlock": { + "cidr": "34.188.0.0/14" + } + }, + { + "ipBlock": { + "cidr": "35.187.128.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.190.96.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.190.240.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.191.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "35.199.128.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.201.32.0/21" + } + }, + { + "ipBlock": { + "cidr": "35.201.40.0/24" + } + }, + { + "ipBlock": { + "cidr": "35.201.42.0/23" + } + }, + { + "ipBlock": { + "cidr": "35.201.44.0/22" + } + }, + { + "ipBlock": { + "cidr": "35.201.48.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.203.192.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.203.208.0/23" + } + }, + { + "ipBlock": { + "cidr": "35.203.220.0/22" + } + }, + { + "ipBlock": { + "cidr": "35.203.224.0/21" + } + }, + { + "ipBlock": { + "cidr": "35.203.240.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.206.0.0/21" + } + }, + { + "ipBlock": { + "cidr": "35.206.8.0/23" + } + }, + { + "ipBlock": { + "cidr": "35.206.12.0/22" + } + }, + { + "ipBlock": { + "cidr": "35.206.16.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.218.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "35.219.192.0/19" + } + }, + { + "ipBlock": { + "cidr": "35.220.28.0/23" + } + }, + { + "ipBlock": { + "cidr": "35.220.30.0/24" + } + }, + { + "ipBlock": { + "cidr": "35.229.0.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.230.192.0/19" + } + }, + { + "ipBlock": { + "cidr": "35.230.224.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.235.128.0/19" + } + }, + { + "ipBlock": { + "cidr": "35.235.160.0/21" + } + }, + { + "ipBlock": { + "cidr": "35.235.168.0/22" + } + }, + { + "ipBlock": { + "cidr": "35.235.176.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.235.192.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.235.208.0/21" + } + }, + { + "ipBlock": { + "cidr": "35.235.224.0/19" + } + }, + { + "ipBlock": { + "cidr": "35.242.28.0/23" + } + }, + { + "ipBlock": { + "cidr": "35.242.30.0/24" + } + }, + { + "ipBlock": { + "cidr": "35.243.16.0/20" + } + }, + { + "ipBlock": { + "cidr": "35.243.48.0/21" + } + }, + { + "ipBlock": { + "cidr": "64.15.112.0/20" + } + }, + { + "ipBlock": { + "cidr": "64.233.160.0/19" + } + }, + { + "ipBlock": { + "cidr": "66.102.0.0/20" + } + }, + { + "ipBlock": { + "cidr": "66.249.64.0/19" + } + }, + { + "ipBlock": { + "cidr": "70.32.128.0/19" + } + }, + { + "ipBlock": { + "cidr": "72.14.192.0/18" + } + }, + { + "ipBlock": { + "cidr": "74.114.24.0/21" + } + }, + { + "ipBlock": { + "cidr": "74.125.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "104.154.0.0/20" + } + }, + { + "ipBlock": { + "cidr": "104.154.112.0/24" + } + }, + { + "ipBlock": { + "cidr": "104.154.122.0/23" + } + }, + { + "ipBlock": { + "cidr": "104.154.124.0/22" + } + }, + { + "ipBlock": { + "cidr": "104.155.240.0/20" + } + }, + { + "ipBlock": { + "cidr": "104.196.64.0/24" + } + }, + { + "ipBlock": { + "cidr": "104.196.72.0/21" + } + }, + { + "ipBlock": { + "cidr": "104.196.80.0/20" + } + }, + { + "ipBlock": { + "cidr": "104.199.64.0/23" + } + }, + { + "ipBlock": { + "cidr": "104.199.240.0/23" + } + }, + { + "ipBlock": { + "cidr": "104.237.160.0/19" + } + }, + { + "ipBlock": { + "cidr": "107.178.192.0/20" + } + }, + { + "ipBlock": { + "cidr": "107.178.224.0/20" + } + }, + { + "ipBlock": { + "cidr": "108.170.192.0/18" + } + }, + { + "ipBlock": { + "cidr": "108.177.0.0/17" + } + }, + { + "ipBlock": { + "cidr": "130.211.0.0/22" + } + }, + { + "ipBlock": { + "cidr": "136.22.2.0/23" + } + }, + { + "ipBlock": { + "cidr": "136.22.4.0/23" + } + }, + { + "ipBlock": { + "cidr": "136.22.8.0/21" + } + }, + { + "ipBlock": { + "cidr": "136.22.16.0/20" + } + }, + { + "ipBlock": { + "cidr": "136.22.160.0/20" + } + }, + { + "ipBlock": { + "cidr": "136.22.176.0/21" + } + }, + { + "ipBlock": { + "cidr": "136.22.184.0/23" + } + }, + { + "ipBlock": { + "cidr": "136.22.186.0/24" + } + }, + { + "ipBlock": { + "cidr": "136.23.0.0/18" + } + }, + { + "ipBlock": { + "cidr": "136.76.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "136.78.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "136.120.0.0/22" + } + }, + { + "ipBlock": { + "cidr": "136.121.8.0/21" + } + }, + { + "ipBlock": { + "cidr": "136.124.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "142.250.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "146.148.0.0/23" + } + }, + { + "ipBlock": { + "cidr": "152.238.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "162.120.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "172.102.8.0/21" + } + }, + { + "ipBlock": { + "cidr": "172.110.32.0/21" + } + }, + { + "ipBlock": { + "cidr": "172.217.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "172.253.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "173.194.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "177.176.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "177.178.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "177.208.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "179.193.160.0/19" + } + }, + { + "ipBlock": { + "cidr": "185.150.148.0/22" + } + }, + { + "ipBlock": { + "cidr": "186.242.64.0/18" + } + }, + { + "ipBlock": { + "cidr": "186.245.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "187.78.0.0/17" + } + }, + { + "ipBlock": { + "cidr": "187.79.0.0/17" + } + }, + { + "ipBlock": { + "cidr": "187.126.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "189.24.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "189.48.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "189.49.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "189.70.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "189.82.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "189.105.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "189.106.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "191.0.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "191.2.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "191.40.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "191.44.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "191.45.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "191.46.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "191.212.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "191.216.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "191.218.0.0/17" + } + }, + { + "ipBlock": { + "cidr": "191.220.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "192.104.160.0/23" + } + }, + { + "ipBlock": { + "cidr": "192.178.0.0/15" + } + }, + { + "ipBlock": { + "cidr": "193.186.4.0/24" + } + }, + { + "ipBlock": { + "cidr": "199.36.154.0/23" + } + }, + { + "ipBlock": { + "cidr": "199.36.156.0/24" + } + }, + { + "ipBlock": { + "cidr": "199.192.112.0/23" + } + }, + { + "ipBlock": { + "cidr": "199.192.114.0/24" + } + }, + { + "ipBlock": { + "cidr": "199.223.237.0/24" + } + }, + { + "ipBlock": { + "cidr": "199.223.238.0/23" + } + }, + { + "ipBlock": { + "cidr": "200.226.0.0/16" + } + }, + { + "ipBlock": { + "cidr": "207.223.160.0/20" + } + }, + { + "ipBlock": { + "cidr": "208.65.152.0/22" + } + }, + { + "ipBlock": { + "cidr": "208.68.108.0/22" + } + }, + { + "ipBlock": { + "cidr": "208.81.188.0/22" + } + }, + { + "ipBlock": { + "cidr": "208.117.224.0/19" + } + }, + { + "ipBlock": { + "cidr": "209.85.128.0/17" + } + }, + { + "ipBlock": { + "cidr": "209.107.176.0/20" + } + }, + { + "ipBlock": { + "cidr": "216.58.192.0/19" + } + }, + { + "ipBlock": { + "cidr": "216.73.80.0/20" + } + }, + { + "ipBlock": { + "cidr": "216.239.32.0/19" + } + }, + { + "ipBlock": { + "cidr": "216.252.220.0/22" + } + } + ], + "ports": [ + { + "protocol": "TCP", + "port": 443 + } + ] + } + ] + } +} diff --git a/docker/kubernetes/shared-notification/refresh-google-api-network.py b/docker/kubernetes/shared-notification/refresh-google-api-network.py new file mode 100644 index 0000000..219a1f5 --- /dev/null +++ b/docker/kubernetes/shared-notification/refresh-google-api-network.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Render Google API IPv4 egress; never applies changes to Kubernetes.""" + +import argparse +import ipaddress +import json +from pathlib import Path +import urllib.request + + +def fetch(name): + with urllib.request.urlopen(f"https://www.gstatic.com/ipranges/{name}.json", timeout=20) as response: + return json.load(response) + + +def api_networks(google, cloud): + networks = [ipaddress.ip_network(p["ipv4Prefix"]) for p in google["prefixes"] if "ipv4Prefix" in p] + for prefix in cloud["prefixes"]: + if "ipv4Prefix" not in prefix: + continue + excluded = ipaddress.ip_network(prefix["ipv4Prefix"]) + remaining = [] + for network in networks: + if not network.overlaps(excluded): + remaining.append(network) + elif not network.subnet_of(excluded): + remaining.extend(network.address_exclude(excluded)) + networks = remaining + return list(ipaddress.collapse_addresses(networks)) + + +def render(google, cloud): + networks = api_networks(google, cloud) + if not networks or any(not network.is_global for network in networks): + raise ValueError("Expected nonempty public Google API IPv4 ranges") + labels = {"app.kubernetes.io/name": "hrynco-shared-notification", + "app.kubernetes.io/managed-by": "hrynco-shared-notification"} + return { + "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", + "metadata": {"name": "web-worker-to-google-apis", "namespace": "shared-notification", + "labels": labels, "annotations": { + "hrynco.com/google-ranges-date": google["creationTime"], + "hrynco.com/cloud-ranges-date": cloud["creationTime"]}}, + "spec": {"podSelector": {"matchLabels": labels, "matchExpressions": [ + {"key": "app.kubernetes.io/component", "operator": "In", "values": ["web", "worker"]}]}, + "policyTypes": ["Egress"], "egress": [{ + "to": [{"ipBlock": {"cidr": str(network)}} for network in networks], + "ports": [{"protocol": "TCP", "port": 443}]}]}, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + policy = render(fetch("goog"), fetch("cloud")) + args.output.write_text(json.dumps(policy, indent=2) + "\n", encoding="utf-8") + print(f"Rendered {len(policy['spec']['egress'][0]['to'])} IPv4 ranges; review before applying.") + + +if __name__ == "__main__": + main() diff --git a/docs/production-operations.md b/docs/production-operations.md new file mode 100644 index 0000000..bda8205 --- /dev/null +++ b/docs/production-operations.md @@ -0,0 +1,564 @@ +# Notification Service: production operations and recovery + +Verified against the running Kubernetes service on **2026-10-24**. Commands below +distinguish read-only checks, service interruptions and destructive recovery. Do not +paste passwords, service-account JSON, tokens, Secret YAML or database dumps into +tickets, chat, Git or CI logs. The administrative UI has no application authentication; +keep it behind the existing loopback SSH tunnel. + +## 1. Actual environment and source of truth + +| Item | Actual value | +| --- | --- | +| Windows source checkout | `C:\projects\HrynCo\hrynco-notification-service` | +| SSH target | `devops@192.168.2.44` (`mini-pc-2`; Linux username is lowercase) | +| Windows SSH key | `C:\Users\agryn\.ssh\id_ed25519_devops`, unlocked in the owner's ssh-agent | +| Kubernetes CLI | `/snap/microk8s/current/kubectl` | +| Kubeconfig / context | `/home/devops/.kube/cluster-admin.conf` / `microk8s` | +| Namespace / CNI | `shared-notification` / Calico `v3.29.3` | +| Server package | `/home/devops/notification-package` | +| Private files / local backups | `/home/devops/shared-notification-private` | +| External storage | `/mnt/shared/notification-service`, XFS, UUID `c2c31640-a247-4988-bc7b-9ee9789464d1` | +| Local admin URL | `http://localhost:15200/admin/services` | +| Web service | `web:8080`, ClusterIP; no public Ingress or NodePort | +| PostgreSQL | `postgres:5432`, database `notification_service` | +| RabbitMQ | `rabbitmq:5672`, vhost `/notification`, management port `15672` | +| Seq | `http://seq.observability.svc.cluster.local:8080` | +| Sandbox SMTP / UI | `mailpit:1025` / `mailpit:8025`; no external relay | + +The repository's `docker/kubernetes/shared-notification/` owns the package. The +server's `config.local.json` supplies the verified host/storage and namespace values; +`artifacts.local.json` still describes the older bootstrap application images. +`notification.py` owns the original foundation and credential bootstrap. Its +`MIGRATIONS`, sandbox SQL and application artifact file are **not yet reconciled with +the six-migration release below**. Do not run its full `bootstrap` or `deploy` command +over this production release. It can overwrite current image/configuration intent or +fail on removed `email_channels.service_name`. + +Use the targeted procedures in this document for the existing production resources. +They do not depend on the historical one-off `it1074-deploy.py`. The original +deployment audit is retained privately at +`/home/devops/shared-notification-private/it1074/{before,result}.json`; `before.json` +contains deployment specifications and fingerprints, not a restorable database backup. +The one-off helper's initial GRANT used the wrong table case; do not rerun it. + +Network source of truth is additive: base policies from `notification.py`, existing +`production-network.yaml` for SMTP and production broker access, and +`google-api-network.json` for Google APIs. Never reconcile the sandbox render as the +complete production setup. Reconcile and review all overlays together. + +## 2. Connect and inspect before changing anything + +Run in **Windows PowerShell** (no SSH alias is required or assumed): + +```powershell +ssh -o StrictHostKeyChecking=yes -o HostKeyAlgorithms=ssh-ed25519 -o ConnectTimeout=10 -i C:/Users/agryn/.ssh/id_ed25519_devops devops@192.168.2.44 +``` + +Inside that **Linux SSH session**, define these session-local helpers. All subsequent +`k` and `kn` commands in this document run there, not in Windows PowerShell: + +```sh +k() { /snap/microk8s/current/kubectl --kubeconfig=/home/devops/.kube/cluster-admin.conf --request-timeout=20s "$@"; } +kn() { k --namespace=shared-notification "$@"; } +cd /home/devops/notification-package +k config current-context +k get node mini-pc-2 +kn get deployment,pod,service,pvc,job +k get pv shared-notification-postgres shared-notification-rabbitmq +findmnt -T /mnt/shared/notification-service -o TARGET,SOURCE,FSTYPE,UUID +kn get resourcequota +``` + +Expected: context `microk8s`, node Ready, the exact XFS UUID above, both PVCs Bound. +Stop if the disk is not mounted: do not let an empty directory on the root disk stand +in for persistent storage. PVs use `Retain` and node affinity to `mini-pc-2`: + +| PVC | PV | Path | Capacity | +| --- | --- | --- | --- | +| `postgres-data` | `shared-notification-postgres` | `/mnt/shared/notification-service/postgres` | 10Gi | +| `rabbitmq-data` | `shared-notification-rabbitmq` | `/mnt/shared/notification-service/rabbitmq` | 5Gi | + +Do not delete PVCs, clear PV claimRefs, reformat storage or regenerate credentials to +repair an application timeout. These are singleton workloads, not an HA database/broker. +The quota allows five Deployments and three Jobs; the completed migrator/grants/sandbox +Jobs can occupy all three Job slots. A new fourth Job can fail admission. + +## 3. Current application release and infrastructure images + +Application source: `d4ee8fc0c8036b0b48e1a481ef1a12a6a1c1c269`, merged PR 28. +Image build: TeamCity `141783`, tag `main-34`. Deploy immutable digests: + +```sh +WEB_IMAGE='registry.hrynco.com/hrynco.notification-service.web@sha256:69c51ac1ba7c1101472a0b7d45153544a95565cc544bba49908b1e4079195911' +WORKER_IMAGE='registry.hrynco.com/hrynco.notification-service.worker@sha256:cc93cbfd82e1d7ff4d88845dfa90c8a1188c51d35775432d385fa5f23795a6a3' +MIGRATOR_IMAGE='registry.hrynco.com/hrynco.notification-service.migrator@sha256:9975e05cb075f45a6a629233af4c56194731378f710feb719b205a68bebdb035' +``` + +The repositories include `notification-service` with a hyphen. Old bootstrap images +used `notificationservice`; do not accidentally choose those. These digests are OCI +indexes; the matching amd64 manifests are web `d69a6d7cdbc8c08d5a1234933a88579fd176a72b8c64fa02167c42d988eed0be`, +worker `92d454a4139de55f5619070ed6b070c94db05749bbd3aca5d0e91d5a11ae8875`, +migrator `a94766802609f4348adba4bdef4bc528275f317c3015b10bb97c8d530749adbd`. +Registry manifest inspection must accept OCI indexes; a Docker-only Accept header can +return a misleading 404. + +Retained infrastructure digests (do not upgrade during application recovery): + +| Image repository | Digest | +| --- | --- | +| `registry.hrynco.com/mirror/postgres` | `sha256:e31e3d5327d1806f6177827c9710643e4f35f7ab3f14d26d05332753d3e95ee0` | +| `registry.hrynco.com/mirror/rabbitmq` | `sha256:ad995f859fce23a0ccd91c24cfac89d8f8cc57de4d708b58dcecea2697ad0d8c` | +| `registry.hrynco.com/mirror/mailpit` | `sha256:13de4ffbd28f3089be6dff32afbc8210edb5a593d738fa877ae7272cc73aeb67` | + +Read-only image and readiness verification: + +```sh +kn get deployment web worker -o custom-columns='NAME:.metadata.name,DESIRED:.spec.replicas,READY:.status.readyReplicas,IMAGE:.spec.template.spec.containers[*].image' +kn get pods -o custom-columns='NAME:.metadata.name,READY:.status.containerStatuses[*].ready,RESTARTS:.status.containerStatuses[*].restartCount,IMAGEID:.status.containerStatuses[*].imageID' +kn rollout status deployment/web --timeout=240s +kn rollout status deployment/worker --timeout=240s +``` + +Web readiness queries `/api/v1/email-channels?serviceName=Invemory.Staging.Sandbox` on +8080, every 10 seconds with a one-second timeout. Cold start under the 250m CPU limit +can initially time out. Worker has no HTTP/readiness probe: **Ready is not proof of +consumer readiness**. Verify RabbitMQ separately. + +## 4. Configuration and credentials + +Both Web and Worker use `notification-config` through `envFrom`: +`DOTNET_ENVIRONMENT=Production`, `ASPNETCORE_ENVIRONMENT=Production`, +`ASPNETCORE_HTTP_PORTS=8080`, RabbitMQ host `rabbitmq`, port `5672`, vhost +`/notification`, Seq URL above, Serilog service `HrynCo.NotificationService` and +environment `Shared`. Inspect nonsecret settings with: + +```sh +kn get configmap notification-config -o yaml +kn get secret +``` + +`notification-private` keys: `ownerConnectionString`, `runtimeConnectionString`, +`postgresPassword`, `runtimePassword`, `rabbitAdminPassword`, `rabbitWorkerPassword`, +`rabbitPublisherPassword`, `seqApiKey`. Web/Worker use the runtime connection; +migrations use owner connection. Worker username is `notification_worker`. +`registry-pull` contains `.dockerconfigjson`; `rabbit-definitions` contains +`definitions.json` with sensitive password hashes. Do not print their values. + +The server private directory retains `credentials.json` and production publisher +password/hash files. Registry authentication and Seq source files are under +`/home/devops/notification-private/registry-auth.json` and `seq-key.txt`. +Recover original credentials from a protected backup if lost; changing Kubernetes +Secrets does not automatically change passwords in existing database/broker volumes. +Do not run bootstrap to generate replacements against retained data. + +Firebase service-account JSON is stored in PostgreSQL `push_channels.credentials` +(JSONB), without application encryption, by explicit owner decision. Database backups +therefore contain private keys. Edit forms never return the saved JSON; leaving the +field blank retains it. A project change requires matching new credentials. Current +channel is attached to `Invemory-Prod`, Firebase project technical ID +`storemate-hrynco-com`. Do not copy local development keys into production. + +`Check Firebase authorization` obtains an OAuth token using saved credentials when +the project is unchanged. It does not send a push, prove FCM send permission, or prove +device display. `Send test push` is a separate explicitly authorized action. +The standalone test sends a Firebase `notification` payload without Invemory binding +data. On the currently installed Invemory Android build it displays through the FCM +SDK in background. In foreground, `InvemoryMessagingService.onMessageReceived` rejects +the empty data map because `bindingId`, `version`, `messageId`, route and expiry are +required; it does not call the base Capacitor handler. Thus accepted test messages can +be invisible while Invemory is open. Samsung logs on 2026-10-24 show the FCM fallback +notification channel and removal at 13:04:28 Europe/Kiev; the receiver does not log +its rejection, so no individual foreground callback was proven. Do not diagnose this +as egress failure or rotate tokens solely because foreground display is absent. +The Worker push consumer is **not enabled** by this rollout: configuration and broker +ACLs still cover email only. A saved active channel and successful OAuth check do not +activate end-to-end production push processing. Do not enable it incidentally during +an email service recovery. + +## 5. Database history and runtime grants + +Read-only history check: + +```sh +kn exec deployment/postgres -- psql -U notification_owner -d notification_service -At -c 'SELECT "MigrationId" FROM "__EFMigrationsHistory" ORDER BY "MigrationId"' +``` + +Expected, in order: + +```text +20260501214629_InitialCreate +20260502154249_PendingChanges +20261017072410_EmailTemplateLocalizations +20261023223753_AddPushDeliveries +20261024073202_LinkEmailChannelsToServices +20261024080457_AddPushChannels +``` + +`notification_owner` is currently the PostgreSQL bootstrap superuser; keep it out of +application connections. `notification_runtime` is a login with no superuser, +createdb or createrole privileges. There are **no default ACL entries**: future tables +do not inherit runtime CRUD automatically. For each new migration, grant only the +tables needed by the application and persist the same SQL in `runtime-grants`. +Do not add blanket default privileges as a workaround. + +The exact current push grants are below. `"PushDeliveries"` is case-sensitive; +`push_deliveries` does not exist. This command is idempotent and was verified: + +```sh +kn exec deployment/postgres -- psql -U notification_owner -d notification_service -v ON_ERROR_STOP=1 -c 'GRANT SELECT, INSERT, UPDATE, DELETE ON push_channels, "PushDeliveries" TO notification_runtime;' +kn exec deployment/postgres -- psql -U notification_owner -d notification_service -At -c "SELECT t,has_table_privilege('notification_runtime',t,'SELECT,INSERT,UPDATE,DELETE') FROM unnest(ARRAY['push_channels','\"PushDeliveries\"']) t" +``` + +Both checks must return `t`. Existing runtime CRUD covers `email_channels`, +`email_templates`, `email_channel_usage`, `services`, `service_languages`, +`email_template_definitions`; schema USAGE and database CONNECT are granted, schema +CREATE is revoked from PUBLIC. The live `runtime-grants` ConfigMap already includes +the correct push grant. To persist it again without rotating passwords: + +```sh +python3 - <<'PY' +import json, subprocess +k = ['/snap/microk8s/current/kubectl', '--kubeconfig=/home/devops/.kube/cluster-admin.conf', '-n', 'shared-notification'] +value = json.loads(subprocess.check_output(k + ['get','configmap','runtime-grants','-o','json'])) +grant = 'GRANT SELECT, INSERT, UPDATE, DELETE ON push_channels, "PushDeliveries" TO notification_runtime;' +sql = value['data']['grants.sql'] +if grant not in sql: + subprocess.run(k + ['patch','configmap','runtime-grants','--type=merge','-p',json.dumps({'data':{'grants.sql':sql+'\n'+grant+'\n'}})], check=True) +PY +``` + +Do not execute the entire grants ConfigMap manually without its intended +`RUNTIME_PASSWORD` secret environment: its bootstrap SQL also manages the runtime role +password. Do not rerun the old sandbox job; channel ownership now uses `service_id`. +The migration maps the legacy `StoreMate-Prod` channel to existing `Invemory-Prod`. +The old identifier is historical migration input, not a new channel/service name. + +## 6. Backup before migration or restore + +**Service interruption:** stop Web and Worker writers first. This does not stop +external publishers; coordinate a publisher pause if a consistent broker/database +recovery point is needed. Never purge queues as a substitute. + +```sh +kn scale deployment/worker deployment/web --replicas=0 +kn wait --for=delete pod -l 'app.kubernetes.io/component in (web,worker)' --timeout=120s +``` + +The canonical backup implementation is safe independently of its old deploy schema. +This maintained source command creates a custom-format dump with mode 0600, checks +the `PGDMP` signature and `pg_restore --list`, and writes SHA-256, bytes and migration +history to `manifest.json`. Run from the server package; only these existing functions +are used, not `deploy` or sandbox SQL: + +```sh +python3 - <<'PY' +import types +import notification as n +c = n.Cluster(types.SimpleNamespace(kubectl='/snap/microk8s/current/kubectl', kubeconfig='/home/devops/.kube/cluster-admin.conf')) +with n.lock('/home/devops/shared-notification-private'): + assert all(c.get('deployment', x)['spec']['replicas'] == 0 for x in ('web','worker')) + print(n.backup(c, '/home/devops/shared-notification-private')) +PY +``` + +Record the returned folder. Pre-migration backup from this rollout: +`/home/devops/shared-notification-private/backup-20261024T094457816933Z`. +It has only the **first three migrations**, predates the owner's production push +channel, and is not a backup of today's six-migration state. Take a fresh backup before +any recovery. A local dump does not protect against loss of this host or disk. +No offsite copy or full disaster restore has been proven. + +Verify a selected backup without restoring it (set the exact existing folder): + +```sh +BACKUP_DIR=/home/devops/shared-notification-private/backup-20261024T094457816933Z +export BACKUP_DIR +python3 - <<'PY' +import hashlib,json,os,pathlib +p=pathlib.Path(os.environ['BACKUP_DIR']) +m=json.loads((p/'manifest.json').read_text()); data=(p/'database.dump').read_bytes() +assert len(data)==m['bytes'] and hashlib.sha256(data).hexdigest()==m['sha256'] +assert data.startswith(b'PGDMP') +print('Checksum and length verified; migration history:', m['history']) +PY +kn exec -i deployment/postgres -- pg_restore --list < "$BACKUP_DIR/database.dump" >/dev/null +``` + +Protect an additional recovery export with restrictive permissions. The following +commands contain secrets in their **files**, never print or commit them. They are +operator recovery instructions, not proof that an offsite backup exists: + +```sh +umask 077 +RECOVERY_DIR=/home/devops/shared-notification-private/recovery-$(date -u +%Y%m%dT%H%M%SZ) +mkdir "$RECOVERY_DIR" +kn get secret -o json > "$RECOVERY_DIR/secrets.json" +kn get deployment,service,configmap,networkpolicy,resourcequota,limitrange,pvc -o json > "$RECOVERY_DIR/namespace-resources.json" +k get pv shared-notification-postgres shared-notification-rabbitmq -o json > "$RECOVERY_DIR/pvs.json" +``` + +These are inventories with server-assigned fields, not manifests to blindly apply +to a new cluster. Keep protected copies of the package config, kubeconfig and private +credential files with your backup system. RabbitMQ definitions do not back up queued +messages. A broker volume backup requires a coordinated stopped broker and publisher +maintenance window; no live-volume tar or restore drill was performed here. + +## 7. Recover/redeploy this release on retained infrastructure + +Prerequisites: correct disk/PVCs, original Secrets, PostgreSQL and RabbitMQ Ready, +no concurrent deployment, verified fresh backup, exact pinned image variables from +section 3. Web/Worker remain stopped from section 6. + +If history already contains exactly the six entries above, skip migration. If it is +exactly the original three-entry prefix, use the following maintained recipe from +the package directory. It creates the same bounded migrator Job used for the rollout, +references the existing owner Secret, and replaces only the completed owned migrator +Job to fit the quota. Other histories require inspection; do not guess or edit EF history. + +```sh +export MIGRATOR_IMAGE +python3 - <<'PY' +import os,types +import notification as n +c=n.Cluster(types.SimpleNamespace(kubectl='/snap/microk8s/current/kubectl',kubeconfig='/home/devops/.kube/cluster-admin.conf')) +prefix=['20260501214629_InitialCreate','20260502154249_PendingChanges','20261017072410_EmailTemplateLocalizations'] +target=prefix+['20261023223753_AddPushDeliveries','20261024073202_LinkEmailChannelsToServices','20261024080457_AddPushChannels'] +with n.lock('/home/devops/shared-notification-private'): + assert all(c.get('deployment',x)['spec']['replicas']==0 for x in ('web','worker')) + assert c.history()==prefix, 'Unexpected history; stop and inspect' + old=c.get('job','notification-migrator') + assert old and old['metadata']['labels']['app.kubernetes.io/managed-by']==n.MANAGED and old['status'].get('succeeded')==1 + job=n.job('migrator',os.environ['MIGRATOR_IMAGE'],environment=[n.env('DOTNET_ENVIRONMENT','Production'),n.env('App__ConnectionString',secret='notification-private',key='ownerConnectionString')]) + n.run_job(c,job) + assert c.history()==target + print('All six migrations verified; apply runtime grants before starting writers.') +PY +``` + +Apply and verify the grants in section 5. Inspect migrated channel ownership without +reading credentials: + +```sh +kn exec deployment/postgres -- psql -U notification_owner -d notification_service -At -c 'SELECT s.name,count(*) FROM email_channels c JOIN services s ON s.id=c.service_id GROUP BY s.name ORDER BY s.name' +``` + +Set only application images, retaining Secrets, volumes, selectors and resources: + +```sh +kn set image deployment/web web="$WEB_IMAGE" +kn set image deployment/worker worker="$WORKER_IMAGE" +kn scale deployment/web --replicas=1 +kn rollout status deployment/web --timeout=240s +kn scale deployment/worker --replicas=1 +kn rollout status deployment/worker --timeout=240s +``` + +Then verify actual pod imageIDs, six migrations, grants, consumers, UI and OAuth. +If migration fails, keep writers stopped while inspecting the failed Job and database +history. If rollout fails, inspect events/logs before changing anything else. Never +leave replicas at zero without reporting the specific failure and recovery action. + +## 8. Rollback and destructive database restore + +An image-only rollback is allowed only when the target image supports the **current** +schema. `kubectl rollout undo` can select the old pre-ServiceId binary and is unsafe +after this migration. Prefer reapplying the known six-migration image pair above for +an application/configuration regression. Do not roll back the database merely because +a port-forward or OAuth connection fails. + +**DESTRUCTIVE, NOT EXECUTED ON PRODUCTION:** the following replaces the database with +the explicitly selected backup. It loses writes after that backup and can conflict +with retained broker messages. Obtain a maintenance window, pause publishers, stop +Web/Worker, verify a fresh safety backup, and select the intended recovery point first. +PostgreSQL must be healthy, original roles/passwords retained, and the backup must +pass section 6. Do not run this block for validation or as a routine redeploy. + +```sh +# Linux SSH session; BACKUP_DIR must identify the deliberately selected verified dump. +test -s "$BACKUP_DIR/database.dump" || exit 1 +kn exec deployment/postgres -- dropdb -U notification_owner --force notification_service +kn exec deployment/postgres -- createdb -U notification_owner -O notification_owner notification_service +kn exec -i deployment/postgres -- pg_restore -U notification_owner -d notification_service --exit-on-error --single-transaction < "$BACKUP_DIR/database.dump" +``` + +`dropdb --force` terminates database connections. If restoration fails, leave writers +stopped, retain the failed database for diagnosis and do not replay arbitrary messages. +Inspect restored migration history. With a three-migration backup, follow section 7 +to migrate forward and reapply grants before starting the six-migration application. +With six migrations, verify grants and restart the pinned pair. Inspect broker queues +and externally recorded outcomes before resuming publishers: database restore does +not rewind acknowledgements or external email/push delivery. + +For total node/PV loss, this document does **not** claim a tested bare-metal restore. +Recover the original mounted disk and private files first where possible. Rebuilding a +cluster from exports requires reviewing namespace ownership, PV binding and node paths, +Secret consistency, and sanitized manifests. Do not invoke the stale bootstrap package +blindly. A separately rehearsed full disaster-recovery procedure remains a gap. + +## 9. RabbitMQ and Worker checks + +```sh +kn exec deployment/rabbitmq -- rabbitmq-diagnostics -q check_running +kn exec deployment/rabbitmq -- rabbitmqctl list_queues -p /notification name consumers messages_ready messages_unacknowledged +kn exec deployment/rabbitmq -- rabbitmqctl list_permissions -p /notification +``` + +Expected after recovery: `notification.send-email` has exactly one consumer; +`invemory.production.notification-results` has one consumer from Invemory. Zero queued +messages was the verified baseline, not a requirement to purge legitimate new work. +Development/staging result queues can have zero consumers. Broker nodename is fixed +`rabbit@rabbitmq`; do not change it against the retained data directory. + +Worker read ACL covers `notification.send-email` only. Its configure/write ACLs cover +the existing email result routes, including production/staging/development. The +production publisher is `invemory_production`, staging is `invemory_staging`; an +additional development account exists. Preserve all current accounts and definitions; +do not replay an older definitions Secret to fix a consumer. Imported definitions +hold credentials, not a complete live message backup. The shared default exchange +does not isolate application `ServiceName` values between publishers. + +## 10. Google OAuth/FCM egress and timeout diagnosis + +On 2026-10-24 DNS resolved both Google endpoints, but all four Web/Worker TLS probes +timed out. Base default-deny allowed only internal services and SMTP 587. Applying +`google-api-network.json` fixed all four TLS handshakes immediately without a restart; +the real production UI then returned **OAuth access token obtained**. + +The policy selects only managed Web/Worker pods in `shared-notification`, IPv4 TCP +443 to Google's published API/service ranges. It grants no ingress or arbitrary +internet access. Existing DNS policy allows TCP/UDP 53 only to kube-system CoreDNS. +Calico OSS standard NetworkPolicy does not provide an FQDN allowlist. Google documents +that individual APIs have no stable dedicated IP list: this snapshot uses `goog.json` +minus `cloud.json` (297 CIDRs at verification), excluding customer Cloud ranges but +still including other Google-operated services. This is an IP-level provider boundary, +not a hostname-level boundary. [Google's range method](https://docs.cloud.google.com/vpc/docs/configure-private-google-access#ip-addr-defaults). + +Read-only probes from each actual runtime pod (7-second bound, no credentials): + +```sh +for component in web worker; do + for endpoint in oauth2.googleapis.com fcm.googleapis.com; do + printf '%s %s\n' "$component" "$endpoint" + kn exec deployment/"$component" -- getent ahostsv4 "$endpoint" + kn exec deployment/"$component" -- sh -c 'timeout 7 openssl s_client -connect "$1:443" -servername "$1" -verify_hostname "$1" -verify_return_error -brief