Реализовать наблюдаемость мессенджера и проверить её под нагрузкой

This commit is contained in:
2026-11-06 18:22:53 +03:00
parent 470d9d6f1e
commit b2eca3d259
78 changed files with 10341 additions and 24 deletions
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Create local synthetic credentials once, without printing their values."""
import os
import secrets
from pathlib import Path
root = Path(__file__).resolve().parents[1]
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
password_file = directory / "postgres_password"
if not password_file.exists():
password_file.write_text(secrets.token_urlsafe(32) + "\n")
password = password_file.read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
"grafana_password": secrets.token_urlsafe(24),
"cache_url": "redis://valkey:6379/0",
"broker_url": "redis://broker:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n")
# Compose bind-mount secrets must be readable by the non-root container process.
# The parent directory remains 0700 on the host.
for role in ("root", "api", "worker", "reconciler", "test"):
names = (
("s3_root_user", "s3_root_password")
if role == "root"
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
)
for index, name in enumerate(names):
path = directory / name
if not path.exists():
path.write_text(
(secrets.token_hex(10) if index == 0 else secrets.token_urlsafe(32)) + "\n"
)
for name in (
"postgres_password",
"grafana_password",
"database_url",
"cache_url",
"broker_url",
*[p.name for p in directory.glob("s3_*")],
):
os.chmod(directory / name, 0o644)
env = root / ".env"
if not env.exists():
env.write_text((root / ".env.example").read_text())
print("Local secret files and environment are ready; existing values preserved.")
+82
View File
@@ -0,0 +1,82 @@
"""Read-only discovery and scoped Compose operations for the local lab5 stack."""
import json
import subprocess
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, env=None):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
timeout=90,
).stdout
def inspect(container):
template = (
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
'"nano_cpus":{{.HostConfig.NanoCpus}},'
'"published_ports":{{json .HostConfig.PortBindings}}}'
)
r = subprocess.run(
["docker", "inspect", "--format", template, container],
capture_output=True,
text=True,
check=True,
timeout=5,
)
return json.loads(r.stdout)
def instance(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
capture_output=True,
text=True,
check=True,
timeout=3,
)
value = inspect(container)
value["instance"] = r.stdout.strip()
return value
def instances():
ids = compose("ps", "-q", "api").split()
with ThreadPoolExecutor(max_workers=8) as pool:
return list(pool.map(instance, ids))
def wait_lb(base="http://127.0.0.1:8086", count=2, timeout=45):
deadline, seen = time.monotonic() + timeout, set()
while time.monotonic() < deadline:
try:
current = {value["instance"] for value in instances()}
if len(current) != count:
time.sleep(0.2)
continue
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
seen.add(response.headers["X-Instance-Id"])
if current <= seen:
return sorted(current)
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
pass
time.sleep(0.1)
raise RuntimeError("Expected healthy API replicas were not observed through LB")
+279
View File
@@ -0,0 +1,279 @@
#!/usr/bin/env python3
"""Collect sanitized evidence from the local lab6 stand."""
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
def compose(*args, **kwargs):
return subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, **kwargs)
def api_json(port, path, **params):
url = f"http://127.0.0.1:{port}{path}"
if params:
url += "?" + urlencode(params)
with urlopen(url, timeout=5) as response:
return json.load(response)
def prom(query):
data = api_json(9096, "/api/v1/query", query=query)["data"]["result"]
return float(data[0]["value"][1]) if data else 0
def until(check, timeout=45):
end = time.monotonic() + timeout
while time.monotonic() < end:
try:
result = check()
if result:
return result
except (HTTPError, URLError):
pass
time.sleep(0.5)
raise RuntimeError(f"Timed out waiting for {check.__name__}")
def log_for(request_id):
data = api_json(
3106,
"/loki/api/v1/query_range",
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
start=str(time.time_ns() - 180_000_000_000),
limit=5,
)["data"]["result"]
for stream in data:
for value in stream["values"]:
return value[2] if len(value) > 2 else stream["stream"]
return None
def span_names(trace_id):
data = api_json(3206, "/api/traces/" + trace_id)
return [
(span["name"], span.get("kind"), span["spanId"], span.get("parentSpanId"))
for batch in data.get("batches", data.get("resourceSpans", []))
for group in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", []))
for span in group["spans"]
]
def alerts():
return {
a["labels"]["alertname"]: a["state"]
for a in api_json(9096, "/api/v1/alerts")["data"]["alerts"]
}
def correlation(client, actor, chat, retry=False):
if retry:
compose("stop", "worker", stdout=subprocess.DEVNULL)
try:
attachment = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=observability.png",
202,
actor=actor,
raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"},
schema="Attachment",
)
if retry:
job_attachment = attachment["json"]["id"]
compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-Xq",
"-v",
"ON_ERROR_STOP=1",
"-v",
"attachment=" + job_attachment,
input=(
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) "
"SELECT job_id,'before_effect','transient',1 FROM attachment_jobs "
"WHERE attachment_id=:'attachment';\n"
),
text=True,
stdout=subprocess.DEVNULL,
)
finally:
if retry:
compose("start", "worker", stdout=subprocess.DEVNULL)
client.wait_ready(actor, attachment["json"])
request_id = attachment["headers"]["X-Request-Id"]
log = until(lambda: log_for(request_id))
trace_id = log["trace_id"]
expected = {"jobs.publish", "jobs.consume", "s3.put", "s3.download", "db.transaction"}
def connected():
spans = span_names(trace_id)
names = {row[0] for row in spans}
consumers = sum(row[0] == "jobs.consume" for row in spans)
return spans if expected <= names and consumers >= (2 if retry else 1) else None
spans = until(connected)
by_id = {row[2]: row for row in spans}
consumers = [row for row in spans if row[0] == "jobs.consume"]
assert all(by_id[row[3]][0] == "jobs.publish" for row in consumers)
logs = until(
lambda: api_json(
3106,
"/loki/api/v1/query_range",
query='{service_name="messenger-worker"} | trace_id="' + trace_id + '"',
start=str(time.time_ns() - 180_000_000_000),
limit=5,
)["data"]["result"]
)
assert any(value[1] == "job_processed" for row in logs for value in row["values"])
return {
"request_id": request_id,
"trace_id": trace_id,
"span_names": sorted({r[0] for r in spans}),
"worker_parent": by_id[consumers[0][3]][0],
"worker_attempts": len(consumers),
"api_and_worker_loki": True,
}
def worker_incident(client, actor, chat):
compose("stop", "worker", stdout=subprocess.DEVNULL)
try:
uploads = [client.attachment(actor, chat)[0] for _ in range(3)]
until(lambda: alerts().get("MessengerQueueLag") == "firing")
record = {
"depth_at_firing": prom("max(messenger_queue_depth)"),
"oldest_seconds_at_firing": prom("max(messenger_queue_oldest_seconds)"),
}
assert record["depth_at_firing"] >= 3
finally:
compose("start", "worker", stdout=subprocess.DEVNULL)
for attachment in uploads:
client.wait_ready(actor, attachment)
until(lambda: not prom("max(messenger_queue_depth)") and "MessengerQueueLag" not in alerts())
return {**record, "state": ["firing", "resolved"], "uploads_ready": len(uploads)}
def database_incident(client, actor):
# Independent, bounded transaction. Lock sessions only; readiness can still use the schema.
command = [
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-XqAt",
"-v",
"ON_ERROR_STOP=1",
]
start = time.monotonic()
lock = subprocess.Popen(
command, cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
)
lock.stdin.write(
"BEGIN;\nLOCK TABLE sessions IN ACCESS EXCLUSIVE MODE;\n"
"\\echo locked\nSELECT pg_sleep(42);\nROLLBACK;\n"
)
lock.stdin.close()
try:
assert lock.stdout.readline().strip() == "locked"
attempts = []
while time.monotonic() - start < 34:
attempt = client.call("GET", "/api/v1/users/me", [200, 503], actor=actor, schema="User")
attempts.append(attempt)
states = alerts()
if all(
states.get(a) == "firing"
for a in ("MessengerHttpErrors", "MessengerDatabaseErrors")
):
break
assert any(item["status"] == 503 for item in attempts)
active = alerts()
assert active.get("MessengerDatabaseErrors") == "firing", active
assert active.get("MessengerHttpErrors") == "firing", active
finally:
lock.wait(timeout=46)
lock.stdout.close()
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
# Keep successful traffic to publish fresh data while the alert window slides.
def resolved():
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
return not ({"MessengerDatabaseErrors", "MessengerHttpErrors"} & alerts().keys())
until(resolved)
failure = next(item for item in attempts if item["status"] == 503)
failure_log = until(lambda: log_for(failure["headers"]["X-Request-Id"]))
names = span_names(failure_log["trace_id"])
assert "db.read" in {row[0] for row in names}
return {
"statuses": [item["status"] for item in attempts],
"max_latency_ms": max(item["elapsed"] for item in attempts) * 1000,
"trace_id": failure_log["trace_id"],
"state": ["firing", "resolved"],
"spans": sorted({row[0] for row in names}),
}
def backend_incident(client, actor, chat):
lost = "sum(otelcol_exporter_send_failed_spans_total)"
baseline = prom(lost)
compose("stop", "tempo", stdout=subprocess.DEVNULL)
try:
for _ in range(15):
client.message(actor, chat, "synthetic telemetry outage probe")
time.sleep(0.5)
delta = until(lambda: value if (value := prom(lost) - baseline) > 0 else None)
memory = prom("max(otelcol_process_memory_rss_bytes)")
assert memory > 0
finally:
compose("start", "tempo", stdout=subprocess.DEVNULL)
client.call("GET", "/health/ready", schema="Health")
return {"successful_messages": 15, "failed_export_spans": delta, "collector_rss_bytes": memory}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--incidents", action="store_true")
parser.add_argument("--retry", action="store_true")
args = parser.parse_args()
if args.output.exists():
parser.error("choose a new evidence file")
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
result = {"correlation": correlation(client, alice, chat, retry=args.retry)}
print("PASS: same Tempo trace and Loki logs for upload and worker", flush=True)
if args.incidents:
result["worker_stop"] = worker_incident(client, alice, chat)
print("PASS: queue alert firing and resolved", flush=True)
result["database_lock"] = database_incident(client, alice)
print("PASS: database and HTTP alerts firing and resolved", flush=True)
result["tempo_stop"] = backend_incident(client, alice, chat)
print("PASS: messages persisted during Tempo outage", flush=True)
args.output.write_text(json.dumps(result, indent=2) + "\n")
if __name__ == "__main__":
main()
+33
View File
@@ -0,0 +1,33 @@
"""Local test operator configuration. Credentials stay in ignored files/in memory."""
import os
from pathlib import Path
from psycopg.conninfo import make_conninfo
from messenger.settings import Settings
ROOT = Path(__file__).resolve().parents[1]
def settings(role="worker"):
prefix = "s3_root" if role == "root" else "s3_" + role
access, secret = (
("user", "password") if role == "root" else ("access_key_id", "secret_access_key")
)
return Settings(
make_conninfo(
host="127.0.0.1",
port=int(os.getenv("TEST_PG_PORT", "55467")),
user="messenger",
dbname="messenger",
password=(ROOT / "secrets/postgres_password").read_text().strip(),
),
"redis://127.0.0.1:" + os.getenv("TEST_CACHE_PORT", "56386") + "/0",
s3_endpoint="http://127.0.0.1:" + os.getenv("TEST_S3_PORT", "59006"),
s3_access_key=(ROOT / ("secrets/" + prefix + "_" + access)).read_text().strip(),
s3_secret_key=(ROOT / ("secrets/" + prefix + "_" + secret)).read_text().strip(),
broker_url="redis://127.0.0.1:" + os.getenv("TEST_BROKER_PORT", "56387") + "/0",
job_timeout=int(os.getenv("TEST_JOB_TIMEOUT_SECONDS", "5")),
fault_injection=True,
)
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# All running application/state services; no migration, down -v, flush or seed.
docker compose --env-file .env restart --timeout 7 api postgres valkey s3 broker relay worker lb
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Sequential process replacement; at least one existing API stays running."""
import argparse
import json
import subprocess
from cluster import compose, instances, wait_lb
def replace_all(base="http://127.0.0.1:8084"):
original = instances()
if len(original) < 2:
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
wait_lb(base, len(original))
replacements = []
for current in original:
subprocess.run(
["docker", "stop", "--time", "5", current["id"]],
check=True,
capture_output=True,
timeout=8,
)
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
compose(
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
)
aliases = wait_lb(base, len(original))
if current["instance"] in aliases:
raise RuntimeError("Old API process is still serving after replacement")
replacements.append(
{
"old_container": current["id"],
"old_instance": current["instance"],
"live_instances": aliases,
}
)
return replacements
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
args = parser.parse_args()
print(json.dumps(replace_all(args.base_url)))
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Map a process alias to ONE container belonging to this Compose service."""
import json
import os
import re
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def alias(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
cwd=ROOT,
capture_output=True,
text=True,
timeout=3,
check=True,
)
return container, r.stdout.strip()
def main():
target = os.environ.get("TARGET_INSTANCE_ID", "")
if not re.fullmatch(r"[a-f0-9]{32}", target):
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
ids = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
with ThreadPoolExecutor(max_workers=8) as pool:
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
if len(matches) != 1:
raise SystemExit("Expected exactly one matching API in this project")
subprocess.run(
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
)
print(json.dumps({"stopped_container": matches[0], "instance": target}))
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop-one.py
+184
View File
@@ -0,0 +1,184 @@
#!/usr/bin/env python3
"""Fixed offered-load comparison. Stops only named services in the current lab6 project."""
import argparse
import concurrent.futures
import json
import math
import os
import re
import subprocess
import time
from pathlib import Path
from urllib.request import urlopen
from observe import Client, compose, prom, until
MODES = (("off", "0", "0"), ("full", "1", "1"), ("half", "1", "0.5"), ("quarter", "1", "0.25"))
COUNTERS = {
"accepted_spans": "sum(otelcol_receiver_accepted_spans_total)",
"sent_spans": "sum(otelcol_exporter_sent_spans_total)",
"failed_spans": "sum(otelcol_exporter_send_failed_spans_total)",
"accepted_logs": "sum(otelcol_receiver_accepted_log_records_total)",
"accepted_metric_points": "sum(otelcol_receiver_accepted_metric_points_total)",
}
def configure(enabled, ratio):
compose(
"up",
"-d",
"--no-build",
"--no-deps",
"--force-recreate",
"api",
"worker",
"relay",
env={**os.environ, "TELEMETRY_ENABLED": enabled, "OTEL_SAMPLE_RATIO": ratio},
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def ready():
try:
with urlopen("http://127.0.0.1:8086/health/ready", timeout=2) as response:
return response.status == 200
except OSError:
return False
until(ready)
def snapshot():
with urlopen("http://127.0.0.1:3206/metrics", timeout=4) as response:
metrics = response.read().decode()
received_bytes = sum(
float(value)
for value in re.findall(
r"^tempo_distributor_bytes_received_total\{[^}]+\} (\S+)", metrics, re.M
)
)
return {
"trace_ingest_proto_bytes": received_bytes,
**{name: prom(expression) for name, expression in COUNTERS.items()},
}
def workload(client, actor, chat, seconds, rate):
def operation(n):
if n % 45 == 0:
started = time.perf_counter()
client.attachment(actor, chat)
return (time.perf_counter() - started) * 1000
if n % 5 == 0:
response = client.message(actor, chat, "synthetic cost experiment")
else:
response = client.call(
"GET",
f"/api/v1/chats/{chat['id']}/messages?limit=20",
actor=actor,
schema="MessagePage",
)
return response["elapsed"] * 1000
start = time.perf_counter()
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
futures = []
for number in range(seconds * rate):
delay = start + number / rate - time.perf_counter()
if delay > 0:
time.sleep(delay)
futures.append(pool.submit(operation, number))
observations = sorted(future.result() for future in futures)
elapsed = time.perf_counter() - start
return {
"requests": len(observations),
"elapsed_s": elapsed,
"latency_ms": {
f"p{int(q * 100)}": observations[math.ceil(q * len(observations)) - 1]
for q in (0.50, 0.95, 0.99)
},
}
def stats():
services = ("api", "worker", "relay", "otel-collector", "prometheus", "loki", "tempo")
ids = compose("ps", "-q", *services, capture_output=True, text=True).stdout.split()
raw = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
)
result = []
for line in raw.splitlines():
row = json.loads(line)
result.append(
{
"container": row["Name"],
"cpu_percent": float(row["CPUPerc"].rstrip("%")),
"memory": row["MemUsage"].split(" / ")[0],
}
)
return result
def run_one(mode, seconds, rate):
label, enabled, ratio = mode
configure(enabled, ratio)
client = Client("http://127.0.0.1:8086")
actor, _, _, chat = client.bootstrap()
for _ in range(25):
client.message(actor, chat, "seed")
workload(client, actor, chat, 3, rate)
time.sleep(3)
initial = snapshot()
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
run = pool.submit(workload, client, actor, chat, seconds, rate)
samples = []
for _ in range(2):
time.sleep(2)
samples.append(stats())
receipt = run.result()
# Collector emits batches and Prometheus scrapes every two seconds.
time.sleep(4)
final = snapshot()
return {
"mode": label,
"sample_ratio": float(ratio),
"load": receipt,
"container_samples": samples,
"ingest": {k: final[k] - v for k, v in initial.items()},
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--seconds", type=int, default=10)
parser.add_argument("--rate", type=int, default=18)
parser.add_argument("--repeats", type=int, default=2)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if args.output.exists() or args.seconds < 8 or args.rate < 1 or args.repeats < 1:
parser.error("use a new output and a positive load for at least 8 s")
results = []
try:
for repeat in range(args.repeats):
for mode in MODES if repeat % 2 == 0 else MODES[::-1]:
row = run_one(mode, args.seconds, args.rate)
print(row["mode"], row["load"]["latency_ms"], row["ingest"], flush=True)
results.append(row)
args.output.write_text(
json.dumps(
{
"seconds": args.seconds,
"offered_requests_per_s": args.rate,
"runs": results,
},
indent=2,
)
+ "\n"
)
finally:
configure("1", "1")
if __name__ == "__main__":
main()
+172
View File
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Two-phase real-age GC check: seed, run other checks, then verify after >=300 seconds."""
import argparse
import hashlib
import io
import json
import sys
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose
from pipeline_lab import settings
from messenger.cleanup import collect
from messenger.pipeline import Pipeline, intent
from messenger.repository import PostgresRepository
from messenger.storage import ObjectStore
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("action", choices=("seed", "verify"))
parser.add_argument("--seed", type=Path, required=True)
parser.add_argument("--output", type=Path)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
args = parser.parse_args()
config = settings("reconciler")
db = PostgresRepository(config.database_url, pool_size=3)
db.open()
store = ObjectStore(config)
try:
if args.action == "seed":
check(not args.seed.exists(), "Seed already exists")
compose("stop", "--timeout", "2", "worker", "relay")
try:
c = Client(args.base_url)
alice, _, _, room = c.bootstrap()
active, data = c.attachment(alice, room)
failed, _ = c.attachment(alice, room)
orphan = str(uuid4())
Pipeline(db, config).reserve(orphan, "staging/" + orphan)
uploader = ObjectStore(settings("api"))
try:
uploader.put(
"staging/" + orphan,
io.BytesIO(data),
len(data),
"image/png",
hashlib.sha256(data).hexdigest(),
)
finally:
uploader.close()
with db.connection(write=True) as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (active["id"],)
).fetchone()
conn.execute(
"UPDATE attachment_jobs SET "
"next_attempt_at=clock_timestamp()+interval '1 hour' "
"WHERE job_id=%s",
(j["job_id"],),
)
conn.execute(
"UPDATE job_outbox SET due_at=clock_timestamp()+interval '1 "
"hour' WHERE job_id=%s",
(j["job_id"],),
)
f = conn.execute(
"SELECT job_id FROM attachment_jobs WHERE attachment_id=%s", (failed["id"],)
).fetchone()
conn.execute(
"INSERT INTO pipeline_faults(job_id,phase,action) VALUES "
"(%s,'before_effect','permanent')",
(f["job_id"],),
)
seed = {
"created_at": datetime.now(UTC).isoformat(),
"active_id": active["id"],
"failed_id": failed["id"],
"orphan_id": orphan,
}
args.seed.write_text(json.dumps(seed, indent=2) + "\n")
print(
"PASS: active job, failed candidate and unaccepted orphan seeded; no "
"credentials saved"
)
finally:
compose("start", "worker", "relay")
else:
check(args.output is not None and not args.output.exists(), "Choose a new output file")
seed = json.loads(args.seed.read_text())
# Real S3 timestamps and failed retention; no modified clocks.
while True:
with db.connection() as conn:
f = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],)
).fetchone()
check(f["state"] == "failed", "Failed seed did not reach terminal state")
age = (datetime.now(UTC) - f["updated_at"]).total_seconds()
if age >= 301:
break
check(age >= 0, "Clock mismatch")
time.sleep(min(1, 301 - age))
with db.connection() as conn:
ready = conn.execute(
"SELECT * FROM attachments WHERE storage_state='ready' LIMIT 1"
).fetchone()
check(ready is not None, "Need a published attachment to protect")
keys = [ready["object_key"], *[v["object_key"] for v in ready["variants"]]]
before = {key: store.head(key)["ETag"] for key in keys}
preview = collect(db, store, 300, False)
result = collect(db, store, 300, True)
check(
store.head("staging/" + seed["active_id"]) is not None, "Active staging was deleted"
)
check(store.head("staging/" + seed["orphan_id"]) is None, "Orphan not deleted")
check(
store.head("staging/" + seed["failed_id"]) is None,
"Expired failed source not deleted",
)
check(
all(store.head(k)["ETag"] == tag for k, tag in before.items()),
"Published result changed",
)
with db.connection(write=True) as conn:
expired = conn.execute(
"SELECT source_expired FROM attachment_jobs WHERE attachment_id=%s",
(seed["failed_id"],),
).fetchone()
check(expired["source_expired"], "Expired source not registered")
j = conn.execute(
"UPDATE attachment_jobs SET next_attempt_at=clock_timestamp() WHERE "
"attachment_id=%s RETURNING job_id",
(seed["active_id"],),
).fetchone()
intent(conn, j["job_id"])
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
if db.attachment(seed["active_id"])["storage_state"] == "ready":
break
time.sleep(0.2)
check(
db.attachment(seed["active_id"])["storage_state"] == "ready",
"Protected work failed to complete",
)
proof = {
"retention_seconds": 300,
"failed_real_age_seconds": age,
"seed": seed,
"dry_run": preview,
"apply": result,
"active_protected_then_ready": True,
"orphan_deleted": True,
"failed_source_expired": True,
"published_objects_unchanged": len(keys),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(proof, indent=2) + "\n")
print("PASS: real-age cleanup preserves active work and published results")
finally:
store.close()
db.close()
if __name__ == "__main__":
main()
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Real LB requests during each replica failure and sequential replacement."""
import argparse
import json
import os
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from resilience import history # noqa: E402
def percentile(values, fraction):
values = sorted(values)
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
class Traffic:
def __init__(self, base, actor, room, workers=6):
self.base, self.actor, self.room = base, actor, room
self.prefix = uuid4().hex
self.stop = threading.Event()
self.lock = threading.Lock()
self.events = []
self.executor = ThreadPoolExecutor(max_workers=workers)
self.workers = workers
def worker(self, worker):
client, iteration = Client(self.base, timeout=3), 0
while not self.stop.is_set():
started = time.monotonic()
write = iteration % 10 == 9
try:
if write:
key = f"{self.prefix}:{worker}:{iteration}"
response = client.message(self.actor, self.room, key, key=key)
else:
response = client.call(
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
)
event = {
"at": time.monotonic(),
"ok": True,
"instance": response["headers"]["X-Instance-Id"],
"latency_ms": (time.monotonic() - started) * 1000,
"message": response["json"] if write else None,
}
iteration += 1
except Failure:
event = {
"at": time.monotonic(),
"ok": False,
"latency_ms": (time.monotonic() - started) * 1000,
}
with self.lock:
self.events.append(event)
def __enter__(self):
self.started = time.monotonic()
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
return self
def __exit__(self, *args):
self.stop.set()
self.executor.shutdown(wait=True)
for future in self.futures:
future.result()
self.finished = time.monotonic()
def report(self, action_start, action_end):
events = sorted(self.events, key=lambda row: row["at"])
ok = [row for row in events if row["ok"]]
after = [row for row in ok if row["at"] >= action_end]
check(bool(after), "No successful request after action")
elapsed = self.finished - self.started
latency = [row["latency_ms"] for row in ok]
return {
"duration_seconds": elapsed,
"workers": self.workers,
"requests": len(events),
"successes": len(ok),
"errors": len(events) - len(ok),
"successful_rps": len(ok) / elapsed,
"error_fraction": (len(events) - len(ok)) / len(events),
"first_success_after_action_seconds": after[0]["at"] - action_start,
"action_seconds": action_end - action_start,
"p50_ms": percentile(latency, 0.5),
"p95_ms": percentile(latency, 0.95),
"p99_ms": percentile(latency, 0.99),
"instances": sorted({row["instance"] for row in ok}),
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
"confirmed_writes": sum(row["message"] is not None for row in ok),
}
def verify_messages(self, client, actor, room):
actual = history(client, actor, room)
by_id = {value["id"]: value for value in actual}
check(len(by_id) == len(actual), "History has duplicate IDs")
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
for event in self.events:
if event["ok"] and event["message"]:
check(
by_id.get(event["message"]["id"]) == event["message"],
"A confirmed message was lost or changed",
)
def public_test(target, base):
result = subprocess.run(
[
"make",
target,
"ACTION_SCRIPT=scripts/"
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
"BASE_URL=" + base,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=150,
)
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
parser.add_argument(
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
)
args = parser.parse_args()
if args.output.exists():
raise SystemExit("Use a new evidence filename")
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
baseline = instances()
check(len(baseline) == 2, "This verification expects the default two API replicas")
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
wait_lb(args.base_url, 2)
result = {"initial_instances": baseline, "failures": []}
client = Client(args.base_url, timeout=4)
alice, bob, eve, room = client.bootstrap()
key = uuid4().hex
def same_request(_):
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
return response["json"], response["headers"]["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
responses = list(pool.map(same_request, range(30)))
check(
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
)
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
result["concurrent_replays"] = 30
result["concurrent_effects"] = 1
try:
for selected in baseline:
selected = instance(selected["id"])
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
timeout=10,
)
action_end = time.monotonic()
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
time.sleep(3)
report = traffic.report(action_start, action_end)
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
report["stopped"] = selected
traffic.verify_messages(client, bob, room)
result["failures"].append(report)
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
wait_lb(args.base_url, 2)
print("Verified failure and restoration of one API", flush=True)
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
replaced = subprocess.run(
[
sys.executable,
str(ROOT / "scripts/rolling-update.py"),
"--base-url",
args.base_url,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=120,
)
action_end = time.monotonic()
time.sleep(3)
traffic.verify_messages(client, bob, room)
result["rolling"] = traffic.report(action_start, action_end)
result["rolling"]["replacements"] = json.loads(replaced.stdout)
result["rolling"]["confirmed_messages_preserved"] = True
print("Verified sequential replacement under traffic", flush=True)
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
wait_lb(args.base_url, 2)
before = {
service: inspect(compose("ps", "-q", service).strip())
for service in ("postgres", "valkey", "s3", "broker", "lb")
}
(ROOT / "evidence/persistence.txt").write_text(
public_test("test-persistence", args.base_url)
)
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
check(
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
"Persistence hook did not restart all state/LB services",
)
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
result["final_instances"] = instances()
logs = compose("logs", "--no-color", "api", "lb")
check(
not any(
a["token"] in logs or a["registration"]["password"] in logs
for a in (alice, bob, eve)
),
"Credential appeared in logs",
)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print("PASS: real replica failures, replay, rolling replacement and persistence")
finally:
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
if __name__ == "__main__":
main()
+513
View File
@@ -0,0 +1,513 @@
#!/usr/bin/env python3
"""Real-process fault matrix for the versioned pipeline; test Compose override required."""
import argparse
import hashlib
import io
import json
import subprocess
import sys
import time
from pathlib import Path
from uuid import uuid4
from botocore.exceptions import ClientError
from cluster import ROOT, compose, inspect, instances, wait_lb
from PIL import Image
from pipeline_lab import settings
from messenger.broker import STREAM, Broker, Job
from messenger.pipeline import Pipeline
from messenger.repository import PostgresRepository, stamp
from messenger.storage import ObjectStore
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def until(fn, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = fn()
if result:
return result
time.sleep(0.2)
raise AssertionError("Timed out waiting for a durable pipeline condition")
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
parser.add_argument("--output", type=Path, default=ROOT / "evidence/pipeline.json")
args = parser.parse_args()
if args.output.exists():
parser.error("Output already exists; choose a new evidence file")
config = settings()
db = PostgresRepository(config.database_url, pool_size=4)
db.open()
broker, store = Broker(config), ObjectStore(config)
c = Client(args.base_url)
alice, bob, eve, room = c.bootstrap()
records, accepted = [], []
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
def query(sql, params=()):
with db.connection() as conn:
return conn.execute(sql, params).fetchall()
def job(a, version=1):
return query(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s",
(a["id"], version),
)[0]
def audit(j):
return query(
"SELECT event,details,created_at FROM pipeline_audit WHERE job_id=%s ORDER BY sequence",
(j["job_id"],),
)
def injected(j, phase):
return any(
e["event"] == "fault_injected" and e["details"]["phase"] == phase for e in audit(j)
)
def fault(j, phase, action, count=1):
with db.connection(write=True) as conn:
conn.execute(
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) VALUES (%s,%s,%s,%s)",
(j["job_id"], phase, action, count),
)
def upload():
a, original = c.attachment(alice, room)
check(original == data, "Unexpected fixture")
accepted.append(a["id"])
return a
def ready(a):
value = c.wait_ready(bob, a)
path = "/api/v1/attachments/" + a["id"]
check(c.call("GET", path + "/content", actor=bob)["bytes"] == data, "Original changed")
for v in value["variants"]:
content = c.call("GET", path + "/content?variant=" + v["name"], actor=bob)["bytes"]
check(hashlib.sha256(content).hexdigest() == v["sha256"], "Variant checksum mismatch")
with Image.open(io.BytesIO(content)) as im:
check(im.size == (v["width"], v["height"]), "Variant dimensions mismatch")
check(len(value["variants"]) == 3, "Missing variants")
check(
len(query("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (a["id"],))) <= 1,
"Unexpected duplicate link",
)
return value
def saved(name, a, start, **details):
j = job(a, details.get("version", 1))
current = query(
"SELECT storage_state,pipeline_version FROM attachments WHERE id=%s", (a["id"],)
)[0]
keys = [
o["Key"]
for o in store.client.list_objects_v2(
Bucket=store.bucket,
Prefix="objects/" + a["id"] + "/v" + str(current["pipeline_version"]) + "/",
).get("Contents", [])
]
check(len(keys) == 4, "Repeated deliveries multiplied the published variant set")
record = {
"scenario": name,
"attachment_id": a["id"],
"job_id": str(j["job_id"]),
"elapsed_seconds": time.monotonic() - start,
"final_state": current["storage_state"],
"attempts": j["attempt"],
"final_version": current["pipeline_version"],
"published_objects": len(keys),
"audit": audit(j),
**details,
}
records.append(record)
print("PASS: " + name, flush=True)
def envelope(j, attempt=None):
return Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=str(j["attachment_id"]),
pipeline_version=j["pipeline_version"],
attempt=attempt or max(1, j["attempt"]),
created_at=stamp(j["created_at"]),
)
try:
# Actual COMMIT/publish gap: no relay exists while the acknowledged upload is stored.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
j = job(a)
check(
query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
"published_at"
]
is None,
"Gap was not established",
)
message = c.message(alice, room, "", attachments=[a["id"]])["json"]
c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", 409, actor=bob)
before = instances()
compose("restart", "--timeout", "3", "api")
wait_lb(args.base_url)
after = instances()
check(
{r["instance"] for r in before}.isdisjoint({r["instance"] for r in after}),
"API did not restart",
)
compose("start", "relay")
until(
lambda: query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
"published_at"
]
)
entry_ids = [
id
for id, fields in broker.client.xrange(STREAM)
if json.loads(fields["job"])["job_id"] == str(j["job_id"])
]
check(bool(entry_ids), "Broker has no accepted work")
broker_id = compose("ps", "-q", "broker").strip()
broker_before = inspect(broker_id)
subprocess.run(
["docker", "kill", "--signal", "KILL", broker_id], check=True, capture_output=True
)
subprocess.run(["docker", "start", broker_id], check=True, capture_output=True)
def broker_restored():
try:
return broker.client.ping()
except Exception:
return False
until(broker_restored)
check(
set(entry_ids) <= {id for id, _ in broker.client.xrange(STREAM)},
"Durable stream lost confirmed work",
)
compose("start", "worker")
ready(a)
history = c.call(
"GET", "/api/v1/chats/" + room["id"] + "/messages", actor=bob, schema="MessagePage"
)["json"]
check(message in history["items"], "Queued attachment message lost")
saved(
"commit_publish_gap_api_restart_broker_sigkill",
a,
start,
api_before=before,
api_after=after,
broker_before=broker_before,
broker_after=inspect(broker_id),
broker_entry_ids=entry_ids,
message_id=message["id"],
)
# A publisher can lose its COMMIT after XADD; same logical ID is sent twice.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, "after_publish", "crash")
compose("start", "relay")
until(lambda: injected(j, "after_publish"))
until(
lambda: (
sum(
json.loads(f["job"])["job_id"] == str(j["job_id"])
for _, f in broker.client.xrange(STREAM)
)
>= 2
)
)
compose("start", "worker")
ready(a)
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
saved("publish_before_outbox_commit_duplicate", a, start)
for phase in ("before_effect", "after_effect", "after_commit"):
compose("stop", "--timeout", "2", "worker")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, phase, "crash")
compose("start", "worker")
ready(a)
until(lambda j=j, phase=phase: injected(j, phase))
if phase == "after_commit":
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
check(job(a)["attempt"] == 1, "Committed job was processed twice")
else:
check(job(a)["attempt"] == 2, "Expired lease did not recover exactly once")
saved("worker_crash_" + phase, a, start)
for action, count, expected in (
("transient", 2, "ready"),
("hang", 1, "ready"),
("permanent", 1, "failed"),
("transient", 5, "failed"),
):
compose("stop", "--timeout", "2", "worker")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, "before_effect", action, count)
compose("start", "worker")
path = "/api/v1/attachments/" + a["id"]
if expected == "failed":
until(
lambda path=path: (
c.call("GET", path, actor=bob, schema="Attachment")["json"]["status"]
== "failed"
)
)
terminal = job(a)
check(terminal["attempt"] == count, "Wrong retry budget")
c.call("GET", path + "/content", 409, actor=bob)
c.call(
"POST",
"/api/v1/chats/" + room["id"] + "/messages",
409,
actor=alice,
body={"text": "", "attachment_ids": [a["id"]]},
headers={"Idempotency-Key": uuid4().hex},
)
# Explicit CLI replay, same logical job; no public administrator endpoint.
replay = json.loads(
compose(
"exec",
"-T",
"worker",
"python",
"-m",
"messenger.operator",
"replay",
a["id"],
)
)
check(replay["job_id"] == str(j["job_id"]), "Replay changed logical job")
else:
terminal = None
ready(a)
events = audit(j)
delays = [e["details"]["delay_seconds"] for e in events if e["event"] == "queued"]
check(all(0.25 <= delay <= 30 for delay in delays), "Invalid bounded jitter")
rejected = subprocess.run(
[
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"worker",
"python",
"-m",
"messenger.operator",
"replay",
a["id"],
],
cwd=ROOT,
capture_output=True,
)
check(rejected.returncode != 0, "Ready replay was accepted")
saved(
action + "_" + str(count) + ("_failed_then_explicit_replay" if terminal else ""),
a,
start,
terminal_before_replay={k: terminal[k] for k in ("state", "attempt", "error_code")}
if terminal
else None,
backoff_delays=delays,
)
# Reorder and stale v1 delivered after v2; exercise the DB publication fence directly too.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
old = job(a)
old_envelope = envelope(old, 1)
outcome, old_claim = Pipeline(db, config).claim(old_envelope)
check(outcome == "claimed", "Cannot establish old writer")
Pipeline(db, config).replay(a["id"], new_version=True)
new = job(a, 2)
# Publish v2 first, then v1, plus duplicates through both worker slots.
for value in (envelope(new, 1), old_envelope, envelope(new, 1), old_envelope):
broker.publish(value)
compose("start", "worker", "relay")
current = ready(a)
row = db.attachment(a["id"])
check(
not Pipeline(db, config).publish(old_claim, "objects/old-writer", []),
"Old writer overwrote v2",
)
for _ in range(20):
broker.publish(old_envelope)
broker.publish(envelope(new, 1))
until(lambda: sum(e["event"] == "stale_delivery" for e in audit(old)) >= 21)
until(lambda: sum(e["event"] == "duplicate_ready" for e in audit(new)) >= 20)
check(
db.attachment(a["id"])["object_key"] == row["object_key"],
"Stale delivery changed current pointer",
)
check(
c.call("GET", "/api/v1/attachments/" + a["id"], actor=bob, schema="Attachment")["json"]
== current,
"Old version mutated ready metadata",
)
saved(
"reorder_stale_v1_after_v2_and_publication_fence",
a,
start,
version=2,
old_audit=audit(old),
duplicates=44,
)
# ACL remains enforced through both API replicas; S3 is private and role-scoped.
a = {"id": accepted[0]}
for suffix in ("", "/content", "/content?variant=crop"):
c.call("GET", "/api/v1/attachments/" + a["id"] + suffix, 404, actor=eve)
# Direct bucket access and write/delete privilege checks use no public credentials.
import boto3
from botocore import UNSIGNED
from botocore.config import Config
first = db.attachment(accepted[0])
unsigned = boto3.client(
"s3", endpoint_url=config.s3_endpoint, config=Config(signature_version=UNSIGNED)
)
try:
unsigned.get_object(Bucket=store.bucket, Key=first["object_key"])
raise AssertionError("Anonymous S3 read succeeded")
except ClientError as exc:
check(
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong anonymous status"
)
finally:
unsigned.close()
denied = []
for role, action in (
("api", "put_object"),
("api", "delete_object"),
("worker", "delete_object"),
("reconciler", "put_object"),
):
scoped = ObjectStore(settings(role))
try:
kw = {"Bucket": store.bucket, "Key": "objects/" + uuid4().hex + "/forbidden"}
if action == "put_object":
kw["Body"] = b"forbidden"
try:
getattr(scoped.client, action)(**kw)
raise AssertionError("IAM action unexpectedly allowed")
except ClientError as exc:
check(
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403,
"Wrong IAM status",
)
denied.append(role + ":" + action)
finally:
scoped.close()
# Stop the actual uploader, then read original and thumbnail through another replica.
response = c.call(
"POST",
"/api/v1/chats/" + room["id"] + "/attachments?filename=cross.png",
202,
actor=alice,
raw=data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
cross = response["json"]
accepted.append(cross["id"])
ready(cross)
uploader = response["headers"]["X-Instance-Id"]
process = next(v for v in instances() if v["instance"] == uploader)
subprocess.run(
["docker", "stop", "--time", "3", process["id"]], check=True, capture_output=True
)
try:
wait_lb(args.base_url, count=1)
download = c.call("GET", "/api/v1/attachments/" + cross["id"] + "/content", actor=bob)
check(
download["bytes"] == data and download["headers"]["X-Instance-Id"] != uploader,
"Other replica could not serve the original",
)
c.call(
"GET",
"/api/v1/attachments/" + cross["id"] + "/content?variant=thumbnail",
actor=bob,
)
cross_proof = {
"uploader": uploader,
"stopped": inspect(process["id"]),
"reader": download["headers"]["X-Instance-Id"],
}
finally:
subprocess.run(["docker", "start", process["id"]], check=True, capture_output=True)
wait_lb(args.base_url)
logs = compose("logs", "--no-color", "api", "worker", "relay", "lb")
check(
not any(
actor["token"] in logs or actor["registration"]["password"] in logs
for actor in (alice, bob, eve)
),
"Credential in logs",
)
states = query(
"SELECT id,storage_state FROM attachments WHERE id=ANY(%s::uuid[])", (accepted,)
)
check(
len(states) == len(accepted) and all(s["storage_state"] == "ready" for s in states),
"Confirmed work lost",
)
stats = compose(
"exec",
"-T",
"worker",
"python",
"-c",
"import json;from pathlib import Path;print(json.dumps({"
"n:Path('/sys/fs/cgroup/'+n).read_text().strip() "
"for n in ('memory.current','memory.peak','memory.events','cpu.stat')}))",
)
result = {
"anonymous_s3_status": 403,
"denied_iam_actions": denied,
"cross_replica": cross_proof,
"track": 2,
"delivery": "at-least-once",
"effect_key": ["attachment_id", "pipeline_version", "variant"],
"worker_concurrency": config.worker_concurrency,
"prefetch_per_slot": 1,
"job_timeout_seconds": config.job_timeout,
"lease_seconds": config.lease_seconds,
"max_attempts": config.job_max_attempts,
"scenarios": records,
"accepted": len(accepted),
"lost_confirmed": 0,
"stale_overwrites": 0,
"worker_cgroup": json.loads(stats),
"logs_credentials_absent": True,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n")
print("PASS: all acknowledged work ready; no stale overwrites", flush=True)
finally:
compose("start", "worker", "relay", "broker", "api")
broker.close()
store.close()
db.close()
if __name__ == "__main__":
main()