test(pipeline): verify durable cleanup and retain successful HTTP evidence

This commit is contained in:
2026-11-15 22:45:10 +03:00
parent bb4e8a2254
commit a275cf2632
10 changed files with 678 additions and 38 deletions
+48 -9
View File
@@ -80,6 +80,7 @@ def main():
"API_IMAGE": "mtusi-messenger:lab5-clean-" + identity,
"PUBLIC_PORT": public_port,
"VCS_REF": sha,
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
@@ -114,6 +115,7 @@ def main():
cwd=clone,
env=env,
)
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
run(["make", "bootstrap"], cwd=clone, env=env)
try:
run(
@@ -141,19 +143,39 @@ def main():
cwd=clone,
env=env,
)
run(
for command_args in [
[
".venv/bin/python",
"-m",
"scripts.verify_storage",
"--output",
"evidence/clean-storage.json",
"scripts.verify_cleanup",
"seed",
"--seed",
"evidence/clean-gc-seed.json",
],
cwd=clone,
env=env,
[
".venv/bin/python",
"-m",
"scripts.verify_pipeline",
"--output",
"evidence/clean-pipeline.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"verify",
"--seed",
"evidence/clean-gc-seed.json",
"--output",
"evidence/clean-cleanup.json",
],
]:
run(command_args, cwd=clone, env=env)
receipt["pipeline"] = json.loads(
(clone / "evidence/clean-pipeline.json").read_text()
)
receipt["storage"] = json.loads(
(clone / "evidence/clean-storage.json").read_text()
receipt["cleanup"] = json.loads(
(clone / "evidence/clean-cleanup.json").read_text()
)
receipt["cluster"] = json.loads(
(clone / "evidence/clean-cluster.json").read_text()
@@ -172,6 +194,21 @@ def main():
"import hashlib,json,pathlib; print(json.dumps({p.name:"
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
)
ids += run(
[
"docker",
"compose",
"ps",
"-q",
"worker",
"relay",
"reconciler",
"controller",
],
cwd=clone,
env=env,
).split()
assert len(ids) == 6
for cid in ids:
actual = json.loads(
run(
@@ -191,7 +228,9 @@ def main():
== sha
)
assert not info["HostConfig"]["PortBindings"]
receipt["both_replicas_runtime_hashes_and_revision_match"] = True
receipt[
"api_worker_relay_reconciler_runtime_hashes_and_revision_match"
] = True
receipt["fsck"] = run(
["git", "fsck", "--full"], cwd=clone, env=env
)
+223
View File
@@ -0,0 +1,223 @@
"""Real S3 retention test: seed is separate from verification output."""
import argparse
import json
import time
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
from scripts.verify_pipeline import inside, operator, wait
def seed(path):
if path.exists():
raise ValueError("Choose a new seed file")
ready()
compose("stop", "--timeout", "1", "reconciler")
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c:
user, _, session = actor(c)
headers = auth(session)
result = c.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Cleanup synthetic seed",
"member_ids": [user["id"]],
},
)
assert result.status_code == 201
chat = result.json()["id"]
r = c.post(
f"/api/v1/chats/{chat}/attachments",
headers={**headers, "Content-Type": "image/png"},
params={"filename": "retained.png"},
content=raw,
)
assert r.status_code == 202
attachment = r.json()["id"]
wait(
lambda: (
c.get(
"/api/v1/attachments/" + attachment, headers=headers
).json()["status"]
== "ready"
)
)
job = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
data = {
"seed_id": uuid4().hex,
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"ready_attachment": attachment,
"ready_job": job,
"chat": chat,
"owner": user["id"],
}
# Real old v1 objects will become obsolete after v2 is ready.
second = operator(job, version=2)
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
)
)
data["version2_job"] = second
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
data["failed_id"] = str(uuid4())
data["active_id"] = str(uuid4())
data["reservation_id"] = str(uuid4())
data["expired_reservation_id"] = str(uuid4())
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
data["active_source"] = f"objects/staging/{data['active_id']}/original"
data["reservation_key"] = (
f"objects/staging/{data['reservation_id']}/original"
)
data["expired_reservation_key"] = (
f"objects/staging/{data['expired_reservation_id']}/original"
)
inside(f"""from pathlib import Path
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
with db.connection() as c, c.transaction():
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
key='objects/staging/'+identity+'/original'
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
print('true')""")
data["seeded_unix"] = time.time()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: retention seed created; real S3 age must reach300s", flush=True
)
def verify(seed_path, output):
if output.exists():
raise ValueError("Choose a new output path")
data = json.loads(seed_path.read_text())
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"seed": seed_path.name,
"complete": False,
}
try:
compose("stop", "--timeout", "1", "reconciler")
def age():
return inside(
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
)
while age() < 302:
time.sleep(min(5, 303 - age()))
report["real_s3_age_seconds"] = age()
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
inside(
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
)
oldkeys = [
data["orphan"],
data["failed_source"],
data["expired_reservation_key"],
] + [
f"objects/{data['ready_attachment']}/v1/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
protected = [data["active_source"], data["reservation_key"], fresh] + [
f"objects/{data['ready_attachment']}/v2/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
before = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(before.values())
dry = inside(
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
)
candidates = {a["key"] for a in dry["actions"]}
assert set(oldkeys) <= candidates and not set(protected) & candidates
afterdry = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert afterdry == before
applied = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
repeat = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
assert not repeat["actions"]
after = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(not after[k] for k in oldkeys) and all(
after[k] for k in protected
)
failed = inside(
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
)
assert failed["error_code"] == "source_expired"
try:
operator(failed["id"])
except RuntimeError:
replay_refused = True
else:
raise AssertionError("Expired source replay must fail")
# Expired reservation was removed, so a delayed API cannot commit its handoff.
fenced = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
)
assert fenced
report.update(
dry_run=dry,
apply=applied,
repeat=repeat,
before=before,
after=after,
dry_run_unchanged=True,
active_current_and_fresh_protected=True,
expired_source_replay_refused=replay_refused,
expired_writer_fenced=fenced,
complete=True,
)
finally:
# Remove only this fixture's artificial active reservation/job; never real jobs.
inside(
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
)
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
flush=True,
)
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["seed", "verify"])
p.add_argument("--seed", required=True, type=Path)
p.add_argument("--output", type=Path)
a = p.parse_args()
if a.action == "seed":
seed(a.seed)
else:
if a.output is None:
p.error("output required")
verify(a.seed, a.output)
if __name__ == "__main__":
main()
+79 -3
View File
@@ -30,7 +30,7 @@ db,store,queue=database(),ObjectStore(),broker()
def inside(code):
cid = compose("ps", "-q", "api").split()[0]
cid = compose("ps", "-q", "controller").split()[0]
return json.loads(
command(
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
@@ -123,8 +123,19 @@ def main():
try:
ready()
# Check instrumentation is really enabled and production default is off.
assert inside(
"import os\nprint(json.dumps(os.getenv('PIPELINE_FAULTS')=='1'))"
cid = compose("ps", "-q", "api").split()[0]
assert (
command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import os;print(os.getenv('PIPELINE_FAULTS'))",
]
)
== "1"
)
stop("reconciler")
with httpx.Client(
@@ -503,6 +514,71 @@ def main():
save(
"malformed_notification_rejected", {"durable_diagnostic": True}
)
# Private bucket, bounded IAM, and download after uploader process dies.
a, j = upload()
confirm(a, j)
key = f"objects/{a['id']}/v1/original"
security = inside(f"""from minio.error import S3Error
from urllib.error import HTTPError
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
except HTTPError as e: anonymous=e.code
else: raise AssertionError('Anonymous access allowed')
denials={{}}
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
try: fn()
except S3Error as e: denials[name]=e.code
else: raise AssertionError('IAM allowed forbidden action')
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
assert security["anonymous"] == 403 and set(
security["iam"].values()
) == {"AccessDenied"}
selected = None
for cid in compose("ps", "-q", "api").split():
alias = command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
]
)
uploader = inside(
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
)
if alias == uploader:
selected = cid
assert selected
command(["docker", "stop", "--time", "1", selected])
ready()
wait(
lambda: (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 200
)
)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.content == raw
and downloaded.headers["X-Instance-Id"] != uploader
)
start("api")
ready()
save(
"private_iam_and_other_api_download",
{
"security": security,
"attachment": a["id"],
"uploader": uploader,
"downloader": downloaded.headers["X-Instance-Id"],
"sha256": hashlib.sha256(raw).hexdigest(),
},
)
report["worker_metrics"] = inside(
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
)