test(pipeline): verify durable cleanup and retain successful HTTP evidence
This commit is contained in:
+48
-9
@@ -80,6 +80,7 @@ def main():
|
||||
"API_IMAGE": "mtusi-messenger:lab5-clean-" + identity,
|
||||
"PUBLIC_PORT": public_port,
|
||||
"VCS_REF": sha,
|
||||
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
@@ -114,6 +115,7 @@ def main():
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
|
||||
run(["make", "bootstrap"], cwd=clone, env=env)
|
||||
try:
|
||||
run(
|
||||
@@ -141,19 +143,39 @@ def main():
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
run(
|
||||
for command_args in [
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_storage",
|
||||
"--output",
|
||||
"evidence/clean-storage.json",
|
||||
"scripts.verify_cleanup",
|
||||
"seed",
|
||||
"--seed",
|
||||
"evidence/clean-gc-seed.json",
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_pipeline",
|
||||
"--output",
|
||||
"evidence/clean-pipeline.json",
|
||||
],
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cleanup",
|
||||
"verify",
|
||||
"--seed",
|
||||
"evidence/clean-gc-seed.json",
|
||||
"--output",
|
||||
"evidence/clean-cleanup.json",
|
||||
],
|
||||
]:
|
||||
run(command_args, cwd=clone, env=env)
|
||||
receipt["pipeline"] = json.loads(
|
||||
(clone / "evidence/clean-pipeline.json").read_text()
|
||||
)
|
||||
receipt["storage"] = json.loads(
|
||||
(clone / "evidence/clean-storage.json").read_text()
|
||||
receipt["cleanup"] = json.loads(
|
||||
(clone / "evidence/clean-cleanup.json").read_text()
|
||||
)
|
||||
receipt["cluster"] = json.loads(
|
||||
(clone / "evidence/clean-cluster.json").read_text()
|
||||
@@ -172,6 +194,21 @@ def main():
|
||||
"import hashlib,json,pathlib; print(json.dumps({p.name:"
|
||||
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
|
||||
)
|
||||
ids += run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"ps",
|
||||
"-q",
|
||||
"worker",
|
||||
"relay",
|
||||
"reconciler",
|
||||
"controller",
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
).split()
|
||||
assert len(ids) == 6
|
||||
for cid in ids:
|
||||
actual = json.loads(
|
||||
run(
|
||||
@@ -191,7 +228,9 @@ def main():
|
||||
== sha
|
||||
)
|
||||
assert not info["HostConfig"]["PortBindings"]
|
||||
receipt["both_replicas_runtime_hashes_and_revision_match"] = True
|
||||
receipt[
|
||||
"api_worker_relay_reconciler_runtime_hashes_and_revision_match"
|
||||
] = True
|
||||
receipt["fsck"] = run(
|
||||
["git", "fsck", "--full"], cwd=clone, env=env
|
||||
)
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Real S3 retention test: seed is separate from verification output."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
||||
from scripts.verify_pipeline import inside, operator, wait
|
||||
|
||||
|
||||
def seed(path):
|
||||
if path.exists():
|
||||
raise ValueError("Choose a new seed file")
|
||||
ready()
|
||||
compose("stop", "--timeout", "1", "reconciler")
|
||||
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c:
|
||||
user, _, session = actor(c)
|
||||
headers = auth(session)
|
||||
result = c.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Cleanup synthetic seed",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert result.status_code == 201
|
||||
chat = result.json()["id"]
|
||||
r = c.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**headers, "Content-Type": "image/png"},
|
||||
params={"filename": "retained.png"},
|
||||
content=raw,
|
||||
)
|
||||
assert r.status_code == 202
|
||||
attachment = r.json()["id"]
|
||||
wait(
|
||||
lambda: (
|
||||
c.get(
|
||||
"/api/v1/attachments/" + attachment, headers=headers
|
||||
).json()["status"]
|
||||
== "ready"
|
||||
)
|
||||
)
|
||||
job = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
|
||||
)
|
||||
data = {
|
||||
"seed_id": uuid4().hex,
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"ready_attachment": attachment,
|
||||
"ready_job": job,
|
||||
"chat": chat,
|
||||
"owner": user["id"],
|
||||
}
|
||||
# Real old v1 objects will become obsolete after v2 is ready.
|
||||
second = operator(job, version=2)
|
||||
wait(
|
||||
lambda: inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
|
||||
)
|
||||
)
|
||||
data["version2_job"] = second
|
||||
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
|
||||
data["failed_id"] = str(uuid4())
|
||||
data["active_id"] = str(uuid4())
|
||||
data["reservation_id"] = str(uuid4())
|
||||
data["expired_reservation_id"] = str(uuid4())
|
||||
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
|
||||
data["active_source"] = f"objects/staging/{data['active_id']}/original"
|
||||
data["reservation_key"] = (
|
||||
f"objects/staging/{data['reservation_id']}/original"
|
||||
)
|
||||
data["expired_reservation_key"] = (
|
||||
f"objects/staging/{data['expired_reservation_id']}/original"
|
||||
)
|
||||
inside(f"""from pathlib import Path
|
||||
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
|
||||
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
|
||||
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
|
||||
with db.connection() as c, c.transaction():
|
||||
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
|
||||
key='objects/staging/'+identity+'/original'
|
||||
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
|
||||
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
|
||||
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
|
||||
print('true')""")
|
||||
data["seeded_unix"] = time.time()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: retention seed created; real S3 age must reach300s", flush=True
|
||||
)
|
||||
|
||||
|
||||
def verify(seed_path, output):
|
||||
if output.exists():
|
||||
raise ValueError("Choose a new output path")
|
||||
data = json.loads(seed_path.read_text())
|
||||
report = {
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"seed": seed_path.name,
|
||||
"complete": False,
|
||||
}
|
||||
try:
|
||||
compose("stop", "--timeout", "1", "reconciler")
|
||||
|
||||
def age():
|
||||
return inside(
|
||||
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
|
||||
)
|
||||
|
||||
while age() < 302:
|
||||
time.sleep(min(5, 303 - age()))
|
||||
report["real_s3_age_seconds"] = age()
|
||||
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
|
||||
inside(
|
||||
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
|
||||
)
|
||||
oldkeys = [
|
||||
data["orphan"],
|
||||
data["failed_source"],
|
||||
data["expired_reservation_key"],
|
||||
] + [
|
||||
f"objects/{data['ready_attachment']}/v1/{v}"
|
||||
for v in ["original", "thumbnail", "crop", "optimized"]
|
||||
]
|
||||
protected = [data["active_source"], data["reservation_key"], fresh] + [
|
||||
f"objects/{data['ready_attachment']}/v2/{v}"
|
||||
for v in ["original", "thumbnail", "crop", "optimized"]
|
||||
]
|
||||
before = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert all(before.values())
|
||||
dry = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
|
||||
)
|
||||
candidates = {a["key"] for a in dry["actions"]}
|
||||
assert set(oldkeys) <= candidates and not set(protected) & candidates
|
||||
afterdry = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert afterdry == before
|
||||
applied = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
||||
)
|
||||
repeat = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
||||
)
|
||||
assert not repeat["actions"]
|
||||
after = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert all(not after[k] for k in oldkeys) and all(
|
||||
after[k] for k in protected
|
||||
)
|
||||
failed = inside(
|
||||
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
|
||||
)
|
||||
assert failed["error_code"] == "source_expired"
|
||||
try:
|
||||
operator(failed["id"])
|
||||
except RuntimeError:
|
||||
replay_refused = True
|
||||
else:
|
||||
raise AssertionError("Expired source replay must fail")
|
||||
# Expired reservation was removed, so a delayed API cannot commit its handoff.
|
||||
fenced = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
|
||||
)
|
||||
assert fenced
|
||||
report.update(
|
||||
dry_run=dry,
|
||||
apply=applied,
|
||||
repeat=repeat,
|
||||
before=before,
|
||||
after=after,
|
||||
dry_run_unchanged=True,
|
||||
active_current_and_fresh_protected=True,
|
||||
expired_source_replay_refused=replay_refused,
|
||||
expired_writer_fenced=fenced,
|
||||
complete=True,
|
||||
)
|
||||
finally:
|
||||
# Remove only this fixture's artificial active reservation/job; never real jobs.
|
||||
inside(
|
||||
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("action", choices=["seed", "verify"])
|
||||
p.add_argument("--seed", required=True, type=Path)
|
||||
p.add_argument("--output", type=Path)
|
||||
a = p.parse_args()
|
||||
if a.action == "seed":
|
||||
seed(a.seed)
|
||||
else:
|
||||
if a.output is None:
|
||||
p.error("output required")
|
||||
verify(a.seed, a.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -30,7 +30,7 @@ db,store,queue=database(),ObjectStore(),broker()
|
||||
|
||||
|
||||
def inside(code):
|
||||
cid = compose("ps", "-q", "api").split()[0]
|
||||
cid = compose("ps", "-q", "controller").split()[0]
|
||||
return json.loads(
|
||||
command(
|
||||
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
|
||||
@@ -123,8 +123,19 @@ def main():
|
||||
try:
|
||||
ready()
|
||||
# Check instrumentation is really enabled and production default is off.
|
||||
assert inside(
|
||||
"import os\nprint(json.dumps(os.getenv('PIPELINE_FAULTS')=='1'))"
|
||||
cid = compose("ps", "-q", "api").split()[0]
|
||||
assert (
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import os;print(os.getenv('PIPELINE_FAULTS'))",
|
||||
]
|
||||
)
|
||||
== "1"
|
||||
)
|
||||
stop("reconciler")
|
||||
with httpx.Client(
|
||||
@@ -503,6 +514,71 @@ def main():
|
||||
save(
|
||||
"malformed_notification_rejected", {"durable_diagnostic": True}
|
||||
)
|
||||
# Private bucket, bounded IAM, and download after uploader process dies.
|
||||
a, j = upload()
|
||||
confirm(a, j)
|
||||
key = f"objects/{a['id']}/v1/original"
|
||||
security = inside(f"""from minio.error import S3Error
|
||||
from urllib.error import HTTPError
|
||||
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
|
||||
except HTTPError as e: anonymous=e.code
|
||||
else: raise AssertionError('Anonymous access allowed')
|
||||
denials={{}}
|
||||
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
|
||||
try: fn()
|
||||
except S3Error as e: denials[name]=e.code
|
||||
else: raise AssertionError('IAM allowed forbidden action')
|
||||
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
|
||||
assert security["anonymous"] == 403 and set(
|
||||
security["iam"].values()
|
||||
) == {"AccessDenied"}
|
||||
selected = None
|
||||
for cid in compose("ps", "-q", "api").split():
|
||||
alias = command(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
|
||||
]
|
||||
)
|
||||
uploader = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
|
||||
)
|
||||
if alias == uploader:
|
||||
selected = cid
|
||||
assert selected
|
||||
command(["docker", "stop", "--time", "1", selected])
|
||||
ready()
|
||||
wait(
|
||||
lambda: (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 200
|
||||
)
|
||||
)
|
||||
downloaded = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
)
|
||||
assert (
|
||||
downloaded.content == raw
|
||||
and downloaded.headers["X-Instance-Id"] != uploader
|
||||
)
|
||||
start("api")
|
||||
ready()
|
||||
save(
|
||||
"private_iam_and_other_api_download",
|
||||
{
|
||||
"security": security,
|
||||
"attachment": a["id"],
|
||||
"uploader": uploader,
|
||||
"downloader": downloaded.headers["X-Instance-Id"],
|
||||
"sha256": hashlib.sha256(raw).hexdigest(),
|
||||
},
|
||||
)
|
||||
report["worker_metrics"] = inside(
|
||||
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user