Files
mtusi2026lab6/scripts/verify_cleanup.py
T

173 lines
7.2 KiB
Python

#!/usr/bin/env python3
"""Two-phase real-age GC check: seed, run other checks, then verify after >=300 seconds."""
import argparse
import hashlib
import io
import json
import sys
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose
from pipeline_lab import settings
from messenger.cleanup import collect
from messenger.pipeline import Pipeline, intent
from messenger.repository import PostgresRepository
from messenger.storage import ObjectStore
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("action", choices=("seed", "verify"))
parser.add_argument("--seed", type=Path, required=True)
parser.add_argument("--output", type=Path)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
args = parser.parse_args()
config = settings("reconciler")
db = PostgresRepository(config.database_url, pool_size=3)
db.open()
store = ObjectStore(config)
try:
if args.action == "seed":
check(not args.seed.exists(), "Seed already exists")
compose("stop", "--timeout", "2", "worker", "relay")
try:
c = Client(args.base_url)
alice, _, _, room = c.bootstrap()
active, data = c.attachment(alice, room)
failed, _ = c.attachment(alice, room)
orphan = str(uuid4())
Pipeline(db, config).reserve(orphan, "staging/" + orphan)
uploader = ObjectStore(settings("api"))
try:
uploader.put(
"staging/" + orphan,
io.BytesIO(data),
len(data),
"image/png",
hashlib.sha256(data).hexdigest(),
)
finally:
uploader.close()
with db.connection(write=True) as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (active["id"],)
).fetchone()
conn.execute(
"UPDATE attachment_jobs SET "
"next_attempt_at=clock_timestamp()+interval '1 hour' "
"WHERE job_id=%s",
(j["job_id"],),
)
conn.execute(
"UPDATE job_outbox SET due_at=clock_timestamp()+interval '1 "
"hour' WHERE job_id=%s",
(j["job_id"],),
)
f = conn.execute(
"SELECT job_id FROM attachment_jobs WHERE attachment_id=%s", (failed["id"],)
).fetchone()
conn.execute(
"INSERT INTO pipeline_faults(job_id,phase,action) VALUES "
"(%s,'before_effect','permanent')",
(f["job_id"],),
)
seed = {
"created_at": datetime.now(UTC).isoformat(),
"active_id": active["id"],
"failed_id": failed["id"],
"orphan_id": orphan,
}
args.seed.write_text(json.dumps(seed, indent=2) + "\n")
print(
"PASS: active job, failed candidate and unaccepted orphan seeded; no "
"credentials saved"
)
finally:
compose("start", "worker", "relay")
else:
check(args.output is not None and not args.output.exists(), "Choose a new output file")
seed = json.loads(args.seed.read_text())
# Real S3 timestamps and failed retention; no modified clocks.
while True:
with db.connection() as conn:
f = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],)
).fetchone()
check(f["state"] == "failed", "Failed seed did not reach terminal state")
age = (datetime.now(UTC) - f["updated_at"]).total_seconds()
if age >= 301:
break
check(age >= 0, "Clock mismatch")
time.sleep(min(1, 301 - age))
with db.connection() as conn:
ready = conn.execute(
"SELECT * FROM attachments WHERE storage_state='ready' LIMIT 1"
).fetchone()
check(ready is not None, "Need a published attachment to protect")
keys = [ready["object_key"], *[v["object_key"] for v in ready["variants"]]]
before = {key: store.head(key)["ETag"] for key in keys}
preview = collect(db, store, 300, False)
result = collect(db, store, 300, True)
check(
store.head("staging/" + seed["active_id"]) is not None, "Active staging was deleted"
)
check(store.head("staging/" + seed["orphan_id"]) is None, "Orphan not deleted")
check(
store.head("staging/" + seed["failed_id"]) is None,
"Expired failed source not deleted",
)
check(
all(store.head(k)["ETag"] == tag for k, tag in before.items()),
"Published result changed",
)
with db.connection(write=True) as conn:
expired = conn.execute(
"SELECT source_expired FROM attachment_jobs WHERE attachment_id=%s",
(seed["failed_id"],),
).fetchone()
check(expired["source_expired"], "Expired source not registered")
j = conn.execute(
"UPDATE attachment_jobs SET next_attempt_at=clock_timestamp() WHERE "
"attachment_id=%s RETURNING job_id",
(seed["active_id"],),
).fetchone()
intent(conn, j["job_id"])
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
if db.attachment(seed["active_id"])["storage_state"] == "ready":
break
time.sleep(0.2)
check(
db.attachment(seed["active_id"])["storage_state"] == "ready",
"Protected work failed to complete",
)
proof = {
"retention_seconds": 300,
"failed_real_age_seconds": age,
"seed": seed,
"dry_run": preview,
"apply": result,
"active_protected_then_ready": True,
"orphan_deleted": True,
"failed_source_expired": True,
"published_objects_unchanged": len(keys),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(proof, indent=2) + "\n")
print("PASS: real-age cleanup preserves active work and published results")
finally:
store.close()
db.close()
if __name__ == "__main__":
main()