173 lines
7.2 KiB
Python
173 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Two-phase real-age GC check: seed, run other checks, then verify after >=300 seconds."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from cluster import ROOT, compose
|
|
from pipeline_lab import settings
|
|
|
|
from messenger.cleanup import collect
|
|
from messenger.pipeline import Pipeline, intent
|
|
from messenger.repository import PostgresRepository
|
|
from messenger.storage import ObjectStore
|
|
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, check # noqa: E402
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(__doc__)
|
|
parser.add_argument("action", choices=("seed", "verify"))
|
|
parser.add_argument("--seed", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path)
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
|
|
args = parser.parse_args()
|
|
config = settings("reconciler")
|
|
db = PostgresRepository(config.database_url, pool_size=3)
|
|
db.open()
|
|
store = ObjectStore(config)
|
|
try:
|
|
if args.action == "seed":
|
|
check(not args.seed.exists(), "Seed already exists")
|
|
compose("stop", "--timeout", "2", "worker", "relay")
|
|
try:
|
|
c = Client(args.base_url)
|
|
alice, _, _, room = c.bootstrap()
|
|
active, data = c.attachment(alice, room)
|
|
failed, _ = c.attachment(alice, room)
|
|
orphan = str(uuid4())
|
|
Pipeline(db, config).reserve(orphan, "staging/" + orphan)
|
|
uploader = ObjectStore(settings("api"))
|
|
try:
|
|
uploader.put(
|
|
"staging/" + orphan,
|
|
io.BytesIO(data),
|
|
len(data),
|
|
"image/png",
|
|
hashlib.sha256(data).hexdigest(),
|
|
)
|
|
finally:
|
|
uploader.close()
|
|
with db.connection(write=True) as conn:
|
|
j = conn.execute(
|
|
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (active["id"],)
|
|
).fetchone()
|
|
conn.execute(
|
|
"UPDATE attachment_jobs SET "
|
|
"next_attempt_at=clock_timestamp()+interval '1 hour' "
|
|
"WHERE job_id=%s",
|
|
(j["job_id"],),
|
|
)
|
|
conn.execute(
|
|
"UPDATE job_outbox SET due_at=clock_timestamp()+interval '1 "
|
|
"hour' WHERE job_id=%s",
|
|
(j["job_id"],),
|
|
)
|
|
f = conn.execute(
|
|
"SELECT job_id FROM attachment_jobs WHERE attachment_id=%s", (failed["id"],)
|
|
).fetchone()
|
|
conn.execute(
|
|
"INSERT INTO pipeline_faults(job_id,phase,action) VALUES "
|
|
"(%s,'before_effect','permanent')",
|
|
(f["job_id"],),
|
|
)
|
|
seed = {
|
|
"created_at": datetime.now(UTC).isoformat(),
|
|
"active_id": active["id"],
|
|
"failed_id": failed["id"],
|
|
"orphan_id": orphan,
|
|
}
|
|
args.seed.write_text(json.dumps(seed, indent=2) + "\n")
|
|
print(
|
|
"PASS: active job, failed candidate and unaccepted orphan seeded; no "
|
|
"credentials saved"
|
|
)
|
|
finally:
|
|
compose("start", "worker", "relay")
|
|
else:
|
|
check(args.output is not None and not args.output.exists(), "Choose a new output file")
|
|
seed = json.loads(args.seed.read_text())
|
|
# Real S3 timestamps and failed retention; no modified clocks.
|
|
while True:
|
|
with db.connection() as conn:
|
|
f = conn.execute(
|
|
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],)
|
|
).fetchone()
|
|
check(f["state"] == "failed", "Failed seed did not reach terminal state")
|
|
age = (datetime.now(UTC) - f["updated_at"]).total_seconds()
|
|
if age >= 301:
|
|
break
|
|
check(age >= 0, "Clock mismatch")
|
|
time.sleep(min(1, 301 - age))
|
|
with db.connection() as conn:
|
|
ready = conn.execute(
|
|
"SELECT * FROM attachments WHERE storage_state='ready' LIMIT 1"
|
|
).fetchone()
|
|
check(ready is not None, "Need a published attachment to protect")
|
|
keys = [ready["object_key"], *[v["object_key"] for v in ready["variants"]]]
|
|
before = {key: store.head(key)["ETag"] for key in keys}
|
|
preview = collect(db, store, 300, False)
|
|
result = collect(db, store, 300, True)
|
|
check(
|
|
store.head("staging/" + seed["active_id"]) is not None, "Active staging was deleted"
|
|
)
|
|
check(store.head("staging/" + seed["orphan_id"]) is None, "Orphan not deleted")
|
|
check(
|
|
store.head("staging/" + seed["failed_id"]) is None,
|
|
"Expired failed source not deleted",
|
|
)
|
|
check(
|
|
all(store.head(k)["ETag"] == tag for k, tag in before.items()),
|
|
"Published result changed",
|
|
)
|
|
with db.connection(write=True) as conn:
|
|
expired = conn.execute(
|
|
"SELECT source_expired FROM attachment_jobs WHERE attachment_id=%s",
|
|
(seed["failed_id"],),
|
|
).fetchone()
|
|
check(expired["source_expired"], "Expired source not registered")
|
|
j = conn.execute(
|
|
"UPDATE attachment_jobs SET next_attempt_at=clock_timestamp() WHERE "
|
|
"attachment_id=%s RETURNING job_id",
|
|
(seed["active_id"],),
|
|
).fetchone()
|
|
intent(conn, j["job_id"])
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
if db.attachment(seed["active_id"])["storage_state"] == "ready":
|
|
break
|
|
time.sleep(0.2)
|
|
check(
|
|
db.attachment(seed["active_id"])["storage_state"] == "ready",
|
|
"Protected work failed to complete",
|
|
)
|
|
proof = {
|
|
"retention_seconds": 300,
|
|
"failed_real_age_seconds": age,
|
|
"seed": seed,
|
|
"dry_run": preview,
|
|
"apply": result,
|
|
"active_protected_then_ready": True,
|
|
"orphan_deleted": True,
|
|
"failed_source_expired": True,
|
|
"published_objects_unchanged": len(keys),
|
|
}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(proof, indent=2) + "\n")
|
|
print("PASS: real-age cleanup preserves active work and published results")
|
|
finally:
|
|
store.close()
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|