230 lines
9.8 KiB
Python
230 lines
9.8 KiB
Python
"""Real S3 retention test: seed is separate from verification output."""
|
|
|
|
import argparse
|
|
import json
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
|
|
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
|
from scripts.tls_client import FreshTrustTransport
|
|
from scripts.verify_pipeline import inside, operator, wait
|
|
|
|
|
|
def seed(path):
|
|
if path.exists():
|
|
raise ValueError("Choose a new seed file")
|
|
ready()
|
|
compose("stop", "--timeout", "1", "reconciler")
|
|
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
with httpx.Client(
|
|
transport=FreshTrustTransport(),
|
|
base_url=BASE,
|
|
trust_env=False,
|
|
timeout=20,
|
|
) as c:
|
|
user, _, session = actor(c)
|
|
headers = auth(session)
|
|
result = c.post(
|
|
"/api/v1/chats",
|
|
headers=headers,
|
|
json={
|
|
"title": "Cleanup synthetic seed",
|
|
"member_ids": [user["id"]],
|
|
},
|
|
)
|
|
assert result.status_code == 201
|
|
chat = result.json()["id"]
|
|
r = c.post(
|
|
f"/api/v1/chats/{chat}/attachments",
|
|
headers={**headers, "Content-Type": "image/png"},
|
|
params={"filename": "retained.png"},
|
|
content=raw,
|
|
)
|
|
assert r.status_code == 202
|
|
attachment = r.json()["id"]
|
|
wait(
|
|
lambda: (
|
|
c.get(
|
|
"/api/v1/attachments/" + attachment, headers=headers
|
|
).json()["status"]
|
|
== "ready"
|
|
)
|
|
)
|
|
job = inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
|
|
)
|
|
data = {
|
|
"seed_id": uuid4().hex,
|
|
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
|
"date": datetime.now(timezone.utc).isoformat(),
|
|
"ready_attachment": attachment,
|
|
"ready_job": job,
|
|
"chat": chat,
|
|
"owner": user["id"],
|
|
}
|
|
# Real old v1 objects will become obsolete after v2 is ready.
|
|
second = operator(job, version=2)
|
|
wait(
|
|
lambda: inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
|
|
)
|
|
)
|
|
data["version2_job"] = second
|
|
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
|
|
data["failed_id"] = str(uuid4())
|
|
data["active_id"] = str(uuid4())
|
|
data["reservation_id"] = str(uuid4())
|
|
data["expired_reservation_id"] = str(uuid4())
|
|
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
|
|
data["active_source"] = f"objects/staging/{data['active_id']}/original"
|
|
data["reservation_key"] = (
|
|
f"objects/staging/{data['reservation_id']}/original"
|
|
)
|
|
data["expired_reservation_key"] = (
|
|
f"objects/staging/{data['expired_reservation_id']}/original"
|
|
)
|
|
inside(f"""from pathlib import Path
|
|
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
|
|
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
|
|
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
|
|
with db.connection() as c, c.transaction():
|
|
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
|
|
key='objects/staging/'+identity+'/original'
|
|
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
|
|
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
|
|
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
|
|
print('true')""")
|
|
data["seeded_unix"] = time.time()
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
|
print(
|
|
"PASS: retention seed created; real S3 age must reach300s", flush=True
|
|
)
|
|
|
|
|
|
def verify(seed_path, output):
|
|
if output.exists():
|
|
raise ValueError("Choose a new output path")
|
|
data = json.loads(seed_path.read_text())
|
|
report = {
|
|
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
|
"seed": seed_path.name,
|
|
"complete": False,
|
|
}
|
|
try:
|
|
compose("stop", "--timeout", "1", "reconciler")
|
|
|
|
def age():
|
|
return inside(
|
|
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
|
|
)
|
|
|
|
while age() < 302:
|
|
time.sleep(min(5, 303 - age()))
|
|
report["real_s3_age_seconds"] = age()
|
|
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
|
|
inside(
|
|
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
|
|
)
|
|
oldkeys = [
|
|
data["orphan"],
|
|
data["failed_source"],
|
|
data["expired_reservation_key"],
|
|
] + [
|
|
f"objects/{data['ready_attachment']}/v1/{v}"
|
|
for v in ["original", "thumbnail", "crop", "optimized"]
|
|
]
|
|
protected = [data["active_source"], data["reservation_key"], fresh] + [
|
|
f"objects/{data['ready_attachment']}/v2/{v}"
|
|
for v in ["original", "thumbnail", "crop", "optimized"]
|
|
]
|
|
before = inside(
|
|
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
|
)
|
|
assert all(before.values())
|
|
dry = inside(
|
|
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
|
|
)
|
|
candidates = {a["key"] for a in dry["actions"]}
|
|
assert set(oldkeys) <= candidates and not set(protected) & candidates
|
|
afterdry = inside(
|
|
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
|
)
|
|
assert afterdry == before
|
|
applied = inside(
|
|
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
|
)
|
|
repeat = inside(
|
|
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
|
)
|
|
assert not repeat["actions"]
|
|
after = inside(
|
|
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
|
)
|
|
assert all(not after[k] for k in oldkeys) and all(
|
|
after[k] for k in protected
|
|
)
|
|
failed = inside(
|
|
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
|
|
)
|
|
assert failed["error_code"] == "source_expired"
|
|
try:
|
|
operator(failed["id"])
|
|
except RuntimeError:
|
|
replay_refused = True
|
|
else:
|
|
raise AssertionError("Expired source replay must fail")
|
|
# Expired reservation was removed, so a delayed API cannot commit its handoff.
|
|
fenced = inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
|
|
)
|
|
assert fenced
|
|
report.update(
|
|
dry_run=dry,
|
|
apply=applied,
|
|
repeat=repeat,
|
|
before=before,
|
|
after=after,
|
|
dry_run_unchanged=True,
|
|
active_current_and_fresh_protected=True,
|
|
expired_source_replay_refused=replay_refused,
|
|
expired_writer_fenced=fenced,
|
|
complete=True,
|
|
)
|
|
finally:
|
|
# Remove only this fixture's artificial active reservation/job; never real jobs.
|
|
inside(
|
|
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
|
|
)
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print(
|
|
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("action", choices=["seed", "verify"])
|
|
p.add_argument("--seed", required=True, type=Path)
|
|
p.add_argument("--output", type=Path)
|
|
a = p.parse_args()
|
|
if a.action == "seed":
|
|
seed(a.seed)
|
|
else:
|
|
if a.output is None:
|
|
p.error("output required")
|
|
verify(a.seed, a.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|