diff --git a/scripts/verify_storage.py b/scripts/verify_storage.py new file mode 100644 index 00000000..1626728d --- /dev/null +++ b/scripts/verify_storage.py @@ -0,0 +1,495 @@ +"""Operator checks on the disposable teaching stand, never production.""" + +import argparse +import hashlib +import io +import json +import socket +import statistics +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +import httpx +from PIL import Image + +from scripts import cluster +from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql + +PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() +PREFIX = """import hashlib,io,json,time,urllib.request,urllib.error +from src.object_store import ObjectStore,BUCKET +from src.database import Database +from src.settings import secret +from src.reconcile import reconcile +s=ObjectStore(); d=Database(url=secret('DATABASE_URL'),size=2); d.open() +""" + + +def inside(code): + value = command( + ["docker", "compose", "exec", "-T", "api", "python", "-"], + input=PREFIX + code + "\ns.close(); d.close()\n", + ) + return json.loads(value) + + +def upload(client, session, chat, data=PNG, *, name=None): + return client.post( + f"/api/v1/chats/{chat['id']}/attachments", + headers={**auth(session), "Content-Type": "image/png"}, + params={"filename": name or uuid4().hex + ".png"}, + content=data, + ) + + +def state(identity): + return sql("SELECT state FROM attachments WHERE id='" + identity + "'") + + +def key(identity): + return "objects/" + identity + "/original" + + +def run_reconcile(apply=False): + return inside( + "print(json.dumps(reconcile(d,s,apply=" + str(apply) + ",grace=300)))" + ) + + +def age(identity): + sql( + "UPDATE attachments SET created_at=clock_timestamp()-interval '301 seconds', " + "expires_at=clock_timestamp()-interval '180 seconds' WHERE id='" + + identity + + "'" + ) + + +def percentiles(values): + values = sorted(values) + return { + f"p{p}_ms": round( + values[min(len(values) - 1, int(p * len(values) / 100))], 3 + ) + for p in [50, 95, 99] + } + + +def concurrency(client, session, chat): + stream = io.BytesIO() + Image.effect_noise((1700, 1700), 100).convert("RGB").save(stream, "PNG") + image = stream.getvalue() + assert len(image) < 10 * 1024 * 1024 + ids = compose("ps", "-q", "api").split() + samples, stop = [], threading.Event() + code = ( + "import json,pathlib; p=pathlib.Path('/sys/fs/cgroup'); " + "print(json.dumps({k:int((p/k).read_text()) for k in ['memory.current','memory.peak']}))" + ) + + def sample(): + while not stop.is_set(): + for cid in ids: + samples.append( + { + "at": round(time.monotonic() - started, 3), + "container": cid[:12], + **json.loads( + command( + ["docker", "exec", cid, "python", "-c", code] + ) + ), + } + ) + stop.wait(0.3) + + barrier = threading.Barrier(10) + + def one(_): + with httpx.Client(base_url=BASE, trust_env=False, timeout=60) as c: + barrier.wait(timeout=30) + begin = time.monotonic() + r = upload(c, session, chat, image) + latency = (time.monotonic() - begin) * 1000 + assert r.status_code == 201, (r.status_code, r.text) + a = r.json() + assert a["sha256"] == hashlib.sha256(image).hexdigest() + data = c.get( + f"/api/v1/attachments/{a['id']}/content", headers=auth(session) + ) + assert data.status_code == 200 and data.content == image + return { + "id": a["id"], + "latency_ms": latency, + "instance": r.headers["x-instance-id"], + } + + cpu_before = { + cid: command(["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"]) + for cid in ids + } + started = time.monotonic() + thread = threading.Thread(target=sample) + thread.start() + try: + with ThreadPoolExecutor(max_workers=10) as pool: + results = list(pool.map(one, range(10))) + finally: + stop.set() + thread.join(timeout=30) + elapsed = time.monotonic() - started + assert samples + return { + "concurrency": 10, + "requests": 10, + "successes": len(results), + "errors": 0, + "bytes_per_image": len(image), + "pixels": 1700 * 1700, + "duration_seconds": round(elapsed, 3), + "successful_rps": round(10 / elapsed, 3), + "latency_scope": "upload only; duration includes verified downloads and memory sampler", + **percentiles([r["latency_ms"] for r in results]), + "cpu_before": cpu_before, + "cpu_after": { + cid: command( + ["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"] + ) + for cid in ids + }, + "memory_samples": samples, + "results": results, + "window_max_memory_current": max(s["memory.current"] for s in samples), + "lifetime_memory_peak": max(s["memory.peak"] for s in samples), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/storage-repeat.json") + parser.add_argument("--research", action="store_true") + args = parser.parse_args() + output = Path(args.output) + assert not output.exists(), "Choose a fresh evidence path" + output.parent.mkdir(parents=True, exist_ok=True) + result = { + "source_commit": command(["git", "rev-parse", "HEAD"]), + "date": datetime.now(timezone.utc).isoformat(), + "complete": False, + } + ready() + compose("stop", "reconciler") + orphan = "objects/verification-" + uuid4().hex + "/orphan" + fresh = "objects/verification-" + uuid4().hex + "/fresh" + research_prefix = "objects/research-" + uuid4().hex + "/" + try: + # Its server LastModified must really age; no clock manipulation in S3. + inside( + f"s.put(key={orphan!r},stream=io.BytesIO(b'orphan'),size=6,content_type='application/octet-stream',sha256=hashlib.sha256(b'orphan').hexdigest()); print('true')" + ) + orphan_started = time.monotonic() + with httpx.Client( + base_url=BASE, trust_env=False, timeout=30 + ) as client: + user, _, session = actor(client) + r = client.post( + "/api/v1/chats", + headers=auth(session), + json={ + "title": "Storage verification", + "member_ids": [user["id"]], + }, + ) + assert r.status_code == 201 + chat = r.json() + r = upload(client, session, chat) + assert r.status_code == 201 + saved, uploader = r.json(), r.headers["x-instance-id"] + target = next( + c + for c in cluster.containers() + if cluster.alias(c["Id"]) == uploader + ) + command(["docker", "stop", "--time", "10", target["Id"]]) + try: + time.sleep(3) + download = client.get( + f"/api/v1/attachments/{saved['id']}/content", + headers=auth(session), + ) + assert download.status_code == 200 and download.content == PNG + assert download.headers["x-instance-id"] != uploader + result["upload_replica_stopped_downloaded_elsewhere"] = { + "stopped_container": target["Id"][:12], + "uploader": uploader, + "downloader": download.headers["x-instance-id"], + "sha256": saved["sha256"], + } + finally: + compose( + "up", + "-d", + "--no-build", + "--no-recreate", + "--wait", + "api", + "lb", + ) + ready() + private_code = f""" +checks={{}} +try: + urllib.request.urlopen('http://s3:9000/messenger/{key(saved["id"])}',timeout=3) + raise AssertionError('public object') +except urllib.error.HTTPError as e: checks['anonymous_get']=e.code; assert e.code==403 +for name,operation in [('list_other_bucket',lambda:list(s.client.list_objects('other-private'))),('create_bucket',lambda:s.client.make_bucket('forbidden-bucket')),('write_outside_prefix',lambda:s.client.put_object(BUCKET,'forbidden',io.BytesIO(b'x'),1))]: + try: operation(); raise AssertionError(name) + except Exception as e: + assert getattr(e,'code',None)=='AccessDenied',type(e).__name__; checks[name]='AccessDenied' +try: + urllib.request.urlopen('http://s3:9001/',timeout=2); raise AssertionError('console exposed') +except urllib.error.URLError: checks['console_internal_network']='unreachable' +print(json.dumps(checks)) +""" + result["private_bucket_and_iam"] = inside(private_code) + compose("run", "--rm", "--no-deps", "s3-bootstrap") + assert ( + client.get( + f"/api/v1/attachments/{saved['id']}/content", + headers=auth(session), + ).content + == PNG + ) + result["bootstrap_repeat_preserved_original"] = True + # A partial HTTP body must never create a durable intent or object. + before = int(sql("SELECT count(*) FROM attachments")) + objects_before_abort = inside("print(len(list(s.objects())))") + parsed = httpx.URL(BASE) + with socket.create_connection( + (parsed.host, parsed.port), timeout=3 + ) as raw: + request = ( + f"POST /api/v1/chats/{chat['id']}/attachments?filename=partial.png HTTP/1.1\r\n" + f"Host: {parsed.host}\r\nAuthorization: Bearer {session['token']}\r\n" + "Content-Type: image/png\r\nContent-Length: 100000\r\n\r\n" + ).encode() + PNG[:20] + raw.sendall(request) + time.sleep(1) + assert int(sql("SELECT count(*) FROM attachments")) == before + assert ( + inside("print(len(list(s.objects())))") == objects_before_abort + ) + result["aborted_upload_has_no_intent_or_bytes"] = True + # Failure before any S3 PUT. + name = uuid4().hex + ".png" + sql( + "CREATE FUNCTION student_upload_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic'; END $$; CREATE TRIGGER student_upload_fault BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION student_upload_fault()" + ) + objects_before = inside("print(len(list(s.objects())))") + try: + assert ( + upload(client, session, chat, name=name).status_code == 500 + ) + assert ( + int( + sql( + "SELECT count(*) FROM attachments WHERE filename='" + + name + + "'" + ) + ) + == 0 + ) + assert ( + inside("print(len(list(s.objects())))") == objects_before + ) + finally: + sql( + "DROP TRIGGER student_upload_fault ON attachments; DROP FUNCTION student_upload_fault()" + ) + result["pg_intent_failure_has_no_bytes"] = True + # S3 PUT succeeded, then ready UPDATE fails: durable pending is recoverable. + name = uuid4().hex + ".png" + sql( + "CREATE FUNCTION student_ready_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.state='ready' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; CREATE TRIGGER student_ready_fault BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION student_ready_fault()" + ) + try: + assert ( + upload(client, session, chat, name=name).status_code == 500 + ) + finally: + sql( + "DROP TRIGGER student_ready_fault ON attachments; DROP FUNCTION student_ready_fault()" + ) + pending = sql( + "SELECT id FROM attachments WHERE filename='" + name + "'" + ) + assert state(pending) == "pending" + assert inside( + f"print(json.dumps(s.head(key={key(pending)!r}) is not None))" + ) + age(pending) + # Stop S3; bypass LB health exclusion only from inside the selected API. + compose("stop", "s3") + name = uuid4().hex + ".png" + try: + direct = f""" +request=urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/{chat["id"]}/attachments?filename={name}',data=bytes.fromhex({PNG.hex()!r}),headers={{'Authorization':{auth(session)["Authorization"]!r},'Content-Type':'image/png'}}) +try: urllib.request.urlopen(request,timeout=15); raise AssertionError('S3 down accepted') +except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':json.loads(e.read())['error']['code']}})) +""" + observed = inside(direct) + assert observed == {"status": 503, "error": "unavailable"} + missing_pending = sql( + "SELECT id FROM attachments WHERE filename='" + name + "'" + ) + assert state(missing_pending) == "pending" + result["s3_unavailable"] = observed + finally: + compose( + "up", "-d", "--no-build", "--no-recreate", "--wait", "s3" + ) + ready() + age(missing_pending) + # Used object missing: preserve metadata ID and the Message link. + used = upload(client, session, chat).json() + sent = client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**auth(session), "Idempotency-Key": uuid4().hex}, + json={"text": "", "attachment_ids": [used["id"]]}, + ) + assert sent.status_code == 201 + inside(f"s.delete(key={key(used['id'])!r}); print('true')") + age(used["id"]) + age(saved["id"]) + result["concurrent_uploads"] = concurrency(client, session, chat) + while time.monotonic() - orphan_started < 302: + print( + "Waiting for genuine S3 LastModified grace: " + + str(round(302 - (time.monotonic() - orphan_started))) + + "s", + flush=True, + ) + time.sleep(min(20, 302 - (time.monotonic() - orphan_started))) + inside( + f"s.put(key={fresh!r},stream=io.BytesIO(b'fresh'),size=5,content_type='application/octet-stream',sha256=hashlib.sha256(b'fresh').hexdigest()); print('true')" + ) + before_states = [ + state(i) + for i in [pending, missing_pending, used["id"], saved["id"]] + ] + dry = run_reconcile() + assert [ + state(i) + for i in [pending, missing_pending, used["id"], saved["id"]] + ] == before_states + assert inside( + f"print(json.dumps(s.head(key={orphan!r}) is not None))" + ) + for action in [ + "recover_pending", + "fail_pending_missing", + "mark_missing", + "delete_orphan", + ]: + assert dry["actions"].get(action, 0) >= 1, dry + applied = run_reconcile(True) + assert state(pending) == "ready" + assert ( + state(missing_pending) == "failed" + and state(used["id"]) == "failed" + ) + assert state(saved["id"]) == "ready" + assert ( + int( + sql( + "SELECT count(*) FROM message_attachments WHERE attachment_id='" + + used["id"] + + "'" + ) + ) + == 1 + ) + assert ( + client.get( + f"/api/v1/attachments/{used['id']}", headers=auth(session) + ).status_code + == 409 + ) + assert ( + client.get( + f"/api/v1/attachments/{pending}/content", + headers=auth(session), + ).content + == PNG + ) + assert inside( + f"print(json.dumps(s.head(key={orphan!r}) is None and s.head(key={fresh!r}) is not None and s.head(key={key(saved['id'])!r}) is not None))" + ) + repeat = run_reconcile(True) + assert not repeat["applied"].get("delete_orphan") + assert repeat["linked_protected"] >= 1 + result["reconciliation"] = { + "dry_run": dry, + "apply": applied, + "repeat": repeat, + "s3_grace_wait_seconds": round( + time.monotonic() - orphan_started, 3 + ), + "fresh_and_ready_unlinked_protected": True, + "missing_used_id_and_link_preserved": True, + "dry_run_unchanged": True, + } + if args.research: + cells = [] + for count in [0, 100, 1100]: + if count: + start = 0 if count == 100 else 100 + inside( + f"\nfor i in range({start},{count}):\n s.put(key={research_prefix!r}+str(i),stream=io.BytesIO(b'x'),size=1,content_type='application/octet-stream',sha256=hashlib.sha256(b'x').hexdigest())\nprint('true')" + ) + for repetition in range(3): + measured = run_reconcile() + cells.append( + { + "extra_objects": count, + "repetition": repetition + 1, + **measured, + } + ) + result["cost_research"] = { + "cells": cells, + "repeats": 3, + "method": "same running stand; 0/100/1100 fresh one-byte objects; dry-run; serial ascending dataset; no warmup", + "median_seconds": { + str(n): statistics.median( + c["duration_seconds"] + for c in cells + if c["extra_objects"] == n + ) + for n in [0, 100, 1100] + }, + } + result["complete"] = True + finally: + # Only keys created by this invocation; no broad bucket deletion. + try: + inside( + f"\nfor o in s.objects():\n if o.object_name.startswith({research_prefix!r}) or o.object_name in [{orphan!r},{fresh!r}]: s.delete(key=o.object_name)\nprint('true')" + ) + finally: + compose("up", "-d", "--no-build", "--no-recreate", "reconciler") + output.write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n" + ) + print( + "PASS: private S3, failover, bounded concurrent uploads and two-store recovery" + ) + + +if __name__ == "__main__": + main()