"""Operator checks on the disposable teaching stand, never production.""" import argparse import hashlib import io import json import socket import statistics import threading import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path from uuid import uuid4 import httpx from PIL import Image from scripts import cluster from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() PREFIX = """import hashlib,io,json,time,urllib.request,urllib.error from src.object_store import ObjectStore,BUCKET from src.database import Database from src.settings import secret from src.reconcile import reconcile s=ObjectStore(); d=Database(url=secret('DATABASE_URL'),size=2); d.open() """ def inside(code): containers = compose("ps", "-q", "api").split() assert containers, "No running API in this project" value = command( ["docker", "exec", "-i", containers[0], "python", "-"], input=PREFIX + code + "\ns.close(); d.close()\n", ) return json.loads(value) def upload(client, session, chat, data=PNG, *, name=None): return client.post( f"/api/v1/chats/{chat['id']}/attachments", headers={**auth(session), "Content-Type": "image/png"}, params={"filename": name or uuid4().hex + ".png"}, content=data, ) def state(identity): return sql("SELECT state FROM attachments WHERE id='" + identity + "'") def key(identity): return "objects/" + identity + "/original" def run_reconcile(apply=False): return inside( "print(json.dumps(reconcile(d,s,apply=" + str(apply) + ",grace=300)))" ) def age(identity): sql( "UPDATE attachments SET created_at=clock_timestamp()-interval '301 seconds', " "expires_at=clock_timestamp()-interval '180 seconds' WHERE id='" + identity + "'" ) def percentiles(values): values = sorted(values) return { f"p{p}_ms": round( values[min(len(values) - 1, int(p * len(values) / 100))], 3 ) for p in [50, 95, 99] } def concurrency(client, session, chat): stream = io.BytesIO() Image.effect_noise((1700, 1700), 100).convert("RGB").save(stream, "PNG") image = stream.getvalue() assert len(image) < 10 * 1024 * 1024 ids = compose("ps", "-q", "api").split() samples, stop = [], threading.Event() code = ( "import json,pathlib; p=pathlib.Path('/sys/fs/cgroup'); " "print(json.dumps({k:int((p/k).read_text()) for k in ['memory.current','memory.peak']}))" ) def sample(): while not stop.is_set(): for cid in ids: samples.append( { "at": round(time.monotonic() - started, 3), "container": cid[:12], **json.loads( command( ["docker", "exec", cid, "python", "-c", code] ) ), } ) stop.wait(0.3) barrier = threading.Barrier(10) def one(_): with httpx.Client(base_url=BASE, trust_env=False, timeout=60) as c: barrier.wait(timeout=30) begin = time.monotonic() r = upload(c, session, chat, image) latency = (time.monotonic() - begin) * 1000 assert r.status_code == 201, (r.status_code, r.text) a = r.json() assert a["sha256"] == hashlib.sha256(image).hexdigest() data = c.get( f"/api/v1/attachments/{a['id']}/content", headers=auth(session) ) assert data.status_code == 200 and data.content == image return { "id": a["id"], "latency_ms": latency, "instance": r.headers["x-instance-id"], } cpu_before = { cid: command(["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"]) for cid in ids } started = time.monotonic() thread = threading.Thread(target=sample) thread.start() try: with ThreadPoolExecutor(max_workers=10) as pool: results = list(pool.map(one, range(10))) finally: stop.set() thread.join(timeout=30) elapsed = time.monotonic() - started assert samples return { "concurrency": 10, "requests": 10, "successes": len(results), "errors": 0, "bytes_per_image": len(image), "pixels": 1700 * 1700, "duration_seconds": round(elapsed, 3), "successful_rps": round(10 / elapsed, 3), "latency_scope": "upload only; duration includes verified downloads and memory sampler", **percentiles([r["latency_ms"] for r in results]), "cpu_before": cpu_before, "cpu_after": { cid: command( ["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"] ) for cid in ids }, "memory_samples": samples, "results": results, "window_max_memory_current": max(s["memory.current"] for s in samples), "lifetime_memory_peak": max(s["memory.peak"] for s in samples), } def main(): parser = argparse.ArgumentParser() parser.add_argument("--output", default="evidence/storage-repeat.json") parser.add_argument("--research", action="store_true") args = parser.parse_args() output = Path(args.output) assert not output.exists(), "Choose a fresh evidence path" output.parent.mkdir(parents=True, exist_ok=True) result = { "source_commit": command(["git", "rev-parse", "HEAD"]), "date": datetime.now(timezone.utc).isoformat(), "complete": False, } ready() from scripts.verify_cluster import snapshot result["runtime"] = snapshot() compose("stop", "reconciler") orphan = "objects/verification-" + uuid4().hex + "/orphan" fresh = "objects/verification-" + uuid4().hex + "/fresh" research_prefix = "objects/research-" + uuid4().hex + "/" try: # Its server LastModified must really age; no clock manipulation in S3. inside( f"s.put(key={orphan!r},stream=io.BytesIO(b'orphan'),size=6,content_type='application/octet-stream',sha256=hashlib.sha256(b'orphan').hexdigest()); print('true')" ) orphan_started = time.monotonic() with httpx.Client( base_url=BASE, trust_env=False, timeout=30 ) as client: user, _, session = actor(client) r = client.post( "/api/v1/chats", headers=auth(session), json={ "title": "Storage verification", "member_ids": [user["id"]], }, ) assert r.status_code == 201 chat = r.json() r = upload(client, session, chat) assert r.status_code == 201 saved, uploader = r.json(), r.headers["x-instance-id"] failed_old = upload(client, session, chat).json() sql( "UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='" + failed_old["id"] + "'" ) age(failed_old["id"]) grace_started = time.monotonic() target = next( c for c in cluster.containers() if cluster.alias(c["Id"]) == uploader ) command(["docker", "stop", "--time", "10", target["Id"]]) try: time.sleep(3) download = client.get( f"/api/v1/attachments/{saved['id']}/content", headers=auth(session), ) assert download.status_code == 200 and download.content == PNG assert download.headers["x-instance-id"] != uploader result["upload_replica_stopped_downloaded_elsewhere"] = { "stopped_container": target["Id"][:12], "uploader": uploader, "downloader": download.headers["x-instance-id"], "sha256": saved["sha256"], } finally: compose( "up", "-d", "--no-build", "--no-recreate", "--wait", "api", "lb", ) ready() private_code = f""" from src.errors import DomainError checks={{}} try: s.head(key='forbidden'); raise AssertionError('HEAD outside IAM prefix') except DomainError as e: assert e.status==503; checks['denied_head_is_not_missing']=e.status try: urllib.request.urlopen('http://s3:9000/messenger/{key(saved["id"])}',timeout=3) raise AssertionError('public object') except urllib.error.HTTPError as e: checks['anonymous_get']=e.code; assert e.code==403 for name,operation in [('list_other_bucket',lambda:list(s.client.list_objects('other-private'))),('create_bucket',lambda:s.client.make_bucket('forbidden-bucket')),('write_outside_prefix',lambda:s.client.put_object(BUCKET,'forbidden',io.BytesIO(b'x'),1))]: try: operation(); raise AssertionError(name) except Exception as e: assert getattr(e,'code',None)=='AccessDenied',type(e).__name__; checks[name]='AccessDenied' try: urllib.request.urlopen('http://s3:9001/',timeout=2); raise AssertionError('console exposed') except urllib.error.URLError: checks['console_internal_network']='unreachable' print(json.dumps(checks)) """ result["private_bucket_and_iam"] = inside(private_code) compose("run", "--rm", "--no-deps", "s3-bootstrap") assert ( client.get( f"/api/v1/attachments/{saved['id']}/content", headers=auth(session), ).content == PNG ) result["bootstrap_repeat_preserved_original"] = True # A partial HTTP body must never create a durable intent or object. before = int(sql("SELECT count(*) FROM attachments")) objects_before_abort = inside("print(len(list(s.objects())))") parsed = httpx.URL(BASE) with socket.create_connection( (parsed.host, parsed.port), timeout=3 ) as raw: request = ( ( f"POST /api/v1/chats/{chat['id']}/attachments?filename=partial.png HTTP/1.1\r\n" f"Host: {parsed.host}\r\nAuthorization: Bearer {session['token']}\r\n" "Content-Type: image/png\r\nContent-Length: 10000000\r\n\r\n" ).encode() + PNG + b"x" * (2 * 1024 * 1024) ) raw.sendall(request) time.sleep(1) assert int(sql("SELECT count(*) FROM attachments")) == before assert ( inside("print(len(list(s.objects())))") == objects_before_abort ) result["aborted_upload_has_no_intent_or_bytes"] = True # Failure before any S3 PUT. name = uuid4().hex + ".png" objects_before = inside("print(len(list(s.objects())))") sql( "CREATE FUNCTION student_upload_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic'; END $$; CREATE TRIGGER student_upload_fault BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION student_upload_fault()" ) try: assert ( upload(client, session, chat, name=name).status_code == 500 ) assert ( int( sql( "SELECT count(*) FROM attachments WHERE filename='" + name + "'" ) ) == 0 ) assert ( inside("print(len(list(s.objects())))") == objects_before ) finally: sql( "DROP TRIGGER student_upload_fault ON attachments; DROP FUNCTION student_upload_fault()" ) result["pg_intent_failure_has_no_bytes"] = True # S3 PUT succeeded, then ready UPDATE fails: durable pending is recoverable. name = uuid4().hex + ".png" sql( "CREATE FUNCTION student_ready_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.state='ready' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; CREATE TRIGGER student_ready_fault BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION student_ready_fault()" ) try: assert ( upload(client, session, chat, name=name).status_code == 500 ) finally: sql( "DROP TRIGGER student_ready_fault ON attachments; DROP FUNCTION student_ready_fault()" ) pending = sql( "SELECT id FROM attachments WHERE filename='" + name + "'" ) assert state(pending) == "pending" assert inside( f"print(json.dumps(s.head(key={key(pending)!r}) is not None))" ) age(pending) # Stop S3; bypass LB health exclusion only from inside the selected API. compose("stop", "s3") name = uuid4().hex + ".png" try: direct = f""" request=urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/{chat["id"]}/attachments?filename={name}',data=bytes.fromhex({PNG.hex()!r}),headers={{'Authorization':{auth(session)["Authorization"]!r},'Content-Type':'image/png'}}) try: urllib.request.urlopen(request,timeout=15); raise AssertionError('S3 down accepted') except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':json.loads(e.read())['error']['code']}})) """ observed = inside(direct) assert observed == {"status": 503, "error": "unavailable"} missing_pending = sql( "SELECT id FROM attachments WHERE filename='" + name + "'" ) assert state(missing_pending) == "pending" result["s3_unavailable"] = observed finally: compose( "up", "-d", "--no-build", "--no-recreate", "--wait", "s3" ) ready() age(missing_pending) # Used object missing: preserve metadata ID and the Message link. used = upload(client, session, chat).json() sent = client.post( f"/api/v1/chats/{chat['id']}/messages", headers={**auth(session), "Idempotency-Key": uuid4().hex}, json={"text": "", "attachment_ids": [used["id"]]}, ) assert sent.status_code == 201 inside(f"s.delete(key={key(used['id'])!r}); print('true')") age(used["id"]) age(saved["id"]) # Fresh cgroups make memory.peak specific to this experiment. compose("restart", "api") cluster.wait_count(2) ready() result["concurrent_uploads"] = concurrency(client, session, chat) result["concurrent_uploads"][ "api_restarted_before_memory_measurement" ] = True while time.monotonic() - grace_started < 302: print( "Waiting for genuine S3 LastModified grace: " + str(round(302 - (time.monotonic() - grace_started))) + "s", flush=True, ) time.sleep( min(20, max(0, 302 - (time.monotonic() - grace_started))) ) inside( f"s.put(key={fresh!r},stream=io.BytesIO(b'fresh'),size=5,content_type='application/octet-stream',sha256=hashlib.sha256(b'fresh').hexdigest()); print('true')" ) failed_fresh = upload(client, session, chat).json() sql( "UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='" + failed_fresh["id"] + "'" ) age(failed_fresh["id"]) before_states = [ state(i) for i in [pending, missing_pending, used["id"], saved["id"]] ] dry = run_reconcile() assert [ state(i) for i in [pending, missing_pending, used["id"], saved["id"]] ] == before_states assert inside( f"print(json.dumps(s.head(key={orphan!r}) is not None))" ) for action in [ "recover_pending", "fail_pending_missing", "mark_missing", "delete_orphan", "delete_failed_object", ]: assert dry["actions"].get(action, 0) >= 1, dry applied = run_reconcile(True) assert state(pending) == "ready" assert ( state(missing_pending) == "failed" and state(used["id"]) == "failed" ) assert state(saved["id"]) == "ready" assert ( int( sql( "SELECT count(*) FROM message_attachments WHERE attachment_id='" + used["id"] + "'" ) ) == 1 ) assert ( client.get( f"/api/v1/attachments/{used['id']}", headers=auth(session) ).status_code == 409 ) assert ( client.get( f"/api/v1/attachments/{pending}/content", headers=auth(session), ).content == PNG ) assert inside( f"print(json.dumps(s.head(key={orphan!r}) is None and s.head(key={fresh!r}) is not None and s.head(key={key(saved['id'])!r}) is not None))" ) assert inside( f"print(json.dumps(s.head(key={key(failed_old['id'])!r}) is None and s.head(key={key(failed_fresh['id'])!r}) is not None))" ) repeat = run_reconcile(True) assert not repeat["applied"] assert repeat["linked_protected"] >= 1 result["reconciliation"] = { "dry_run": dry, "apply": applied, "repeat": repeat, "s3_grace_wait_seconds": round( time.monotonic() - orphan_started, 3 ), "fresh_and_ready_unlinked_protected": True, "old_failed_unlinked_deleted_and_fresh_failed_protected": True, "missing_used_id_and_link_preserved": True, "dry_run_unchanged": True, } if args.research: cells = [] for count in [0, 100, 1100]: if count: start = 0 if count == 100 else 100 inside( f"\nfor i in range({start},{count}):\n s.put(key={research_prefix!r}+str(i),stream=io.BytesIO(b'x'),size=1,content_type='application/octet-stream',sha256=hashlib.sha256(b'x').hexdigest())\nprint('true')" ) for repetition in range(3): measured = run_reconcile() cells.append( { "extra_objects": count, "repetition": repetition + 1, **measured, } ) result["cost_research"] = { "cells": cells, "repeats": 3, "method": "same running stand; 0/100/1100 fresh one-byte objects; dry-run; serial ascending dataset; no warmup", "median_seconds": { str(n): statistics.median( c["duration_seconds"] for c in cells if c["extra_objects"] == n ) for n in [0, 100, 1100] }, } result["complete"] = True finally: # Only keys created by this invocation; no broad bucket deletion. try: inside( f"\nfor o in s.objects():\n if o.object_name.startswith({research_prefix!r}) or o.object_name in [{orphan!r},{fresh!r}]: s.delete(key=o.object_name)\nprint('true')" ) finally: compose("up", "-d", "--no-build", "--no-recreate", "reconciler") output.write_text( json.dumps(result, ensure_ascii=False, indent=2) + "\n" ) print( "PASS: private S3, failover, bounded concurrent uploads and two-store recovery" ) if __name__ == "__main__": main()