#!/usr/bin/env python3 """Private S3, 10 concurrent proxy uploads, failure recovery and reconciliation cost.""" import argparse import hashlib import io import json import os import socket import subprocess import sys import time import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime from pathlib import Path from uuid import uuid4 from botocore.exceptions import ClientError from cluster import compose, inspect, instances, wait_lb from PIL import Image from storage_lab import ROOT, connection, store from messenger.reconcile import Reconciler sys.path.insert(0, str(ROOT / "tests")) from client import Client, check # noqa: E402 PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() def put_orphan(admin): key = "originals/" + str(uuid4()) admin.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest()) return key def seed(path): admin = store("root") try: key = put_orphan(admin) path.write_text( json.dumps({"orphan": key, "created_at": datetime.now(UTC).isoformat()}) + "\n" ) print("Seeded one orphan; its real S3 LastModified must age 300 seconds.") finally: admin.close() def memory(cids): result = {} for cid in cids: raw = subprocess.check_output( [ "docker", "exec", cid, "sh", "-c", "cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak " "/sys/fs/cgroup/cpu.stat", ], text=True, ) lines = raw.splitlines() current, peak = map(int, lines[:2]) counters = {k: int(v) for k, v in (line.split() for line in lines[2:])} result[cid] = {"current_bytes": current, "peak_bytes": peak, "cpu": counters} return result def pressure(client, actor, chat): results = [] for side in (600, 1700): image = Image.frombytes("RGB", (side, side), os.urandom(side * side * 3)) output = io.BytesIO() image.save(output, format="PNG", compress_level=0) data = output.getvalue() check(len(data) <= 10485760, "Fixture accidentally oversized") # Recreate API only, so kernel memory.peak includes this trial and startup. compose("up", "-d", "--no-deps", "--force-recreate", "--scale", "api=2", "api") wait_lb(count=2, base=client.base) cids = [row["id"] for row in instances()] resource_started = time.monotonic() baseline = memory(cids) import threading barrier = threading.Barrier(10) def one(i, data=data, barrier=barrier): c = Client(client.base, timeout=30) barrier.wait(timeout=10) started = time.monotonic() value = c.call( "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=load-{i}.png", expected=201, actor=actor, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", ) check(value["json"]["sha256"] == hashlib.sha256(data).hexdigest(), "Checksum mismatch") return { "duration_ms": (time.monotonic() - started) * 1000, "instance": value["headers"]["X-Instance-Id"], "id": value["json"]["id"], } started = time.monotonic() with ThreadPoolExecutor(max_workers=10) as pool: values = list(pool.map(one, range(10))) elapsed = time.monotonic() - started peak = memory(cids) resource_seconds = time.monotonic() - resource_started cpu_cores = ( sum(peak[k]["cpu"]["usage_usec"] - baseline[k]["cpu"]["usage_usec"] for k in cids) / 1e6 / resource_seconds ) values_ms = sorted(row["duration_ms"] for row in values) results.append( { "clients": 10, "bytes_each": len(data), "pixels": side * side, "duration_seconds": elapsed, "successful_rps": 10 / elapsed, "resource_window_seconds": resource_seconds, "mean_api_cpu_cores_including_probes": cpu_cores, "errors": 0, "p50_ms": values_ms[4], "p95_ms": values_ms[9], "p99_ms": values_ms[9], "before": baseline, "after": peak, "responses": values, } ) print(f"PASS: 10 simultaneous uploads, {len(data)} bytes each", flush=True) return results def private(admin, api, chat, actor, client): value, _ = client.attachment(actor, chat) with connection() as conn: row = conn.execute("SELECT * FROM attachments WHERE id=%s", (value["id"],)).fetchone() # HTTP from a different container, no Authorization, no signature, no API. code = ( "import urllib.request,urllib.error; " f'u="http://s3:9000/{api.bucket}/{row["object_key"]}"; ' "\ntry:\n r=urllib.request.urlopen(u); print(r.status)" "\nexcept urllib.error.HTTPError as e: print(e.code)" ) cid = instances()[0]["id"] status = int(subprocess.check_output(["docker", "exec", cid, "python", "-c", code], text=True)) check(status == 403, "Anonymous S3 GET succeeded") for method, kwargs in [ ("delete_object", {"Bucket": api.bucket, "Key": row["object_key"]}), ("list_objects_v2", {"Bucket": "unrelated-bucket"}), ]: try: getattr(api.client, method)(**kwargs) except ClientError as exc: check(exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Expected IAM 403") else: raise RuntimeError("API identity exceeded its policy") return { "anonymous_get_status": status, "api_delete_denied": True, "other_bucket_denied": True, "attachment": value["id"], } def cross_replica(client, actor, reader, chat): value = client.call( "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=switch.png", expected=201, actor=actor, raw=PNG, headers={"Content-Type": "image/png"}, schema="Attachment", ) alias = value["headers"]["X-Instance-Id"] selected = next(row for row in instances() if row["instance"] == alias) try: subprocess.run( [str(ROOT / "scripts/stop-one.sh")], check=True, capture_output=True, env={**os.environ, "TARGET_INSTANCE_ID": alias}, timeout=10, ) stopped = inspect(selected["id"]) check(not stopped["running"], "Uploading process was not stopped") time.sleep(1) result = client.call( "GET", "/api/v1/attachments/" + value["json"]["id"] + "/content", actor=reader ) check( result["bytes"] == PNG and result["headers"]["X-Instance-Id"] != alias, "Another API failed to serve original", ) return { "uploader": selected, "after_stop": stopped, "downloader_instance": result["headers"]["X-Instance-Id"], "original_preserved": True, } finally: subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True) wait_lb(base=client.base, count=2) def interruption(client, actor, chat): with connection() as conn: before = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"] # Close in the middle of a correctly declared body; no durable intent must exist yet. s = socket.create_connection(("127.0.0.1", 8084), timeout=3) header = ( f"POST /api/v1/chats/{chat['id']}/attachments?filename=interrupted.png HTTP/1.1\r\n" "Host: localhost\r\nContent-Type: image/png\r\nContent-Length: 1048576\r\n" f"Authorization: Bearer {actor['token']}\r\nConnection: close\r\n\r\n" ) s.sendall(header.encode() + PNG[:32]) s.close() time.sleep(0.5) with connection() as conn: after = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"] check(before == after, "Interrupted inbound body created an intent") return {"before_rows": before, "after_rows": after, "aborted_body_no_intent": True} def outage(client, actor, chat): att, _ = client.attachment(actor, chat) before = inspect(compose("ps", "-q", "s3").strip()) compose("stop", "s3") started = time.monotonic() try: statuses = [] for method, path, raw in [ ("GET", "/api/v1/attachments/" + att["id"] + "/content", None), ("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=unavailable.png", PNG), ]: request = urllib.request.Request( client.base + path, data=raw, method=method, headers={"Authorization": "Bearer " + actor["token"], "Content-Type": "image/png"}, ) try: response = urllib.request.urlopen(request, timeout=8) except urllib.error.HTTPError as exc: response = exc with response: statuses.append(response.code) check(response.code == 503, "S3 outage must produce 503") failed = len(statuses) finally: compose("start", "s3") wait_lb(base=client.base, count=2) after = inspect(compose("ps", "-q", "s3").strip()) actual = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=actor)[ "bytes" ] check(actual == PNG, "S3 restart lost original") return { "before": before, "after": after, "failed_operations": failed, "statuses": statuses, "stop_start_and_ready_seconds": time.monotonic() - started, "original_preserved": True, } def reconciliation(seed_path, admin, reconciler, client, actor, chat): saved = json.loads(seed_path.read_text()) orphan = saved["orphan"] head = admin.head(orphan) check(head is not None, "Seed orphan is missing") age = time.time() - head["LastModified"].timestamp() check(age >= 300, "Seed must be at least 300s old; keep it while other checks run") fresh = put_orphan(admin) used, _ = client.attachment(actor, chat) client.message(actor, chat, "used attachment", attachments=[used["id"]]) with connection() as conn: conn.execute( "UPDATE attachments SET updated_at=clock_timestamp()-interval '10 minutes' WHERE id=%s", (used["id"],), ) dry = Reconciler(conn, reconciler).run() check( any( a["action"] == "delete_orphan" and a["id"] == orphan.split("/")[1] for a in dry["actions"] ), "Dry run did not detect old orphan", ) check(admin.head(orphan) is not None, "Dry-run mutated S3") applied = Reconciler(conn, reconciler, apply=True).run() repeated = Reconciler(conn, reconciler, apply=True).run() check(admin.head(orphan) is None and admin.head(fresh) is not None, "Grace safety failed") row = conn.execute("SELECT * FROM attachments WHERE id=%s", (used["id"],)).fetchone() check( row["storage_state"] == "ready" and admin.head(row["object_key"]) is not None, "Used attachment was removed", ) check(repeated["actions"] == [], "Repeated apply failed to converge") # Remove a USED object's bytes, preserve its metadata/link, and repair after re-upload. admin.client.delete_object(Bucket=admin.bucket, Key=row["object_key"]) missing_dry = Reconciler(conn, reconciler).run() check( conn.execute( "SELECT storage_state FROM attachments WHERE id=%s", (used["id"],) ).fetchone()["storage_state"] == "ready", "Dry run mutated metadata", ) missing_apply = Reconciler(conn, reconciler, apply=True).run() check( conn.execute( "SELECT storage_state FROM attachments WHERE id=%s", (used["id"],) ).fetchone()["storage_state"] == "missing", "Missing original was not marked", ) check( conn.execute( "SELECT count(*) AS n FROM message_attachments WHERE attachment_id=%s", (used["id"],), ).fetchone()["n"] == 1, "A used relation was removed", ) admin.put( row["object_key"], io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest(), ) healing = Reconciler(conn, reconciler, apply=True).run() check( client.call("GET", "/api/v1/attachments/" + used["id"] + "/content", actor=actor)[ "bytes" ] == PNG, "Restored original is wrong", ) return { "old_orphan_age_seconds": age, "dry_run": dry, "apply": applied, "repeat": repeated, "missing_dry_run": missing_dry, "missing_apply": missing_apply, "healing": healing, "fresh_preserved": True, "used_metadata_and_link_preserved": True, } def scan_cost(admin, reconciler): results = [] created = [] try: for target in (0, 100, 1100): while len(created) < target: created.append(put_orphan(admin)) repeats = [] for _ in range(3): with connection() as conn: repeats.append(Reconciler(conn, reconciler).run()) results.append({"extra_fresh_objects": target, "repeats": repeats}) return results finally: for key in created: admin.client.delete_object(Bucket=admin.bucket, Key=key) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--seed", type=Path) parser.add_argument("--resume-basics", type=Path) parser.add_argument("--seed-file", type=Path, default=ROOT / "evidence/storage-seed.json") parser.add_argument("--output", type=Path, default=ROOT / "evidence/storage.json") args = parser.parse_args() if args.seed: seed(args.seed) return if args.output.exists(): raise SystemExit("Choose a new evidence filename") os.environ["NO_PROXY"] = os.environ["no_proxy"] = "localhost,127.0.0.1" admin, api, rec = store("root"), store("api"), store("reconciler") try: client = Client("http://127.0.0.1:8084", timeout=30) alice, bob, eve, chat = client.bootstrap() result = {"started_at": datetime.now(UTC).isoformat()} if args.resume_basics: result.update(json.loads(args.resume_basics.read_text())) else: result["privacy"] = private(admin, api, chat, alice, client) result["cross_replica"] = cross_replica(client, alice, bob, chat) result["interruption"] = interruption(client, alice, chat) result["uploads"] = pressure(client, alice, chat) result["outage"] = outage(client, alice, chat) result["reconciliation"] = reconciliation(args.seed_file, admin, rec, client, alice, chat) result["scan_cost"] = scan_cost(admin, rec) result["finished_at"] = datetime.now(UTC).isoformat() args.output.write_text(json.dumps(result, indent=2) + "\n") print( "PASS: private S3, upload memory bound, interruption, outage and reconciler", flush=True ) finally: for value in (admin, api, rec): value.close() if __name__ == "__main__": main()