Files
mtusi2026lab4/scripts/verify_storage.py
T

526 lines
21 KiB
Python

"""Operator checks on the disposable teaching stand, never production."""
import argparse
import hashlib
import io
import json
import socket
import statistics
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from PIL import Image
from scripts import cluster
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
PREFIX = """import hashlib,io,json,time,urllib.request,urllib.error
from src.object_store import ObjectStore,BUCKET
from src.database import Database
from src.settings import secret
from src.reconcile import reconcile
s=ObjectStore(); d=Database(url=secret('DATABASE_URL'),size=2); d.open()
"""
def inside(code):
value = command(
["docker", "compose", "exec", "-T", "api", "python", "-"],
input=PREFIX + code + "\ns.close(); d.close()\n",
)
return json.loads(value)
def upload(client, session, chat, data=PNG, *, name=None):
return client.post(
f"/api/v1/chats/{chat['id']}/attachments",
headers={**auth(session), "Content-Type": "image/png"},
params={"filename": name or uuid4().hex + ".png"},
content=data,
)
def state(identity):
return sql("SELECT state FROM attachments WHERE id='" + identity + "'")
def key(identity):
return "objects/" + identity + "/original"
def run_reconcile(apply=False):
return inside(
"print(json.dumps(reconcile(d,s,apply=" + str(apply) + ",grace=300)))"
)
def age(identity):
sql(
"UPDATE attachments SET created_at=clock_timestamp()-interval '301 seconds', "
"expires_at=clock_timestamp()-interval '180 seconds' WHERE id='"
+ identity
+ "'"
)
def percentiles(values):
values = sorted(values)
return {
f"p{p}_ms": round(
values[min(len(values) - 1, int(p * len(values) / 100))], 3
)
for p in [50, 95, 99]
}
def concurrency(client, session, chat):
stream = io.BytesIO()
Image.effect_noise((1700, 1700), 100).convert("RGB").save(stream, "PNG")
image = stream.getvalue()
assert len(image) < 10 * 1024 * 1024
ids = compose("ps", "-q", "api").split()
samples, stop = [], threading.Event()
code = (
"import json,pathlib; p=pathlib.Path('/sys/fs/cgroup'); "
"print(json.dumps({k:int((p/k).read_text()) for k in ['memory.current','memory.peak']}))"
)
def sample():
while not stop.is_set():
for cid in ids:
samples.append(
{
"at": round(time.monotonic() - started, 3),
"container": cid[:12],
**json.loads(
command(
["docker", "exec", cid, "python", "-c", code]
)
),
}
)
stop.wait(0.3)
barrier = threading.Barrier(10)
def one(_):
with httpx.Client(base_url=BASE, trust_env=False, timeout=60) as c:
barrier.wait(timeout=30)
begin = time.monotonic()
r = upload(c, session, chat, image)
latency = (time.monotonic() - begin) * 1000
assert r.status_code == 201, (r.status_code, r.text)
a = r.json()
assert a["sha256"] == hashlib.sha256(image).hexdigest()
data = c.get(
f"/api/v1/attachments/{a['id']}/content", headers=auth(session)
)
assert data.status_code == 200 and data.content == image
return {
"id": a["id"],
"latency_ms": latency,
"instance": r.headers["x-instance-id"],
}
cpu_before = {
cid: command(["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"])
for cid in ids
}
started = time.monotonic()
thread = threading.Thread(target=sample)
thread.start()
try:
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(one, range(10)))
finally:
stop.set()
thread.join(timeout=30)
elapsed = time.monotonic() - started
assert samples
return {
"concurrency": 10,
"requests": 10,
"successes": len(results),
"errors": 0,
"bytes_per_image": len(image),
"pixels": 1700 * 1700,
"duration_seconds": round(elapsed, 3),
"successful_rps": round(10 / elapsed, 3),
"latency_scope": "upload only; duration includes verified downloads and memory sampler",
**percentiles([r["latency_ms"] for r in results]),
"cpu_before": cpu_before,
"cpu_after": {
cid: command(
["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"]
)
for cid in ids
},
"memory_samples": samples,
"results": results,
"window_max_memory_current": max(s["memory.current"] for s in samples),
"lifetime_memory_peak": max(s["memory.peak"] for s in samples),
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/storage-repeat.json")
parser.add_argument("--research", action="store_true")
args = parser.parse_args()
output = Path(args.output)
assert not output.exists(), "Choose a fresh evidence path"
output.parent.mkdir(parents=True, exist_ok=True)
result = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"complete": False,
}
ready()
from scripts.verify_cluster import snapshot
result["runtime"] = snapshot()
compose("stop", "reconciler")
orphan = "objects/verification-" + uuid4().hex + "/orphan"
fresh = "objects/verification-" + uuid4().hex + "/fresh"
research_prefix = "objects/research-" + uuid4().hex + "/"
try:
# Its server LastModified must really age; no clock manipulation in S3.
inside(
f"s.put(key={orphan!r},stream=io.BytesIO(b'orphan'),size=6,content_type='application/octet-stream',sha256=hashlib.sha256(b'orphan').hexdigest()); print('true')"
)
orphan_started = time.monotonic()
with httpx.Client(
base_url=BASE, trust_env=False, timeout=30
) as client:
user, _, session = actor(client)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={
"title": "Storage verification",
"member_ids": [user["id"]],
},
)
assert r.status_code == 201
chat = r.json()
r = upload(client, session, chat)
assert r.status_code == 201
saved, uploader = r.json(), r.headers["x-instance-id"]
failed_old = upload(client, session, chat).json()
sql(
"UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='"
+ failed_old["id"]
+ "'"
)
age(failed_old["id"])
grace_started = time.monotonic()
target = next(
c
for c in cluster.containers()
if cluster.alias(c["Id"]) == uploader
)
command(["docker", "stop", "--time", "10", target["Id"]])
try:
time.sleep(3)
download = client.get(
f"/api/v1/attachments/{saved['id']}/content",
headers=auth(session),
)
assert download.status_code == 200 and download.content == PNG
assert download.headers["x-instance-id"] != uploader
result["upload_replica_stopped_downloaded_elsewhere"] = {
"stopped_container": target["Id"][:12],
"uploader": uploader,
"downloader": download.headers["x-instance-id"],
"sha256": saved["sha256"],
}
finally:
compose(
"up",
"-d",
"--no-build",
"--no-recreate",
"--wait",
"api",
"lb",
)
ready()
private_code = f"""
from src.errors import DomainError
checks={{}}
try:
s.head(key='forbidden'); raise AssertionError('HEAD outside IAM prefix')
except DomainError as e:
assert e.status==503; checks['denied_head_is_not_missing']=e.status
try:
urllib.request.urlopen('http://s3:9000/messenger/{key(saved["id"])}',timeout=3)
raise AssertionError('public object')
except urllib.error.HTTPError as e: checks['anonymous_get']=e.code; assert e.code==403
for name,operation in [('list_other_bucket',lambda:list(s.client.list_objects('other-private'))),('create_bucket',lambda:s.client.make_bucket('forbidden-bucket')),('write_outside_prefix',lambda:s.client.put_object(BUCKET,'forbidden',io.BytesIO(b'x'),1))]:
try: operation(); raise AssertionError(name)
except Exception as e:
assert getattr(e,'code',None)=='AccessDenied',type(e).__name__; checks[name]='AccessDenied'
try:
urllib.request.urlopen('http://s3:9001/',timeout=2); raise AssertionError('console exposed')
except urllib.error.URLError: checks['console_internal_network']='unreachable'
print(json.dumps(checks))
"""
result["private_bucket_and_iam"] = inside(private_code)
compose("run", "--rm", "--no-deps", "s3-bootstrap")
assert (
client.get(
f"/api/v1/attachments/{saved['id']}/content",
headers=auth(session),
).content
== PNG
)
result["bootstrap_repeat_preserved_original"] = True
# A partial HTTP body must never create a durable intent or object.
before = int(sql("SELECT count(*) FROM attachments"))
objects_before_abort = inside("print(len(list(s.objects())))")
parsed = httpx.URL(BASE)
with socket.create_connection(
(parsed.host, parsed.port), timeout=3
) as raw:
request = (
f"POST /api/v1/chats/{chat['id']}/attachments?filename=partial.png HTTP/1.1\r\n"
f"Host: {parsed.host}\r\nAuthorization: Bearer {session['token']}\r\n"
"Content-Type: image/png\r\nContent-Length: 100000\r\n\r\n"
).encode() + PNG[:20]
raw.sendall(request)
time.sleep(1)
assert int(sql("SELECT count(*) FROM attachments")) == before
assert (
inside("print(len(list(s.objects())))") == objects_before_abort
)
result["aborted_upload_has_no_intent_or_bytes"] = True
# Failure before any S3 PUT.
name = uuid4().hex + ".png"
sql(
"CREATE FUNCTION student_upload_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic'; END $$; CREATE TRIGGER student_upload_fault BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION student_upload_fault()"
)
objects_before = inside("print(len(list(s.objects())))")
try:
assert (
upload(client, session, chat, name=name).status_code == 500
)
assert (
int(
sql(
"SELECT count(*) FROM attachments WHERE filename='"
+ name
+ "'"
)
)
== 0
)
assert (
inside("print(len(list(s.objects())))") == objects_before
)
finally:
sql(
"DROP TRIGGER student_upload_fault ON attachments; DROP FUNCTION student_upload_fault()"
)
result["pg_intent_failure_has_no_bytes"] = True
# S3 PUT succeeded, then ready UPDATE fails: durable pending is recoverable.
name = uuid4().hex + ".png"
sql(
"CREATE FUNCTION student_ready_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.state='ready' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; CREATE TRIGGER student_ready_fault BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION student_ready_fault()"
)
try:
assert (
upload(client, session, chat, name=name).status_code == 500
)
finally:
sql(
"DROP TRIGGER student_ready_fault ON attachments; DROP FUNCTION student_ready_fault()"
)
pending = sql(
"SELECT id FROM attachments WHERE filename='" + name + "'"
)
assert state(pending) == "pending"
assert inside(
f"print(json.dumps(s.head(key={key(pending)!r}) is not None))"
)
age(pending)
# Stop S3; bypass LB health exclusion only from inside the selected API.
compose("stop", "s3")
name = uuid4().hex + ".png"
try:
direct = f"""
request=urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/{chat["id"]}/attachments?filename={name}',data=bytes.fromhex({PNG.hex()!r}),headers={{'Authorization':{auth(session)["Authorization"]!r},'Content-Type':'image/png'}})
try: urllib.request.urlopen(request,timeout=15); raise AssertionError('S3 down accepted')
except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':json.loads(e.read())['error']['code']}}))
"""
observed = inside(direct)
assert observed == {"status": 503, "error": "unavailable"}
missing_pending = sql(
"SELECT id FROM attachments WHERE filename='" + name + "'"
)
assert state(missing_pending) == "pending"
result["s3_unavailable"] = observed
finally:
compose(
"up", "-d", "--no-build", "--no-recreate", "--wait", "s3"
)
ready()
age(missing_pending)
# Used object missing: preserve metadata ID and the Message link.
used = upload(client, session, chat).json()
sent = client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers={**auth(session), "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [used["id"]]},
)
assert sent.status_code == 201
inside(f"s.delete(key={key(used['id'])!r}); print('true')")
age(used["id"])
age(saved["id"])
result["concurrent_uploads"] = concurrency(client, session, chat)
while time.monotonic() - grace_started < 302:
print(
"Waiting for genuine S3 LastModified grace: "
+ str(round(302 - (time.monotonic() - grace_started)))
+ "s",
flush=True,
)
time.sleep(
min(20, max(0, 302 - (time.monotonic() - grace_started)))
)
inside(
f"s.put(key={fresh!r},stream=io.BytesIO(b'fresh'),size=5,content_type='application/octet-stream',sha256=hashlib.sha256(b'fresh').hexdigest()); print('true')"
)
failed_fresh = upload(client, session, chat).json()
sql(
"UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='"
+ failed_fresh["id"]
+ "'"
)
age(failed_fresh["id"])
before_states = [
state(i)
for i in [pending, missing_pending, used["id"], saved["id"]]
]
dry = run_reconcile()
assert [
state(i)
for i in [pending, missing_pending, used["id"], saved["id"]]
] == before_states
assert inside(
f"print(json.dumps(s.head(key={orphan!r}) is not None))"
)
for action in [
"recover_pending",
"fail_pending_missing",
"mark_missing",
"delete_orphan",
"delete_failed_object",
]:
assert dry["actions"].get(action, 0) >= 1, dry
applied = run_reconcile(True)
assert state(pending) == "ready"
assert (
state(missing_pending) == "failed"
and state(used["id"]) == "failed"
)
assert state(saved["id"]) == "ready"
assert (
int(
sql(
"SELECT count(*) FROM message_attachments WHERE attachment_id='"
+ used["id"]
+ "'"
)
)
== 1
)
assert (
client.get(
f"/api/v1/attachments/{used['id']}", headers=auth(session)
).status_code
== 409
)
assert (
client.get(
f"/api/v1/attachments/{pending}/content",
headers=auth(session),
).content
== PNG
)
assert inside(
f"print(json.dumps(s.head(key={orphan!r}) is None and s.head(key={fresh!r}) is not None and s.head(key={key(saved['id'])!r}) is not None))"
)
assert inside(
f"print(json.dumps(s.head(key={key(failed_old['id'])!r}) is None and s.head(key={key(failed_fresh['id'])!r}) is not None))"
)
repeat = run_reconcile(True)
assert not repeat["applied"].get("delete_orphan")
assert repeat["linked_protected"] >= 1
result["reconciliation"] = {
"dry_run": dry,
"apply": applied,
"repeat": repeat,
"s3_grace_wait_seconds": round(
time.monotonic() - orphan_started, 3
),
"fresh_and_ready_unlinked_protected": True,
"old_failed_unlinked_deleted_and_fresh_failed_protected": True,
"missing_used_id_and_link_preserved": True,
"dry_run_unchanged": True,
}
if args.research:
cells = []
for count in [0, 100, 1100]:
if count:
start = 0 if count == 100 else 100
inside(
f"\nfor i in range({start},{count}):\n s.put(key={research_prefix!r}+str(i),stream=io.BytesIO(b'x'),size=1,content_type='application/octet-stream',sha256=hashlib.sha256(b'x').hexdigest())\nprint('true')"
)
for repetition in range(3):
measured = run_reconcile()
cells.append(
{
"extra_objects": count,
"repetition": repetition + 1,
**measured,
}
)
result["cost_research"] = {
"cells": cells,
"repeats": 3,
"method": "same running stand; 0/100/1100 fresh one-byte objects; dry-run; serial ascending dataset; no warmup",
"median_seconds": {
str(n): statistics.median(
c["duration_seconds"]
for c in cells
if c["extra_objects"] == n
)
for n in [0, 100, 1100]
},
}
result["complete"] = True
finally:
# Only keys created by this invocation; no broad bucket deletion.
try:
inside(
f"\nfor o in s.objects():\n if o.object_name.startswith({research_prefix!r}) or o.object_name in [{orphan!r},{fresh!r}]: s.delete(key=o.object_name)\nprint('true')"
)
finally:
compose("up", "-d", "--no-build", "--no-recreate", "reconciler")
output.write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: private S3, failover, bounded concurrent uploads and two-store recovery"
)
if __name__ == "__main__":
main()