forked from mtusi2026/mtusi2026lab4
539 lines
21 KiB
Python
539 lines
21 KiB
Python
"""Operator checks on the disposable teaching stand, never production."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import socket
|
|
import statistics
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
from PIL import Image
|
|
|
|
from scripts import cluster
|
|
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
|
|
|
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
PREFIX = """import hashlib,io,json,time,urllib.request,urllib.error
|
|
from src.object_store import ObjectStore,BUCKET
|
|
from src.database import Database
|
|
from src.settings import secret
|
|
from src.reconcile import reconcile
|
|
s=ObjectStore(); d=Database(url=secret('DATABASE_URL'),size=2); d.open()
|
|
"""
|
|
|
|
|
|
def inside(code):
|
|
containers = compose("ps", "-q", "api").split()
|
|
assert containers, "No running API in this project"
|
|
value = command(
|
|
["docker", "exec", "-i", containers[0], "python", "-"],
|
|
input=PREFIX + code + "\ns.close(); d.close()\n",
|
|
)
|
|
return json.loads(value)
|
|
|
|
|
|
def upload(client, session, chat, data=PNG, *, name=None):
|
|
return client.post(
|
|
f"/api/v1/chats/{chat['id']}/attachments",
|
|
headers={**auth(session), "Content-Type": "image/png"},
|
|
params={"filename": name or uuid4().hex + ".png"},
|
|
content=data,
|
|
)
|
|
|
|
|
|
def state(identity):
|
|
return sql("SELECT state FROM attachments WHERE id='" + identity + "'")
|
|
|
|
|
|
def key(identity):
|
|
return "objects/" + identity + "/original"
|
|
|
|
|
|
def run_reconcile(apply=False):
|
|
return inside(
|
|
"print(json.dumps(reconcile(d,s,apply=" + str(apply) + ",grace=300)))"
|
|
)
|
|
|
|
|
|
def age(identity):
|
|
sql(
|
|
"UPDATE attachments SET created_at=clock_timestamp()-interval '301 seconds', "
|
|
"expires_at=clock_timestamp()-interval '180 seconds' WHERE id='"
|
|
+ identity
|
|
+ "'"
|
|
)
|
|
|
|
|
|
def percentiles(values):
|
|
values = sorted(values)
|
|
return {
|
|
f"p{p}_ms": round(
|
|
values[min(len(values) - 1, int(p * len(values) / 100))], 3
|
|
)
|
|
for p in [50, 95, 99]
|
|
}
|
|
|
|
|
|
def concurrency(client, session, chat):
|
|
stream = io.BytesIO()
|
|
Image.effect_noise((1700, 1700), 100).convert("RGB").save(stream, "PNG")
|
|
image = stream.getvalue()
|
|
assert len(image) < 10 * 1024 * 1024
|
|
ids = compose("ps", "-q", "api").split()
|
|
samples, stop = [], threading.Event()
|
|
code = (
|
|
"import json,pathlib; p=pathlib.Path('/sys/fs/cgroup'); "
|
|
"print(json.dumps({k:int((p/k).read_text()) for k in ['memory.current','memory.peak']}))"
|
|
)
|
|
|
|
def sample():
|
|
while not stop.is_set():
|
|
for cid in ids:
|
|
samples.append(
|
|
{
|
|
"at": round(time.monotonic() - started, 3),
|
|
"container": cid[:12],
|
|
**json.loads(
|
|
command(
|
|
["docker", "exec", cid, "python", "-c", code]
|
|
)
|
|
),
|
|
}
|
|
)
|
|
stop.wait(0.3)
|
|
|
|
barrier = threading.Barrier(10)
|
|
|
|
def one(_):
|
|
with httpx.Client(base_url=BASE, trust_env=False, timeout=60) as c:
|
|
barrier.wait(timeout=30)
|
|
begin = time.monotonic()
|
|
r = upload(c, session, chat, image)
|
|
latency = (time.monotonic() - begin) * 1000
|
|
assert r.status_code == 201, (r.status_code, r.text)
|
|
a = r.json()
|
|
assert a["sha256"] == hashlib.sha256(image).hexdigest()
|
|
data = c.get(
|
|
f"/api/v1/attachments/{a['id']}/content", headers=auth(session)
|
|
)
|
|
assert data.status_code == 200 and data.content == image
|
|
return {
|
|
"id": a["id"],
|
|
"latency_ms": latency,
|
|
"instance": r.headers["x-instance-id"],
|
|
}
|
|
|
|
cpu_before = {
|
|
cid: command(["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"])
|
|
for cid in ids
|
|
}
|
|
started = time.monotonic()
|
|
thread = threading.Thread(target=sample)
|
|
thread.start()
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
results = list(pool.map(one, range(10)))
|
|
finally:
|
|
stop.set()
|
|
thread.join(timeout=30)
|
|
elapsed = time.monotonic() - started
|
|
assert samples
|
|
return {
|
|
"concurrency": 10,
|
|
"requests": 10,
|
|
"successes": len(results),
|
|
"errors": 0,
|
|
"bytes_per_image": len(image),
|
|
"pixels": 1700 * 1700,
|
|
"duration_seconds": round(elapsed, 3),
|
|
"successful_rps": round(10 / elapsed, 3),
|
|
"latency_scope": "upload only; duration includes verified downloads and memory sampler",
|
|
**percentiles([r["latency_ms"] for r in results]),
|
|
"cpu_before": cpu_before,
|
|
"cpu_after": {
|
|
cid: command(
|
|
["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"]
|
|
)
|
|
for cid in ids
|
|
},
|
|
"memory_samples": samples,
|
|
"results": results,
|
|
"window_max_memory_current": max(s["memory.current"] for s in samples),
|
|
"lifetime_memory_peak": max(s["memory.peak"] for s in samples),
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output", default="evidence/storage-repeat.json")
|
|
parser.add_argument("--research", action="store_true")
|
|
args = parser.parse_args()
|
|
output = Path(args.output)
|
|
assert not output.exists(), "Choose a fresh evidence path"
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
result = {
|
|
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
|
"date": datetime.now(timezone.utc).isoformat(),
|
|
"complete": False,
|
|
}
|
|
ready()
|
|
from scripts.verify_cluster import snapshot
|
|
|
|
result["runtime"] = snapshot()
|
|
compose("stop", "reconciler")
|
|
orphan = "objects/verification-" + uuid4().hex + "/orphan"
|
|
fresh = "objects/verification-" + uuid4().hex + "/fresh"
|
|
research_prefix = "objects/research-" + uuid4().hex + "/"
|
|
try:
|
|
# Its server LastModified must really age; no clock manipulation in S3.
|
|
inside(
|
|
f"s.put(key={orphan!r},stream=io.BytesIO(b'orphan'),size=6,content_type='application/octet-stream',sha256=hashlib.sha256(b'orphan').hexdigest()); print('true')"
|
|
)
|
|
orphan_started = time.monotonic()
|
|
with httpx.Client(
|
|
base_url=BASE, trust_env=False, timeout=30
|
|
) as client:
|
|
user, _, session = actor(client)
|
|
r = client.post(
|
|
"/api/v1/chats",
|
|
headers=auth(session),
|
|
json={
|
|
"title": "Storage verification",
|
|
"member_ids": [user["id"]],
|
|
},
|
|
)
|
|
assert r.status_code == 201
|
|
chat = r.json()
|
|
r = upload(client, session, chat)
|
|
assert r.status_code == 201
|
|
saved, uploader = r.json(), r.headers["x-instance-id"]
|
|
failed_old = upload(client, session, chat).json()
|
|
sql(
|
|
"UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='"
|
|
+ failed_old["id"]
|
|
+ "'"
|
|
)
|
|
age(failed_old["id"])
|
|
grace_started = time.monotonic()
|
|
target = next(
|
|
c
|
|
for c in cluster.containers()
|
|
if cluster.alias(c["Id"]) == uploader
|
|
)
|
|
command(["docker", "stop", "--time", "10", target["Id"]])
|
|
try:
|
|
time.sleep(3)
|
|
download = client.get(
|
|
f"/api/v1/attachments/{saved['id']}/content",
|
|
headers=auth(session),
|
|
)
|
|
assert download.status_code == 200 and download.content == PNG
|
|
assert download.headers["x-instance-id"] != uploader
|
|
result["upload_replica_stopped_downloaded_elsewhere"] = {
|
|
"stopped_container": target["Id"][:12],
|
|
"uploader": uploader,
|
|
"downloader": download.headers["x-instance-id"],
|
|
"sha256": saved["sha256"],
|
|
}
|
|
finally:
|
|
compose(
|
|
"up",
|
|
"-d",
|
|
"--no-build",
|
|
"--no-recreate",
|
|
"--wait",
|
|
"api",
|
|
"lb",
|
|
)
|
|
ready()
|
|
private_code = f"""
|
|
from src.errors import DomainError
|
|
checks={{}}
|
|
try:
|
|
s.head(key='forbidden'); raise AssertionError('HEAD outside IAM prefix')
|
|
except DomainError as e:
|
|
assert e.status==503; checks['denied_head_is_not_missing']=e.status
|
|
try:
|
|
urllib.request.urlopen('http://s3:9000/messenger/{key(saved["id"])}',timeout=3)
|
|
raise AssertionError('public object')
|
|
except urllib.error.HTTPError as e: checks['anonymous_get']=e.code; assert e.code==403
|
|
for name,operation in [('list_other_bucket',lambda:list(s.client.list_objects('other-private'))),('create_bucket',lambda:s.client.make_bucket('forbidden-bucket')),('write_outside_prefix',lambda:s.client.put_object(BUCKET,'forbidden',io.BytesIO(b'x'),1))]:
|
|
try: operation(); raise AssertionError(name)
|
|
except Exception as e:
|
|
assert getattr(e,'code',None)=='AccessDenied',type(e).__name__; checks[name]='AccessDenied'
|
|
try:
|
|
urllib.request.urlopen('http://s3:9001/',timeout=2); raise AssertionError('console exposed')
|
|
except urllib.error.URLError: checks['console_internal_network']='unreachable'
|
|
print(json.dumps(checks))
|
|
"""
|
|
result["private_bucket_and_iam"] = inside(private_code)
|
|
compose("run", "--rm", "--no-deps", "s3-bootstrap")
|
|
assert (
|
|
client.get(
|
|
f"/api/v1/attachments/{saved['id']}/content",
|
|
headers=auth(session),
|
|
).content
|
|
== PNG
|
|
)
|
|
result["bootstrap_repeat_preserved_original"] = True
|
|
# A partial HTTP body must never create a durable intent or object.
|
|
before = int(sql("SELECT count(*) FROM attachments"))
|
|
objects_before_abort = inside("print(len(list(s.objects())))")
|
|
parsed = httpx.URL(BASE)
|
|
with socket.create_connection(
|
|
(parsed.host, parsed.port), timeout=3
|
|
) as raw:
|
|
request = (
|
|
(
|
|
f"POST /api/v1/chats/{chat['id']}/attachments?filename=partial.png HTTP/1.1\r\n"
|
|
f"Host: {parsed.host}\r\nAuthorization: Bearer {session['token']}\r\n"
|
|
"Content-Type: image/png\r\nContent-Length: 10000000\r\n\r\n"
|
|
).encode()
|
|
+ PNG
|
|
+ b"x" * (2 * 1024 * 1024)
|
|
)
|
|
raw.sendall(request)
|
|
time.sleep(1)
|
|
assert int(sql("SELECT count(*) FROM attachments")) == before
|
|
assert (
|
|
inside("print(len(list(s.objects())))") == objects_before_abort
|
|
)
|
|
result["aborted_upload_has_no_intent_or_bytes"] = True
|
|
# Failure before any S3 PUT.
|
|
name = uuid4().hex + ".png"
|
|
objects_before = inside("print(len(list(s.objects())))")
|
|
sql(
|
|
"CREATE FUNCTION student_upload_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic'; END $$; CREATE TRIGGER student_upload_fault BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION student_upload_fault()"
|
|
)
|
|
try:
|
|
assert (
|
|
upload(client, session, chat, name=name).status_code == 500
|
|
)
|
|
assert (
|
|
int(
|
|
sql(
|
|
"SELECT count(*) FROM attachments WHERE filename='"
|
|
+ name
|
|
+ "'"
|
|
)
|
|
)
|
|
== 0
|
|
)
|
|
assert (
|
|
inside("print(len(list(s.objects())))") == objects_before
|
|
)
|
|
finally:
|
|
sql(
|
|
"DROP TRIGGER student_upload_fault ON attachments; DROP FUNCTION student_upload_fault()"
|
|
)
|
|
result["pg_intent_failure_has_no_bytes"] = True
|
|
# S3 PUT succeeded, then ready UPDATE fails: durable pending is recoverable.
|
|
name = uuid4().hex + ".png"
|
|
sql(
|
|
"CREATE FUNCTION student_ready_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.state='ready' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; CREATE TRIGGER student_ready_fault BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION student_ready_fault()"
|
|
)
|
|
try:
|
|
assert (
|
|
upload(client, session, chat, name=name).status_code == 500
|
|
)
|
|
finally:
|
|
sql(
|
|
"DROP TRIGGER student_ready_fault ON attachments; DROP FUNCTION student_ready_fault()"
|
|
)
|
|
pending = sql(
|
|
"SELECT id FROM attachments WHERE filename='" + name + "'"
|
|
)
|
|
assert state(pending) == "pending"
|
|
assert inside(
|
|
f"print(json.dumps(s.head(key={key(pending)!r}) is not None))"
|
|
)
|
|
age(pending)
|
|
# Stop S3; bypass LB health exclusion only from inside the selected API.
|
|
compose("stop", "s3")
|
|
name = uuid4().hex + ".png"
|
|
try:
|
|
direct = f"""
|
|
request=urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/{chat["id"]}/attachments?filename={name}',data=bytes.fromhex({PNG.hex()!r}),headers={{'Authorization':{auth(session)["Authorization"]!r},'Content-Type':'image/png'}})
|
|
try: urllib.request.urlopen(request,timeout=15); raise AssertionError('S3 down accepted')
|
|
except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':json.loads(e.read())['error']['code']}}))
|
|
"""
|
|
observed = inside(direct)
|
|
assert observed == {"status": 503, "error": "unavailable"}
|
|
missing_pending = sql(
|
|
"SELECT id FROM attachments WHERE filename='" + name + "'"
|
|
)
|
|
assert state(missing_pending) == "pending"
|
|
result["s3_unavailable"] = observed
|
|
finally:
|
|
compose(
|
|
"up", "-d", "--no-build", "--no-recreate", "--wait", "s3"
|
|
)
|
|
ready()
|
|
age(missing_pending)
|
|
# Used object missing: preserve metadata ID and the Message link.
|
|
used = upload(client, session, chat).json()
|
|
sent = client.post(
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
headers={**auth(session), "Idempotency-Key": uuid4().hex},
|
|
json={"text": "", "attachment_ids": [used["id"]]},
|
|
)
|
|
assert sent.status_code == 201
|
|
inside(f"s.delete(key={key(used['id'])!r}); print('true')")
|
|
age(used["id"])
|
|
age(saved["id"])
|
|
# Fresh cgroups make memory.peak specific to this experiment.
|
|
compose("restart", "api")
|
|
cluster.wait_count(2)
|
|
ready()
|
|
result["concurrent_uploads"] = concurrency(client, session, chat)
|
|
result["concurrent_uploads"][
|
|
"api_restarted_before_memory_measurement"
|
|
] = True
|
|
while time.monotonic() - grace_started < 302:
|
|
print(
|
|
"Waiting for genuine S3 LastModified grace: "
|
|
+ str(round(302 - (time.monotonic() - grace_started)))
|
|
+ "s",
|
|
flush=True,
|
|
)
|
|
time.sleep(
|
|
min(20, max(0, 302 - (time.monotonic() - grace_started)))
|
|
)
|
|
inside(
|
|
f"s.put(key={fresh!r},stream=io.BytesIO(b'fresh'),size=5,content_type='application/octet-stream',sha256=hashlib.sha256(b'fresh').hexdigest()); print('true')"
|
|
)
|
|
failed_fresh = upload(client, session, chat).json()
|
|
sql(
|
|
"UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='"
|
|
+ failed_fresh["id"]
|
|
+ "'"
|
|
)
|
|
age(failed_fresh["id"])
|
|
before_states = [
|
|
state(i)
|
|
for i in [pending, missing_pending, used["id"], saved["id"]]
|
|
]
|
|
dry = run_reconcile()
|
|
assert [
|
|
state(i)
|
|
for i in [pending, missing_pending, used["id"], saved["id"]]
|
|
] == before_states
|
|
assert inside(
|
|
f"print(json.dumps(s.head(key={orphan!r}) is not None))"
|
|
)
|
|
for action in [
|
|
"recover_pending",
|
|
"fail_pending_missing",
|
|
"mark_missing",
|
|
"delete_orphan",
|
|
"delete_failed_object",
|
|
]:
|
|
assert dry["actions"].get(action, 0) >= 1, dry
|
|
applied = run_reconcile(True)
|
|
assert state(pending) == "ready"
|
|
assert (
|
|
state(missing_pending) == "failed"
|
|
and state(used["id"]) == "failed"
|
|
)
|
|
assert state(saved["id"]) == "ready"
|
|
assert (
|
|
int(
|
|
sql(
|
|
"SELECT count(*) FROM message_attachments WHERE attachment_id='"
|
|
+ used["id"]
|
|
+ "'"
|
|
)
|
|
)
|
|
== 1
|
|
)
|
|
assert (
|
|
client.get(
|
|
f"/api/v1/attachments/{used['id']}", headers=auth(session)
|
|
).status_code
|
|
== 409
|
|
)
|
|
assert (
|
|
client.get(
|
|
f"/api/v1/attachments/{pending}/content",
|
|
headers=auth(session),
|
|
).content
|
|
== PNG
|
|
)
|
|
assert inside(
|
|
f"print(json.dumps(s.head(key={orphan!r}) is None and s.head(key={fresh!r}) is not None and s.head(key={key(saved['id'])!r}) is not None))"
|
|
)
|
|
assert inside(
|
|
f"print(json.dumps(s.head(key={key(failed_old['id'])!r}) is None and s.head(key={key(failed_fresh['id'])!r}) is not None))"
|
|
)
|
|
repeat = run_reconcile(True)
|
|
assert not repeat["applied"]
|
|
assert repeat["linked_protected"] >= 1
|
|
result["reconciliation"] = {
|
|
"dry_run": dry,
|
|
"apply": applied,
|
|
"repeat": repeat,
|
|
"s3_grace_wait_seconds": round(
|
|
time.monotonic() - orphan_started, 3
|
|
),
|
|
"fresh_and_ready_unlinked_protected": True,
|
|
"old_failed_unlinked_deleted_and_fresh_failed_protected": True,
|
|
"missing_used_id_and_link_preserved": True,
|
|
"dry_run_unchanged": True,
|
|
}
|
|
if args.research:
|
|
cells = []
|
|
for count in [0, 100, 1100]:
|
|
if count:
|
|
start = 0 if count == 100 else 100
|
|
inside(
|
|
f"\nfor i in range({start},{count}):\n s.put(key={research_prefix!r}+str(i),stream=io.BytesIO(b'x'),size=1,content_type='application/octet-stream',sha256=hashlib.sha256(b'x').hexdigest())\nprint('true')"
|
|
)
|
|
for repetition in range(3):
|
|
measured = run_reconcile()
|
|
cells.append(
|
|
{
|
|
"extra_objects": count,
|
|
"repetition": repetition + 1,
|
|
**measured,
|
|
}
|
|
)
|
|
result["cost_research"] = {
|
|
"cells": cells,
|
|
"repeats": 3,
|
|
"method": "same running stand; 0/100/1100 fresh one-byte objects; dry-run; serial ascending dataset; no warmup",
|
|
"median_seconds": {
|
|
str(n): statistics.median(
|
|
c["duration_seconds"]
|
|
for c in cells
|
|
if c["extra_objects"] == n
|
|
)
|
|
for n in [0, 100, 1100]
|
|
},
|
|
}
|
|
result["complete"] = True
|
|
finally:
|
|
# Only keys created by this invocation; no broad bucket deletion.
|
|
try:
|
|
inside(
|
|
f"\nfor o in s.objects():\n if o.object_name.startswith({research_prefix!r}) or o.object_name in [{orphan!r},{fresh!r}]: s.delete(key=o.object_name)\nprint('true')"
|
|
)
|
|
finally:
|
|
compose("up", "-d", "--no-build", "--no-recreate", "reconciler")
|
|
output.write_text(
|
|
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print(
|
|
"PASS: private S3, failover, bounded concurrent uploads and two-store recovery"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|