Files
mtusi2026lab4/scripts/verify_storage.py
T

425 lines
16 KiB
Python

#!/usr/bin/env python3
"""Private S3, 10 concurrent proxy uploads, failure recovery and reconciliation cost."""
import argparse
import hashlib
import io
import json
import os
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
from botocore.exceptions import ClientError
from cluster import compose, inspect, instances, wait_lb
from PIL import Image
from storage_lab import ROOT, connection, store
from messenger.reconcile import Reconciler
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
def put_orphan(admin):
key = "originals/" + str(uuid4())
admin.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
return key
def seed(path):
admin = store("root")
try:
key = put_orphan(admin)
path.write_text(
json.dumps({"orphan": key, "created_at": datetime.now(UTC).isoformat()}) + "\n"
)
print("Seeded one orphan; its real S3 LastModified must age 300 seconds.")
finally:
admin.close()
def memory(cids):
result = {}
for cid in cids:
raw = subprocess.check_output(
[
"docker",
"exec",
cid,
"sh",
"-c",
"cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak "
"/sys/fs/cgroup/cpu.stat",
],
text=True,
)
lines = raw.splitlines()
current, peak = map(int, lines[:2])
counters = {k: int(v) for k, v in (line.split() for line in lines[2:])}
result[cid] = {"current_bytes": current, "peak_bytes": peak, "cpu": counters}
return result
def pressure(client, actor, chat):
results = []
for side in (600, 1700):
image = Image.frombytes("RGB", (side, side), os.urandom(side * side * 3))
output = io.BytesIO()
image.save(output, format="PNG", compress_level=0)
data = output.getvalue()
check(len(data) <= 10485760, "Fixture accidentally oversized")
# Recreate API only, so kernel memory.peak includes this trial and startup.
compose("up", "-d", "--no-deps", "--force-recreate", "--scale", "api=2", "api")
wait_lb(count=2, base=client.base)
cids = [row["id"] for row in instances()]
resource_started = time.monotonic()
baseline = memory(cids)
import threading
barrier = threading.Barrier(10)
def one(i, data=data, barrier=barrier):
c = Client(client.base, timeout=30)
barrier.wait(timeout=10)
started = time.monotonic()
value = c.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=load-{i}.png",
expected=201,
actor=actor,
raw=data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
check(value["json"]["sha256"] == hashlib.sha256(data).hexdigest(), "Checksum mismatch")
return {
"duration_ms": (time.monotonic() - started) * 1000,
"instance": value["headers"]["X-Instance-Id"],
"id": value["json"]["id"],
}
started = time.monotonic()
with ThreadPoolExecutor(max_workers=10) as pool:
values = list(pool.map(one, range(10)))
elapsed = time.monotonic() - started
peak = memory(cids)
resource_seconds = time.monotonic() - resource_started
cpu_cores = (
sum(peak[k]["cpu"]["usage_usec"] - baseline[k]["cpu"]["usage_usec"] for k in cids)
/ 1e6
/ resource_seconds
)
values_ms = sorted(row["duration_ms"] for row in values)
results.append(
{
"clients": 10,
"bytes_each": len(data),
"pixels": side * side,
"duration_seconds": elapsed,
"successful_rps": 10 / elapsed,
"resource_window_seconds": resource_seconds,
"mean_api_cpu_cores_including_probes": cpu_cores,
"errors": 0,
"p50_ms": values_ms[4],
"p95_ms": values_ms[9],
"p99_ms": values_ms[9],
"before": baseline,
"after": peak,
"responses": values,
}
)
print(f"PASS: 10 simultaneous uploads, {len(data)} bytes each", flush=True)
return results
def private(admin, api, chat, actor, client):
value, _ = client.attachment(actor, chat)
with connection() as conn:
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (value["id"],)).fetchone()
# HTTP from a different container, no Authorization, no signature, no API.
code = (
"import urllib.request,urllib.error; "
f'u="http://s3:9000/{api.bucket}/{row["object_key"]}"; '
"\ntry:\n r=urllib.request.urlopen(u); print(r.status)"
"\nexcept urllib.error.HTTPError as e: print(e.code)"
)
cid = instances()[0]["id"]
status = int(subprocess.check_output(["docker", "exec", cid, "python", "-c", code], text=True))
check(status == 403, "Anonymous S3 GET succeeded")
for method, kwargs in [
("delete_object", {"Bucket": api.bucket, "Key": row["object_key"]}),
("list_objects_v2", {"Bucket": "unrelated-bucket"}),
]:
try:
getattr(api.client, method)(**kwargs)
except ClientError as exc:
check(exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Expected IAM 403")
else:
raise RuntimeError("API identity exceeded its policy")
return {
"anonymous_get_status": status,
"api_delete_denied": True,
"other_bucket_denied": True,
"attachment": value["id"],
}
def cross_replica(client, actor, reader, chat):
value = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=switch.png",
expected=201,
actor=actor,
raw=PNG,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
alias = value["headers"]["X-Instance-Id"]
selected = next(row for row in instances() if row["instance"] == alias)
try:
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
check=True,
capture_output=True,
env={**os.environ, "TARGET_INSTANCE_ID": alias},
timeout=10,
)
stopped = inspect(selected["id"])
check(not stopped["running"], "Uploading process was not stopped")
time.sleep(1)
result = client.call(
"GET", "/api/v1/attachments/" + value["json"]["id"] + "/content", actor=reader
)
check(
result["bytes"] == PNG and result["headers"]["X-Instance-Id"] != alias,
"Another API failed to serve original",
)
return {
"uploader": selected,
"after_stop": stopped,
"downloader_instance": result["headers"]["X-Instance-Id"],
"original_preserved": True,
}
finally:
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
wait_lb(base=client.base, count=2)
def interruption(client, actor, chat):
with connection() as conn:
before = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"]
# Close in the middle of a correctly declared body; no durable intent must exist yet.
s = socket.create_connection(("127.0.0.1", 8084), timeout=3)
header = (
f"POST /api/v1/chats/{chat['id']}/attachments?filename=interrupted.png HTTP/1.1\r\n"
"Host: localhost\r\nContent-Type: image/png\r\nContent-Length: 1048576\r\n"
f"Authorization: Bearer {actor['token']}\r\nConnection: close\r\n\r\n"
)
s.sendall(header.encode() + PNG[:32])
s.close()
time.sleep(0.5)
with connection() as conn:
after = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"]
check(before == after, "Interrupted inbound body created an intent")
return {"before_rows": before, "after_rows": after, "aborted_body_no_intent": True}
def outage(client, actor, chat):
att, _ = client.attachment(actor, chat)
before = inspect(compose("ps", "-q", "s3").strip())
compose("stop", "s3")
started = time.monotonic()
try:
statuses = []
for method, path, raw in [
("GET", "/api/v1/attachments/" + att["id"] + "/content", None),
("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=unavailable.png", PNG),
]:
request = urllib.request.Request(
client.base + path,
data=raw,
method=method,
headers={"Authorization": "Bearer " + actor["token"], "Content-Type": "image/png"},
)
try:
response = urllib.request.urlopen(request, timeout=8)
except urllib.error.HTTPError as exc:
response = exc
with response:
statuses.append(response.code)
check(response.code == 503, "S3 outage must produce 503")
failed = len(statuses)
finally:
compose("start", "s3")
wait_lb(base=client.base, count=2)
after = inspect(compose("ps", "-q", "s3").strip())
actual = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=actor)[
"bytes"
]
check(actual == PNG, "S3 restart lost original")
return {
"before": before,
"after": after,
"failed_operations": failed,
"statuses": statuses,
"stop_start_and_ready_seconds": time.monotonic() - started,
"original_preserved": True,
}
def reconciliation(seed_path, admin, reconciler, client, actor, chat):
saved = json.loads(seed_path.read_text())
orphan = saved["orphan"]
head = admin.head(orphan)
check(head is not None, "Seed orphan is missing")
age = time.time() - head["LastModified"].timestamp()
check(age >= 300, "Seed must be at least 300s old; keep it while other checks run")
fresh = put_orphan(admin)
used, _ = client.attachment(actor, chat)
client.message(actor, chat, "used attachment", attachments=[used["id"]])
with connection() as conn:
conn.execute(
"UPDATE attachments SET updated_at=clock_timestamp()-interval '10 minutes' WHERE id=%s",
(used["id"],),
)
dry = Reconciler(conn, reconciler).run()
check(
any(
a["action"] == "delete_orphan" and a["id"] == orphan.split("/")[1]
for a in dry["actions"]
),
"Dry run did not detect old orphan",
)
check(admin.head(orphan) is not None, "Dry-run mutated S3")
applied = Reconciler(conn, reconciler, apply=True).run()
repeated = Reconciler(conn, reconciler, apply=True).run()
check(admin.head(orphan) is None and admin.head(fresh) is not None, "Grace safety failed")
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (used["id"],)).fetchone()
check(
row["storage_state"] == "ready" and admin.head(row["object_key"]) is not None,
"Used attachment was removed",
)
check(repeated["actions"] == [], "Repeated apply failed to converge")
# Remove a USED object's bytes, preserve its metadata/link, and repair after re-upload.
admin.client.delete_object(Bucket=admin.bucket, Key=row["object_key"])
missing_dry = Reconciler(conn, reconciler).run()
check(
conn.execute(
"SELECT storage_state FROM attachments WHERE id=%s", (used["id"],)
).fetchone()["storage_state"]
== "ready",
"Dry run mutated metadata",
)
missing_apply = Reconciler(conn, reconciler, apply=True).run()
check(
conn.execute(
"SELECT storage_state FROM attachments WHERE id=%s", (used["id"],)
).fetchone()["storage_state"]
== "missing",
"Missing original was not marked",
)
check(
conn.execute(
"SELECT count(*) AS n FROM message_attachments WHERE attachment_id=%s",
(used["id"],),
).fetchone()["n"]
== 1,
"A used relation was removed",
)
admin.put(
row["object_key"],
io.BytesIO(PNG),
len(PNG),
"image/png",
hashlib.sha256(PNG).hexdigest(),
)
healing = Reconciler(conn, reconciler, apply=True).run()
check(
client.call("GET", "/api/v1/attachments/" + used["id"] + "/content", actor=actor)[
"bytes"
]
== PNG,
"Restored original is wrong",
)
return {
"old_orphan_age_seconds": age,
"dry_run": dry,
"apply": applied,
"repeat": repeated,
"missing_dry_run": missing_dry,
"missing_apply": missing_apply,
"healing": healing,
"fresh_preserved": True,
"used_metadata_and_link_preserved": True,
}
def scan_cost(admin, reconciler):
results = []
created = []
try:
for target in (0, 100, 1100):
while len(created) < target:
created.append(put_orphan(admin))
repeats = []
for _ in range(3):
with connection() as conn:
repeats.append(Reconciler(conn, reconciler).run())
results.append({"extra_fresh_objects": target, "repeats": repeats})
return results
finally:
for key in created:
admin.client.delete_object(Bucket=admin.bucket, Key=key)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--seed", type=Path)
parser.add_argument("--resume-basics", type=Path)
parser.add_argument("--seed-file", type=Path, default=ROOT / "evidence/storage-seed.json")
parser.add_argument("--output", type=Path, default=ROOT / "evidence/storage.json")
args = parser.parse_args()
if args.seed:
seed(args.seed)
return
if args.output.exists():
raise SystemExit("Choose a new evidence filename")
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "localhost,127.0.0.1"
admin, api, rec = store("root"), store("api"), store("reconciler")
try:
client = Client("http://127.0.0.1:8084", timeout=30)
alice, bob, eve, chat = client.bootstrap()
result = {"started_at": datetime.now(UTC).isoformat()}
if args.resume_basics:
result.update(json.loads(args.resume_basics.read_text()))
else:
result["privacy"] = private(admin, api, chat, alice, client)
result["cross_replica"] = cross_replica(client, alice, bob, chat)
result["interruption"] = interruption(client, alice, chat)
result["uploads"] = pressure(client, alice, chat)
result["outage"] = outage(client, alice, chat)
result["reconciliation"] = reconciliation(args.seed_file, admin, rec, client, alice, chat)
result["scan_cost"] = scan_cost(admin, rec)
result["finished_at"] = datetime.now(UTC).isoformat()
args.output.write_text(json.dumps(result, indent=2) + "\n")
print(
"PASS: private S3, upload memory bound, interruption, outage and reconciler", flush=True
)
finally:
for value in (admin, api, rec):
value.close()
if __name__ == "__main__":
main()