forked from mtusi2026/mtusi2026lab4
425 lines
16 KiB
Python
425 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Private S3, 10 concurrent proxy uploads, failure recovery and reconciliation cost."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from botocore.exceptions import ClientError
|
|
from cluster import compose, inspect, instances, wait_lb
|
|
from PIL import Image
|
|
from storage_lab import ROOT, connection, store
|
|
|
|
from messenger.reconcile import Reconciler
|
|
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, check # noqa: E402
|
|
|
|
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
|
def put_orphan(admin):
|
|
key = "originals/" + str(uuid4())
|
|
admin.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
|
|
return key
|
|
|
|
|
|
def seed(path):
|
|
admin = store("root")
|
|
try:
|
|
key = put_orphan(admin)
|
|
path.write_text(
|
|
json.dumps({"orphan": key, "created_at": datetime.now(UTC).isoformat()}) + "\n"
|
|
)
|
|
print("Seeded one orphan; its real S3 LastModified must age 300 seconds.")
|
|
finally:
|
|
admin.close()
|
|
|
|
|
|
def memory(cids):
|
|
result = {}
|
|
for cid in cids:
|
|
raw = subprocess.check_output(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
cid,
|
|
"sh",
|
|
"-c",
|
|
"cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak "
|
|
"/sys/fs/cgroup/cpu.stat",
|
|
],
|
|
text=True,
|
|
)
|
|
lines = raw.splitlines()
|
|
current, peak = map(int, lines[:2])
|
|
counters = {k: int(v) for k, v in (line.split() for line in lines[2:])}
|
|
result[cid] = {"current_bytes": current, "peak_bytes": peak, "cpu": counters}
|
|
return result
|
|
|
|
|
|
def pressure(client, actor, chat):
|
|
results = []
|
|
for side in (600, 1700):
|
|
image = Image.frombytes("RGB", (side, side), os.urandom(side * side * 3))
|
|
output = io.BytesIO()
|
|
image.save(output, format="PNG", compress_level=0)
|
|
data = output.getvalue()
|
|
check(len(data) <= 10485760, "Fixture accidentally oversized")
|
|
# Recreate API only, so kernel memory.peak includes this trial and startup.
|
|
compose("up", "-d", "--no-deps", "--force-recreate", "--scale", "api=2", "api")
|
|
wait_lb(count=2, base=client.base)
|
|
cids = [row["id"] for row in instances()]
|
|
resource_started = time.monotonic()
|
|
baseline = memory(cids)
|
|
import threading
|
|
|
|
barrier = threading.Barrier(10)
|
|
|
|
def one(i, data=data, barrier=barrier):
|
|
c = Client(client.base, timeout=30)
|
|
barrier.wait(timeout=10)
|
|
started = time.monotonic()
|
|
value = c.call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/attachments?filename=load-{i}.png",
|
|
expected=201,
|
|
actor=actor,
|
|
raw=data,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)
|
|
check(value["json"]["sha256"] == hashlib.sha256(data).hexdigest(), "Checksum mismatch")
|
|
return {
|
|
"duration_ms": (time.monotonic() - started) * 1000,
|
|
"instance": value["headers"]["X-Instance-Id"],
|
|
"id": value["json"]["id"],
|
|
}
|
|
|
|
started = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
values = list(pool.map(one, range(10)))
|
|
elapsed = time.monotonic() - started
|
|
peak = memory(cids)
|
|
resource_seconds = time.monotonic() - resource_started
|
|
cpu_cores = (
|
|
sum(peak[k]["cpu"]["usage_usec"] - baseline[k]["cpu"]["usage_usec"] for k in cids)
|
|
/ 1e6
|
|
/ resource_seconds
|
|
)
|
|
values_ms = sorted(row["duration_ms"] for row in values)
|
|
results.append(
|
|
{
|
|
"clients": 10,
|
|
"bytes_each": len(data),
|
|
"pixels": side * side,
|
|
"duration_seconds": elapsed,
|
|
"successful_rps": 10 / elapsed,
|
|
"resource_window_seconds": resource_seconds,
|
|
"mean_api_cpu_cores_including_probes": cpu_cores,
|
|
"errors": 0,
|
|
"p50_ms": values_ms[4],
|
|
"p95_ms": values_ms[9],
|
|
"p99_ms": values_ms[9],
|
|
"before": baseline,
|
|
"after": peak,
|
|
"responses": values,
|
|
}
|
|
)
|
|
print(f"PASS: 10 simultaneous uploads, {len(data)} bytes each", flush=True)
|
|
return results
|
|
|
|
|
|
def private(admin, api, chat, actor, client):
|
|
value, _ = client.attachment(actor, chat)
|
|
with connection() as conn:
|
|
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (value["id"],)).fetchone()
|
|
# HTTP from a different container, no Authorization, no signature, no API.
|
|
code = (
|
|
"import urllib.request,urllib.error; "
|
|
f'u="http://s3:9000/{api.bucket}/{row["object_key"]}"; '
|
|
"\ntry:\n r=urllib.request.urlopen(u); print(r.status)"
|
|
"\nexcept urllib.error.HTTPError as e: print(e.code)"
|
|
)
|
|
cid = instances()[0]["id"]
|
|
status = int(subprocess.check_output(["docker", "exec", cid, "python", "-c", code], text=True))
|
|
check(status == 403, "Anonymous S3 GET succeeded")
|
|
for method, kwargs in [
|
|
("delete_object", {"Bucket": api.bucket, "Key": row["object_key"]}),
|
|
("list_objects_v2", {"Bucket": "unrelated-bucket"}),
|
|
]:
|
|
try:
|
|
getattr(api.client, method)(**kwargs)
|
|
except ClientError as exc:
|
|
check(exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Expected IAM 403")
|
|
else:
|
|
raise RuntimeError("API identity exceeded its policy")
|
|
return {
|
|
"anonymous_get_status": status,
|
|
"api_delete_denied": True,
|
|
"other_bucket_denied": True,
|
|
"attachment": value["id"],
|
|
}
|
|
|
|
|
|
def cross_replica(client, actor, reader, chat):
|
|
value = client.call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/attachments?filename=switch.png",
|
|
expected=201,
|
|
actor=actor,
|
|
raw=PNG,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)
|
|
alias = value["headers"]["X-Instance-Id"]
|
|
selected = next(row for row in instances() if row["instance"] == alias)
|
|
try:
|
|
subprocess.run(
|
|
[str(ROOT / "scripts/stop-one.sh")],
|
|
check=True,
|
|
capture_output=True,
|
|
env={**os.environ, "TARGET_INSTANCE_ID": alias},
|
|
timeout=10,
|
|
)
|
|
stopped = inspect(selected["id"])
|
|
check(not stopped["running"], "Uploading process was not stopped")
|
|
time.sleep(1)
|
|
result = client.call(
|
|
"GET", "/api/v1/attachments/" + value["json"]["id"] + "/content", actor=reader
|
|
)
|
|
check(
|
|
result["bytes"] == PNG and result["headers"]["X-Instance-Id"] != alias,
|
|
"Another API failed to serve original",
|
|
)
|
|
return {
|
|
"uploader": selected,
|
|
"after_stop": stopped,
|
|
"downloader_instance": result["headers"]["X-Instance-Id"],
|
|
"original_preserved": True,
|
|
}
|
|
finally:
|
|
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
|
|
wait_lb(base=client.base, count=2)
|
|
|
|
|
|
def interruption(client, actor, chat):
|
|
with connection() as conn:
|
|
before = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"]
|
|
# Close in the middle of a correctly declared body; no durable intent must exist yet.
|
|
s = socket.create_connection(("127.0.0.1", 8084), timeout=3)
|
|
header = (
|
|
f"POST /api/v1/chats/{chat['id']}/attachments?filename=interrupted.png HTTP/1.1\r\n"
|
|
"Host: localhost\r\nContent-Type: image/png\r\nContent-Length: 1048576\r\n"
|
|
f"Authorization: Bearer {actor['token']}\r\nConnection: close\r\n\r\n"
|
|
)
|
|
s.sendall(header.encode() + PNG[:32])
|
|
s.close()
|
|
time.sleep(0.5)
|
|
with connection() as conn:
|
|
after = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"]
|
|
check(before == after, "Interrupted inbound body created an intent")
|
|
return {"before_rows": before, "after_rows": after, "aborted_body_no_intent": True}
|
|
|
|
|
|
def outage(client, actor, chat):
|
|
att, _ = client.attachment(actor, chat)
|
|
before = inspect(compose("ps", "-q", "s3").strip())
|
|
compose("stop", "s3")
|
|
started = time.monotonic()
|
|
try:
|
|
statuses = []
|
|
for method, path, raw in [
|
|
("GET", "/api/v1/attachments/" + att["id"] + "/content", None),
|
|
("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=unavailable.png", PNG),
|
|
]:
|
|
request = urllib.request.Request(
|
|
client.base + path,
|
|
data=raw,
|
|
method=method,
|
|
headers={"Authorization": "Bearer " + actor["token"], "Content-Type": "image/png"},
|
|
)
|
|
try:
|
|
response = urllib.request.urlopen(request, timeout=8)
|
|
except urllib.error.HTTPError as exc:
|
|
response = exc
|
|
with response:
|
|
statuses.append(response.code)
|
|
check(response.code == 503, "S3 outage must produce 503")
|
|
failed = len(statuses)
|
|
finally:
|
|
compose("start", "s3")
|
|
wait_lb(base=client.base, count=2)
|
|
after = inspect(compose("ps", "-q", "s3").strip())
|
|
actual = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=actor)[
|
|
"bytes"
|
|
]
|
|
check(actual == PNG, "S3 restart lost original")
|
|
return {
|
|
"before": before,
|
|
"after": after,
|
|
"failed_operations": failed,
|
|
"statuses": statuses,
|
|
"stop_start_and_ready_seconds": time.monotonic() - started,
|
|
"original_preserved": True,
|
|
}
|
|
|
|
|
|
def reconciliation(seed_path, admin, reconciler, client, actor, chat):
|
|
saved = json.loads(seed_path.read_text())
|
|
orphan = saved["orphan"]
|
|
head = admin.head(orphan)
|
|
check(head is not None, "Seed orphan is missing")
|
|
age = time.time() - head["LastModified"].timestamp()
|
|
check(age >= 300, "Seed must be at least 300s old; keep it while other checks run")
|
|
fresh = put_orphan(admin)
|
|
used, _ = client.attachment(actor, chat)
|
|
client.message(actor, chat, "used attachment", attachments=[used["id"]])
|
|
with connection() as conn:
|
|
conn.execute(
|
|
"UPDATE attachments SET updated_at=clock_timestamp()-interval '10 minutes' WHERE id=%s",
|
|
(used["id"],),
|
|
)
|
|
dry = Reconciler(conn, reconciler).run()
|
|
check(
|
|
any(
|
|
a["action"] == "delete_orphan" and a["id"] == orphan.split("/")[1]
|
|
for a in dry["actions"]
|
|
),
|
|
"Dry run did not detect old orphan",
|
|
)
|
|
check(admin.head(orphan) is not None, "Dry-run mutated S3")
|
|
applied = Reconciler(conn, reconciler, apply=True).run()
|
|
repeated = Reconciler(conn, reconciler, apply=True).run()
|
|
check(admin.head(orphan) is None and admin.head(fresh) is not None, "Grace safety failed")
|
|
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (used["id"],)).fetchone()
|
|
check(
|
|
row["storage_state"] == "ready" and admin.head(row["object_key"]) is not None,
|
|
"Used attachment was removed",
|
|
)
|
|
check(repeated["actions"] == [], "Repeated apply failed to converge")
|
|
# Remove a USED object's bytes, preserve its metadata/link, and repair after re-upload.
|
|
admin.client.delete_object(Bucket=admin.bucket, Key=row["object_key"])
|
|
missing_dry = Reconciler(conn, reconciler).run()
|
|
check(
|
|
conn.execute(
|
|
"SELECT storage_state FROM attachments WHERE id=%s", (used["id"],)
|
|
).fetchone()["storage_state"]
|
|
== "ready",
|
|
"Dry run mutated metadata",
|
|
)
|
|
missing_apply = Reconciler(conn, reconciler, apply=True).run()
|
|
check(
|
|
conn.execute(
|
|
"SELECT storage_state FROM attachments WHERE id=%s", (used["id"],)
|
|
).fetchone()["storage_state"]
|
|
== "missing",
|
|
"Missing original was not marked",
|
|
)
|
|
check(
|
|
conn.execute(
|
|
"SELECT count(*) AS n FROM message_attachments WHERE attachment_id=%s",
|
|
(used["id"],),
|
|
).fetchone()["n"]
|
|
== 1,
|
|
"A used relation was removed",
|
|
)
|
|
admin.put(
|
|
row["object_key"],
|
|
io.BytesIO(PNG),
|
|
len(PNG),
|
|
"image/png",
|
|
hashlib.sha256(PNG).hexdigest(),
|
|
)
|
|
healing = Reconciler(conn, reconciler, apply=True).run()
|
|
check(
|
|
client.call("GET", "/api/v1/attachments/" + used["id"] + "/content", actor=actor)[
|
|
"bytes"
|
|
]
|
|
== PNG,
|
|
"Restored original is wrong",
|
|
)
|
|
return {
|
|
"old_orphan_age_seconds": age,
|
|
"dry_run": dry,
|
|
"apply": applied,
|
|
"repeat": repeated,
|
|
"missing_dry_run": missing_dry,
|
|
"missing_apply": missing_apply,
|
|
"healing": healing,
|
|
"fresh_preserved": True,
|
|
"used_metadata_and_link_preserved": True,
|
|
}
|
|
|
|
|
|
def scan_cost(admin, reconciler):
|
|
results = []
|
|
created = []
|
|
try:
|
|
for target in (0, 100, 1100):
|
|
while len(created) < target:
|
|
created.append(put_orphan(admin))
|
|
repeats = []
|
|
for _ in range(3):
|
|
with connection() as conn:
|
|
repeats.append(Reconciler(conn, reconciler).run())
|
|
results.append({"extra_fresh_objects": target, "repeats": repeats})
|
|
return results
|
|
finally:
|
|
for key in created:
|
|
admin.client.delete_object(Bucket=admin.bucket, Key=key)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--seed", type=Path)
|
|
parser.add_argument("--resume-basics", type=Path)
|
|
parser.add_argument("--seed-file", type=Path, default=ROOT / "evidence/storage-seed.json")
|
|
parser.add_argument("--output", type=Path, default=ROOT / "evidence/storage.json")
|
|
args = parser.parse_args()
|
|
if args.seed:
|
|
seed(args.seed)
|
|
return
|
|
if args.output.exists():
|
|
raise SystemExit("Choose a new evidence filename")
|
|
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "localhost,127.0.0.1"
|
|
admin, api, rec = store("root"), store("api"), store("reconciler")
|
|
try:
|
|
client = Client("http://127.0.0.1:8084", timeout=30)
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
result = {"started_at": datetime.now(UTC).isoformat()}
|
|
if args.resume_basics:
|
|
result.update(json.loads(args.resume_basics.read_text()))
|
|
else:
|
|
result["privacy"] = private(admin, api, chat, alice, client)
|
|
result["cross_replica"] = cross_replica(client, alice, bob, chat)
|
|
result["interruption"] = interruption(client, alice, chat)
|
|
result["uploads"] = pressure(client, alice, chat)
|
|
result["outage"] = outage(client, alice, chat)
|
|
result["reconciliation"] = reconciliation(args.seed_file, admin, rec, client, alice, chat)
|
|
result["scan_cost"] = scan_cost(admin, rec)
|
|
result["finished_at"] = datetime.now(UTC).isoformat()
|
|
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
|
print(
|
|
"PASS: private S3, upload memory bound, interruption, outage and reconciler", flush=True
|
|
)
|
|
finally:
|
|
for value in (admin, api, rec):
|
|
value.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|