test(storage): add crash recovery and concurrent upload experiments
This commit is contained in:
@@ -0,0 +1,495 @@
|
||||
"""Operator checks on the disposable teaching stand, never production."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import socket
|
||||
import statistics
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
from scripts import cluster
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
PREFIX = """import hashlib,io,json,time,urllib.request,urllib.error
|
||||
from src.object_store import ObjectStore,BUCKET
|
||||
from src.database import Database
|
||||
from src.settings import secret
|
||||
from src.reconcile import reconcile
|
||||
s=ObjectStore(); d=Database(url=secret('DATABASE_URL'),size=2); d.open()
|
||||
"""
|
||||
|
||||
|
||||
def inside(code):
|
||||
value = command(
|
||||
["docker", "compose", "exec", "-T", "api", "python", "-"],
|
||||
input=PREFIX + code + "\ns.close(); d.close()\n",
|
||||
)
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
def upload(client, session, chat, data=PNG, *, name=None):
|
||||
return client.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments",
|
||||
headers={**auth(session), "Content-Type": "image/png"},
|
||||
params={"filename": name or uuid4().hex + ".png"},
|
||||
content=data,
|
||||
)
|
||||
|
||||
|
||||
def state(identity):
|
||||
return sql("SELECT state FROM attachments WHERE id='" + identity + "'")
|
||||
|
||||
|
||||
def key(identity):
|
||||
return "objects/" + identity + "/original"
|
||||
|
||||
|
||||
def run_reconcile(apply=False):
|
||||
return inside(
|
||||
"print(json.dumps(reconcile(d,s,apply=" + str(apply) + ",grace=300)))"
|
||||
)
|
||||
|
||||
|
||||
def age(identity):
|
||||
sql(
|
||||
"UPDATE attachments SET created_at=clock_timestamp()-interval '301 seconds', "
|
||||
"expires_at=clock_timestamp()-interval '180 seconds' WHERE id='"
|
||||
+ identity
|
||||
+ "'"
|
||||
)
|
||||
|
||||
|
||||
def percentiles(values):
|
||||
values = sorted(values)
|
||||
return {
|
||||
f"p{p}_ms": round(
|
||||
values[min(len(values) - 1, int(p * len(values) / 100))], 3
|
||||
)
|
||||
for p in [50, 95, 99]
|
||||
}
|
||||
|
||||
|
||||
def concurrency(client, session, chat):
|
||||
stream = io.BytesIO()
|
||||
Image.effect_noise((1700, 1700), 100).convert("RGB").save(stream, "PNG")
|
||||
image = stream.getvalue()
|
||||
assert len(image) < 10 * 1024 * 1024
|
||||
ids = compose("ps", "-q", "api").split()
|
||||
samples, stop = [], threading.Event()
|
||||
code = (
|
||||
"import json,pathlib; p=pathlib.Path('/sys/fs/cgroup'); "
|
||||
"print(json.dumps({k:int((p/k).read_text()) for k in ['memory.current','memory.peak']}))"
|
||||
)
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
for cid in ids:
|
||||
samples.append(
|
||||
{
|
||||
"at": round(time.monotonic() - started, 3),
|
||||
"container": cid[:12],
|
||||
**json.loads(
|
||||
command(
|
||||
["docker", "exec", cid, "python", "-c", code]
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
stop.wait(0.3)
|
||||
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def one(_):
|
||||
with httpx.Client(base_url=BASE, trust_env=False, timeout=60) as c:
|
||||
barrier.wait(timeout=30)
|
||||
begin = time.monotonic()
|
||||
r = upload(c, session, chat, image)
|
||||
latency = (time.monotonic() - begin) * 1000
|
||||
assert r.status_code == 201, (r.status_code, r.text)
|
||||
a = r.json()
|
||||
assert a["sha256"] == hashlib.sha256(image).hexdigest()
|
||||
data = c.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=auth(session)
|
||||
)
|
||||
assert data.status_code == 200 and data.content == image
|
||||
return {
|
||||
"id": a["id"],
|
||||
"latency_ms": latency,
|
||||
"instance": r.headers["x-instance-id"],
|
||||
}
|
||||
|
||||
cpu_before = {
|
||||
cid: command(["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"])
|
||||
for cid in ids
|
||||
}
|
||||
started = time.monotonic()
|
||||
thread = threading.Thread(target=sample)
|
||||
thread.start()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results = list(pool.map(one, range(10)))
|
||||
finally:
|
||||
stop.set()
|
||||
thread.join(timeout=30)
|
||||
elapsed = time.monotonic() - started
|
||||
assert samples
|
||||
return {
|
||||
"concurrency": 10,
|
||||
"requests": 10,
|
||||
"successes": len(results),
|
||||
"errors": 0,
|
||||
"bytes_per_image": len(image),
|
||||
"pixels": 1700 * 1700,
|
||||
"duration_seconds": round(elapsed, 3),
|
||||
"successful_rps": round(10 / elapsed, 3),
|
||||
"latency_scope": "upload only; duration includes verified downloads and memory sampler",
|
||||
**percentiles([r["latency_ms"] for r in results]),
|
||||
"cpu_before": cpu_before,
|
||||
"cpu_after": {
|
||||
cid: command(
|
||||
["docker", "exec", cid, "cat", "/sys/fs/cgroup/cpu.stat"]
|
||||
)
|
||||
for cid in ids
|
||||
},
|
||||
"memory_samples": samples,
|
||||
"results": results,
|
||||
"window_max_memory_current": max(s["memory.current"] for s in samples),
|
||||
"lifetime_memory_peak": max(s["memory.peak"] for s in samples),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/storage-repeat.json")
|
||||
parser.add_argument("--research", action="store_true")
|
||||
args = parser.parse_args()
|
||||
output = Path(args.output)
|
||||
assert not output.exists(), "Choose a fresh evidence path"
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
result = {
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"complete": False,
|
||||
}
|
||||
ready()
|
||||
compose("stop", "reconciler")
|
||||
orphan = "objects/verification-" + uuid4().hex + "/orphan"
|
||||
fresh = "objects/verification-" + uuid4().hex + "/fresh"
|
||||
research_prefix = "objects/research-" + uuid4().hex + "/"
|
||||
try:
|
||||
# Its server LastModified must really age; no clock manipulation in S3.
|
||||
inside(
|
||||
f"s.put(key={orphan!r},stream=io.BytesIO(b'orphan'),size=6,content_type='application/octet-stream',sha256=hashlib.sha256(b'orphan').hexdigest()); print('true')"
|
||||
)
|
||||
orphan_started = time.monotonic()
|
||||
with httpx.Client(
|
||||
base_url=BASE, trust_env=False, timeout=30
|
||||
) as client:
|
||||
user, _, session = actor(client)
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={
|
||||
"title": "Storage verification",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
chat = r.json()
|
||||
r = upload(client, session, chat)
|
||||
assert r.status_code == 201
|
||||
saved, uploader = r.json(), r.headers["x-instance-id"]
|
||||
target = next(
|
||||
c
|
||||
for c in cluster.containers()
|
||||
if cluster.alias(c["Id"]) == uploader
|
||||
)
|
||||
command(["docker", "stop", "--time", "10", target["Id"]])
|
||||
try:
|
||||
time.sleep(3)
|
||||
download = client.get(
|
||||
f"/api/v1/attachments/{saved['id']}/content",
|
||||
headers=auth(session),
|
||||
)
|
||||
assert download.status_code == 200 and download.content == PNG
|
||||
assert download.headers["x-instance-id"] != uploader
|
||||
result["upload_replica_stopped_downloaded_elsewhere"] = {
|
||||
"stopped_container": target["Id"][:12],
|
||||
"uploader": uploader,
|
||||
"downloader": download.headers["x-instance-id"],
|
||||
"sha256": saved["sha256"],
|
||||
}
|
||||
finally:
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-build",
|
||||
"--no-recreate",
|
||||
"--wait",
|
||||
"api",
|
||||
"lb",
|
||||
)
|
||||
ready()
|
||||
private_code = f"""
|
||||
checks={{}}
|
||||
try:
|
||||
urllib.request.urlopen('http://s3:9000/messenger/{key(saved["id"])}',timeout=3)
|
||||
raise AssertionError('public object')
|
||||
except urllib.error.HTTPError as e: checks['anonymous_get']=e.code; assert e.code==403
|
||||
for name,operation in [('list_other_bucket',lambda:list(s.client.list_objects('other-private'))),('create_bucket',lambda:s.client.make_bucket('forbidden-bucket')),('write_outside_prefix',lambda:s.client.put_object(BUCKET,'forbidden',io.BytesIO(b'x'),1))]:
|
||||
try: operation(); raise AssertionError(name)
|
||||
except Exception as e:
|
||||
assert getattr(e,'code',None)=='AccessDenied',type(e).__name__; checks[name]='AccessDenied'
|
||||
try:
|
||||
urllib.request.urlopen('http://s3:9001/',timeout=2); raise AssertionError('console exposed')
|
||||
except urllib.error.URLError: checks['console_internal_network']='unreachable'
|
||||
print(json.dumps(checks))
|
||||
"""
|
||||
result["private_bucket_and_iam"] = inside(private_code)
|
||||
compose("run", "--rm", "--no-deps", "s3-bootstrap")
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{saved['id']}/content",
|
||||
headers=auth(session),
|
||||
).content
|
||||
== PNG
|
||||
)
|
||||
result["bootstrap_repeat_preserved_original"] = True
|
||||
# A partial HTTP body must never create a durable intent or object.
|
||||
before = int(sql("SELECT count(*) FROM attachments"))
|
||||
objects_before_abort = inside("print(len(list(s.objects())))")
|
||||
parsed = httpx.URL(BASE)
|
||||
with socket.create_connection(
|
||||
(parsed.host, parsed.port), timeout=3
|
||||
) as raw:
|
||||
request = (
|
||||
f"POST /api/v1/chats/{chat['id']}/attachments?filename=partial.png HTTP/1.1\r\n"
|
||||
f"Host: {parsed.host}\r\nAuthorization: Bearer {session['token']}\r\n"
|
||||
"Content-Type: image/png\r\nContent-Length: 100000\r\n\r\n"
|
||||
).encode() + PNG[:20]
|
||||
raw.sendall(request)
|
||||
time.sleep(1)
|
||||
assert int(sql("SELECT count(*) FROM attachments")) == before
|
||||
assert (
|
||||
inside("print(len(list(s.objects())))") == objects_before_abort
|
||||
)
|
||||
result["aborted_upload_has_no_intent_or_bytes"] = True
|
||||
# Failure before any S3 PUT.
|
||||
name = uuid4().hex + ".png"
|
||||
sql(
|
||||
"CREATE FUNCTION student_upload_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic'; END $$; CREATE TRIGGER student_upload_fault BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION student_upload_fault()"
|
||||
)
|
||||
objects_before = inside("print(len(list(s.objects())))")
|
||||
try:
|
||||
assert (
|
||||
upload(client, session, chat, name=name).status_code == 500
|
||||
)
|
||||
assert (
|
||||
int(
|
||||
sql(
|
||||
"SELECT count(*) FROM attachments WHERE filename='"
|
||||
+ name
|
||||
+ "'"
|
||||
)
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert (
|
||||
inside("print(len(list(s.objects())))") == objects_before
|
||||
)
|
||||
finally:
|
||||
sql(
|
||||
"DROP TRIGGER student_upload_fault ON attachments; DROP FUNCTION student_upload_fault()"
|
||||
)
|
||||
result["pg_intent_failure_has_no_bytes"] = True
|
||||
# S3 PUT succeeded, then ready UPDATE fails: durable pending is recoverable.
|
||||
name = uuid4().hex + ".png"
|
||||
sql(
|
||||
"CREATE FUNCTION student_ready_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.state='ready' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; CREATE TRIGGER student_ready_fault BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION student_ready_fault()"
|
||||
)
|
||||
try:
|
||||
assert (
|
||||
upload(client, session, chat, name=name).status_code == 500
|
||||
)
|
||||
finally:
|
||||
sql(
|
||||
"DROP TRIGGER student_ready_fault ON attachments; DROP FUNCTION student_ready_fault()"
|
||||
)
|
||||
pending = sql(
|
||||
"SELECT id FROM attachments WHERE filename='" + name + "'"
|
||||
)
|
||||
assert state(pending) == "pending"
|
||||
assert inside(
|
||||
f"print(json.dumps(s.head(key={key(pending)!r}) is not None))"
|
||||
)
|
||||
age(pending)
|
||||
# Stop S3; bypass LB health exclusion only from inside the selected API.
|
||||
compose("stop", "s3")
|
||||
name = uuid4().hex + ".png"
|
||||
try:
|
||||
direct = f"""
|
||||
request=urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/{chat["id"]}/attachments?filename={name}',data=bytes.fromhex({PNG.hex()!r}),headers={{'Authorization':{auth(session)["Authorization"]!r},'Content-Type':'image/png'}})
|
||||
try: urllib.request.urlopen(request,timeout=15); raise AssertionError('S3 down accepted')
|
||||
except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':json.loads(e.read())['error']['code']}}))
|
||||
"""
|
||||
observed = inside(direct)
|
||||
assert observed == {"status": 503, "error": "unavailable"}
|
||||
missing_pending = sql(
|
||||
"SELECT id FROM attachments WHERE filename='" + name + "'"
|
||||
)
|
||||
assert state(missing_pending) == "pending"
|
||||
result["s3_unavailable"] = observed
|
||||
finally:
|
||||
compose(
|
||||
"up", "-d", "--no-build", "--no-recreate", "--wait", "s3"
|
||||
)
|
||||
ready()
|
||||
age(missing_pending)
|
||||
# Used object missing: preserve metadata ID and the Message link.
|
||||
used = upload(client, session, chat).json()
|
||||
sent = client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers={**auth(session), "Idempotency-Key": uuid4().hex},
|
||||
json={"text": "", "attachment_ids": [used["id"]]},
|
||||
)
|
||||
assert sent.status_code == 201
|
||||
inside(f"s.delete(key={key(used['id'])!r}); print('true')")
|
||||
age(used["id"])
|
||||
age(saved["id"])
|
||||
result["concurrent_uploads"] = concurrency(client, session, chat)
|
||||
while time.monotonic() - orphan_started < 302:
|
||||
print(
|
||||
"Waiting for genuine S3 LastModified grace: "
|
||||
+ str(round(302 - (time.monotonic() - orphan_started)))
|
||||
+ "s",
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(min(20, 302 - (time.monotonic() - orphan_started)))
|
||||
inside(
|
||||
f"s.put(key={fresh!r},stream=io.BytesIO(b'fresh'),size=5,content_type='application/octet-stream',sha256=hashlib.sha256(b'fresh').hexdigest()); print('true')"
|
||||
)
|
||||
before_states = [
|
||||
state(i)
|
||||
for i in [pending, missing_pending, used["id"], saved["id"]]
|
||||
]
|
||||
dry = run_reconcile()
|
||||
assert [
|
||||
state(i)
|
||||
for i in [pending, missing_pending, used["id"], saved["id"]]
|
||||
] == before_states
|
||||
assert inside(
|
||||
f"print(json.dumps(s.head(key={orphan!r}) is not None))"
|
||||
)
|
||||
for action in [
|
||||
"recover_pending",
|
||||
"fail_pending_missing",
|
||||
"mark_missing",
|
||||
"delete_orphan",
|
||||
]:
|
||||
assert dry["actions"].get(action, 0) >= 1, dry
|
||||
applied = run_reconcile(True)
|
||||
assert state(pending) == "ready"
|
||||
assert (
|
||||
state(missing_pending) == "failed"
|
||||
and state(used["id"]) == "failed"
|
||||
)
|
||||
assert state(saved["id"]) == "ready"
|
||||
assert (
|
||||
int(
|
||||
sql(
|
||||
"SELECT count(*) FROM message_attachments WHERE attachment_id='"
|
||||
+ used["id"]
|
||||
+ "'"
|
||||
)
|
||||
)
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{used['id']}", headers=auth(session)
|
||||
).status_code
|
||||
== 409
|
||||
)
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{pending}/content",
|
||||
headers=auth(session),
|
||||
).content
|
||||
== PNG
|
||||
)
|
||||
assert inside(
|
||||
f"print(json.dumps(s.head(key={orphan!r}) is None and s.head(key={fresh!r}) is not None and s.head(key={key(saved['id'])!r}) is not None))"
|
||||
)
|
||||
repeat = run_reconcile(True)
|
||||
assert not repeat["applied"].get("delete_orphan")
|
||||
assert repeat["linked_protected"] >= 1
|
||||
result["reconciliation"] = {
|
||||
"dry_run": dry,
|
||||
"apply": applied,
|
||||
"repeat": repeat,
|
||||
"s3_grace_wait_seconds": round(
|
||||
time.monotonic() - orphan_started, 3
|
||||
),
|
||||
"fresh_and_ready_unlinked_protected": True,
|
||||
"missing_used_id_and_link_preserved": True,
|
||||
"dry_run_unchanged": True,
|
||||
}
|
||||
if args.research:
|
||||
cells = []
|
||||
for count in [0, 100, 1100]:
|
||||
if count:
|
||||
start = 0 if count == 100 else 100
|
||||
inside(
|
||||
f"\nfor i in range({start},{count}):\n s.put(key={research_prefix!r}+str(i),stream=io.BytesIO(b'x'),size=1,content_type='application/octet-stream',sha256=hashlib.sha256(b'x').hexdigest())\nprint('true')"
|
||||
)
|
||||
for repetition in range(3):
|
||||
measured = run_reconcile()
|
||||
cells.append(
|
||||
{
|
||||
"extra_objects": count,
|
||||
"repetition": repetition + 1,
|
||||
**measured,
|
||||
}
|
||||
)
|
||||
result["cost_research"] = {
|
||||
"cells": cells,
|
||||
"repeats": 3,
|
||||
"method": "same running stand; 0/100/1100 fresh one-byte objects; dry-run; serial ascending dataset; no warmup",
|
||||
"median_seconds": {
|
||||
str(n): statistics.median(
|
||||
c["duration_seconds"]
|
||||
for c in cells
|
||||
if c["extra_objects"] == n
|
||||
)
|
||||
for n in [0, 100, 1100]
|
||||
},
|
||||
}
|
||||
result["complete"] = True
|
||||
finally:
|
||||
# Only keys created by this invocation; no broad bucket deletion.
|
||||
try:
|
||||
inside(
|
||||
f"\nfor o in s.objects():\n if o.object_name.startswith({research_prefix!r}) or o.object_name in [{orphan!r},{fresh!r}]: s.delete(key=o.object_name)\nprint('true')"
|
||||
)
|
||||
finally:
|
||||
compose("up", "-d", "--no-build", "--no-recreate", "reconciler")
|
||||
output.write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"PASS: private S3, failover, bounded concurrent uploads and two-store recovery"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user