Files
mtusi2026lab5-fork/experiments/src/driver.py
T

694 lines
26 KiB
Python

"""Controlled delivery/fault cases. Called only by an explicit experiment run."""
import datetime as dt
import hashlib
import hmac
import json
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from pathlib import Path
from support import ROOT, digest, dump
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
from image_probe import jpeg_size
class Violation(AssertionError):
pass
class HarnessError(RuntimeError):
pass
def require(condition, message):
if not condition:
raise Violation(message)
class Stack:
def __init__(self, project, config, private, protocol):
self.project, self.config, self.private, self.protocol = (
project,
config,
private,
protocol,
)
self.prefix = [
"docker",
"compose",
"--project-name",
project,
"-f",
str(config),
]
def call(self, *args):
result = subprocess.run(
self.prefix + list(args),
cwd=ROOT,
text=True,
capture_output=True,
timeout=60,
)
if result.returncode:
error = result.stderr
for path in (self.private / "secrets").iterdir():
value = path.read_text().strip()
if len(value) >= 12:
error = error.replace(value, "[REDACTED]")
with (self.private.parent.parent / "command-errors.jsonl").open(
"a"
) as output:
output.write(
json.dumps(
{
"project": self.project,
"args": args,
"exit_code": result.returncode,
"stderr": error,
}
)
+ "\n"
)
raise HarnessError("Compose command failed; see command-errors.jsonl")
return result.stdout.strip()
def wait_broker_ready(self, timeout=60):
"""Wait only during setup; runtime queue failures remain fatal."""
deadline = time.monotonic() + timeout
args = ["exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues",
"name", "messages_ready", "messages_unacknowledged",
"--formatter", "json", "--timeout", "5"]
attempts = 0
while time.monotonic() < deadline:
attempts += 1
try:
result = subprocess.run(
self.prefix + args, cwd=ROOT, text=True, capture_output=True,
timeout=min(15, max(0.1, deadline-time.monotonic())),
)
if result.returncode == 0:
rows = json.loads(result.stdout)
if not isinstance(rows, list):
raise HarnessError("Unexpected RabbitMQ queue response")
dump(self.private.parent.parent / ("broker-readiness-"+self.private.name+".json"), {
"project": self.project, "attempts": attempts,
"ready": True,
})
return
error = result.stderr
# Auth, command syntax and stopped-container errors are not startup races.
if "requires the 'rabbit' app to be running" not in error:
raise HarnessError("RabbitMQ readiness probe failed: " + error.strip())
except subprocess.TimeoutExpired:
error = "RabbitMQ startup probe timed out"
with (self.private.parent.parent / "broker-startup.jsonl").open("a") as output:
output.write(json.dumps({"project": self.project, "attempt": attempts,
"stderr": error}) + "\n")
time.sleep(min(1, max(0, deadline-time.monotonic())))
raise HarnessError("RabbitMQ did not finish startup before the setup deadline")
def sql(self, query):
return self.call(
"exec",
"-T",
"db",
"sh",
"-c",
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"',
"sh",
query,
)
def workers(self):
return self.call("ps", "-aq", "worker").splitlines()
def worker_action(self, action, ids):
for ident in ids:
owner = subprocess.check_output(
[
"docker",
"inspect",
"--format",
'{{index .Config.Labels "com.docker.compose.project"}}',
ident,
],
text=True,
).strip()
if owner != self.project:
raise HarnessError("Container belongs to another project")
if ids:
subprocess.run(
[
"docker",
action,
*(["--time", "5"] if action == "stop" else []),
*ids,
],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def queue_depth(self):
rows = json.loads(
self.call(
"exec",
"-T",
"broker",
"rabbitmqctl",
"-q",
"list_queues",
"name",
"messages_ready",
"messages_unacknowledged",
"--formatter",
"json",
)
)
return sum(
row["messages_ready"] + row["messages_unacknowledged"]
for row in rows
if row["name"] == "attachments.v1"
)
def operator(self, *args):
running = self.call("ps", "--status", "running", "-q", "dispatcher")
mode = (
["exec", "-T", "dispatcher"]
if running
else ["run", "--rm", "--no-deps", "-T", "dispatcher"]
)
return self.call(*mode, "./messenger", *args)
def admin(self, *args):
return json.loads(self.operator("jobs", *args))
def snapshot(self, attachment):
result = self.sql(
"SELECT json_build_object('desired',a.pipeline_version,'published',a.published_pipeline_version,'status',a.status,'jobs',(SELECT json_agg(json_build_object('job_id',j.id,'version',j.pipeline_version,'attempts',j.attempts,'superseded',j.superseded_at,'lease_until',j.lease_until) ORDER BY j.pipeline_version) FROM attachment_jobs j WHERE j.attachment_id=a.id),'attempts_audit',(SELECT COALESCE(json_agg(json_build_object('job_id',t.job_id,'generation',t.generation,'attempt',t.attempt,'started_at',t.started_at,'finished_at',t.finished_at,'error_code',t.error_code,'next_attempt_at',t.next_attempt_at) ORDER BY t.started_at),'[]') FROM attachment_job_attempts t JOIN attachment_jobs x ON x.id=t.job_id WHERE x.attachment_id=a.id),'publications',(SELECT COALESCE(json_agg(json_build_object('id',p.id,'version',p.pipeline_version,'variants',p.variants,'published_at',p.published_at) ORDER BY p.id),'[]') FROM attachment_publications p WHERE p.attachment_id=a.id)) FROM attachments a WHERE a.id='"
+ attachment
+ "'"
)
return json.loads(result)
def messages(self, attachment):
return json.loads(
self.sql(
"SELECT json_agg(json_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',pipeline_version,'attempt',1,'created_at',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')) ORDER BY pipeline_version) FROM attachment_jobs WHERE attachment_id='"
+ attachment
+ "'"
)
)
def publish(self, job):
self.operator("broker-publish", json.dumps(job))
def containers(self):
ids = self.call("ps", "-aq").splitlines()
if not ids:
return []
# Never save Config.Env: it contains the experiment DB password.
rows = json.loads(
subprocess.check_output(["docker", "inspect", *ids], text=True)
)
return [
{
"id": row["Id"],
"image": row["Image"],
"service": row["Config"]["Labels"].get("com.docker.compose.service"),
"state": {
k: row["State"].get(k)
for k in (
"Status",
"ExitCode",
"OOMKilled",
"StartedAt",
"FinishedAt",
)
},
"restarts": row["RestartCount"],
"memory": row["HostConfig"]["Memory"],
"nano_cpus": row["HostConfig"]["NanoCpus"],
}
for row in rows
]
def s3(self, method, key="", query=None):
secret_dir = self.private / "secrets"
access = (secret_dir / "s3_api_access_key_id").read_text().strip()
secret = (secret_dir / "s3_api_secret_access_key").read_text().strip()
host = "127.0.0.1:" + str(self.protocol["ports"]["s3"])
uri = "/messenger-attachments/" + key
query_string = urllib.parse.urlencode(
sorted((query or {}).items()), quote_via=urllib.parse.quote
)
now = dt.datetime.now(dt.timezone.utc)
timestamp, day = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d")
payload = digest(b"")
signed = "host;x-amz-content-sha256;x-amz-date"
headers = (
"host:"
+ host
+ "\nx-amz-content-sha256:"
+ payload
+ "\nx-amz-date:"
+ timestamp
+ "\n"
)
canonical = "\n".join([method, uri, query_string, headers, signed, payload])
scope = day + "/us-east-1/s3/aws4_request"
text = (
"AWS4-HMAC-SHA256\n"
+ timestamp
+ "\n"
+ scope
+ "\n"
+ digest(canonical.encode())
)
signing = ("AWS4" + secret).encode()
for part in (day, "us-east-1", "s3", "aws4_request"):
signing = hmac.new(signing, part.encode(), hashlib.sha256).digest()
signature = hmac.new(signing, text.encode(), hashlib.sha256).hexdigest()
request = urllib.request.Request(
"http://" + host + uri + ("?" + query_string if query_string else ""),
method=method,
headers={
"x-amz-date": timestamp,
"x-amz-content-sha256": payload,
"Authorization": "AWS4-HMAC-SHA256 Credential="
+ access
+ "/"
+ scope
+ ", SignedHeaders="
+ signed
+ ", Signature="
+ signature,
},
)
try:
response = urllib.request.urlopen(request, timeout=15)
except urllib.error.HTTPError as exc:
response = exc
with response:
code, body = response.code, response.read()
if code in (401, 403):
raise HarnessError("S3 measurement identity could not authenticate")
return code, body
def webp_size(data):
require(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "invalid WebP container")
offset = 12
dimensions = None
while offset + 8 <= len(data):
kind = data[offset : offset + 4]
length = int.from_bytes(data[offset + 4 : offset + 8], "little")
body = data[offset + 8 : offset + 8 + length]
require(kind not in (b"EXIF", b"XMP ", b"ICCP"), "WebP metadata leaked")
if kind == b"VP8 ":
dimensions = (
int.from_bytes(body[6:8], "little") & 16383,
int.from_bytes(body[8:10], "little") & 16383,
)
if kind == b"VP8L":
bits = int.from_bytes(body[1:5], "little")
dimensions = ((bits & 16383) + 1, ((bits >> 14) & 16383) + 1)
if kind == b"VP8X":
dimensions = (
int.from_bytes(body[4:7], "little") + 1,
int.from_bytes(body[7:10], "little") + 1,
)
offset += 8 + length + (length % 2)
require(dimensions is not None, "WebP dimensions absent")
return dimensions
class Case:
def __init__(self, stack, profile, scenario, data, directory):
self.s, self.profile, self.scenario, self.data, self.directory = (
stack,
profile,
scenario,
data,
directory,
)
directory.mkdir(parents=True)
self.events = (directory / "events.jsonl").open("x")
self.start = time.monotonic()
self.last = None
self.gaps = []
self.attachment = None
self.verifications = 0
self.c = Client("http://127.0.0.1:" + str(stack.protocol["ports"]["api"]))
def event(self, kind, **values):
now = time.monotonic()
record = {
"utc": dt.datetime.now(dt.timezone.utc).isoformat(),
"elapsed": now - self.start,
"kind": kind,
**values,
}
self.events.write(json.dumps(record, ensure_ascii=False) + "\n")
self.events.flush()
def wait(self, fn, timeout=None, invariant=True):
deadline = time.monotonic() + (
timeout or self.s.protocol["case_timeout_seconds"]
)
while time.monotonic() < deadline:
value = fn()
if value and time.monotonic() <= deadline:
return value
time.sleep(self.s.protocol["poll_interval_seconds"])
if invariant:
raise Violation("completion/quiescence deadline exceeded")
raise HarnessError("required fault/precondition did not occur before deadline")
def observe(self):
now = time.monotonic()
if self.last is not None and now - self.last > 2:
self.gaps.append(now - self.last)
self.last = now
value = self.c.call(
"GET",
"/api/v1/attachments/" + self.attachment,
actor=self.alice,
schema="Attachment",
)["json"]
snapshot = self.s.snapshot(self.attachment)
self.event("observation", api=value, database=snapshot)
return value, snapshot
def ready(self, version):
def check():
value, snapshot = self.observe()
require(value["status"] != "failed", "valid fixture reached failed")
return (
(value, snapshot)
if value["status"] == "ready" and snapshot["published"] == version
else None
)
return self.wait(check)
def arm(self, point):
marker = self.s.private / "faults" / (point + "." + self.attachment)
marker.write_text("experiment " + self.scenario + "\n")
self.event("arm", point=point)
return Path(str(marker) + ".hit")
def upload(self):
self.alice, self.bob, self.eve, chat = self.c.bootstrap()
value = self.c.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=fixture.png",
202,
actor=self.alice,
raw=self.data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)["json"]
self.attachment = value["id"]
require(
value["sha256"] == digest(self.data), "accepted original checksum mismatch"
)
self.event("accepted_202", attachment_id=self.attachment)
require(
len(self.s.messages(self.attachment)) == 1,
"accepted upload has no unique durable job",
)
def upgrade(self):
require(
self.s.admin(
"upgrade", self.attachment, "2", "Protocol v1 " + self.scenario
)["changed"],
"upgrade intent was not persisted",
)
self.event("upgrade_accepted", version=2)
def publish(self, job, count):
for _ in range(count):
self.s.publish(job)
self.event(
"explicit_delivery",
job_id=job["job_id"],
version=job["pipeline_version"],
count=count,
)
def verify(self, version):
value, snapshot = self.ready(version)
self.verifications += 1
observation = "verify-" + str(self.verifications).zfill(3) + "-v" + str(version)
objects = self.directory / "objects" / observation
objects.mkdir(parents=True)
require(
snapshot["desired"] == version, "desired version differs from publication"
)
original = self.c.call(
"GET", "/api/v1/attachments/" + self.attachment + "/content", actor=self.bob
)["bytes"]
require(original == self.data, "original changed")
require(
len(value["variants"]) == 3
and {v["name"] for v in value["variants"]}
== {"thumbnail", "crop", "optimized"},
"incomplete or duplicate variant set",
)
manifest = {}
for item in value["variants"]:
name = item["name"]
key = (
"variants/"
+ self.attachment
+ "/v"
+ str(version)
+ "/"
+ name
+ (".webp" if name == "optimized" else ".jpg")
)
response = self.c.call(
"GET",
"/api/v1/attachments/" + self.attachment + "/content?variant=" + name,
actor=self.bob,
)
require(
digest(response["bytes"]) == item["sha256"]
and len(response["bytes"]) == item["size_bytes"],
"variant checksum mismatch",
)
require(
response["headers"].get_content_type() == item["content_type"],
"variant MIME mismatch",
)
try:
dimensions = (
webp_size(response["bytes"])
if name == "optimized"
else jpeg_size(response["bytes"])
)
except Failure as exc:
raise Violation("actual variant decode failed") from exc
w, h = self.profile["width"], self.profile["height"]
limit = 1280 if name == "optimized" else 256
expected = (
(128, 128)
if name == "crop"
else (min(w, limit), max(1, h * limit // w))
if w >= h and w > limit
else (max(1, w * limit // h), limit)
if h > w and h > limit
else (w, h)
)
require(
dimensions == expected == (item["width"], item["height"]),
"actual variant dimensions mismatch",
)
code, body = self.s.s3("GET", key)
require(
code == 200 and body == response["bytes"],
"API exposes a different version than S3",
)
(
objects / (name + (".webp" if name == "optimized" else ".jpg"))
).write_bytes(body)
manifest[key] = digest(body)
code, body = self.s.s3(
"GET",
query={
"list-type": "2",
"prefix": "variants/" + self.attachment + "/v" + str(version) + "/",
},
)
require(code == 200, "cannot list variant set")
require(
len(ET.fromstring(body).findall("{*}Contents")) == 3,
"extra/missing object set",
)
history = snapshot["publications"]
versions = [p["version"] for p in history]
require(
versions == sorted(versions) and len(versions) == len(set(versions)),
"publication duplicated or rolled back",
)
require(versions[-1] == version, "publication history missing active version")
require(
all(
j["version"] == version or j["superseded"] is not None
for j in snapshot["jobs"]
),
"older accepted job has no durable disposition",
)
dump(self.directory / (observation + "-manifest.json"), manifest)
return manifest, snapshot
def execute(self):
self.s.worker_action("stop", self.s.workers())
self.s.call("stop", "dispatcher")
self.upload()
workers = self.s.workers()
require(len(workers) == 2, "two workers are required by protocol")
scenario = self.scenario
version = 1
first = None
if scenario == "commit_publish_gap":
marker = self.arm("after_commit_before_publish")
if scenario != "reorder":
self.s.call("start", "dispatcher")
if scenario == "commit_publish_gap":
self.wait(marker.exists, invariant=False)
if scenario == "duplicates":
self.publish(
self.s.messages(self.attachment)[0],
self.s.protocol["duplicates_before_ready"],
)
if scenario.startswith("crash_"):
point = {
"crash_before_effect": "before_effect",
"crash_after_effect": "after_object_before_commit",
"crash_before_ack": "after_commit_before_ack",
}[scenario]
marker = self.arm(point)
self.s.worker_action("start", [workers[0]])
self.wait(marker.exists, invariant=False)
self.event("fault_hit", point=point)
elif scenario == "reorder":
# No consumer is running; broker order is deliberately v2 then old v1.
self.upgrade()
jobs = self.s.messages(self.attachment)
self.publish(jobs[1], 1)
self.publish(jobs[0], 1)
version = 2
self.s.worker_action("start", workers)
elif scenario == "stale_lease":
marker = self.arm("after_claim")
self.s.worker_action("start", [workers[0]])
self.wait(marker.exists, invariant=False)
self.s.call("start", "dispatcher")
self.upgrade()
self.s.worker_action("start", [workers[1]])
first, _ = self.verify(2)
version = 2
release = (
self.s.private
/ "faults"
/ ("after_claim." + self.attachment + ".release")
)
release.write_text("release\n")
self.event("release_old_lease")
self.wait(
Path(str(release) + ".observed").exists, timeout=5, invariant=False
)
self.wait(
(
self.s.private
/ "faults"
/ ("stale_rejected." + self.attachment + ".observed")
).exists,
timeout=5,
invariant=False,
)
else:
self.s.worker_action("start", workers)
if scenario == "old_after_v2":
old_manifest, _ = self.verify(1)
old = self.s.messages(self.attachment)[0]
self.upgrade()
version = 2
first, _ = self.verify(2)
if not any(
old_manifest.get(key.replace("/v2/", "/v1/")) != value
for key, value in first.items()
):
raise HarnessError(
"Version fingerprints are indistinguishable for this fixture"
)
self.publish(old, self.s.protocol["duplicates_after_ready"])
for key, checksum in old_manifest.items():
code, body = self.s.s3("GET", key)
require(
code == 200 and digest(body) == checksum,
"v1 immutable object changed",
)
if scenario == "duplicates":
first, before = self.verify(1)
self.publish(
self.s.messages(self.attachment)[0],
self.s.protocol["duplicates_after_ready"],
)
if scenario != "reorder":
self.s.call("start", "dispatcher")
self.s.worker_action("start", workers)
self.wait(lambda: self.s.queue_depth() == 0)
# Let an already-delivered stale attempt leave its pause and hit the fence.
if scenario == "stale_lease":
time.sleep(1)
final, snapshot = self.verify(version)
if first is not None:
require(first == final, "old delivery changed published bytes")
if scenario in ("duplicates", "old_after_v2"):
require(
all(j["attempts"] == 1 for j in snapshot["jobs"]),
"terminal duplicate started another attempt",
)
if scenario == "reorder":
self.s.call("start", "dispatcher")
self.event("case_complete", version=version)
return {
"attachment_id": self.attachment,
"version": version,
"snapshot": snapshot,
"manifest": final,
"observation_gaps": self.gaps,
"elapsed_seconds": time.monotonic() - self.start,
}
def recover(self):
for marker in (self.s.private / "faults").glob("after_claim.*.hit"):
Path(str(marker)[:-4] + ".release").write_text("cleanup release\n")
self.s.call("start", "dispatcher")
self.s.worker_action("start", self.s.workers())
self.wait(
lambda: (
self.s.sql(
"SELECT count(*) FROM attachments WHERE status IN ('queued','processing')"
)
== "0"
and self.s.queue_depth() == 0
)
)
def close(self):
self.events.close()