forked from mtusi2026/mtusi2026lab6
315 lines
11 KiB
Python
315 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade-4 fault injection and media pipeline verification."""
|
|
|
|
import hashlib
|
|
import json
|
|
import struct
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from client import Client, arguments, check, run
|
|
from image_probe import jpeg_size
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def compose(*args, capture=False, check_result=True):
|
|
command = ["docker", "compose"]
|
|
if (ROOT / ".env").exists():
|
|
command += ["--env-file", ".env"]
|
|
command += list(args)
|
|
return subprocess.run(
|
|
command,
|
|
cwd=ROOT,
|
|
check=check_result,
|
|
text=capture,
|
|
stdout=subprocess.PIPE if capture else subprocess.DEVNULL,
|
|
stderr=subprocess.PIPE if capture else subprocess.DEVNULL,
|
|
)
|
|
|
|
|
|
def sql(statement):
|
|
result = compose(
|
|
"exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
|
"-v", "ON_ERROR_STOP=1", "-Atc", statement,
|
|
capture=True,
|
|
)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def restart_count(service):
|
|
container_id = compose("ps", "-q", service, capture=True).stdout.strip()
|
|
check(bool(container_id), service + " container is not running")
|
|
result = subprocess.run(
|
|
["docker", "inspect", "--format", "{{.RestartCount}}", container_id],
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
)
|
|
return int(result.stdout.strip())
|
|
|
|
|
|
def wait_until(predicate, timeout, description):
|
|
deadline = time.monotonic() + timeout
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
last = predicate()
|
|
if last:
|
|
return last
|
|
time.sleep(0.2)
|
|
raise AssertionError(description + "; last=" + repr(last))
|
|
|
|
|
|
def wait_failed(client, actor, attachment, timeout=30):
|
|
current = attachment
|
|
|
|
def poll():
|
|
nonlocal current
|
|
current = client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + attachment["id"],
|
|
actor=actor,
|
|
schema="Attachment",
|
|
)["json"]
|
|
return current if current["status"] == "failed" else None
|
|
|
|
return wait_until(poll, timeout, "attachment did not become failed")
|
|
|
|
|
|
def webp_size(data):
|
|
check(
|
|
len(data) >= 30 and data[:4] == b"RIFF" and data[8:12] == b"WEBP",
|
|
"optimized output is not a WebP RIFF container",
|
|
)
|
|
chunk = data[12:16]
|
|
payload = data[20:]
|
|
if chunk == b"VP8 ":
|
|
marker = payload.find(b"\x9d\x01\x2a")
|
|
check(marker >= 0 and len(payload) >= marker + 7, "invalid VP8 frame")
|
|
width, height = struct.unpack_from("<HH", payload, marker + 3)
|
|
return width & 0x3FFF, height & 0x3FFF
|
|
if chunk == b"VP8L":
|
|
check(len(payload) >= 5 and payload[0] == 0x2F, "invalid VP8L frame")
|
|
bits = int.from_bytes(payload[1:5], "little")
|
|
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
|
|
if chunk == b"VP8X":
|
|
check(len(payload) >= 10, "invalid VP8X frame")
|
|
width = int.from_bytes(payload[4:7], "little") + 1
|
|
height = int.from_bytes(payload[7:10], "little") + 1
|
|
return width, height
|
|
raise AssertionError("unsupported WebP primary chunk " + repr(chunk))
|
|
|
|
|
|
def verify_variants(client, actor, ready, original):
|
|
check(ready["status"] == "ready", "attachment is not ready")
|
|
by_name = {item["name"]: item for item in ready["variants"]}
|
|
check(
|
|
set(by_name) == {"thumbnail", "crop", "optimized"},
|
|
"ready attachment must contain exactly three level-4 variants",
|
|
)
|
|
expected = {
|
|
"thumbnail": ("image/jpeg", (256, 160)),
|
|
"crop": ("image/jpeg", (128, 128)),
|
|
"optimized": ("image/webp", (320, 200)),
|
|
}
|
|
for name, (content_type, dimensions) in expected.items():
|
|
metadata = by_name[name]
|
|
check(
|
|
(metadata["width"], metadata["height"]) == dimensions,
|
|
name + " metadata dimensions mismatch",
|
|
)
|
|
downloaded = client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + ready["id"] + "/content?variant=" + name,
|
|
actor=actor,
|
|
)
|
|
check(
|
|
downloaded["headers"].get_content_type() == content_type,
|
|
name + " content type mismatch",
|
|
)
|
|
check(
|
|
len(downloaded["bytes"]) == metadata["size_bytes"] and
|
|
hashlib.sha256(downloaded["bytes"]).hexdigest() == metadata["sha256"],
|
|
name + " checksum/size mismatch",
|
|
)
|
|
actual = (
|
|
webp_size(downloaded["bytes"])
|
|
if name == "optimized"
|
|
else jpeg_size(downloaded["bytes"])
|
|
)
|
|
check(actual == dimensions, name + " actual dimensions mismatch")
|
|
downloaded_original = client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + ready["id"] + "/content",
|
|
actor=actor,
|
|
)["bytes"]
|
|
check(downloaded_original == original, "original bytes changed")
|
|
check(
|
|
sql("SELECT staging_key IS NULL FROM attachments WHERE id='" + ready["id"] + "'") == "t",
|
|
"successful processing did not safely clear staging",
|
|
)
|
|
|
|
|
|
def jobctl(*args, expect_success=True):
|
|
result = compose(
|
|
"run", "--rm", "--no-deps", "--entrypoint", "/app/jobctl",
|
|
"dispatcher", *args,
|
|
capture=True,
|
|
check_result=False,
|
|
)
|
|
if expect_success:
|
|
check(result.returncode == 0, "jobctl failed: " + result.stderr)
|
|
else:
|
|
check(result.returncode != 0, "unsafe jobctl command unexpectedly succeeded")
|
|
return result
|
|
|
|
|
|
def main():
|
|
args = arguments("Lab5 grade-4 fault injection scenario")
|
|
client = Client(args.base_url, args.ca_file)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
|
|
# Media pipeline: crop and optimized are real files with deterministic
|
|
# metadata; optimized is not enlarged beyond the 320x200 fixture.
|
|
baseline, original = client.attachment(alice, chat)
|
|
ready = client.wait_ready(bob, baseline)
|
|
verify_variants(client, bob, ready, original)
|
|
|
|
# Crash after DB commit but before publish. The outbox row has no marker,
|
|
# dispatcher restarts, publishes it, and no operator creates a second job.
|
|
compose("stop", "worker")
|
|
wait_until(
|
|
lambda: sql("SELECT count(*) FROM attachment_jobs WHERE status='queued' AND published_at IS NULL") == "0",
|
|
10,
|
|
"outbox was not initially drained",
|
|
)
|
|
dispatcher_restarts = restart_count("dispatcher")
|
|
compose("exec", "-T", "dispatcher", "touch", "/tmp/mtusi-fault-before-publish")
|
|
outbox_attachment, outbox_original = client.attachment(alice, chat)
|
|
wait_until(
|
|
lambda: restart_count("dispatcher") > dispatcher_restarts,
|
|
15,
|
|
"dispatcher fault marker did not crash the process",
|
|
)
|
|
check(
|
|
sql("SELECT count(*) FROM attachment_jobs WHERE attachment_id='" + outbox_attachment["id"] + "'") == "1",
|
|
"outbox recovery created another logical job",
|
|
)
|
|
wait_until(
|
|
lambda: sql("SELECT published_at IS NOT NULL FROM attachment_jobs WHERE attachment_id='" + outbox_attachment["id"] + "'") == "t",
|
|
15,
|
|
"outbox row was not published after dispatcher restart",
|
|
)
|
|
compose("start", "worker")
|
|
outbox_ready = client.wait_ready(bob, outbox_attachment)
|
|
verify_variants(client, bob, outbox_ready, outbox_original)
|
|
|
|
# Crash after all object writes and before the result transaction/ack.
|
|
# Expire the lease explicitly to make the deterministic test fast.
|
|
worker_restarts = restart_count("worker")
|
|
compose("exec", "-T", "worker", "touch", "/tmp/mtusi-fault-after-objects")
|
|
crashed_attachment, crashed_original = client.attachment(alice, chat)
|
|
wait_until(
|
|
lambda: restart_count("worker") > worker_restarts,
|
|
20,
|
|
"worker fault marker did not crash the process",
|
|
)
|
|
sql(
|
|
"UPDATE attachment_jobs SET lease_until=CURRENT_TIMESTAMP-INTERVAL '1 second' "
|
|
"WHERE attachment_id='" + crashed_attachment["id"] + "' AND status='processing'"
|
|
)
|
|
crashed_ready = client.wait_ready(bob, crashed_attachment)
|
|
verify_variants(client, bob, crashed_ready, crashed_original)
|
|
check(
|
|
int(sql(
|
|
"SELECT count(*) FROM attachment_job_events e JOIN attachment_jobs j USING(job_id) "
|
|
"WHERE j.attachment_id='" + crashed_attachment["id"] + "' AND e.event_type='lease_recovered'"
|
|
)) >= 1,
|
|
"expired processing lease was not audited as recovered",
|
|
)
|
|
check(
|
|
sql(
|
|
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" +
|
|
crashed_attachment["id"] + "'"
|
|
) == "3",
|
|
"crash/redelivery multiplied variants",
|
|
)
|
|
|
|
# A trusted metadata integrity mismatch is permanent and fails on attempt 1.
|
|
compose("stop", "worker")
|
|
permanent, _ = client.attachment(alice, chat)
|
|
sql(
|
|
"UPDATE attachments SET sha256=decode(repeat('00',32),'hex') WHERE id='" +
|
|
permanent["id"] + "'"
|
|
)
|
|
compose("start", "worker")
|
|
permanent_failed = wait_failed(client, bob, permanent)
|
|
check(permanent_failed["error_code"] == "invalid_image", "permanent error code mismatch")
|
|
check(
|
|
sql(
|
|
"SELECT attempt || ':' || status FROM attachment_jobs WHERE attachment_id='" +
|
|
permanent["id"] + "'"
|
|
) == "1:failed",
|
|
"permanent failure was retried",
|
|
)
|
|
|
|
# A transient missing staging object consumes the bounded retry budget,
|
|
# appears in the failed registry, and can be replayed only after staging is
|
|
# restored. Replaying the resulting succeeded job must be rejected.
|
|
compose("stop", "worker")
|
|
transient, transient_original = client.attachment(alice, chat)
|
|
transient_id = transient["id"]
|
|
job_id = sql(
|
|
"SELECT job_id::text FROM attachment_jobs WHERE attachment_id='" + transient_id + "'"
|
|
)
|
|
original_staging = sql(
|
|
"SELECT staging_key FROM attachments WHERE id='" + transient_id + "'"
|
|
)
|
|
sql(
|
|
"UPDATE attachments SET staging_key='staging/" + transient_id +
|
|
"/missing' WHERE id='" + transient_id + "'"
|
|
)
|
|
compose("start", "worker")
|
|
transient_failed = wait_failed(client, bob, transient, timeout=40)
|
|
check(transient_failed["error_code"] == "retry_exhausted", "transient final code mismatch")
|
|
check(
|
|
sql("SELECT attempt || ':' || status FROM attachment_jobs WHERE job_id='" + job_id + "'") ==
|
|
"5:failed",
|
|
"transient failure did not stop at max attempts",
|
|
)
|
|
check(
|
|
sql(
|
|
"SELECT count(*) FROM attachment_job_events WHERE job_id='" + job_id +
|
|
"' AND event_type='retry_scheduled'"
|
|
) == "4",
|
|
"transient retry audit count mismatch",
|
|
)
|
|
failed_registry = json.loads(jobctl("list-failed").stdout)
|
|
check(any(item["job_id"] == job_id for item in failed_registry), "failed registry omitted job")
|
|
|
|
sql(
|
|
"UPDATE attachments SET staging_key='" + original_staging +
|
|
"' WHERE id='" + transient_id + "'"
|
|
)
|
|
replay = json.loads(jobctl("replay", job_id).stdout)
|
|
check(replay["status"] == "queued" and replay["replay_count"] == 1, "replay was not audited")
|
|
replay_ready = client.wait_ready(bob, transient_failed)
|
|
verify_variants(client, bob, replay_ready, transient_original)
|
|
jobctl("replay", job_id, expect_success=False)
|
|
check(
|
|
sql(
|
|
"SELECT replay_count || ':' || status FROM attachment_jobs WHERE job_id='" + job_id + "'"
|
|
) == "1:succeeded",
|
|
"rejected replay changed a successful job",
|
|
)
|
|
|
|
print(
|
|
"PASS: level-4 variants, outbox crash recovery, lease recovery, "
|
|
"backoff/failed registry and safe replay verified."
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|