Files
mtusi2026lab5/tests/level4_resilience.py
T
2026-11-10 16:21:46 +03:00

315 lines
11 KiB
Python

#!/usr/bin/env python3
"""Grade-4 fault injection and media pipeline verification."""
import hashlib
import json
import struct
import subprocess
import time
from pathlib import Path
from client import Client, arguments, check, run
from image_probe import jpeg_size
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, capture=False, check_result=True):
command = ["docker", "compose"]
if (ROOT / ".env").exists():
command += ["--env-file", ".env"]
command += list(args)
return subprocess.run(
command,
cwd=ROOT,
check=check_result,
text=capture,
stdout=subprocess.PIPE if capture else subprocess.DEVNULL,
stderr=subprocess.PIPE if capture else subprocess.DEVNULL,
)
def sql(statement):
result = compose(
"exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-v", "ON_ERROR_STOP=1", "-Atc", statement,
capture=True,
)
return result.stdout.strip()
def restart_count(service):
container_id = compose("ps", "-q", service, capture=True).stdout.strip()
check(bool(container_id), service + " container is not running")
result = subprocess.run(
["docker", "inspect", "--format", "{{.RestartCount}}", container_id],
check=True,
text=True,
stdout=subprocess.PIPE,
)
return int(result.stdout.strip())
def wait_until(predicate, timeout, description):
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
last = predicate()
if last:
return last
time.sleep(0.2)
raise AssertionError(description + "; last=" + repr(last))
def wait_failed(client, actor, attachment, timeout=30):
current = attachment
def poll():
nonlocal current
current = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"],
actor=actor,
schema="Attachment",
)["json"]
return current if current["status"] == "failed" else None
return wait_until(poll, timeout, "attachment did not become failed")
def webp_size(data):
check(
len(data) >= 30 and data[:4] == b"RIFF" and data[8:12] == b"WEBP",
"optimized output is not a WebP RIFF container",
)
chunk = data[12:16]
payload = data[20:]
if chunk == b"VP8 ":
marker = payload.find(b"\x9d\x01\x2a")
check(marker >= 0 and len(payload) >= marker + 7, "invalid VP8 frame")
width, height = struct.unpack_from("<HH", payload, marker + 3)
return width & 0x3FFF, height & 0x3FFF
if chunk == b"VP8L":
check(len(payload) >= 5 and payload[0] == 0x2F, "invalid VP8L frame")
bits = int.from_bytes(payload[1:5], "little")
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
if chunk == b"VP8X":
check(len(payload) >= 10, "invalid VP8X frame")
width = int.from_bytes(payload[4:7], "little") + 1
height = int.from_bytes(payload[7:10], "little") + 1
return width, height
raise AssertionError("unsupported WebP primary chunk " + repr(chunk))
def verify_variants(client, actor, ready, original):
check(ready["status"] == "ready", "attachment is not ready")
by_name = {item["name"]: item for item in ready["variants"]}
check(
set(by_name) == {"thumbnail", "crop", "optimized"},
"ready attachment must contain exactly three level-4 variants",
)
expected = {
"thumbnail": ("image/jpeg", (256, 160)),
"crop": ("image/jpeg", (128, 128)),
"optimized": ("image/webp", (320, 200)),
}
for name, (content_type, dimensions) in expected.items():
metadata = by_name[name]
check(
(metadata["width"], metadata["height"]) == dimensions,
name + " metadata dimensions mismatch",
)
downloaded = client.call(
"GET",
"/api/v1/attachments/" + ready["id"] + "/content?variant=" + name,
actor=actor,
)
check(
downloaded["headers"].get_content_type() == content_type,
name + " content type mismatch",
)
check(
len(downloaded["bytes"]) == metadata["size_bytes"] and
hashlib.sha256(downloaded["bytes"]).hexdigest() == metadata["sha256"],
name + " checksum/size mismatch",
)
actual = (
webp_size(downloaded["bytes"])
if name == "optimized"
else jpeg_size(downloaded["bytes"])
)
check(actual == dimensions, name + " actual dimensions mismatch")
downloaded_original = client.call(
"GET",
"/api/v1/attachments/" + ready["id"] + "/content",
actor=actor,
)["bytes"]
check(downloaded_original == original, "original bytes changed")
check(
sql("SELECT staging_key IS NULL FROM attachments WHERE id='" + ready["id"] + "'") == "t",
"successful processing did not safely clear staging",
)
def jobctl(*args, expect_success=True):
result = compose(
"run", "--rm", "--no-deps", "--entrypoint", "/app/jobctl",
"dispatcher", *args,
capture=True,
check_result=False,
)
if expect_success:
check(result.returncode == 0, "jobctl failed: " + result.stderr)
else:
check(result.returncode != 0, "unsafe jobctl command unexpectedly succeeded")
return result
def main():
args = arguments("Lab5 grade-4 fault injection scenario")
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
# Media pipeline: crop and optimized are real files with deterministic
# metadata; optimized is not enlarged beyond the 320x200 fixture.
baseline, original = client.attachment(alice, chat)
ready = client.wait_ready(bob, baseline)
verify_variants(client, bob, ready, original)
# Crash after DB commit but before publish. The outbox row has no marker,
# dispatcher restarts, publishes it, and no operator creates a second job.
compose("stop", "worker")
wait_until(
lambda: sql("SELECT count(*) FROM attachment_jobs WHERE status='queued' AND published_at IS NULL") == "0",
10,
"outbox was not initially drained",
)
dispatcher_restarts = restart_count("dispatcher")
compose("exec", "-T", "dispatcher", "touch", "/tmp/mtusi-fault-before-publish")
outbox_attachment, outbox_original = client.attachment(alice, chat)
wait_until(
lambda: restart_count("dispatcher") > dispatcher_restarts,
15,
"dispatcher fault marker did not crash the process",
)
check(
sql("SELECT count(*) FROM attachment_jobs WHERE attachment_id='" + outbox_attachment["id"] + "'") == "1",
"outbox recovery created another logical job",
)
wait_until(
lambda: sql("SELECT published_at IS NOT NULL FROM attachment_jobs WHERE attachment_id='" + outbox_attachment["id"] + "'") == "t",
15,
"outbox row was not published after dispatcher restart",
)
compose("start", "worker")
outbox_ready = client.wait_ready(bob, outbox_attachment)
verify_variants(client, bob, outbox_ready, outbox_original)
# Crash after all object writes and before the result transaction/ack.
# Expire the lease explicitly to make the deterministic test fast.
worker_restarts = restart_count("worker")
compose("exec", "-T", "worker", "touch", "/tmp/mtusi-fault-after-objects")
crashed_attachment, crashed_original = client.attachment(alice, chat)
wait_until(
lambda: restart_count("worker") > worker_restarts,
20,
"worker fault marker did not crash the process",
)
sql(
"UPDATE attachment_jobs SET lease_until=CURRENT_TIMESTAMP-INTERVAL '1 second' "
"WHERE attachment_id='" + crashed_attachment["id"] + "' AND status='processing'"
)
crashed_ready = client.wait_ready(bob, crashed_attachment)
verify_variants(client, bob, crashed_ready, crashed_original)
check(
int(sql(
"SELECT count(*) FROM attachment_job_events e JOIN attachment_jobs j USING(job_id) "
"WHERE j.attachment_id='" + crashed_attachment["id"] + "' AND e.event_type='lease_recovered'"
)) >= 1,
"expired processing lease was not audited as recovered",
)
check(
sql(
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" +
crashed_attachment["id"] + "'"
) == "3",
"crash/redelivery multiplied variants",
)
# A trusted metadata integrity mismatch is permanent and fails on attempt 1.
compose("stop", "worker")
permanent, _ = client.attachment(alice, chat)
sql(
"UPDATE attachments SET sha256=decode(repeat('00',32),'hex') WHERE id='" +
permanent["id"] + "'"
)
compose("start", "worker")
permanent_failed = wait_failed(client, bob, permanent)
check(permanent_failed["error_code"] == "invalid_image", "permanent error code mismatch")
check(
sql(
"SELECT attempt || ':' || status FROM attachment_jobs WHERE attachment_id='" +
permanent["id"] + "'"
) == "1:failed",
"permanent failure was retried",
)
# A transient missing staging object consumes the bounded retry budget,
# appears in the failed registry, and can be replayed only after staging is
# restored. Replaying the resulting succeeded job must be rejected.
compose("stop", "worker")
transient, transient_original = client.attachment(alice, chat)
transient_id = transient["id"]
job_id = sql(
"SELECT job_id::text FROM attachment_jobs WHERE attachment_id='" + transient_id + "'"
)
original_staging = sql(
"SELECT staging_key FROM attachments WHERE id='" + transient_id + "'"
)
sql(
"UPDATE attachments SET staging_key='staging/" + transient_id +
"/missing' WHERE id='" + transient_id + "'"
)
compose("start", "worker")
transient_failed = wait_failed(client, bob, transient, timeout=40)
check(transient_failed["error_code"] == "retry_exhausted", "transient final code mismatch")
check(
sql("SELECT attempt || ':' || status FROM attachment_jobs WHERE job_id='" + job_id + "'") ==
"5:failed",
"transient failure did not stop at max attempts",
)
check(
sql(
"SELECT count(*) FROM attachment_job_events WHERE job_id='" + job_id +
"' AND event_type='retry_scheduled'"
) == "4",
"transient retry audit count mismatch",
)
failed_registry = json.loads(jobctl("list-failed").stdout)
check(any(item["job_id"] == job_id for item in failed_registry), "failed registry omitted job")
sql(
"UPDATE attachments SET staging_key='" + original_staging +
"' WHERE id='" + transient_id + "'"
)
replay = json.loads(jobctl("replay", job_id).stdout)
check(replay["status"] == "queued" and replay["replay_count"] == 1, "replay was not audited")
replay_ready = client.wait_ready(bob, transient_failed)
verify_variants(client, bob, replay_ready, transient_original)
jobctl("replay", job_id, expect_success=False)
check(
sql(
"SELECT replay_count || ':' || status FROM attachment_jobs WHERE job_id='" + job_id + "'"
) == "1:succeeded",
"rejected replay changed a successful job",
)
print(
"PASS: level-4 variants, outbox crash recovery, lease recovery, "
"backoff/failed registry and safe replay verified."
)
if __name__ == "__main__":
run(main)