Files
mtusi2026lab2/student-tests/tools/forward_runtime.py
T

387 lines
13 KiB
Python

import hashlib
import json
import os
import struct
import sys
import time
from io import BytesIO
from pathlib import Path
from typing import Any
from PIL import Image
from scripts import pipeline_suite as ps
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
from png_trns_fixtures import boundary_png, png_with_trns
ROOT = Path(__file__).resolve().parents[2]
MODE = sys.argv.pop(1)
OUT = Path(os.environ.get("FORWARD_EVIDENCE_DIR", "evidence/forward-published")) / (MODE + ".json")
if OUT.exists():
raise SystemExit("Result exists; do not repeat completed evidence")
SUITE = ps.Suite(ps.arguments())
STACK = SUITE.stack
REPORT: dict[str, Any] = {
"mode": MODE,
"source_sha": ps.run(["git", "rev-parse", "HEAD"]).strip(),
"started_at": ps.now(),
"cases": [],
}
LIMITS = RenderLimits(10485760, 40000000)
def save():
REPORT["accepted_ids"] = SUITE.api.accepted if SUITE.api else []
OUT.write_text(json.dumps(REPORT, ensure_ascii=False, indent=2) + "\n")
def case(name, function):
row = {"name": name, "started_at": ps.now()}
REPORT["cases"].append(row)
save()
began = time.monotonic()
try:
row["observed"] = function()
row["result"] = "PASS"
except Exception as error:
row.update(result="FAIL", error=str(error))
row["elapsed_seconds"] = round(time.monotonic() - began, 3)
save()
print(name, row["result"], row.get("error", ""), flush=True)
def containers():
return STACK.dc("ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}").splitlines()
def outage():
assert SUITE.api is not None
SUITE.drained()
before_dlq = STACK.queues()[ps.DLQ]
STACK.stop("worker")
att = SUITE.api.fixture()
ident = att["id"]
ps.poll(
lambda: all(r["published"] for r in STACK.sql("outbox", id=ident)), 30, "published outbox"
)
jobs = STACK.sql("jobs", id=ident)
REPORT["outage_checkpoint"] = {
"attachment_id": ident,
"jobs_before": jobs,
"dlq_before": before_dlq,
}
save()
STACK.stop("postgres")
REPORT["outage_checkpoint"]["stopped_containers"] = containers()
save()
since = ps.now()
worker_id = STACK.dc("ps", "-aq", "worker").strip()
assert worker_id
ps.run(["docker", "start", worker_id])
def requeues():
logs = STACK.dc("logs", "--no-color", "--since", since, "worker")
rows = [
line
for line in logs.splitlines()
if "job.requeued" in line and jobs[0]["job_id"] in line
]
return rows if len(rows) >= 12 else None
try:
lines = ps.poll(requeues, 90, "12 real database failures", 1)
OUT.with_name(OUT.stem + "-worker.txt").write_text("\n".join(lines) + "\n")
during = STACK.queues()
finally:
STACK.up("postgres")
ps.poll(lambda: ps.http_ready(SUITE.args.base_url), 30, "API after PostgreSQL")
ready = SUITE.api.wait(ident)
SUITE.verify(ready)
SUITE.drained()
after = STACK.sql("jobs", id=ident)
assert (
after[0]["job_id"] == jobs[0]["job_id"] and after[0]["attempt"] == jobs[0]["attempt"] == 1
)
assert after[0]["status"] == "done"
assert STACK.queues()[ps.DLQ] == before_dlq
assert len(STACK.sql("variants", id=ident)) == 3
return {
"attachment_id": ident,
"jobs_before": jobs,
"jobs_after": after,
"requeues": len(lines),
"queues_during": during,
"queues_after": STACK.queues(),
"replay": False,
"stopped_containers": REPORT["outage_checkpoint"]["stopped_containers"],
}
def png(name, data, reference):
assert SUITE.api is not None
att = SUITE.api.upload(data, name=name + ".png")
ident = att["id"]
results = []
for version in (1, 2):
if version == 2:
STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
ps.poll(lambda: SUITE.record(ident)["published_version"] == 2, 90, "published v2")
ready = SUITE.api.wait(ident)
SUITE.verify(ready)
assert SUITE.api.content(ident)["bytes"] == data
expected = {r.name: r for r in render_variants(reference, LIMITS, version)}
variants = []
for variant in ready["variants"]:
content = SUITE.api.content(ident, variant["name"])["bytes"]
assert content == expected[variant["name"]].data, (
variant["name"] + " differs from independent white reference"
)
with Image.open(BytesIO(content)) as image:
assert not image.getexif()
assert not any(key in image.info for key in ("icc_profile", "xmp", "transparency"))
variants.append(
{
"name": variant["name"],
"sha256": hashlib.sha256(content).hexdigest(),
"bytes": len(content),
"dimensions": [variant["width"], variant["height"]],
}
)
results.append({"version": version, "variants": variants})
before = SUITE.snapshot(ident)
STACK.publish(SUITE.payload(ident, 1))
STACK.publish(SUITE.payload(ident, 2))
SUITE.drained()
assert SUITE.snapshot(ident) == before
assert len(STACK.sql("variants", id=ident)) == 6
return {
"attachment_id": ident,
"input_sha256": hashlib.sha256(data).hexdigest(),
"input_bytes": len(data),
"versions": results,
"stale_and_duplicate_noop": True,
}
FIXTURES = ROOT / "build/forward-png"
FIXTURES.mkdir(parents=True, exist_ok=True)
for fixture_name, fixture_depth, fixture_key in (
("gray2", 2, (1,)),
("gray4", 4, (5,)),
("gray16", 16, (12345,)),
("rgb16", 16, (12345, 23456, 34567)),
):
(FIXTURES / (fixture_name + "-40mp.png")).write_bytes(boundary_png(fixture_depth, fixture_key))
ps.SQL["timed_events"] = (
"SELECT attachment_id,job_id,kind,at FROM job_events "
"WHERE attachment_id = ANY(:'ids'::uuid[]) AND kind IN ('claimed','done') ORDER BY id"
)
def without_trns(data):
result, offset = data[:8], 8
while offset < len(data):
size = struct.unpack(">I", data[offset : offset + 4])[0] + 12
if data[offset + 4 : offset + 8] != b"tRNS":
result += data[offset : offset + size]
offset += size
return result
def verify_large(ident, data, transparent):
assert SUITE.api is not None
ready = SUITE.api.wait(ident)
SUITE.verify(ready)
assert SUITE.api.content(ident)["bytes"] == data
variants = []
for variant in ready["variants"]:
content = SUITE.api.content(ident, variant["name"])["bytes"]
with Image.open(BytesIO(content)) as image:
left = image.getpixel((5, image.height // 2))
right = image.getpixel((image.width - 6, image.height // 2))
assert (left == (255, 255, 255)) == transparent
assert right != (255, 255, 255)
assert not image.getexif()
variants.append({**variant, "left_pixel": left, "right_pixel": right})
return variants
def duplicate_check(ident):
before = SUITE.snapshot(ident)
STACK.publish(SUITE.payload(ident, 1))
STACK.publish(SUITE.payload(ident, 2))
SUITE.drained()
assert SUITE.snapshot(ident) == before
assert len(STACK.sql("variants", id=ident)) == 6
def large(name, data, transparent=True):
assert SUITE.api is not None
att = SUITE.api.upload(data, name=name + ".png")
ident = att["id"]
versions = [{"version": 1, "variants": verify_large(ident, data, transparent)}]
STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
ps.poll(lambda: SUITE.record(ident)["published_version"] == 2, 90, "v2 publication")
versions.append({"version": 2, "variants": verify_large(ident, data, transparent)})
duplicate_check(ident)
return {
"attachment_id": ident,
"input_sha256": hashlib.sha256(data).hexdigest(),
"bytes": len(data),
"versions": versions,
"memory_peak": int(STACK.cgroup("worker", "memory.peak")),
"memory_max": int(STACK.cgroup("worker", "memory.max")),
}
def concurrent():
assert SUITE.api is not None
data = (FIXTURES / "rgb16-40mp.png").read_bytes()
SUITE.drained()
STACK.stop("worker")
try:
ids = [SUITE.api.upload(data, name="concurrent-rgb16.png")["id"] for _ in range(2)]
SUITE.queued(2)
finally:
ps.run(["docker", "start", STACK.dc("ps", "-aq", "worker").strip()])
versions = []
for version in (1, 2):
if version == 2:
STACK.stop("worker")
try:
for ident in ids:
STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
SUITE.queued(2)
finally:
ps.run(["docker", "start", STACK.dc("ps", "-aq", "worker").strip()])
for ident in ids:
ps.poll(
lambda ident=ident, version=version: (
SUITE.record(ident)["published_version"] == version
),
90,
"concurrent publication",
)
variants = {ident: verify_large(ident, data, True) for ident in ids}
events = STACK.sql("timed_events", ids="{" + ",".join(ids) + "}")
job_ids = {
row["job_id"]
for ident in ids
for row in STACK.sql("jobs", id=ident)
if row["pipeline_version"] == version
}
selected = [e for e in events if e["job_id"] in job_ids]
claims = [e["at"] for e in selected if e["kind"] == "claimed"]
done = [e["at"] for e in selected if e["kind"] == "done"]
assert len(claims) == len(done) == 2
assert max(claims) < min(done), selected
versions.append(
{
"version": version,
"events": selected,
"processing_overlapped": True,
"variants": variants,
}
)
for ident in ids:
duplicate_check(ident)
peak, limit = (int(STACK.cgroup("worker", name)) for name in ("memory.peak", "memory.max"))
assert peak < limit
return {
"attachment_ids": ids,
"versions": versions,
"memory_peak": peak,
"memory_max": limit,
"memory_events": STACK.cgroup("worker", "memory.events"),
}
def forced(**overrides):
data = (FIXTURES / "rgb16-40mp.png").read_bytes()
result = SUITE.limited(data, **overrides)
STACK.restore()
result["next_normal"] = SUITE.smoke_pipeline()
return result
try:
SUITE.detect_api()
SUITE.api = ps.Api(SUITE.args)
dead_before = STACK.queues()[ps.DLQ][0]
REPORT["conditions"] = {
"host_load": os.getloadavg(),
"containers": containers(),
"running": ps.run(["docker", "ps", "--format", "{{.Names}} {{.Status}}"]),
"worker_limits": STACK.dc(
"exec",
"-T",
"worker",
"printenv",
"RENDER_MEMORY_MB",
"RENDER_TIMEOUT_SECONDS",
"WORKER_CONCURRENCY",
),
}
save()
if MODE == "postgres-outage":
case("published_outbox_database_outage", outage)
elif MODE == "png-http":
for name, depth, color in [
("gray1", 1, (0,)),
("gray2", 2, (1,)),
("gray4", 4, (5,)),
("gray8", 8, (85,)),
("gray16", 16, (12345,)),
("rgb8", 8, (255, 0, 0)),
("rgb16", 16, (12345, 23456, 34567)),
]:
case(
name,
lambda name=name, depth=depth, color=color: png(name, *png_with_trns(depth, color)),
)
case(
"rgb16-adam7-filter4-exif6",
lambda: png(
"adam7",
*png_with_trns(
16,
(12345, 23456, 34567),
size=(65, 63),
interlace=1,
filtering=4,
orientation=6,
),
),
)
elif MODE == "png-boundary":
for name in ("gray2", "gray4", "gray16", "rgb16"):
case(
name, lambda name=name: large(name, (FIXTURES / (name + "-40mp.png")).read_bytes())
)
case(
"gray16-without-trns",
lambda: large(
"gray16-opaque", without_trns((FIXTURES / "gray16-40mp.png").read_bytes()), False
),
)
case("concurrent-rgb16", concurrent)
case("forced-timeout", lambda: forced(FAULT_RENDER_TIMEOUT_SECONDS="0.01"))
case("forced-memory", lambda: forced(FAULT_RENDER_MEMORY_MB="200"))
ids = "{" + ",".join(SUITE.api.accepted) + "}"
ps.poll(lambda: not STACK.sql("staging_left", ids=ids), 45, "staging sweep")
SUITE.drained()
REPORT["invariants"] = SUITE.invariants(dead_before)
finally:
STACK.up("postgres")
STACK.restore()
REPORT["finished_at"] = ps.now()
REPORT["result"] = (
"PASS"
if REPORT["cases"]
and all(x.get("result") == "PASS" for x in REPORT["cases"] + REPORT.get("invariants", []))
else "FAIL"
)
save()
raise SystemExit(0 if REPORT["result"] == "PASS" else 1)