import hashlib import json import os import struct import sys import time from io import BytesIO from pathlib import Path from typing import Any from PIL import Image from scripts import pipeline_suite as ps from messenger.attachments.infrastructure.images import RenderLimits, render_variants from png_trns_fixtures import boundary_png, png_with_trns ROOT = Path(__file__).resolve().parents[2] MODE = sys.argv.pop(1) OUT = Path(os.environ.get("FORWARD_EVIDENCE_DIR", "evidence/forward-published")) / (MODE + ".json") if OUT.exists(): raise SystemExit("Result exists; do not repeat completed evidence") SUITE = ps.Suite(ps.arguments()) STACK = SUITE.stack REPORT: dict[str, Any] = { "mode": MODE, "source_sha": ps.run(["git", "rev-parse", "HEAD"]).strip(), "started_at": ps.now(), "cases": [], } LIMITS = RenderLimits(10485760, 40000000) def save(): REPORT["accepted_ids"] = SUITE.api.accepted if SUITE.api else [] OUT.write_text(json.dumps(REPORT, ensure_ascii=False, indent=2) + "\n") def case(name, function): row = {"name": name, "started_at": ps.now()} REPORT["cases"].append(row) save() began = time.monotonic() try: row["observed"] = function() row["result"] = "PASS" except Exception as error: row.update(result="FAIL", error=str(error)) row["elapsed_seconds"] = round(time.monotonic() - began, 3) save() print(name, row["result"], row.get("error", ""), flush=True) def containers(): return STACK.dc("ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}").splitlines() def outage(): assert SUITE.api is not None SUITE.drained() before_dlq = STACK.queues()[ps.DLQ] STACK.stop("worker") att = SUITE.api.fixture() ident = att["id"] ps.poll( lambda: all(r["published"] for r in STACK.sql("outbox", id=ident)), 30, "published outbox" ) jobs = STACK.sql("jobs", id=ident) REPORT["outage_checkpoint"] = { "attachment_id": ident, "jobs_before": jobs, "dlq_before": before_dlq, } save() STACK.stop("postgres") REPORT["outage_checkpoint"]["stopped_containers"] = containers() save() since = ps.now() worker_id = STACK.dc("ps", "-aq", "worker").strip() assert worker_id ps.run(["docker", "start", worker_id]) def requeues(): logs = STACK.dc("logs", "--no-color", "--since", since, "worker") rows = [ line for line in logs.splitlines() if "job.requeued" in line and jobs[0]["job_id"] in line ] return rows if len(rows) >= 12 else None try: lines = ps.poll(requeues, 90, "12 real database failures", 1) OUT.with_name(OUT.stem + "-worker.txt").write_text("\n".join(lines) + "\n") during = STACK.queues() finally: STACK.up("postgres") ps.poll(lambda: ps.http_ready(SUITE.args.base_url), 30, "API after PostgreSQL") ready = SUITE.api.wait(ident) SUITE.verify(ready) SUITE.drained() after = STACK.sql("jobs", id=ident) assert ( after[0]["job_id"] == jobs[0]["job_id"] and after[0]["attempt"] == jobs[0]["attempt"] == 1 ) assert after[0]["status"] == "done" assert STACK.queues()[ps.DLQ] == before_dlq assert len(STACK.sql("variants", id=ident)) == 3 return { "attachment_id": ident, "jobs_before": jobs, "jobs_after": after, "requeues": len(lines), "queues_during": during, "queues_after": STACK.queues(), "replay": False, "stopped_containers": REPORT["outage_checkpoint"]["stopped_containers"], } def png(name, data, reference): assert SUITE.api is not None att = SUITE.api.upload(data, name=name + ".png") ident = att["id"] results = [] for version in (1, 2): if version == 2: STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2") ps.poll(lambda: SUITE.record(ident)["published_version"] == 2, 90, "published v2") ready = SUITE.api.wait(ident) SUITE.verify(ready) assert SUITE.api.content(ident)["bytes"] == data expected = {r.name: r for r in render_variants(reference, LIMITS, version)} variants = [] for variant in ready["variants"]: content = SUITE.api.content(ident, variant["name"])["bytes"] assert content == expected[variant["name"]].data, ( variant["name"] + " differs from independent white reference" ) with Image.open(BytesIO(content)) as image: assert not image.getexif() assert not any(key in image.info for key in ("icc_profile", "xmp", "transparency")) variants.append( { "name": variant["name"], "sha256": hashlib.sha256(content).hexdigest(), "bytes": len(content), "dimensions": [variant["width"], variant["height"]], } ) results.append({"version": version, "variants": variants}) before = SUITE.snapshot(ident) STACK.publish(SUITE.payload(ident, 1)) STACK.publish(SUITE.payload(ident, 2)) SUITE.drained() assert SUITE.snapshot(ident) == before assert len(STACK.sql("variants", id=ident)) == 6 return { "attachment_id": ident, "input_sha256": hashlib.sha256(data).hexdigest(), "input_bytes": len(data), "versions": results, "stale_and_duplicate_noop": True, } FIXTURES = ROOT / "build/forward-png" FIXTURES.mkdir(parents=True, exist_ok=True) for fixture_name, fixture_depth, fixture_key in ( ("gray2", 2, (1,)), ("gray4", 4, (5,)), ("gray16", 16, (12345,)), ("rgb16", 16, (12345, 23456, 34567)), ): (FIXTURES / (fixture_name + "-40mp.png")).write_bytes(boundary_png(fixture_depth, fixture_key)) ps.SQL["timed_events"] = ( "SELECT attachment_id,job_id,kind,at FROM job_events " "WHERE attachment_id = ANY(:'ids'::uuid[]) AND kind IN ('claimed','done') ORDER BY id" ) def without_trns(data): result, offset = data[:8], 8 while offset < len(data): size = struct.unpack(">I", data[offset : offset + 4])[0] + 12 if data[offset + 4 : offset + 8] != b"tRNS": result += data[offset : offset + size] offset += size return result def verify_large(ident, data, transparent): assert SUITE.api is not None ready = SUITE.api.wait(ident) SUITE.verify(ready) assert SUITE.api.content(ident)["bytes"] == data variants = [] for variant in ready["variants"]: content = SUITE.api.content(ident, variant["name"])["bytes"] with Image.open(BytesIO(content)) as image: left = image.getpixel((5, image.height // 2)) right = image.getpixel((image.width - 6, image.height // 2)) assert (left == (255, 255, 255)) == transparent assert right != (255, 255, 255) assert not image.getexif() variants.append({**variant, "left_pixel": left, "right_pixel": right}) return variants def duplicate_check(ident): before = SUITE.snapshot(ident) STACK.publish(SUITE.payload(ident, 1)) STACK.publish(SUITE.payload(ident, 2)) SUITE.drained() assert SUITE.snapshot(ident) == before assert len(STACK.sql("variants", id=ident)) == 6 def large(name, data, transparent=True): assert SUITE.api is not None att = SUITE.api.upload(data, name=name + ".png") ident = att["id"] versions = [{"version": 1, "variants": verify_large(ident, data, transparent)}] STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2") ps.poll(lambda: SUITE.record(ident)["published_version"] == 2, 90, "v2 publication") versions.append({"version": 2, "variants": verify_large(ident, data, transparent)}) duplicate_check(ident) return { "attachment_id": ident, "input_sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data), "versions": versions, "memory_peak": int(STACK.cgroup("worker", "memory.peak")), "memory_max": int(STACK.cgroup("worker", "memory.max")), } def concurrent(): assert SUITE.api is not None data = (FIXTURES / "rgb16-40mp.png").read_bytes() SUITE.drained() STACK.stop("worker") try: ids = [SUITE.api.upload(data, name="concurrent-rgb16.png")["id"] for _ in range(2)] SUITE.queued(2) finally: ps.run(["docker", "start", STACK.dc("ps", "-aq", "worker").strip()]) versions = [] for version in (1, 2): if version == 2: STACK.stop("worker") try: for ident in ids: STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2") SUITE.queued(2) finally: ps.run(["docker", "start", STACK.dc("ps", "-aq", "worker").strip()]) for ident in ids: ps.poll( lambda ident=ident, version=version: ( SUITE.record(ident)["published_version"] == version ), 90, "concurrent publication", ) variants = {ident: verify_large(ident, data, True) for ident in ids} events = STACK.sql("timed_events", ids="{" + ",".join(ids) + "}") job_ids = { row["job_id"] for ident in ids for row in STACK.sql("jobs", id=ident) if row["pipeline_version"] == version } selected = [e for e in events if e["job_id"] in job_ids] claims = [e["at"] for e in selected if e["kind"] == "claimed"] done = [e["at"] for e in selected if e["kind"] == "done"] assert len(claims) == len(done) == 2 assert max(claims) < min(done), selected versions.append( { "version": version, "events": selected, "processing_overlapped": True, "variants": variants, } ) for ident in ids: duplicate_check(ident) peak, limit = (int(STACK.cgroup("worker", name)) for name in ("memory.peak", "memory.max")) assert peak < limit return { "attachment_ids": ids, "versions": versions, "memory_peak": peak, "memory_max": limit, "memory_events": STACK.cgroup("worker", "memory.events"), } def forced(**overrides): data = (FIXTURES / "rgb16-40mp.png").read_bytes() result = SUITE.limited(data, **overrides) STACK.restore() result["next_normal"] = SUITE.smoke_pipeline() return result try: SUITE.detect_api() SUITE.api = ps.Api(SUITE.args) dead_before = STACK.queues()[ps.DLQ][0] REPORT["conditions"] = { "host_load": os.getloadavg(), "containers": containers(), "running": ps.run(["docker", "ps", "--format", "{{.Names}} {{.Status}}"]), "worker_limits": STACK.dc( "exec", "-T", "worker", "printenv", "RENDER_MEMORY_MB", "RENDER_TIMEOUT_SECONDS", "WORKER_CONCURRENCY", ), } save() if MODE == "postgres-outage": case("published_outbox_database_outage", outage) elif MODE == "png-http": for name, depth, color in [ ("gray1", 1, (0,)), ("gray2", 2, (1,)), ("gray4", 4, (5,)), ("gray8", 8, (85,)), ("gray16", 16, (12345,)), ("rgb8", 8, (255, 0, 0)), ("rgb16", 16, (12345, 23456, 34567)), ]: case( name, lambda name=name, depth=depth, color=color: png(name, *png_with_trns(depth, color)), ) case( "rgb16-adam7-filter4-exif6", lambda: png( "adam7", *png_with_trns( 16, (12345, 23456, 34567), size=(65, 63), interlace=1, filtering=4, orientation=6, ), ), ) elif MODE == "png-boundary": for name in ("gray2", "gray4", "gray16", "rgb16"): case( name, lambda name=name: large(name, (FIXTURES / (name + "-40mp.png")).read_bytes()) ) case( "gray16-without-trns", lambda: large( "gray16-opaque", without_trns((FIXTURES / "gray16-40mp.png").read_bytes()), False ), ) case("concurrent-rgb16", concurrent) case("forced-timeout", lambda: forced(FAULT_RENDER_TIMEOUT_SECONDS="0.01")) case("forced-memory", lambda: forced(FAULT_RENDER_MEMORY_MB="200")) ids = "{" + ",".join(SUITE.api.accepted) + "}" ps.poll(lambda: not STACK.sql("staging_left", ids=ids), 45, "staging sweep") SUITE.drained() REPORT["invariants"] = SUITE.invariants(dead_before) finally: STACK.up("postgres") STACK.restore() REPORT["finished_at"] = ps.now() REPORT["result"] = ( "PASS" if REPORT["cases"] and all(x.get("result") == "PASS" for x in REPORT["cases"] + REPORT.get("invariants", [])) else "FAIL" ) save() raise SystemExit(0 if REPORT["result"] == "PASS" else 1)