387 lines
13 KiB
Python
387 lines
13 KiB
Python
import hashlib
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
import time
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from PIL import Image
|
|
from scripts import pipeline_suite as ps
|
|
|
|
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
|
|
from png_trns_fixtures import boundary_png, png_with_trns
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
MODE = sys.argv.pop(1)
|
|
OUT = Path(os.environ.get("FORWARD_EVIDENCE_DIR", "evidence/forward-published")) / (MODE + ".json")
|
|
if OUT.exists():
|
|
raise SystemExit("Result exists; do not repeat completed evidence")
|
|
SUITE = ps.Suite(ps.arguments())
|
|
STACK = SUITE.stack
|
|
REPORT: dict[str, Any] = {
|
|
"mode": MODE,
|
|
"source_sha": ps.run(["git", "rev-parse", "HEAD"]).strip(),
|
|
"started_at": ps.now(),
|
|
"cases": [],
|
|
}
|
|
LIMITS = RenderLimits(10485760, 40000000)
|
|
|
|
|
|
def save():
|
|
REPORT["accepted_ids"] = SUITE.api.accepted if SUITE.api else []
|
|
OUT.write_text(json.dumps(REPORT, ensure_ascii=False, indent=2) + "\n")
|
|
|
|
|
|
def case(name, function):
|
|
row = {"name": name, "started_at": ps.now()}
|
|
REPORT["cases"].append(row)
|
|
save()
|
|
began = time.monotonic()
|
|
try:
|
|
row["observed"] = function()
|
|
row["result"] = "PASS"
|
|
except Exception as error:
|
|
row.update(result="FAIL", error=str(error))
|
|
row["elapsed_seconds"] = round(time.monotonic() - began, 3)
|
|
save()
|
|
print(name, row["result"], row.get("error", ""), flush=True)
|
|
|
|
|
|
def containers():
|
|
return STACK.dc("ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}").splitlines()
|
|
|
|
|
|
def outage():
|
|
assert SUITE.api is not None
|
|
SUITE.drained()
|
|
before_dlq = STACK.queues()[ps.DLQ]
|
|
STACK.stop("worker")
|
|
att = SUITE.api.fixture()
|
|
ident = att["id"]
|
|
ps.poll(
|
|
lambda: all(r["published"] for r in STACK.sql("outbox", id=ident)), 30, "published outbox"
|
|
)
|
|
jobs = STACK.sql("jobs", id=ident)
|
|
REPORT["outage_checkpoint"] = {
|
|
"attachment_id": ident,
|
|
"jobs_before": jobs,
|
|
"dlq_before": before_dlq,
|
|
}
|
|
save()
|
|
STACK.stop("postgres")
|
|
REPORT["outage_checkpoint"]["stopped_containers"] = containers()
|
|
save()
|
|
since = ps.now()
|
|
worker_id = STACK.dc("ps", "-aq", "worker").strip()
|
|
assert worker_id
|
|
ps.run(["docker", "start", worker_id])
|
|
|
|
def requeues():
|
|
logs = STACK.dc("logs", "--no-color", "--since", since, "worker")
|
|
rows = [
|
|
line
|
|
for line in logs.splitlines()
|
|
if "job.requeued" in line and jobs[0]["job_id"] in line
|
|
]
|
|
return rows if len(rows) >= 12 else None
|
|
|
|
try:
|
|
lines = ps.poll(requeues, 90, "12 real database failures", 1)
|
|
OUT.with_name(OUT.stem + "-worker.txt").write_text("\n".join(lines) + "\n")
|
|
during = STACK.queues()
|
|
finally:
|
|
STACK.up("postgres")
|
|
ps.poll(lambda: ps.http_ready(SUITE.args.base_url), 30, "API after PostgreSQL")
|
|
ready = SUITE.api.wait(ident)
|
|
SUITE.verify(ready)
|
|
SUITE.drained()
|
|
after = STACK.sql("jobs", id=ident)
|
|
assert (
|
|
after[0]["job_id"] == jobs[0]["job_id"] and after[0]["attempt"] == jobs[0]["attempt"] == 1
|
|
)
|
|
assert after[0]["status"] == "done"
|
|
assert STACK.queues()[ps.DLQ] == before_dlq
|
|
assert len(STACK.sql("variants", id=ident)) == 3
|
|
return {
|
|
"attachment_id": ident,
|
|
"jobs_before": jobs,
|
|
"jobs_after": after,
|
|
"requeues": len(lines),
|
|
"queues_during": during,
|
|
"queues_after": STACK.queues(),
|
|
"replay": False,
|
|
"stopped_containers": REPORT["outage_checkpoint"]["stopped_containers"],
|
|
}
|
|
|
|
|
|
def png(name, data, reference):
|
|
assert SUITE.api is not None
|
|
att = SUITE.api.upload(data, name=name + ".png")
|
|
ident = att["id"]
|
|
results = []
|
|
for version in (1, 2):
|
|
if version == 2:
|
|
STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
|
|
ps.poll(lambda: SUITE.record(ident)["published_version"] == 2, 90, "published v2")
|
|
ready = SUITE.api.wait(ident)
|
|
SUITE.verify(ready)
|
|
assert SUITE.api.content(ident)["bytes"] == data
|
|
expected = {r.name: r for r in render_variants(reference, LIMITS, version)}
|
|
variants = []
|
|
for variant in ready["variants"]:
|
|
content = SUITE.api.content(ident, variant["name"])["bytes"]
|
|
assert content == expected[variant["name"]].data, (
|
|
variant["name"] + " differs from independent white reference"
|
|
)
|
|
with Image.open(BytesIO(content)) as image:
|
|
assert not image.getexif()
|
|
assert not any(key in image.info for key in ("icc_profile", "xmp", "transparency"))
|
|
variants.append(
|
|
{
|
|
"name": variant["name"],
|
|
"sha256": hashlib.sha256(content).hexdigest(),
|
|
"bytes": len(content),
|
|
"dimensions": [variant["width"], variant["height"]],
|
|
}
|
|
)
|
|
results.append({"version": version, "variants": variants})
|
|
before = SUITE.snapshot(ident)
|
|
STACK.publish(SUITE.payload(ident, 1))
|
|
STACK.publish(SUITE.payload(ident, 2))
|
|
SUITE.drained()
|
|
assert SUITE.snapshot(ident) == before
|
|
assert len(STACK.sql("variants", id=ident)) == 6
|
|
return {
|
|
"attachment_id": ident,
|
|
"input_sha256": hashlib.sha256(data).hexdigest(),
|
|
"input_bytes": len(data),
|
|
"versions": results,
|
|
"stale_and_duplicate_noop": True,
|
|
}
|
|
|
|
|
|
FIXTURES = ROOT / "build/forward-png"
|
|
FIXTURES.mkdir(parents=True, exist_ok=True)
|
|
for fixture_name, fixture_depth, fixture_key in (
|
|
("gray2", 2, (1,)),
|
|
("gray4", 4, (5,)),
|
|
("gray16", 16, (12345,)),
|
|
("rgb16", 16, (12345, 23456, 34567)),
|
|
):
|
|
(FIXTURES / (fixture_name + "-40mp.png")).write_bytes(boundary_png(fixture_depth, fixture_key))
|
|
ps.SQL["timed_events"] = (
|
|
"SELECT attachment_id,job_id,kind,at FROM job_events "
|
|
"WHERE attachment_id = ANY(:'ids'::uuid[]) AND kind IN ('claimed','done') ORDER BY id"
|
|
)
|
|
|
|
|
|
def without_trns(data):
|
|
result, offset = data[:8], 8
|
|
while offset < len(data):
|
|
size = struct.unpack(">I", data[offset : offset + 4])[0] + 12
|
|
if data[offset + 4 : offset + 8] != b"tRNS":
|
|
result += data[offset : offset + size]
|
|
offset += size
|
|
return result
|
|
|
|
|
|
def verify_large(ident, data, transparent):
|
|
assert SUITE.api is not None
|
|
ready = SUITE.api.wait(ident)
|
|
SUITE.verify(ready)
|
|
assert SUITE.api.content(ident)["bytes"] == data
|
|
variants = []
|
|
for variant in ready["variants"]:
|
|
content = SUITE.api.content(ident, variant["name"])["bytes"]
|
|
with Image.open(BytesIO(content)) as image:
|
|
left = image.getpixel((5, image.height // 2))
|
|
right = image.getpixel((image.width - 6, image.height // 2))
|
|
assert (left == (255, 255, 255)) == transparent
|
|
assert right != (255, 255, 255)
|
|
assert not image.getexif()
|
|
variants.append({**variant, "left_pixel": left, "right_pixel": right})
|
|
return variants
|
|
|
|
|
|
def duplicate_check(ident):
|
|
before = SUITE.snapshot(ident)
|
|
STACK.publish(SUITE.payload(ident, 1))
|
|
STACK.publish(SUITE.payload(ident, 2))
|
|
SUITE.drained()
|
|
assert SUITE.snapshot(ident) == before
|
|
assert len(STACK.sql("variants", id=ident)) == 6
|
|
|
|
|
|
def large(name, data, transparent=True):
|
|
assert SUITE.api is not None
|
|
att = SUITE.api.upload(data, name=name + ".png")
|
|
ident = att["id"]
|
|
versions = [{"version": 1, "variants": verify_large(ident, data, transparent)}]
|
|
STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
|
|
ps.poll(lambda: SUITE.record(ident)["published_version"] == 2, 90, "v2 publication")
|
|
versions.append({"version": 2, "variants": verify_large(ident, data, transparent)})
|
|
duplicate_check(ident)
|
|
return {
|
|
"attachment_id": ident,
|
|
"input_sha256": hashlib.sha256(data).hexdigest(),
|
|
"bytes": len(data),
|
|
"versions": versions,
|
|
"memory_peak": int(STACK.cgroup("worker", "memory.peak")),
|
|
"memory_max": int(STACK.cgroup("worker", "memory.max")),
|
|
}
|
|
|
|
|
|
def concurrent():
|
|
assert SUITE.api is not None
|
|
data = (FIXTURES / "rgb16-40mp.png").read_bytes()
|
|
SUITE.drained()
|
|
STACK.stop("worker")
|
|
try:
|
|
ids = [SUITE.api.upload(data, name="concurrent-rgb16.png")["id"] for _ in range(2)]
|
|
SUITE.queued(2)
|
|
finally:
|
|
ps.run(["docker", "start", STACK.dc("ps", "-aq", "worker").strip()])
|
|
versions = []
|
|
for version in (1, 2):
|
|
if version == 2:
|
|
STACK.stop("worker")
|
|
try:
|
|
for ident in ids:
|
|
STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
|
|
SUITE.queued(2)
|
|
finally:
|
|
ps.run(["docker", "start", STACK.dc("ps", "-aq", "worker").strip()])
|
|
for ident in ids:
|
|
ps.poll(
|
|
lambda ident=ident, version=version: (
|
|
SUITE.record(ident)["published_version"] == version
|
|
),
|
|
90,
|
|
"concurrent publication",
|
|
)
|
|
variants = {ident: verify_large(ident, data, True) for ident in ids}
|
|
events = STACK.sql("timed_events", ids="{" + ",".join(ids) + "}")
|
|
job_ids = {
|
|
row["job_id"]
|
|
for ident in ids
|
|
for row in STACK.sql("jobs", id=ident)
|
|
if row["pipeline_version"] == version
|
|
}
|
|
selected = [e for e in events if e["job_id"] in job_ids]
|
|
claims = [e["at"] for e in selected if e["kind"] == "claimed"]
|
|
done = [e["at"] for e in selected if e["kind"] == "done"]
|
|
assert len(claims) == len(done) == 2
|
|
assert max(claims) < min(done), selected
|
|
versions.append(
|
|
{
|
|
"version": version,
|
|
"events": selected,
|
|
"processing_overlapped": True,
|
|
"variants": variants,
|
|
}
|
|
)
|
|
for ident in ids:
|
|
duplicate_check(ident)
|
|
peak, limit = (int(STACK.cgroup("worker", name)) for name in ("memory.peak", "memory.max"))
|
|
assert peak < limit
|
|
return {
|
|
"attachment_ids": ids,
|
|
"versions": versions,
|
|
"memory_peak": peak,
|
|
"memory_max": limit,
|
|
"memory_events": STACK.cgroup("worker", "memory.events"),
|
|
}
|
|
|
|
|
|
def forced(**overrides):
|
|
data = (FIXTURES / "rgb16-40mp.png").read_bytes()
|
|
result = SUITE.limited(data, **overrides)
|
|
STACK.restore()
|
|
result["next_normal"] = SUITE.smoke_pipeline()
|
|
return result
|
|
|
|
|
|
try:
|
|
SUITE.detect_api()
|
|
SUITE.api = ps.Api(SUITE.args)
|
|
dead_before = STACK.queues()[ps.DLQ][0]
|
|
REPORT["conditions"] = {
|
|
"host_load": os.getloadavg(),
|
|
"containers": containers(),
|
|
"running": ps.run(["docker", "ps", "--format", "{{.Names}} {{.Status}}"]),
|
|
"worker_limits": STACK.dc(
|
|
"exec",
|
|
"-T",
|
|
"worker",
|
|
"printenv",
|
|
"RENDER_MEMORY_MB",
|
|
"RENDER_TIMEOUT_SECONDS",
|
|
"WORKER_CONCURRENCY",
|
|
),
|
|
}
|
|
save()
|
|
if MODE == "postgres-outage":
|
|
case("published_outbox_database_outage", outage)
|
|
elif MODE == "png-http":
|
|
for name, depth, color in [
|
|
("gray1", 1, (0,)),
|
|
("gray2", 2, (1,)),
|
|
("gray4", 4, (5,)),
|
|
("gray8", 8, (85,)),
|
|
("gray16", 16, (12345,)),
|
|
("rgb8", 8, (255, 0, 0)),
|
|
("rgb16", 16, (12345, 23456, 34567)),
|
|
]:
|
|
case(
|
|
name,
|
|
lambda name=name, depth=depth, color=color: png(name, *png_with_trns(depth, color)),
|
|
)
|
|
case(
|
|
"rgb16-adam7-filter4-exif6",
|
|
lambda: png(
|
|
"adam7",
|
|
*png_with_trns(
|
|
16,
|
|
(12345, 23456, 34567),
|
|
size=(65, 63),
|
|
interlace=1,
|
|
filtering=4,
|
|
orientation=6,
|
|
),
|
|
),
|
|
)
|
|
elif MODE == "png-boundary":
|
|
for name in ("gray2", "gray4", "gray16", "rgb16"):
|
|
case(
|
|
name, lambda name=name: large(name, (FIXTURES / (name + "-40mp.png")).read_bytes())
|
|
)
|
|
case(
|
|
"gray16-without-trns",
|
|
lambda: large(
|
|
"gray16-opaque", without_trns((FIXTURES / "gray16-40mp.png").read_bytes()), False
|
|
),
|
|
)
|
|
case("concurrent-rgb16", concurrent)
|
|
case("forced-timeout", lambda: forced(FAULT_RENDER_TIMEOUT_SECONDS="0.01"))
|
|
case("forced-memory", lambda: forced(FAULT_RENDER_MEMORY_MB="200"))
|
|
ids = "{" + ",".join(SUITE.api.accepted) + "}"
|
|
ps.poll(lambda: not STACK.sql("staging_left", ids=ids), 45, "staging sweep")
|
|
SUITE.drained()
|
|
REPORT["invariants"] = SUITE.invariants(dead_before)
|
|
finally:
|
|
STACK.up("postgres")
|
|
STACK.restore()
|
|
REPORT["finished_at"] = ps.now()
|
|
REPORT["result"] = (
|
|
"PASS"
|
|
if REPORT["cases"]
|
|
and all(x.get("result") == "PASS" for x in REPORT["cases"] + REPORT.get("invariants", []))
|
|
else "FAIL"
|
|
)
|
|
save()
|
|
raise SystemExit(0 if REPORT["result"] == "PASS" else 1)
|