Files
mtusi2026lab5/scripts/make_images.py
ArtyomandClaude Opus 5.5 fe2693870e Add the level 3 pipeline scenarios and the stand evidence
Bring the stdlib suite and image generator of the lab 5 preparation:
make suite-preflight checks services, readiness, schema columns, broker
topology and the faults override, and make level3 runs the grade 3
scenarios (smoke through the pipeline, worker down with API, relay and
broker restarts, duplicate delivery after ready, poison image, S3
outage with recovery, retries up to JOB_MAX_ATTEMPTS) and checks the
invariants into evidence/level3.json. make pipeline-evidence shows the
queued job on the broker while the worker is stopped, its properties
and contract payload, the 409 before ready and SIGTERM of the relay and
the worker; make topology dumps queues, consumers with prefetch,
permissions and users.

The lab 4 failover and S3 outage scenarios and the upload fault suite
now expect 202 and wait for ready, the stack evidence lists the broker
and its client packages, and the CI acceptance job runs the pipeline
scenarios and the SIGTERM check of the new processes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 01:19:13 +03:00

355 lines
11 KiB
Python

#!/usr/bin/env python3
import argparse
import json
import struct
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MAX_BYTES = 10485760
MAX_PIXELS = 40000000
REJECTED_OR_FAILED = {"upload": [202, 415], "final": "failed", "error_code": "invalid_image"}
GIF = (
b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00"
b",\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;"
)
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
def png(width, height, rows, color_type=2, depth=8, *, extra=(), level=6):
packer = zlib.compressobj(level)
body = b"".join(packer.compress(b"\x00" + row) for row in rows) + packer.flush()
header = struct.pack(">IIBBBBB", width, height, depth, color_type, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ b"".join(extra)
+ chunk(b"IDAT", body)
+ chunk(b"IEND", b"")
)
def ramp(value, size):
return value * 255 // max(1, size - 1)
def gradient(width, height, channels):
for y in range(height):
yield bytes(
channel
for x in range(width)
for channel in (ramp(x, width), ramp(y, height), 160, ramp(x + y, width + height))[
:channels
]
)
def blank(width, height, channels, depth=8):
row = bytes(width * channels * depth // 8)
for _ in range(height):
yield row
def padded(data, total):
pad = total - len(data) - 12
return data[:33] + chunk(b"paDd", bytes(pad)) + data[33:]
class Bits:
def __init__(self):
self.out = bytearray()
self.acc = 0
self.size = 0
def put(self, value, length):
self.acc = (self.acc << length) | value
self.size += length
while self.size >= 8:
self.size -= 8
byte = (self.acc >> self.size) & 0xFF
self.out += b"\xff\x00" if byte == 0xFF else bytes((byte,))
self.acc &= (1 << self.size) - 1
def zeros(self, count):
head = min(count, (8 - self.size) % 8)
self.put(0, head)
count -= head
self.out += bytes(count // 8)
self.put(0, count % 8)
def flush(self):
if self.size:
self.put((1 << (8 - self.size)) - 1, 8 - self.size)
return bytes(self.out)
def segment(marker, payload):
return b"\xff" + bytes((marker,)) + struct.pack(">H", len(payload) + 2) + payload
def ycbcr(red, green, blue):
return (
0.299 * red + 0.587 * green + 0.114 * blue,
128 - 0.168736 * red - 0.331264 * green + 0.5 * blue,
128 + 0.5 * red - 0.418688 * green - 0.081312 * blue,
)
def exif(orientation):
entry = struct.pack(">HHIHH", 0x0112, 3, 1, orientation, 0)
tiff = b"MM\x00\x2a\x00\x00\x00\x08" + struct.pack(">H", 1) + entry + struct.pack(">I", 0)
return segment(0xE1, b"Exif\x00\x00" + tiff)
def jpeg(width, height, mode="rgb", orientation=None, uniform=False):
components = {"gray": 1, "rgb": 3, "cmyk": 4}[mode]
head = b"\xff\xd8" + (
exif(orientation)
if orientation
else segment(0xE0, b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00")
)
if mode == "cmyk":
head += segment(0xEE, b"Adobe\x00\x64\x00\x00\x00\x00\x00")
head += segment(0xDB, b"\x00" + bytes([1] * 64))
head += segment(
0xC0,
struct.pack(">BHHB", 8, height, width, components)
+ b"".join(bytes((i + 1, 0x11, 0)) for i in range(components)),
)
head += segment(0xC4, b"\x00" + bytes([0, 0, 0, 12] + [0] * 12) + bytes(range(12)))
head += segment(0xC4, b"\x10" + bytes([1] + [0] * 15) + b"\x00")
head += segment(
0xDA,
bytes((components,))
+ b"".join(bytes((i + 1, 0)) for i in range(components))
+ b"\x00\x3f\x00",
)
columns, rows = (width + 7) // 8, (height + 7) // 8
bits, previous = Bits(), [0] * components
for block in range(1 if uniform else columns * rows):
bx, by = block % columns, block // columns
red, green, blue = ramp(bx, columns), ramp(by, rows), 160
values = {"gray": (red,), "rgb": ycbcr(red, green, blue), "cmyk": (red, green, blue, 64)}[
mode
]
for index, value in enumerate(values):
dc = round(8 * (value - 128))
diff, previous[index] = dc - previous[index], dc
category = abs(diff).bit_length()
bits.put(category, 4)
if category:
bits.put(diff if diff > 0 else diff + (1 << category) - 1, category)
bits.put(0, 1)
if uniform:
bits.zeros((columns * rows - 1) * components * 5)
return head + bits.flush() + b"\xff\xd9"
def fit(width, height, box, upscale=False):
if not upscale and max(width, height) <= box:
return [width, height]
if width >= height:
return [box, max(1, height * box // width)]
return [max(1, width * box // height), box]
def ready(width, height, orientation=1):
if orientation in (5, 6, 7, 8):
width, height = height, width
return {
"upload": [202],
"final": "ready",
"variants": {
"thumbnail": fit(width, height, 256),
"crop": [128, 128],
"optimized": fit(width, height, 1280),
},
}
def catalog(heavy):
small = png(64, 64, gradient(64, 64, 3))
truncated_png = png(320, 200, gradient(320, 200, 3))
idat = truncated_png.index(b"IDAT")
corrupt = bytearray(truncated_png)
corrupt[idat + 40 : idat + 48] = b"\x00" * 8
palette = bytes(channel for index in range(256) for channel in (index, 255 - index, 128))
alpha = bytes(index for index in range(256))
items = [
(
"png-rgb-320x200.png",
"image/png",
lambda: png(320, 200, gradient(320, 200, 3)),
ready(320, 200),
),
(
"png-rgba-300x300.png",
"image/png",
lambda: png(300, 300, gradient(300, 300, 4), 6),
ready(300, 300),
),
(
"png-palette-trns-200x100.png",
"image/png",
lambda: png(
200,
100,
(bytes((x + y) % 256 for x in range(200)) for y in range(100)),
3,
extra=(chunk(b"PLTE", palette), chunk(b"tRNS", alpha)),
),
ready(200, 100),
),
(
"png-gray-17x1000.png",
"image/png",
lambda: png(17, 1000, gradient(17, 1000, 1), 0),
ready(17, 1000),
),
(
"png-gray-alpha-640x480.png",
"image/png",
lambda: png(640, 480, gradient(640, 480, 2), 4),
ready(640, 480),
),
(
"png-gray16-300x200.png",
"image/png",
lambda: png(300, 200, gradient(600, 200, 1), 0, 16),
ready(300, 200),
),
(
"png-rgba16-64x64.png",
"image/png",
lambda: png(64, 64, gradient(512, 64, 1), 6, 16),
ready(64, 64),
),
("png-1x1.png", "image/png", lambda: png(1, 1, gradient(1, 1, 3)), ready(1, 1)),
(
"png-wide-3000x2.png",
"image/png",
lambda: png(3000, 2, gradient(3000, 2, 3)),
ready(3000, 2),
),
(
"png-small-50x30.png",
"image/png",
lambda: png(50, 30, gradient(50, 30, 3)),
ready(50, 30),
),
("jpeg-rgb-1000x750.jpg", "image/jpeg", lambda: jpeg(1000, 750), ready(1000, 750)),
("jpeg-gray-801x601.jpg", "image/jpeg", lambda: jpeg(801, 601, "gray"), ready(801, 601)),
("jpeg-cmyk-400x300.jpg", "image/jpeg", lambda: jpeg(400, 300, "cmyk"), ready(400, 300)),
(
"jpeg-exif6-400x200.jpg",
"image/jpeg",
lambda: jpeg(400, 200, orientation=6),
ready(400, 200, 6),
),
(
"jpeg-exif3-300x200.jpg",
"image/jpeg",
lambda: jpeg(300, 200, orientation=3),
ready(300, 200, 3),
),
("jpeg-rgb-4000x3000.jpg", "image/jpeg", lambda: jpeg(4000, 3000), ready(4000, 3000)),
("png-exactly-10mib.png", "image/png", lambda: padded(small, MAX_BYTES), ready(64, 64)),
(
"png-over-10mib.png",
"image/png",
lambda: padded(small, MAX_BYTES + 1),
{"upload": [413]},
),
(
"png-over-pixels-8001x5000.png",
"image/png",
lambda: png(8001, 5000, blank(8001, 5000, 3), level=9),
{"upload": [413]},
),
(
"jpeg-over-pixels-10000x5000.jpg",
"image/jpeg",
lambda: jpeg(10000, 5000, uniform=True),
{"upload": [413]},
),
("png-truncated.png", "image/png", lambda: truncated_png[: idat + 600], REJECTED_OR_FAILED),
("png-corrupt-idat.png", "image/png", lambda: bytes(corrupt), REJECTED_OR_FAILED),
("jpeg-truncated.jpg", "image/jpeg", lambda: jpeg(1000, 750)[:3000], REJECTED_OR_FAILED),
(
"png-zero-width.png",
"image/png",
zero_width,
{"upload": [400, 415]},
),
(
"not-an-image.png",
"image/png",
lambda: b"plain text, not an image\n" * 8,
{"upload": [415]},
),
("gif-declared-as-png.png", "image/png", lambda: GIF, {"upload": [415]}),
]
if heavy:
items.append(
(
"png-at-pixel-limit-8000x5000.png",
"image/png",
lambda: png(8000, 5000, blank(8000, 5000, 3), level=9),
ready(8000, 5000),
)
)
return items
def zero_width():
header = struct.pack(">IIBBBBB", 0, 10, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(b""))
+ chunk(b"IEND", b"")
)
def build(out, heavy=False, only=None):
out.mkdir(parents=True, exist_ok=True)
manifest = []
for name, content_type, make, expect in catalog(heavy):
if only and name not in only:
continue
data = make()
(out / name).write_bytes(data)
manifest.append(
{"file": name, "content_type": content_type, "size_bytes": len(data), "expect": expect}
)
(out / "manifest.json").write_text(
json.dumps(
{"rules": {"max_bytes": MAX_BYTES, "max_pixels": MAX_PIXELS}, "images": manifest},
indent=2,
)
+ "\n"
)
return manifest
def main():
parser = argparse.ArgumentParser(
description="Generate lab5 test images and manifest.json with expected outcomes"
)
parser.add_argument("--out", default=str(ROOT / "build/lab5-images"))
parser.add_argument(
"--heavy", action="store_true", help="also write a 40 Mpx image at the pixel limit"
)
parser.add_argument("--only", default="", help="comma separated file names")
args = parser.parse_args()
manifest = build(Path(args.out), args.heavy, {x for x in args.only.split(",") if x})
for item in manifest:
print(f"{item['file']:36} {item['size_bytes']:>10} {item['expect']['upload']}")
if __name__ == "__main__":
main()