forked from mtusi2026/mtusi2026lab2
Bring the stdlib suite and image generator of the lab 5 preparation: make suite-preflight checks services, readiness, schema columns, broker topology and the faults override, and make level3 runs the grade 3 scenarios (smoke through the pipeline, worker down with API, relay and broker restarts, duplicate delivery after ready, poison image, S3 outage with recovery, retries up to JOB_MAX_ATTEMPTS) and checks the invariants into evidence/level3.json. make pipeline-evidence shows the queued job on the broker while the worker is stopped, its properties and contract payload, the 409 before ready and SIGTERM of the relay and the worker; make topology dumps queues, consumers with prefetch, permissions and users. The lab 4 failover and S3 outage scenarios and the upload fault suite now expect 202 and wait for ready, the stack evidence lists the broker and its client packages, and the CI acceptance job runs the pipeline scenarios and the SIGTERM check of the new processes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
355 lines
11 KiB
Python
355 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import json
|
|
import struct
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
MAX_BYTES = 10485760
|
|
MAX_PIXELS = 40000000
|
|
REJECTED_OR_FAILED = {"upload": [202, 415], "final": "failed", "error_code": "invalid_image"}
|
|
GIF = (
|
|
b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00"
|
|
b",\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;"
|
|
)
|
|
|
|
|
|
def chunk(kind, data):
|
|
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
|
|
|
|
|
def png(width, height, rows, color_type=2, depth=8, *, extra=(), level=6):
|
|
packer = zlib.compressobj(level)
|
|
body = b"".join(packer.compress(b"\x00" + row) for row in rows) + packer.flush()
|
|
header = struct.pack(">IIBBBBB", width, height, depth, color_type, 0, 0, 0)
|
|
return (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", header)
|
|
+ b"".join(extra)
|
|
+ chunk(b"IDAT", body)
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def ramp(value, size):
|
|
return value * 255 // max(1, size - 1)
|
|
|
|
|
|
def gradient(width, height, channels):
|
|
for y in range(height):
|
|
yield bytes(
|
|
channel
|
|
for x in range(width)
|
|
for channel in (ramp(x, width), ramp(y, height), 160, ramp(x + y, width + height))[
|
|
:channels
|
|
]
|
|
)
|
|
|
|
|
|
def blank(width, height, channels, depth=8):
|
|
row = bytes(width * channels * depth // 8)
|
|
for _ in range(height):
|
|
yield row
|
|
|
|
|
|
def padded(data, total):
|
|
pad = total - len(data) - 12
|
|
return data[:33] + chunk(b"paDd", bytes(pad)) + data[33:]
|
|
|
|
|
|
class Bits:
|
|
def __init__(self):
|
|
self.out = bytearray()
|
|
self.acc = 0
|
|
self.size = 0
|
|
|
|
def put(self, value, length):
|
|
self.acc = (self.acc << length) | value
|
|
self.size += length
|
|
while self.size >= 8:
|
|
self.size -= 8
|
|
byte = (self.acc >> self.size) & 0xFF
|
|
self.out += b"\xff\x00" if byte == 0xFF else bytes((byte,))
|
|
self.acc &= (1 << self.size) - 1
|
|
|
|
def zeros(self, count):
|
|
head = min(count, (8 - self.size) % 8)
|
|
self.put(0, head)
|
|
count -= head
|
|
self.out += bytes(count // 8)
|
|
self.put(0, count % 8)
|
|
|
|
def flush(self):
|
|
if self.size:
|
|
self.put((1 << (8 - self.size)) - 1, 8 - self.size)
|
|
return bytes(self.out)
|
|
|
|
|
|
def segment(marker, payload):
|
|
return b"\xff" + bytes((marker,)) + struct.pack(">H", len(payload) + 2) + payload
|
|
|
|
|
|
def ycbcr(red, green, blue):
|
|
return (
|
|
0.299 * red + 0.587 * green + 0.114 * blue,
|
|
128 - 0.168736 * red - 0.331264 * green + 0.5 * blue,
|
|
128 + 0.5 * red - 0.418688 * green - 0.081312 * blue,
|
|
)
|
|
|
|
|
|
def exif(orientation):
|
|
entry = struct.pack(">HHIHH", 0x0112, 3, 1, orientation, 0)
|
|
tiff = b"MM\x00\x2a\x00\x00\x00\x08" + struct.pack(">H", 1) + entry + struct.pack(">I", 0)
|
|
return segment(0xE1, b"Exif\x00\x00" + tiff)
|
|
|
|
|
|
def jpeg(width, height, mode="rgb", orientation=None, uniform=False):
|
|
components = {"gray": 1, "rgb": 3, "cmyk": 4}[mode]
|
|
head = b"\xff\xd8" + (
|
|
exif(orientation)
|
|
if orientation
|
|
else segment(0xE0, b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00")
|
|
)
|
|
if mode == "cmyk":
|
|
head += segment(0xEE, b"Adobe\x00\x64\x00\x00\x00\x00\x00")
|
|
head += segment(0xDB, b"\x00" + bytes([1] * 64))
|
|
head += segment(
|
|
0xC0,
|
|
struct.pack(">BHHB", 8, height, width, components)
|
|
+ b"".join(bytes((i + 1, 0x11, 0)) for i in range(components)),
|
|
)
|
|
head += segment(0xC4, b"\x00" + bytes([0, 0, 0, 12] + [0] * 12) + bytes(range(12)))
|
|
head += segment(0xC4, b"\x10" + bytes([1] + [0] * 15) + b"\x00")
|
|
head += segment(
|
|
0xDA,
|
|
bytes((components,))
|
|
+ b"".join(bytes((i + 1, 0)) for i in range(components))
|
|
+ b"\x00\x3f\x00",
|
|
)
|
|
columns, rows = (width + 7) // 8, (height + 7) // 8
|
|
bits, previous = Bits(), [0] * components
|
|
for block in range(1 if uniform else columns * rows):
|
|
bx, by = block % columns, block // columns
|
|
red, green, blue = ramp(bx, columns), ramp(by, rows), 160
|
|
values = {"gray": (red,), "rgb": ycbcr(red, green, blue), "cmyk": (red, green, blue, 64)}[
|
|
mode
|
|
]
|
|
for index, value in enumerate(values):
|
|
dc = round(8 * (value - 128))
|
|
diff, previous[index] = dc - previous[index], dc
|
|
category = abs(diff).bit_length()
|
|
bits.put(category, 4)
|
|
if category:
|
|
bits.put(diff if diff > 0 else diff + (1 << category) - 1, category)
|
|
bits.put(0, 1)
|
|
if uniform:
|
|
bits.zeros((columns * rows - 1) * components * 5)
|
|
return head + bits.flush() + b"\xff\xd9"
|
|
|
|
|
|
def fit(width, height, box, upscale=False):
|
|
if not upscale and max(width, height) <= box:
|
|
return [width, height]
|
|
if width >= height:
|
|
return [box, max(1, height * box // width)]
|
|
return [max(1, width * box // height), box]
|
|
|
|
|
|
def ready(width, height, orientation=1):
|
|
if orientation in (5, 6, 7, 8):
|
|
width, height = height, width
|
|
return {
|
|
"upload": [202],
|
|
"final": "ready",
|
|
"variants": {
|
|
"thumbnail": fit(width, height, 256),
|
|
"crop": [128, 128],
|
|
"optimized": fit(width, height, 1280),
|
|
},
|
|
}
|
|
|
|
|
|
def catalog(heavy):
|
|
small = png(64, 64, gradient(64, 64, 3))
|
|
truncated_png = png(320, 200, gradient(320, 200, 3))
|
|
idat = truncated_png.index(b"IDAT")
|
|
corrupt = bytearray(truncated_png)
|
|
corrupt[idat + 40 : idat + 48] = b"\x00" * 8
|
|
palette = bytes(channel for index in range(256) for channel in (index, 255 - index, 128))
|
|
alpha = bytes(index for index in range(256))
|
|
items = [
|
|
(
|
|
"png-rgb-320x200.png",
|
|
"image/png",
|
|
lambda: png(320, 200, gradient(320, 200, 3)),
|
|
ready(320, 200),
|
|
),
|
|
(
|
|
"png-rgba-300x300.png",
|
|
"image/png",
|
|
lambda: png(300, 300, gradient(300, 300, 4), 6),
|
|
ready(300, 300),
|
|
),
|
|
(
|
|
"png-palette-trns-200x100.png",
|
|
"image/png",
|
|
lambda: png(
|
|
200,
|
|
100,
|
|
(bytes((x + y) % 256 for x in range(200)) for y in range(100)),
|
|
3,
|
|
extra=(chunk(b"PLTE", palette), chunk(b"tRNS", alpha)),
|
|
),
|
|
ready(200, 100),
|
|
),
|
|
(
|
|
"png-gray-17x1000.png",
|
|
"image/png",
|
|
lambda: png(17, 1000, gradient(17, 1000, 1), 0),
|
|
ready(17, 1000),
|
|
),
|
|
(
|
|
"png-gray-alpha-640x480.png",
|
|
"image/png",
|
|
lambda: png(640, 480, gradient(640, 480, 2), 4),
|
|
ready(640, 480),
|
|
),
|
|
(
|
|
"png-gray16-300x200.png",
|
|
"image/png",
|
|
lambda: png(300, 200, gradient(600, 200, 1), 0, 16),
|
|
ready(300, 200),
|
|
),
|
|
(
|
|
"png-rgba16-64x64.png",
|
|
"image/png",
|
|
lambda: png(64, 64, gradient(512, 64, 1), 6, 16),
|
|
ready(64, 64),
|
|
),
|
|
("png-1x1.png", "image/png", lambda: png(1, 1, gradient(1, 1, 3)), ready(1, 1)),
|
|
(
|
|
"png-wide-3000x2.png",
|
|
"image/png",
|
|
lambda: png(3000, 2, gradient(3000, 2, 3)),
|
|
ready(3000, 2),
|
|
),
|
|
(
|
|
"png-small-50x30.png",
|
|
"image/png",
|
|
lambda: png(50, 30, gradient(50, 30, 3)),
|
|
ready(50, 30),
|
|
),
|
|
("jpeg-rgb-1000x750.jpg", "image/jpeg", lambda: jpeg(1000, 750), ready(1000, 750)),
|
|
("jpeg-gray-801x601.jpg", "image/jpeg", lambda: jpeg(801, 601, "gray"), ready(801, 601)),
|
|
("jpeg-cmyk-400x300.jpg", "image/jpeg", lambda: jpeg(400, 300, "cmyk"), ready(400, 300)),
|
|
(
|
|
"jpeg-exif6-400x200.jpg",
|
|
"image/jpeg",
|
|
lambda: jpeg(400, 200, orientation=6),
|
|
ready(400, 200, 6),
|
|
),
|
|
(
|
|
"jpeg-exif3-300x200.jpg",
|
|
"image/jpeg",
|
|
lambda: jpeg(300, 200, orientation=3),
|
|
ready(300, 200, 3),
|
|
),
|
|
("jpeg-rgb-4000x3000.jpg", "image/jpeg", lambda: jpeg(4000, 3000), ready(4000, 3000)),
|
|
("png-exactly-10mib.png", "image/png", lambda: padded(small, MAX_BYTES), ready(64, 64)),
|
|
(
|
|
"png-over-10mib.png",
|
|
"image/png",
|
|
lambda: padded(small, MAX_BYTES + 1),
|
|
{"upload": [413]},
|
|
),
|
|
(
|
|
"png-over-pixels-8001x5000.png",
|
|
"image/png",
|
|
lambda: png(8001, 5000, blank(8001, 5000, 3), level=9),
|
|
{"upload": [413]},
|
|
),
|
|
(
|
|
"jpeg-over-pixels-10000x5000.jpg",
|
|
"image/jpeg",
|
|
lambda: jpeg(10000, 5000, uniform=True),
|
|
{"upload": [413]},
|
|
),
|
|
("png-truncated.png", "image/png", lambda: truncated_png[: idat + 600], REJECTED_OR_FAILED),
|
|
("png-corrupt-idat.png", "image/png", lambda: bytes(corrupt), REJECTED_OR_FAILED),
|
|
("jpeg-truncated.jpg", "image/jpeg", lambda: jpeg(1000, 750)[:3000], REJECTED_OR_FAILED),
|
|
(
|
|
"png-zero-width.png",
|
|
"image/png",
|
|
zero_width,
|
|
{"upload": [400, 415]},
|
|
),
|
|
(
|
|
"not-an-image.png",
|
|
"image/png",
|
|
lambda: b"plain text, not an image\n" * 8,
|
|
{"upload": [415]},
|
|
),
|
|
("gif-declared-as-png.png", "image/png", lambda: GIF, {"upload": [415]}),
|
|
]
|
|
if heavy:
|
|
items.append(
|
|
(
|
|
"png-at-pixel-limit-8000x5000.png",
|
|
"image/png",
|
|
lambda: png(8000, 5000, blank(8000, 5000, 3), level=9),
|
|
ready(8000, 5000),
|
|
)
|
|
)
|
|
return items
|
|
|
|
|
|
def zero_width():
|
|
header = struct.pack(">IIBBBBB", 0, 10, 8, 2, 0, 0, 0)
|
|
return (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", header)
|
|
+ chunk(b"IDAT", zlib.compress(b""))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def build(out, heavy=False, only=None):
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
manifest = []
|
|
for name, content_type, make, expect in catalog(heavy):
|
|
if only and name not in only:
|
|
continue
|
|
data = make()
|
|
(out / name).write_bytes(data)
|
|
manifest.append(
|
|
{"file": name, "content_type": content_type, "size_bytes": len(data), "expect": expect}
|
|
)
|
|
(out / "manifest.json").write_text(
|
|
json.dumps(
|
|
{"rules": {"max_bytes": MAX_BYTES, "max_pixels": MAX_PIXELS}, "images": manifest},
|
|
indent=2,
|
|
)
|
|
+ "\n"
|
|
)
|
|
return manifest
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Generate lab5 test images and manifest.json with expected outcomes"
|
|
)
|
|
parser.add_argument("--out", default=str(ROOT / "build/lab5-images"))
|
|
parser.add_argument(
|
|
"--heavy", action="store_true", help="also write a 40 Mpx image at the pixel limit"
|
|
)
|
|
parser.add_argument("--only", default="", help="comma separated file names")
|
|
args = parser.parse_args()
|
|
manifest = build(Path(args.out), args.heavy, {x for x in args.only.split(",") if x})
|
|
for item in manifest:
|
|
print(f"{item['file']:36} {item['size_bytes']:>10} {item['expect']['upload']}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|