diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml index 47bf8f47..629a6e05 100644 --- a/.github/workflows/application.yml +++ b/.github/workflows/application.yml @@ -19,7 +19,7 @@ jobs: - run: make student-test VENV=.venv acceptance: - name: Стенд Compose с RustFS за HAProxy, smoke, вложения, отказ реплики, рестарт и отказы зависимостей + name: Стенд Compose с RabbitMQ, relay и worker за HAProxy, smoke, pipeline, отказ реплики, рестарт и отказы зависимостей runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -34,6 +34,17 @@ jobs: - name: Публичный smoke через балансировщик run: make test + - name: Топология брокера, consumer с prefetch и права клиентов + run: make topology + + - name: Уровень «3» pipeline — worker остановлен, рестарт API и брокера, дубль, S3, исчерпание попыток + run: | + make suite-preflight + make level3 + + - name: Сообщение задания в очереди, 409 до ready и SIGTERM relay и worker + run: make pipeline-evidence + - name: Вложение после остановки принявшей его реплики и при недоступном S3 run: | make scenario SCENARIO=failover-upload @@ -68,12 +79,22 @@ jobs: - name: Наружу опубликован только балансировщик run: | - docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" {print; bad=1} END {exit bad}' + docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" && $1 != "broker" {print; bad=1} END {exit bad}' + docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$1 == "broker" && $2 !~ /^127\.0\.0\.1:/ {print; bad=1} END {exit bad}' - name: Процессы работают не от root run: | docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}' + - name: Корректная остановка relay и worker по SIGTERM + run: | + start=$(date +%s) + docker compose stop -t 10 relay worker + elapsed=$(( $(date +%s) - start )) + codes=$(docker inspect -f '{{.State.ExitCode}}' $(docker compose ps -aq relay worker) | sort -u) + echo "остановка за ${elapsed} с, коды завершения ${codes}" + test "$elapsed" -le 10 && test "$codes" = "0" + - name: Корректная остановка реплик API по SIGTERM run: | start=$(date +%s) diff --git a/Makefile b/Makefile index 12c86f06..c8707a7d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ RATE ?= 100 API_IMAGE ?= mtusi-messenger:lab5 FROM_REV ?= SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ - scripts/upload_memory.py + scripts/upload_memory.py scripts/make_images.py scripts/pipeline_suite.py \ + scripts/pipeline_evidence.py SCENARIO ?= failover-upload PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q @@ -20,7 +21,9 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \ persistence-evidence storage-evidence cache-evidence test-faults seed explain \ limits detection failover-each rolling-update level4 \ - scenario s3-check attachments-evidence upload-faults upload-memory + scenario s3-check attachments-evidence upload-faults upload-memory \ + images queues topology worker-stop worker-start broker-restart suite-preflight level3 \ + pipeline-evidence help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -31,6 +34,7 @@ help: @echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence" @echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда" @echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/" + @echo "lab5: queues, topology, worker-stop/worker-start, broker-restart, images, suite-preflight, level3" check: $(PYTHON) scripts/check_contract.py @@ -168,3 +172,42 @@ explain: seed echo "\$$ make explain # scripts/sql/explain-history.sql"; \ $(PSQL) -f - < scripts/sql/explain-history.sql; } > evidence/explain-history.txt cat evidence/explain-history.txt + +images: + $(PYTHON) scripts/make_images.py + +queues: + sh scripts/broker.sh queues + +topology: + { echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; \ + echo "$$ rabbitmqctl list_queues -p messenger name type durable policy messages"; \ + $(COMPOSE) exec -T broker rabbitmqctl -q list_queues -p messenger name type durable policy messages; \ + echo "$$ rabbitmqctl list_consumers -p messenger"; \ + $(COMPOSE) exec -T broker rabbitmqctl -q list_consumers -p messenger queue_name ack_required prefetch_count; \ + echo "$$ rabbitmqctl list_permissions -p messenger"; \ + $(COMPOSE) exec -T broker rabbitmqctl -q list_permissions -p messenger; \ + echo "$$ rabbitmqctl list_users"; \ + $(COMPOSE) exec -T broker rabbitmqctl -q list_users; \ + echo "$$ docker compose ps"; \ + $(COMPOSE) ps --format '{{.Service}}\t{{.Status}}\t{{.Ports}}'; } > evidence/topology.txt + cat evidence/topology.txt + +worker-stop: + sh scripts/worker.sh stop + +worker-start: + sh scripts/worker.sh start + +broker-restart: + sh scripts/broker.sh restart + +suite-preflight: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --preflight + +level3: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 3 --output evidence/level3.json + +pipeline-evidence: + $(PYTHON) scripts/pipeline_evidence.py --base-url "$(BASE_URL)" > evidence/pipeline-stand.txt; \ + status=$$?; cat evidence/pipeline-stand.txt; exit $$status diff --git a/openspec/changes/add-lab5-async-attachments/tasks.md b/openspec/changes/add-lab5-async-attachments/tasks.md index 1c7e967d..334b3b16 100644 --- a/openspec/changes/add-lab5-async-attachments/tasks.md +++ b/openspec/changes/add-lab5-async-attachments/tasks.md @@ -40,9 +40,9 @@ ## 7. Сценарии и обвязка -- [ ] 7.1 `scripts/pipeline_suite.py`, `scripts/make_images.py` из `.prep/lab5`; цели Makefile `queues`, `worker-stop`, `worker-start`, `broker-restart`, `suite-preflight`, `level3`, `topology`; ruff для новых `scripts/*.py` -- [ ] 7.2 CI-job под лабу 5: брокер, relay и worker в стенде, сценарии уровня «3» -- [ ] 7.3 Проверка группы: `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` с `make restore`, `make suite-preflight`, `make level3` +- [x] 7.1 `scripts/pipeline_suite.py`, `scripts/make_images.py` из `.prep/lab5`; цели Makefile `queues`, `worker-stop`, `worker-start`, `broker-restart`, `suite-preflight`, `level3`, `topology`; ruff для новых `scripts/*.py` +- [x] 7.2 CI-job под лабу 5: брокер, relay и worker в стенде, сценарии уровня «3» +- [x] 7.3 Проверка группы: `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` с `make restore`, `make suite-preflight`, `make level3` ## 8. Отчёт diff --git a/pyproject.toml b/pyproject.toml index d1b68b6c..a035f0ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,8 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "T20", "TID251"] "scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"] +"scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"] +"scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"] "src/messenger/*/infrastructure/**" = ["TID251", "ARG002"] "src/messenger/*/domain/errors.py" = ["N818"] "src/messenger/*/application/errors.py" = ["N818"] diff --git a/scripts/evidence-stack.sh b/scripts/evidence-stack.sh index 0e5575e2..7a4bc96d 100755 --- a/scripts/evidence-stack.sh +++ b/scripts/evidence-stack.sh @@ -36,8 +36,10 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()): echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)" echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)" echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)" + echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)" echo "packages (uv.lock):" - for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow; do + for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \ + faststream aio-pika aiormq dishka-faststream; do printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)" done echo "process users (service uid command):" diff --git a/scripts/lab4_scenarios.py b/scripts/lab4_scenarios.py index 32c35d6f..9ac01c73 100755 --- a/scripts/lab4_scenarios.py +++ b/scripts/lab4_scenarios.py @@ -23,9 +23,9 @@ def compose(*args, stdin=None): return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False) -def upload(c, actor, chat, expected=201): +def upload(c, actor, chat, expected=202): path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png" - schema = "Attachment" if expected == 201 else None + schema = "Attachment" if expected == 202 else None return c.call( "POST", path, @@ -74,6 +74,7 @@ def failover_upload(c, args): alice, bob, _, chat = c.bootstrap() sent = upload(c, alice, chat) att, target = sent["json"], sent["headers"]["X-Instance-Id"] + c.wait_ready(bob, att) started = time.monotonic() hook = subprocess.run( [str(ROOT / "scripts/stop-one.sh")], @@ -109,7 +110,7 @@ def failover_upload(c, args): def s3_outage(c, args): alice, bob, _, chat = c.bootstrap() - att = upload(c, alice, chat)["json"] + att = c.wait_ready(bob, upload(c, alice, chat)["json"]) check( compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed" ) diff --git a/scripts/make_images.py b/scripts/make_images.py new file mode 100644 index 00000000..f4b34596 --- /dev/null +++ b/scripts/make_images.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +import argparse +import json +import struct +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MAX_BYTES = 10485760 +MAX_PIXELS = 40000000 +REJECTED_OR_FAILED = {"upload": [202, 415], "final": "failed", "error_code": "invalid_image"} +GIF = ( + b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00" + b",\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" +) + + +def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + +def png(width, height, rows, color_type=2, depth=8, *, extra=(), level=6): + packer = zlib.compressobj(level) + body = b"".join(packer.compress(b"\x00" + row) for row in rows) + packer.flush() + header = struct.pack(">IIBBBBB", width, height, depth, color_type, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + b"".join(extra) + + chunk(b"IDAT", body) + + chunk(b"IEND", b"") + ) + + +def ramp(value, size): + return value * 255 // max(1, size - 1) + + +def gradient(width, height, channels): + for y in range(height): + yield bytes( + channel + for x in range(width) + for channel in (ramp(x, width), ramp(y, height), 160, ramp(x + y, width + height))[ + :channels + ] + ) + + +def blank(width, height, channels, depth=8): + row = bytes(width * channels * depth // 8) + for _ in range(height): + yield row + + +def padded(data, total): + pad = total - len(data) - 12 + return data[:33] + chunk(b"paDd", bytes(pad)) + data[33:] + + +class Bits: + def __init__(self): + self.out = bytearray() + self.acc = 0 + self.size = 0 + + def put(self, value, length): + self.acc = (self.acc << length) | value + self.size += length + while self.size >= 8: + self.size -= 8 + byte = (self.acc >> self.size) & 0xFF + self.out += b"\xff\x00" if byte == 0xFF else bytes((byte,)) + self.acc &= (1 << self.size) - 1 + + def zeros(self, count): + head = min(count, (8 - self.size) % 8) + self.put(0, head) + count -= head + self.out += bytes(count // 8) + self.put(0, count % 8) + + def flush(self): + if self.size: + self.put((1 << (8 - self.size)) - 1, 8 - self.size) + return bytes(self.out) + + +def segment(marker, payload): + return b"\xff" + bytes((marker,)) + struct.pack(">H", len(payload) + 2) + payload + + +def ycbcr(red, green, blue): + return ( + 0.299 * red + 0.587 * green + 0.114 * blue, + 128 - 0.168736 * red - 0.331264 * green + 0.5 * blue, + 128 + 0.5 * red - 0.418688 * green - 0.081312 * blue, + ) + + +def exif(orientation): + entry = struct.pack(">HHIHH", 0x0112, 3, 1, orientation, 0) + tiff = b"MM\x00\x2a\x00\x00\x00\x08" + struct.pack(">H", 1) + entry + struct.pack(">I", 0) + return segment(0xE1, b"Exif\x00\x00" + tiff) + + +def jpeg(width, height, mode="rgb", orientation=None, uniform=False): + components = {"gray": 1, "rgb": 3, "cmyk": 4}[mode] + head = b"\xff\xd8" + ( + exif(orientation) + if orientation + else segment(0xE0, b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00") + ) + if mode == "cmyk": + head += segment(0xEE, b"Adobe\x00\x64\x00\x00\x00\x00\x00") + head += segment(0xDB, b"\x00" + bytes([1] * 64)) + head += segment( + 0xC0, + struct.pack(">BHHB", 8, height, width, components) + + b"".join(bytes((i + 1, 0x11, 0)) for i in range(components)), + ) + head += segment(0xC4, b"\x00" + bytes([0, 0, 0, 12] + [0] * 12) + bytes(range(12))) + head += segment(0xC4, b"\x10" + bytes([1] + [0] * 15) + b"\x00") + head += segment( + 0xDA, + bytes((components,)) + + b"".join(bytes((i + 1, 0)) for i in range(components)) + + b"\x00\x3f\x00", + ) + columns, rows = (width + 7) // 8, (height + 7) // 8 + bits, previous = Bits(), [0] * components + for block in range(1 if uniform else columns * rows): + bx, by = block % columns, block // columns + red, green, blue = ramp(bx, columns), ramp(by, rows), 160 + values = {"gray": (red,), "rgb": ycbcr(red, green, blue), "cmyk": (red, green, blue, 64)}[ + mode + ] + for index, value in enumerate(values): + dc = round(8 * (value - 128)) + diff, previous[index] = dc - previous[index], dc + category = abs(diff).bit_length() + bits.put(category, 4) + if category: + bits.put(diff if diff > 0 else diff + (1 << category) - 1, category) + bits.put(0, 1) + if uniform: + bits.zeros((columns * rows - 1) * components * 5) + return head + bits.flush() + b"\xff\xd9" + + +def fit(width, height, box, upscale=False): + if not upscale and max(width, height) <= box: + return [width, height] + if width >= height: + return [box, max(1, height * box // width)] + return [max(1, width * box // height), box] + + +def ready(width, height, orientation=1): + if orientation in (5, 6, 7, 8): + width, height = height, width + return { + "upload": [202], + "final": "ready", + "variants": { + "thumbnail": fit(width, height, 256), + "crop": [128, 128], + "optimized": fit(width, height, 1280), + }, + } + + +def catalog(heavy): + small = png(64, 64, gradient(64, 64, 3)) + truncated_png = png(320, 200, gradient(320, 200, 3)) + idat = truncated_png.index(b"IDAT") + corrupt = bytearray(truncated_png) + corrupt[idat + 40 : idat + 48] = b"\x00" * 8 + palette = bytes(channel for index in range(256) for channel in (index, 255 - index, 128)) + alpha = bytes(index for index in range(256)) + items = [ + ( + "png-rgb-320x200.png", + "image/png", + lambda: png(320, 200, gradient(320, 200, 3)), + ready(320, 200), + ), + ( + "png-rgba-300x300.png", + "image/png", + lambda: png(300, 300, gradient(300, 300, 4), 6), + ready(300, 300), + ), + ( + "png-palette-trns-200x100.png", + "image/png", + lambda: png( + 200, + 100, + (bytes((x + y) % 256 for x in range(200)) for y in range(100)), + 3, + extra=(chunk(b"PLTE", palette), chunk(b"tRNS", alpha)), + ), + ready(200, 100), + ), + ( + "png-gray-17x1000.png", + "image/png", + lambda: png(17, 1000, gradient(17, 1000, 1), 0), + ready(17, 1000), + ), + ( + "png-gray-alpha-640x480.png", + "image/png", + lambda: png(640, 480, gradient(640, 480, 2), 4), + ready(640, 480), + ), + ( + "png-gray16-300x200.png", + "image/png", + lambda: png(300, 200, gradient(600, 200, 1), 0, 16), + ready(300, 200), + ), + ( + "png-rgba16-64x64.png", + "image/png", + lambda: png(64, 64, gradient(512, 64, 1), 6, 16), + ready(64, 64), + ), + ("png-1x1.png", "image/png", lambda: png(1, 1, gradient(1, 1, 3)), ready(1, 1)), + ( + "png-wide-3000x2.png", + "image/png", + lambda: png(3000, 2, gradient(3000, 2, 3)), + ready(3000, 2), + ), + ( + "png-small-50x30.png", + "image/png", + lambda: png(50, 30, gradient(50, 30, 3)), + ready(50, 30), + ), + ("jpeg-rgb-1000x750.jpg", "image/jpeg", lambda: jpeg(1000, 750), ready(1000, 750)), + ("jpeg-gray-801x601.jpg", "image/jpeg", lambda: jpeg(801, 601, "gray"), ready(801, 601)), + ("jpeg-cmyk-400x300.jpg", "image/jpeg", lambda: jpeg(400, 300, "cmyk"), ready(400, 300)), + ( + "jpeg-exif6-400x200.jpg", + "image/jpeg", + lambda: jpeg(400, 200, orientation=6), + ready(400, 200, 6), + ), + ( + "jpeg-exif3-300x200.jpg", + "image/jpeg", + lambda: jpeg(300, 200, orientation=3), + ready(300, 200, 3), + ), + ("jpeg-rgb-4000x3000.jpg", "image/jpeg", lambda: jpeg(4000, 3000), ready(4000, 3000)), + ("png-exactly-10mib.png", "image/png", lambda: padded(small, MAX_BYTES), ready(64, 64)), + ( + "png-over-10mib.png", + "image/png", + lambda: padded(small, MAX_BYTES + 1), + {"upload": [413]}, + ), + ( + "png-over-pixels-8001x5000.png", + "image/png", + lambda: png(8001, 5000, blank(8001, 5000, 3), level=9), + {"upload": [413]}, + ), + ( + "jpeg-over-pixels-10000x5000.jpg", + "image/jpeg", + lambda: jpeg(10000, 5000, uniform=True), + {"upload": [413]}, + ), + ("png-truncated.png", "image/png", lambda: truncated_png[: idat + 600], REJECTED_OR_FAILED), + ("png-corrupt-idat.png", "image/png", lambda: bytes(corrupt), REJECTED_OR_FAILED), + ("jpeg-truncated.jpg", "image/jpeg", lambda: jpeg(1000, 750)[:3000], REJECTED_OR_FAILED), + ( + "png-zero-width.png", + "image/png", + zero_width, + {"upload": [400, 415]}, + ), + ( + "not-an-image.png", + "image/png", + lambda: b"plain text, not an image\n" * 8, + {"upload": [415]}, + ), + ("gif-declared-as-png.png", "image/png", lambda: GIF, {"upload": [415]}), + ] + if heavy: + items.append( + ( + "png-at-pixel-limit-8000x5000.png", + "image/png", + lambda: png(8000, 5000, blank(8000, 5000, 3), level=9), + ready(8000, 5000), + ) + ) + return items + + +def zero_width(): + header = struct.pack(">IIBBBBB", 0, 10, 8, 2, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(b"")) + + chunk(b"IEND", b"") + ) + + +def build(out, heavy=False, only=None): + out.mkdir(parents=True, exist_ok=True) + manifest = [] + for name, content_type, make, expect in catalog(heavy): + if only and name not in only: + continue + data = make() + (out / name).write_bytes(data) + manifest.append( + {"file": name, "content_type": content_type, "size_bytes": len(data), "expect": expect} + ) + (out / "manifest.json").write_text( + json.dumps( + {"rules": {"max_bytes": MAX_BYTES, "max_pixels": MAX_PIXELS}, "images": manifest}, + indent=2, + ) + + "\n" + ) + return manifest + + +def main(): + parser = argparse.ArgumentParser( + description="Generate lab5 test images and manifest.json with expected outcomes" + ) + parser.add_argument("--out", default=str(ROOT / "build/lab5-images")) + parser.add_argument( + "--heavy", action="store_true", help="also write a 40 Mpx image at the pixel limit" + ) + parser.add_argument("--only", default="", help="comma separated file names") + args = parser.parse_args() + manifest = build(Path(args.out), args.heavy, {x for x in args.only.split(",") if x}) + for item in manifest: + print(f"{item['file']:36} {item['size_bytes']:>10} {item['expect']['upload']}") + + +if __name__ == "__main__": + main() diff --git a/scripts/pipeline_evidence.py b/scripts/pipeline_evidence.py new file mode 100644 index 00000000..5ed7b9c3 --- /dev/null +++ b/scripts/pipeline_evidence.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, check + +COMPOSE = os.environ.get("COMPOSE", "docker compose").split() +PEEK = ( + 'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/' + 'rabbitmq_ops_password)" exec rabbitmqadmin -V messenger ' + "--table-style ascii get messages -q attachments.process -a ack_requeue_true" +) +QUEUES = "rabbitmqctl -q list_queues -p messenger name messages_ready messages_unacknowledged" + + +def compose(*argv): + done = subprocess.run( + [*COMPOSE, *argv], cwd=ROOT, capture_output=True, text=True, timeout=180, check=False + ) + check(done.returncode == 0, f"{' '.join(argv[:3])}: {done.stderr.strip()[-300:]}") + return done.stdout + + +def ready_messages(): + for line in compose("exec", "-T", "broker", *QUEUES.split()).splitlines(): + name, ready, _ = [*line.split("\t"), "", ""][:3] + if name == "attachments.process": + return int(ready) + return 0 + + +def say(line): + print(line, flush=True) + + +def stop(service): + began = time.monotonic() + compose("stop", "-t", "10", service) + seconds = time.monotonic() - began + container = compose("ps", "-aq", service).split()[0] + code = subprocess.run( + ["docker", "inspect", "-f", "{{.State.ExitCode}}", container], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + say(f"$ docker compose stop -t 10 {service} -> {seconds:.1f} s, exit code {code}") + check(seconds <= 10 and code == "0", f"{service} did not stop cleanly on SIGTERM") + + +def start(service): + compose("up", "-d", "--no-deps", "--wait", "--wait-timeout", "90", service) + say(f"$ docker compose up -d --no-deps --wait {service} -> healthy") + + +def main(): + parser = argparse.ArgumentParser(description="Lab5 stand evidence: queue, message, SIGTERM") + parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + args = parser.parse_args() + client = Client(args.base_url) + alice, bob, _, chat = client.bootstrap() + say(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}") + stop("worker") + attachment, _ = client.attachment(alice, chat) + say(f"upload -> 202 status={attachment['status']} id={attachment['id']}") + path = f"/api/v1/attachments/{attachment['id']}" + for variant in ("original", "thumbnail"): + client.call("GET", f"{path}/content?variant={variant}", 409, actor=bob) + say(f"GET content?variant={variant} while worker is stopped -> 409") + deadline = time.monotonic() + 30 + while ready_messages() < 1: + check(time.monotonic() < deadline, "the job did not reach the broker") + time.sleep(0.5) + say("$ rabbitmqctl list_queues -p messenger name messages_ready messages_unacknowledged") + say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip()) + say("$ rabbitmqadmin get messages -q attachments.process -a ack_requeue_true") + peeked = compose("exec", "-T", "broker", "sh", "-c", PEEK) + say(peeked.rstrip()) + body = json.loads(peeked[peeked.index('{"') : peeked.index("}", peeked.index('{"')) + 1]) + say(f"payload keys: {sorted(body)}") + check(body["attachment_id"] == attachment["id"], "the queued job is for another attachment") + stop("relay") + start("relay") + start("worker") + ready = client.wait_ready(bob, attachment) + names = [(v["name"], v["width"], v["height"]) for v in ready["variants"]] + say(f"after worker start -> status={ready['status']} variants={names}") + deadline = time.monotonic() + 30 + while ready_messages(): + check(time.monotonic() < deadline, "the queue did not drain") + time.sleep(0.5) + say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip()) + say("PASS: pipeline stand evidence") + + +if __name__ == "__main__": + main() diff --git a/scripts/pipeline_suite.py b/scripts/pipeline_suite.py new file mode 100644 index 00000000..0309fd3d --- /dev/null +++ b/scripts/pipeline_suite.py @@ -0,0 +1,945 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import os +import shlex +import struct +import subprocess +import sys +import time +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +import make_images +from client import Client, Failure, check +from image_probe import jpeg_size + +FIXTURE = ROOT / "tests/fixtures/sample.png" +EXCHANGE, ROUTING_KEY = "attachments", "attachment.process.v1" +QUEUE, DLQ = "attachments.process", "attachments.process.dlq" +RABBITMQADMIN = ( + 'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/' + 'rabbitmq_ops_password)" exec rabbitmqadmin --non-interactive -V messenger "$@"' +) +PSQL = 'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"' +ERRORS = (Failure, OSError, ValueError, KeyError, subprocess.TimeoutExpired) +SCENARIOS = ( + ("smoke_pipeline", 3, "upload, ready, variants match metadata"), + ("worker_down_broker_restart", 3, "worker down, API and broker restart"), + ("duplicate_after_ready", 3, "redelivery of a finished job"), + ("poison_image", 3, "permanent error without retries"), + ("s3_outage_recovers", 3, "transient error with retry"), + ("retry_exhausted", 3, "transient error until JOB_MAX_ATTEMPTS"), + ("image_matrix", 4, "limits and variants"), + ("crash_commit_publish", 4, "after COMMIT, before publish"), + ("crash_publish_confirm", 4, "after publish, before confirm"), + ("crash_after_s3_before_commit", 4, "after S3 write, before COMMIT"), + ("crash_after_commit_before_ack", 4, "after COMMIT, before ack"), + ("lease_expiry_fencing", 4, "stalled worker, lease expiry"), + ("replay_failed", 4, "operator replay"), + ("malformed_message_dead_lettered", 4, "message that is not a job goes to the DLQ"), + ("crash_before_side_effect", 5, "after claim, before side effects"), + ("worker_kill_mid_task", 5, "SIGKILL during processing"), + ("duplicate_in_flight", 5, "two copies of one job in flight"), + ("reorder_attempts", 5, "attempt 2 delivered before attempt 1"), + ("reorder_versions", 5, "v2 delivered before v1"), + ("stale_after_new_version", 5, "v1 job after v2 is published"), +) +QUERIES = """ +attachment: + SELECT status, published_version, error_code, staging_key FROM attachments WHERE id = :'id' + +jobs: + SELECT job_id, pipeline_version, status, attempt FROM jobs + WHERE attachment_id = :'id' ORDER BY pipeline_version + +outbox: + SELECT o.payload, o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) + WHERE j.attachment_id = :'id' ORDER BY o.id + +variants: + SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants + WHERE attachment_id = :'id' ORDER BY pipeline_version, name + +events: + SELECT kind, attempt FROM job_events WHERE attachment_id = :'id' ORDER BY id + +nonterminal: + SELECT id, status FROM attachments + WHERE id = ANY(:'ids'::uuid[]) AND status NOT IN ('ready', 'failed') + +known: + SELECT id FROM attachments WHERE id = ANY(:'ids'::uuid[]) + +duplicate_effects: + SELECT attachment_id, pipeline_version, name, count(*) AS copies FROM attachment_variants + WHERE attachment_id = ANY(:'ids'::uuid[]) GROUP BY 1, 2, 3 HAVING count(*) > 1 + +attempts: + SELECT job_id, attempt FROM jobs WHERE attachment_id = ANY(:'ids'::uuid[]) AND attempt > :max + +pending_outbox: + SELECT o.job_id FROM outbox o JOIN jobs j USING (job_id) + WHERE j.attachment_id = ANY(:'ids'::uuid[]) AND o.published_at IS NULL + +unpublished_version: + SELECT a.id, a.published_version, max(j.pipeline_version) AS done FROM attachments a + JOIN jobs j ON j.attachment_id = a.id AND j.status = 'done' WHERE a.id = ANY(:'ids'::uuid[]) + GROUP BY a.id, a.published_version HAVING a.published_version <> max(j.pipeline_version) + +staging_left: + SELECT id FROM attachments + WHERE id = ANY(:'ids'::uuid[]) AND status = 'ready' AND staging_key IS NOT NULL + +object_keys: + SELECT object_key AS key FROM attachment_variants + UNION ALL SELECT original_key FROM attachments WHERE original_key IS NOT NULL + UNION ALL SELECT staging_key FROM attachments WHERE staging_key IS NOT NULL + +columns: + SELECT table_name, column_name FROM information_schema.columns + WHERE table_schema = current_schema() +""" +SQL = { + name.strip(): " ".join(body.split()) + for name, body in (block.split(":\n", 1) for block in QUERIES.strip().split("\n\n")) +} +REQUIRED_COLUMNS = { + "attachments": "id status published_version error_code staging_key original_key", + "attachment_variants": "attachment_id pipeline_version name sha256 size_bytes object_key", + "jobs": "job_id attachment_id pipeline_version status attempt", + "outbox": "job_id payload published_at", + "job_events": "job_id attachment_id kind attempt", +} + + +def now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def run(cmd, stdin=None, env=None, timeout=180): + done = subprocess.run( + cmd, input=stdin, capture_output=True, text=True, env=env, timeout=timeout, check=False + ) + if done.returncode: + raise Failure(f"{' '.join(cmd[:7])}: exit {done.returncode}: {done.stderr.strip()[-400:]}") + return done.stdout + + +def poll(probe, timeout, what, interval=0.5): + deadline = time.monotonic() + timeout + while True: + value = probe() + if value: + return value + if time.monotonic() > deadline: + raise Failure(f"timeout {timeout}s waiting for {what}") + time.sleep(interval) + + +def http_ready(base_url): + try: + with urllib.request.urlopen(base_url.rstrip("/") + "/health/ready", timeout=3) as response: + return response.status == 200 + except OSError: + return False + + +def image_size(data, content_type): + if content_type == "image/jpeg": + return jpeg_size(data) + if content_type == "image/png": + return struct.unpack(">II", data[16:24]) + check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP") + if data[12:16] == b"VP8 ": + width, height = struct.unpack("> 14) & 0x3FFF) + 1 + return int.from_bytes(data[24:27], "little") + 1, int.from_bytes(data[27:30], "little") + 1 + + +class Stack: + def __init__(self, args): + self.args = args + self.base = shlex.split(args.compose) + self.faults = shlex.split(args.compose_faults) + + def dc(self, *argv, faults=False, stdin=None, env=None, timeout=180): + command = [*(self.faults if faults else self.base), *argv] + return run(command, stdin, dict(os.environ, **(env or {})), timeout) + + def up(self, *services, faults=False, env=None): + wait = ["up", "-d", "--no-deps", "--no-recreate", "--wait", "--wait-timeout", "90"] + if faults or env: + wait.remove("--no-recreate") + self.dc(*wait, *services, faults=faults, env=env) + + def launch(self, service, point=""): + variable = "WORKER_FAULT_POINT" if service == self.args.worker else "RELAY_FAULT_POINT" + self.up(service, faults=True, env={variable: point}) + + def restore(self): + self.dc("up", "-d", "--no-deps", "--wait", self.args.relay, self.args.worker) + + def stop(self, *services): + self.dc("stop", "-t", "10", *services) + + def kill(self, service): + self.dc("kill", "-s", "KILL", service) + + def restart(self, *services): + self.dc("restart", "-t", "20", *services) + self.up(*services) + + def state(self, service): + text = self.dc("ps", "-a", "--format", "json", service).strip() + rows = ( + json.loads(text) if text.startswith("[") else [json.loads(x) for x in text.split("\n")] + ) + check(len(rows) == 1, "expected one container for " + service) + return rows[0]["State"], rows[0].get("ExitCode") + + def wait_exit(self, service, timeout=60): + def probe(): + state, code = self.state(service) + return (code,) if state in ("exited", "dead") else None + + return poll(probe, timeout, service + " exit")[0] + + def sql(self, name, **params): + query = f"SELECT coalesce(json_agg(q), '[]'::json) FROM ({SQL[name]}) q;\n" + variables = [x for key, value in params.items() for x in ("-v", f"{key}={value}")] + out = self.dc( + "exec", "-T", self.args.postgres, "sh", "-c", PSQL, "sh", *variables, stdin=query + ) + return json.loads(out) + + def queues(self): + columns = "name messages_ready messages_unacknowledged --formatter json" + listing = f"exec -T {self.args.broker} rabbitmqctl -q list_queues -p messenger {columns}" + rows = json.loads(self.dc(*listing.split())) + return {q["name"]: (q["messages_ready"], q["messages_unacknowledged"]) for q in rows} + + def rabbitmqadmin(self, *argv, stdin=None): + return self.dc( + "exec", "-T", self.args.broker, "sh", "-c", RABBITMQADMIN, "sh", *argv, stdin=stdin + ) + + def publish(self, payload): + properties = {"delivery_mode": 2, "content_type": "application/json", "type": ROUTING_KEY} + properties["message_id"] = payload["job_id"] + command = f"publish message -e {EXCHANGE} -k {ROUTING_KEY} -f - -p".split() + out = self.rabbitmqadmin(*command, json.dumps(properties), stdin=json.dumps(payload)) + check("routed successfully" in out, "broker did not route the injected job") + + def admin(self, *argv): + command = shlex.split(self.args.admin) + list(argv) + out = self.dc("run", "--rm", "--no-deps", "-T", self.args.worker, *command) + return json.loads(out.strip().splitlines()[-1]) + + +class Api: + def __init__(self, args): + self.base_url = args.base_url + self.client = Client(args.base_url, args.ca_file) + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.accepted, self.traces, self.final = [], {}, {} + + def upload(self, data, kind="image/png", name="sample.png", expected=202, final=("ready",)): + path = f"/api/v1/chats/{self.chat['id']}/attachments?filename={quote(name)}" + allowed = [expected] if isinstance(expected, int) else expected + headers = {"Content-Type": kind} + response = self.client.call( + "POST", path, allowed, actor=self.alice, raw=data, headers=headers, schema="Attachment" + ) + if response["status"] != 202: + return None + body = response["json"] + snapshot = ( + body["status"] == "queued" and body["variants"] == [] and body["error_code"] is None + ) + check(snapshot, "202 is not a queued snapshot") + same = body["sha256"] == hashlib.sha256(data).hexdigest() and body["size_bytes"] == len( + data + ) + check(same, "202 checksum or size differs from the uploaded bytes") + self.accepted.append(body["id"]) + self.final[body["id"]] = set(final) + self.traces[body["id"]] = ["queued"] + return body + + def fixture(self, **kwargs): + return self.upload(FIXTURE.read_bytes(), **kwargs) + + def get(self, ident): + path = "/api/v1/attachments/" + ident + body = self.client.call("GET", path, actor=self.bob, schema="Attachment")["json"] + self.traces.setdefault(ident, []).append(body["status"]) + return body + + def wait(self, ident, final=("ready",), timeout=90): + def probe(): + body = self.get(ident) + failed = body["status"] == "failed" and "failed" not in final + check(not failed, f"attachment reached failed: {body['error_code']}") + return body if body["status"] in final else None + + return poll(probe, timeout, f"{ident} {'/'.join(final)}", 0.3) + + def content(self, ident, variant=None, expected=200): + path = f"/api/v1/attachments/{ident}/content" + (f"?variant={variant}" if variant else "") + return self.client.call("GET", path, expected, actor=self.bob) + + def message(self, text, attachments=None, expected=201): + if expected == 201: + return self.client.message(self.alice, self.chat, text, attachments=attachments)["json"] + body = {"text": text, "attachment_ids": attachments or []} + headers = {"Idempotency-Key": os.urandom(8).hex()} + path = f"/api/v1/chats/{self.chat['id']}/messages" + return self.client.call( + "POST", path, expected, actor=self.alice, body=body, headers=headers + ) + + +class Suite: + def __init__(self, args): + self.args = args + self.stack = Stack(args) + self.api = None + self.samples = {} + self.dead_letters = 0 + + def scenarios(self): + return [(name, level, window, getattr(self, name)) for name, level, window in SCENARIOS] + + def drained(self, timeout=60): + poll(lambda: self.stack.queues()[QUEUE] == (0, 0), timeout, "main queue drained") + + def queued(self, count, timeout=30): + poll(lambda: self.stack.queues()[QUEUE][0] >= count, timeout, f"{count} ready messages") + + def record(self, ident): + row = self.stack.sql("attachment", id=ident)[0] + self.samples.setdefault(ident, []).append(row["published_version"]) + self.api.traces.setdefault(ident, []).append(row["status"]) + return row + + def snapshot(self, ident): + body = self.api.get(ident) + public = sorted(body["variants"], key=lambda v: v["name"]) + published = self.record(ident)["published_version"] + rows = self.stack.sql("variants", id=ident) + return {"status": body["status"], "public": public, "rows": rows, "published": published} + + def payload(self, ident, version=1): + rows = self.stack.sql("outbox", id=ident) + found = [r["payload"] for r in rows if r["payload"]["pipeline_version"] == version] + check(bool(found), f"no outbox payload for v{version}") + return found[-1] + + def verify(self, ready, expect=None): + names = sorted(v["name"] for v in ready["variants"]) + check(len(names) == len(set(names)), "duplicate variant names in API") + missing = set(self.args.expect_variants.split(",")) - set(names) + check(not missing, f"missing variants {sorted(missing)}") + original = self.api.content(ready["id"]) + data = original["bytes"] + same = ( + hashlib.sha256(data).hexdigest() == ready["sha256"] and len(data) == ready["size_bytes"] + ) + check(same, "original bytes differ from metadata") + check( + original["headers"].get_content_type() == ready["content_type"], "original type differs" + ) + for variant in ready["variants"]: + name = variant["name"] + got = self.api.content(ready["id"], name) + data, kind = got["bytes"], got["headers"].get_content_type() + check(kind == variant["content_type"], name + " content type differs") + same = len(data) == variant["size_bytes"] + check( + same and hashlib.sha256(data).hexdigest() == variant["sha256"], + name + " bytes differ", + ) + size = (variant["width"], variant["height"]) + check(tuple(image_size(data, kind)) == size, name + " real size differs from metadata") + wanted = (expect or {}).get(name) + slack = 1 if name == "optimized" else 0 + fits = not wanted or all(abs(a - b) <= slack for a, b in zip(wanted, size)) + check(fits, f"{name} is {size}, expected {wanted}") + return names + + def crash(self, service, point, observe): + self.drained() + self.stack.launch(service, point) + ident = self.api.fixture()["id"] + code = self.stack.wait_exit(service) + check(code == 137, f"{service} exited with {code}, expected 137 at {point}") + seen = observe(ident) + self.stack.launch(service) + ready = self.api.wait(ident, timeout=self.args.recovery_timeout) + poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox") + self.drained() + self.verify(ready) + variants = self.stack.sql("variants", id=ident) + return { + "attachment_id": ident, + "exit_code": code, + "after_crash": seen, + "variants": variants, + } + + def smoke_pipeline(self): + att = self.api.fixture() + ready = self.api.wait(att["id"]) + return { + "attachment_id": att["id"], + "variants": self.verify(ready, {"thumbnail": [256, 160]}), + } + + def worker_down_broker_restart(self): + steps = [] + + def mark(step, **extra): + steps.append({"at": now(), "step": step, **extra}) + + self.stack.stop(self.args.worker) + mark("worker stopped") + att = self.api.fixture() + mark("upload 202 queued", attachment_id=att["id"]) + self.api.content(att["id"], expected=409) + mark("content 409 while queued") + message = self.api.message("", [att["id"]]) + self.api.message("text messages still work") + path = f"/api/v1/chats/{self.api.chat['id']}/messages?limit=5" + self.api.client.call("GET", path, actor=self.api.bob, schema="MessagePage") + mark("message with attachment 201, other messages work", message_id=message["id"]) + self.queued(1) + mark("job is in the broker queue", queues=self.stack.queues()) + self.stack.restart(*self.args.api, self.args.relay) + poll(lambda: http_ready(self.api.base_url), 90, "API ready") + mark("API replicas and relay restarted") + self.stack.restart(self.args.broker) + self.queued(1) + mark("broker restarted with its volume, job survived", queues=self.stack.queues()) + self.stack.launch(self.args.worker) + mark("worker started") + ready = self.api.wait(att["id"]) + mark("ready", variants=self.verify(ready)) + return {"attachment_id": att["id"], "timeline": steps} + + def duplicate_after_ready(self): + ident = self.api.fixture()["id"] + self.api.wait(ident) + self.drained() + before = self.snapshot(ident) + self.stack.publish(self.payload(ident)) + self.drained() + time.sleep(1) + check(self.snapshot(ident) == before, "duplicate delivery changed the published result") + return {"attachment_id": ident, "events": self.stack.sql("events", id=ident)} + + def poison_image(self): + data = make_images.png(320, 200, make_images.gradient(320, 200, 3)) + broken = data[: data.index(b"IDAT") + 600] + att = self.api.upload(broken, expected=[202, 415], final=("failed",)) + if att is None: + return {"rejected_by_api": 415} + failed = self.api.wait(att["id"], ("failed",)) + check( + failed["error_code"] == "invalid_image", f"unexpected error_code {failed['error_code']}" + ) + jobs = self.stack.sql("jobs", id=att["id"]) + check(all(j["attempt"] == 1 for j in jobs), "permanent error was retried") + self.api.content(att["id"], expected=409) + self.api.message("", [att["id"]], expected=409) + return {"attachment_id": att["id"], "error_code": failed["error_code"], "jobs": jobs} + + def s3_outage_recovers(self): + self.stack.stop(self.args.worker) + ident = self.api.fixture()["id"] + self.queued(1) + self.stack.stop(self.args.s3) + try: + self.stack.launch(self.args.worker) + poll( + lambda: any(j["attempt"] > 1 for j in self.stack.sql("jobs", id=ident)), 60, "retry" + ) + status = self.record(ident)["status"] + check(status in ("queued", "processing"), "transient error produced " + status) + finally: + self.stack.up(self.args.s3) + poll(lambda: http_ready(self.api.base_url), 90, "API ready") + self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout)) + return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)} + + def exhaust(self): + self.drained() + self.stack.launch(self.args.worker, "transient_error") + ident = self.api.fixture(final=("failed", "ready"))["id"] + failed = self.api.wait(ident, ("failed",), timeout=self.args.recovery_timeout) + self.stack.launch(self.args.worker) + jobs = self.stack.sql("jobs", id=ident) + check(failed["error_code"] == "retry_exhausted", f"unexpected {failed['error_code']}") + check(max(j["attempt"] for j in jobs) == self.args.max_attempts, "attempts != max") + return ident, jobs + + def retry_exhausted(self): + ident, jobs = self.exhaust() + self.api.final[ident] = {"failed"} + self.api.content(ident, expected=409) + self.api.message("", [ident], expected=409) + return {"attachment_id": ident, "jobs": jobs} + + def replay_failed(self): + ident, _ = self.exhaust() + self.api.final[ident] = {"ready"} + first = self.stack.admin("replay", "--attachment-id", ident) + self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout)) + self.drained() + before = self.snapshot(ident) + second = self.stack.admin("replay", "--attachment-id", ident) + self.drained() + check(self.snapshot(ident) == before, "replay of a ready attachment changed the result") + events = self.stack.sql("events", id=ident) + check(any("replay" in e["kind"] for e in events), "replay left no audit event") + return { + "attachment_id": ident, + "replay": first, + "replay_of_ready": second, + "events": events, + } + + def malformed_message_dead_lettered(self): + before = self.stack.queues()[DLQ][0] + self.stack.publish({"schema_version": 2, "job_id": str(uuid.uuid4())}) + poll(lambda: self.stack.queues()[DLQ][0] > before, 30, "dead letter") + self.dead_letters += 1 + return {"dead_letters": self.stack.queues()[DLQ][0] - before} + + def image_matrix(self): + folder = Path(self.args.images) + results = [] + for item in make_images.build(folder, self.args.heavy): + expect, name = item["expect"], item["file"] + ready = expect.get("final") == "ready" + data = (folder / name).read_bytes() + final = ("ready",) if ready else ("failed", "ready") + att = self.api.upload(data, item["content_type"], name, expect["upload"], final) + row = {"file": name, "http": 202 if att else "rejected"} + if att: + done = self.api.wait(att["id"], ("ready", "failed"), self.args.recovery_timeout) + row.update(status=done["status"], error_code=done["error_code"]) + if ready: + check(done["status"] == "ready", name + " did not become ready") + row["variants"] = self.verify(done, expect["variants"]) + else: + self.api.final[att["id"]] = {"failed"} + code = expect.get("error_code") + check(done["error_code"] == code, f"{name} should fail with {code}") + results.append(row) + return {"images": results} + + def crash_commit_publish(self): + def observe(ident): + outbox = self.stack.sql("outbox", id=ident) + check(outbox and not any(r["published"] for r in outbox), "outbox already published") + check(self.stack.queues()[QUEUE] == (0, 0), "job reached the broker before publish") + return {"outbox_published": False, "status": self.record(ident)["status"]} + + return self.crash(self.args.relay, "after_commit_before_publish", observe) + + def crash_publish_confirm(self): + def observe(ident): + outbox = [r["published"] for r in self.stack.sql("outbox", id=ident)] + return {"outbox_published": outbox, "queues": self.stack.queues()} + + return self.crash(self.args.relay, "after_publish_before_confirm", observe) + + def crash_after_s3_before_commit(self): + def observe(ident): + check(not self.stack.sql("variants", id=ident), "variants committed before the crash") + return {"status": self.record(ident)["status"], "variant_rows": 0} + + return self.crash(self.args.worker, "after_s3_write_before_commit", observe) + + def crash_after_commit_before_ack(self): + def observe(ident): + row = self.record(ident) + check(row["status"] == "ready", "result was not committed before the crash point") + variants = self.stack.sql("variants", id=ident) + return {"status": row["status"], "variants": variants, "queues": self.stack.queues()} + + result = self.crash(self.args.worker, "after_commit_before_ack", observe) + same = result["variants"] == result["after_crash"]["variants"] + check(same, "redelivery after COMMIT changed variants") + return result + + def crash_before_side_effect(self): + def observe(ident): + return { + "status": self.record(ident)["status"], + "jobs": self.stack.sql("jobs", id=ident), + } + + return self.crash(self.args.worker, "after_claim_before_side_effect", observe) + + def lease_expiry_fencing(self): + self.drained() + self.stack.launch(self.args.worker, "stall_after_claim") + ident = self.api.fixture()["id"] + self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout)) + before = self.snapshot(ident) + time.sleep(self.args.stall_seconds + 3) + after = self.snapshot(ident) + self.stack.launch(self.args.worker) + check(before == after, "stalled worker overwrote the result after its lease expired") + jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident) + return {"attachment_id": ident, "jobs": jobs, "events": events} + + def worker_kill_mid_task(self): + self.drained() + self.stack.launch(self.args.worker, "stall_after_claim") + ident = self.api.fixture()["id"] + poll(lambda: self.record(ident)["status"] == "processing", 30, "processing") + self.stack.kill(self.args.worker) + code = self.stack.wait_exit(self.args.worker) + self.stack.launch(self.args.worker) + ready = self.api.wait(ident, timeout=self.args.recovery_timeout) + self.drained() + self.verify(ready) + return {"attachment_id": ident, "exit_code": code, "jobs": self.stack.sql("jobs", id=ident)} + + def duplicate_in_flight(self): + self.stack.stop(self.args.worker) + ident = self.api.fixture()["id"] + self.queued(1) + self.stack.publish(self.payload(ident)) + self.queued(2) + self.stack.launch(self.args.worker) + ready = self.api.wait(ident) + self.drained() + self.verify(ready) + variants, events = self.stack.sql("variants", id=ident), self.stack.sql("events", id=ident) + return {"attachment_id": ident, "variants": variants, "events": events} + + def reorder_attempts(self): + self.stack.stop(self.args.worker, self.args.relay) + ident = self.api.fixture()["id"] + original = self.payload(ident) + self.stack.publish({**original, "attempt": original["attempt"] + 1}) + self.queued(1) + self.stack.launch(self.args.relay) + self.queued(2) + self.stack.launch(self.args.worker) + ready = self.api.wait(ident) + self.drained() + self.verify(ready) + jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident) + return {"attachment_id": ident, "jobs": jobs, "events": events} + + def published(self, ident, version): + poll( + lambda: self.record(ident)["published_version"] >= version, + self.args.recovery_timeout, + f"published v{version}", + ) + + def reprocess(self, ident): + return self.stack.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2") + + def reorder_versions(self): + self.stack.stop(self.args.worker, self.args.relay) + ident = self.api.fixture()["id"] + self.reprocess(ident) + self.stack.publish(self.payload(ident, 2)) + self.queued(1) + self.stack.launch(self.args.relay) + self.queued(3) + self.stack.launch(self.args.worker) + self.published(ident, 2) + self.api.wait(ident) + self.drained() + final = self.snapshot(ident) + check(final["published"] == 2, f"published_version rolled back to {final['published']}") + served = {v["name"]: v["sha256"] for v in final["public"]} + stored = {r["name"]: r["sha256"] for r in final["rows"] if r["pipeline_version"] == 2} + check(served == stored, "API serves variants of a version that is not published") + self.verify(self.api.get(ident)) + return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)} + + def stale_after_new_version(self): + ident = self.api.fixture()["id"] + self.api.wait(ident) + self.reprocess(ident) + self.published(ident, 2) + self.drained() + before = self.snapshot(ident) + self.stack.publish(self.payload(ident, 1)) + self.drained() + time.sleep(1) + after = self.snapshot(ident) + check(before == after and after["published"] == 2, "stale v1 job changed the v2 result") + return {"attachment_id": ident, "published_version": after["published"]} + + def invariants(self, dead_before): + ids = self.api.accepted + array = "{" + ",".join(ids) + "}" + if not ids: + return [{"id": "accepted_work", "result": "FAIL", "details": "no upload got 202"}] + + def empty(name, message, **params): + rows = self.stack.sql(name, ids=array, **params) + check(not rows, f"{message}: {rows[:5]}") + return {"rows": 0} + + def terminal(): + empty("nonterminal", "non-terminal attachments left") + wrong = [i for i in ids if self.api.get(i)["status"] not in self.api.final[i]] + check(not wrong, "unexpected final status: " + ", ".join(wrong)) + return {"accepted": len(ids)} + + def no_rollback(): + traces = self.api.traces.items() + broken = [ + i for i, t in traces if "ready" in t and set(t[t.index("ready") :]) != {"ready"} + ] + check(not broken, "ready rolled back: " + ", ".join(broken)) + return {"traced": len(self.api.traces)} + + def monotonic(): + broken = [i for i, values in self.samples.items() if values != sorted(values)] + check(not broken, "published_version went backwards: " + ", ".join(broken)) + empty("unpublished_version", "published_version differs from the newest done job") + return {"sampled": len(self.samples)} + + def served(): + bodies = [self.api.get(i) for i in ids] + ready = [b for b in bodies if b["status"] == "ready"] + for body in ready: + self.verify(body) + return {"verified": len(ready)} + + def orphans(): + empty("pending_outbox", "unpublished outbox rows") + empty("staging_left", "staging kept after ready") + queues = self.stack.queues() + check(queues[QUEUE] == (0, 0), "messages left in " + QUEUE) + expected = dead_before + self.dead_letters + check(queues[DLQ][0] == expected, f"unexpected dead letters: {queues[DLQ]}") + if not self.args.s3_list: + return {"queues": queues, "s3": "SKIP: --s3-list is not set"} + keys = {r["key"] for r in self.stack.sql("object_keys")} + listing = run(shlex.split(self.args.s3_list)).splitlines() + stray = [k for k in (x.split()[-1] for x in listing if x.strip()) if k not in keys] + check(not stray, f"S3 objects without metadata: {stray[:5]}") + return {"queues": queues, "s3_objects": len(listing)} + + def no_loss(): + known = {r["id"] for r in self.stack.sql("known", ids=array)} + check(known == set(ids), "accepted attachments are missing in PostgreSQL") + return {"accepted": len(ids)} + + claims = [ + ("terminal", "every 202 reaches its expected terminal state", terminal), + ( + "single_effect", + "one variant set per (attachment_id, pipeline_version, variant)", + lambda: empty("duplicate_effects", "duplicate effect rows"), + ), + ("no_ready_rollback", "ready never returns to queued or processing", no_rollback), + ("monotonic_publication", "published_version only moves forward", monotonic), + ("bytes_match_metadata", "sha256, size and real dimensions match served bytes", served), + ( + "bounded_attempts", + "attempt <= JOB_MAX_ATTEMPTS", + lambda: empty("attempts", "attempt above limit", max=self.args.max_attempts), + ), + ("no_orphans", "no pending outbox, messages, dead letters, staging, objects", orphans), + ("no_lost_202", "every 202 is present in PostgreSQL and the API", no_loss), + ] + return [self.claim(name, statement, probe) for name, statement, probe in claims] + + @staticmethod + def claim(name, statement, probe): + try: + return {"id": name, "statement": statement, "result": "PASS", "details": probe()} + except ERRORS as error: + return {"id": name, "statement": statement, "result": "FAIL", "details": str(error)} + + def preflight(self): + def services(): + self.detect_api() + present = set(self.stack.dc("config", "--services").split()) + wanted = {*self.args.api, self.args.relay, self.args.worker, self.args.broker} + wanted |= {self.args.postgres, self.args.s3} + check(wanted <= present, f"missing services {sorted(wanted - present)}") + return sorted(wanted) + + def schema(): + columns = {} + for row in self.stack.sql("columns"): + columns.setdefault(row["table_name"], set()).add(row["column_name"]) + needed = {t: set(c.split()) for t, c in REQUIRED_COLUMNS.items()} + missing = {t: sorted(c - columns.get(t, set())) for t, c in needed.items()} + check(not any(missing.values()), f"missing columns {missing}") + return sorted(needed) + + def broker(): + queues = self.stack.queues() + check({QUEUE, DLQ} <= set(queues), "queues are not declared") + check(QUEUE in self.stack.rabbitmqadmin("list", "queues"), "rabbitmqadmin failed") + return queues + + def faults(): + check("FAULT_POINT" in self.stack.dc("config", faults=True), "no FAULT_POINT in faults") + return self.args.compose_faults + + def api(): + check(http_ready(self.args.base_url), "GET /health/ready is not 200") + return self.args.base_url + + steps = [ + ("compose services", services), + ("api ready", api), + ("postgres schema", schema), + ("broker topology and ops CLI", broker), + ("faults override", faults), + ] + return [self.claim(name, name, probe) for name, probe in steps] + + def main(self): + report = { + "suite": "lab5-pipeline-idempotency", + "started_at": now(), + "config": self.config(), + } + if self.args.preflight: + report["preflight"] = self.preflight() + return self.finish(report, report["preflight"]) + results = [] + try: + self.detect_api() + self.api = Api(self.args) + dead_before = self.stack.queues()[DLQ][0] + self.stack.launch(self.args.relay) + self.stack.launch(self.args.worker) + for name, level, window, scenario in self.scenarios(): + if self.selected(name, level): + results.append(self.attempt(name, level, window, scenario)) + report["invariants"] = self.invariants(dead_before) + except ERRORS as error: + report["error"] = str(error) + finally: + self.stack.up(self.args.s3) + self.stack.restore() + report["scenarios"] = results + return self.finish(report, results + report.get("invariants", [])) + + def attempt(self, name, level, window, scenario): + began = time.monotonic() + print("RUN " + name, flush=True) + try: + outcome = {"result": "PASS", "observed": scenario()} + except ERRORS as error: + outcome = {"result": "FAIL", "error": str(error)} + self.heal() + print(f"{outcome['result']} {name}", flush=True) + seconds = round(time.monotonic() - began, 1) + return {"name": name, "level": level, "window": window, "seconds": seconds, **outcome} + + def heal(self): + try: + self.stack.up(self.args.s3, self.args.broker) + self.stack.launch(self.args.relay) + self.stack.launch(self.args.worker) + except ERRORS as error: + print("heal failed: " + str(error), flush=True) + + def detect_api(self): + if not self.args.api: + names = self.stack.dc("config", "--services").split() + api = [x for x in names if x == "api" or (x[:4] == "api-" and x[4:].isdigit())] + self.args.api = api + check(bool(self.args.api), "no API service: expected api or api-N, or pass --api") + + def selected(self, name, level): + return name in self.args.only.split(",") if self.args.only else level <= self.args.level + + def config(self): + keys = ( + "base_url level only expect_variants max_attempts stall_seconds compose compose_faults" + ) + return {key: getattr(self.args, key) for key in keys.split()} + + def finish(self, report, verdicts): + passed = bool(verdicts) and "error" not in report + report["result"] = ( + "PASS" if passed and all(v["result"] == "PASS" for v in verdicts) else "FAIL" + ) + report["finished_at"] = now() + text = json.dumps(report, ensure_ascii=False, indent=2) + "\n" + if self.args.output: + Path(self.args.output).parent.mkdir(parents=True, exist_ok=True) + Path(self.args.output).write_text(text) + else: + print(text) + print(f"{report['result']}: lab5 pipeline suite, report {self.args.output or 'stdout'}") + return 0 if report["result"] == "PASS" else 1 + + +def arguments(): + env = os.environ.get + faults = "docker compose -f compose.yaml -f infra/compose.faults.yaml" + parser = argparse.ArgumentParser(description="Lab5 pipeline fault injection with invariants") + parser.add_argument("--base-url", default=env("BASE_URL", "http://localhost:8080")) + parser.add_argument("--ca-file", default=env("CA_FILE")) + parser.add_argument("--compose", default=env("COMPOSE", "docker compose")) + parser.add_argument("--compose-faults", default=env("COMPOSE_FAULTS", faults)) + parser.add_argument("--admin", default=env("ADMIN_CLI", "python -m messenger.worker.admin")) + parser.add_argument("--api", nargs="+", default=env("API_SERVICES", "").split()) + parser.add_argument("--worker", default="worker") + parser.add_argument("--relay", default="relay") + parser.add_argument("--broker", default="broker") + parser.add_argument("--postgres", default="postgres") + parser.add_argument("--s3", default="s3") + parser.add_argument("--s3-list", default=env("S3_LIST_CMD", ""), help="prints object keys") + parser.add_argument("--level", type=int, choices=[3, 4, 5], default=5) + parser.add_argument("--only", default="", help="comma separated scenario names") + parser.add_argument("--list", action="store_true") + parser.add_argument("--preflight", action="store_true") + parser.add_argument("--expect-variants", default=None) + parser.add_argument("--max-attempts", type=int, default=int(env("JOB_MAX_ATTEMPTS", "5"))) + parser.add_argument( + "--stall-seconds", type=float, default=float(env("FAULT_STALL_SECONDS", "15")) + ) + parser.add_argument("--recovery-timeout", type=float, default=120) + parser.add_argument("--images", default=str(ROOT / "build/lab5-images")) + parser.add_argument("--heavy", action="store_true") + parser.add_argument("--output", default="") + args = parser.parse_args() + if args.expect_variants is None: + args.expect_variants = "thumbnail" if args.level == 3 else "thumbnail,crop,optimized" + return args + + +def main(): + suite = Suite(arguments()) + if suite.args.list: + for name, level, window, _ in suite.scenarios(): + print(f"{name:32} level {level} {window}") + return 0 + return suite.main() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/stack/test_attachment_faults.py b/student-tests/stack/test_attachment_faults.py index b4270ec0..be9af4b7 100644 --- a/student-tests/stack/test_attachment_faults.py +++ b/student-tests/stack/test_attachment_faults.py @@ -37,7 +37,10 @@ HEAD = ( FAULTS = """ CREATE TABLE lab4_fault_seen (id uuid PRIMARY KEY, object_key text, filename text); CREATE FUNCTION lab4_fault_seen() RETURNS trigger LANGUAGE plpgsql AS $$ -BEGIN INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.object_key, NEW.filename); RETURN NEW; END $$; +BEGIN + INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.staging_key, NEW.filename); + RETURN NEW; +END $$; CREATE FUNCTION lab4_fault_commit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename LIKE 'fault-fail-%' THEN @@ -61,7 +64,7 @@ END $$; CREATE TRIGGER lab4_fault_seen AFTER INSERT ON attachments FOR EACH ROW WHEN (NEW.filename LIKE 'fault-%') EXECUTE FUNCTION lab4_fault_seen(); CREATE TRIGGER lab4_fault_commit BEFORE UPDATE ON attachments FOR EACH ROW - WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'ready') + WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'queued') EXECUTE FUNCTION lab4_fault_commit(); """ CLEANUP = """ @@ -94,7 +97,7 @@ def bucket_keys() -> set[str]: "list", "--recursive", "--json", - f"root/{BUCKET}/attachments/", + f"root/{BUCKET}/staging/", ) return {item["key"] for item in json.loads(listing)["items"]} @@ -221,7 +224,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None: created = 0 attachment = await upload(http, alice, chat) exact = await upload(http, alice, chat, png(16, 16, size=LIMIT)) - assert (attachment.status_code, exact.status_code) == (201, 201) + assert (attachment.status_code, exact.status_code) == (202, 202) assert exact.json()["size_bytes"] == LIMIT created += 2 mine = attachment.json()["id"] @@ -319,9 +322,9 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None: 400, ) expect( - "variant=thumbnail", + "variant=preview", await http.get( - f"/api/v1/attachments/{mine}/content?variant=thumbnail", headers=bearer(alice) + f"/api/v1/attachments/{mine}/content?variant=preview", headers=bearer(alice) ), 400, ) @@ -332,7 +335,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None: ) statuses = [entry["status"] for entry in logs(since) if entry.get("event") == "request.handled"] print( - f"строк вложений в чате: {rows} (= ответов 201); ответов в журнале реплик: " + f"строк вложений в чате: {rows} (= ответов 202); ответов в журнале реплик: " f"{len(statuses)}, из них 5xx: {sum(code >= 500 for code in statuses)}" ) assert not [status for status in statuses if status >= 500] @@ -375,7 +378,7 @@ async def crash(http: AsyncClient, token: str, chat: str) -> tuple[str, str]: name = f"fault-crash-{uuid4().hex}.png" sent = asyncio.create_task(upload(http, token, chat, filename=name)) address = await asyncio.to_thread(sql, "SELECT lab4_fault_sleeper()") - assert address, "UPDATE ready не дошёл до триггера" + assert address, "UPDATE queued не дошёл до триггера" victim = replica_at(address) kill(victim) try: @@ -396,6 +399,7 @@ async def test_crash_between_put_and_commit_is_swept_after_grace( s = await scene(http) alice, chat = s["alice"], s["chats"][0] kept = (await upload(http, alice, chat, filename="kept.png")).json() + await waited(lambda: status_of(kept["id"]) == "ready", 60) assert ( await message(http, alice, chat, {"text": "", "attachment_ids": [kept["id"]]}) ).status_code == 201 @@ -465,4 +469,4 @@ async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, fault assert (status, body["error"]["code"]) == (503, "unavailable") assert elapsed <= REQUEST_TIMEOUT + 2 assert seen(name) is None - assert (await upload(http, s["alice"], s["chats"][0])).status_code == 201 + assert (await upload(http, s["alice"], s["chats"][0])).status_code == 202