Add the level 3 pipeline scenarios and the stand evidence

Bring the stdlib suite and image generator of the lab 5 preparation:
make suite-preflight checks services, readiness, schema columns, broker
topology and the faults override, and make level3 runs the grade 3
scenarios (smoke through the pipeline, worker down with API, relay and
broker restarts, duplicate delivery after ready, poison image, S3
outage with recovery, retries up to JOB_MAX_ATTEMPTS) and checks the
invariants into evidence/level3.json. make pipeline-evidence shows the
queued job on the broker while the worker is stopped, its properties
and contract payload, the 409 before ready and SIGTERM of the relay and
the worker; make topology dumps queues, consumers with prefetch,
permissions and users.

The lab 4 failover and S3 outage scenarios and the upload fault suite
now expect 202 and wait for ready, the stack evidence lists the broker
and its client packages, and the CI acceptance job runs the pipeline
scenarios and the SIGTERM check of the new processes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-29 01:19:13 +03:00
co-authored by Claude Opus 5.5
parent c55fb1b998
commit fe2693870e
10 changed files with 1497 additions and 20 deletions
+23 -2
View File
@@ -19,7 +19,7 @@ jobs:
- run: make student-test VENV=.venv
acceptance:
name: Стенд Compose с RustFS за HAProxy, smoke, вложения, отказ реплики, рестарт и отказы зависимостей
name: Стенд Compose с RabbitMQ, relay и worker за HAProxy, smoke, pipeline, отказ реплики, рестарт и отказы зависимостей
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -34,6 +34,17 @@ jobs:
- name: Публичный smoke через балансировщик
run: make test
- name: Топология брокера, consumer с prefetch и права клиентов
run: make topology
- name: Уровень «3» pipeline — worker остановлен, рестарт API и брокера, дубль, S3, исчерпание попыток
run: |
make suite-preflight
make level3
- name: Сообщение задания в очереди, 409 до ready и SIGTERM relay и worker
run: make pipeline-evidence
- name: Вложение после остановки принявшей его реплики и при недоступном S3
run: |
make scenario SCENARIO=failover-upload
@@ -68,12 +79,22 @@ jobs:
- name: Наружу опубликован только балансировщик
run: |
docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" {print; bad=1} END {exit bad}'
docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" && $1 != "broker" {print; bad=1} END {exit bad}'
docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$1 == "broker" && $2 !~ /^127\.0\.0\.1:/ {print; bad=1} END {exit bad}'
- name: Процессы работают не от root
run: |
docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}'
- name: Корректная остановка relay и worker по SIGTERM
run: |
start=$(date +%s)
docker compose stop -t 10 relay worker
elapsed=$(( $(date +%s) - start ))
codes=$(docker inspect -f '{{.State.ExitCode}}' $(docker compose ps -aq relay worker) | sort -u)
echo "остановка за ${elapsed} с, коды завершения ${codes}"
test "$elapsed" -le 10 && test "$codes" = "0"
- name: Корректная остановка реплик API по SIGTERM
run: |
start=$(date +%s)
+45 -2
View File
@@ -11,7 +11,8 @@ RATE ?= 100
API_IMAGE ?= mtusi-messenger:lab5
FROM_REV ?=
SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py
scripts/upload_memory.py scripts/make_images.py scripts/pipeline_suite.py \
scripts/pipeline_evidence.py
SCENARIO ?= failover-upload
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
@@ -20,7 +21,9 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
persistence-evidence storage-evidence cache-evidence test-faults seed explain \
limits detection failover-each rolling-update level4 \
scenario s3-check attachments-evidence upload-faults upload-memory
scenario s3-check attachments-evidence upload-faults upload-memory \
images queues topology worker-stop worker-start broker-restart suite-preflight level3 \
pipeline-evidence
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -31,6 +34,7 @@ help:
@echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence"
@echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда"
@echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/"
@echo "lab5: queues, topology, worker-stop/worker-start, broker-restart, images, suite-preflight, level3"
check:
$(PYTHON) scripts/check_contract.py
@@ -168,3 +172,42 @@ explain: seed
echo "\$$ make explain # scripts/sql/explain-history.sql"; \
$(PSQL) -f - < scripts/sql/explain-history.sql; } > evidence/explain-history.txt
cat evidence/explain-history.txt
images:
$(PYTHON) scripts/make_images.py
queues:
sh scripts/broker.sh queues
topology:
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; \
echo "$$ rabbitmqctl list_queues -p messenger name type durable policy messages"; \
$(COMPOSE) exec -T broker rabbitmqctl -q list_queues -p messenger name type durable policy messages; \
echo "$$ rabbitmqctl list_consumers -p messenger"; \
$(COMPOSE) exec -T broker rabbitmqctl -q list_consumers -p messenger queue_name ack_required prefetch_count; \
echo "$$ rabbitmqctl list_permissions -p messenger"; \
$(COMPOSE) exec -T broker rabbitmqctl -q list_permissions -p messenger; \
echo "$$ rabbitmqctl list_users"; \
$(COMPOSE) exec -T broker rabbitmqctl -q list_users; \
echo "$$ docker compose ps"; \
$(COMPOSE) ps --format '{{.Service}}\t{{.Status}}\t{{.Ports}}'; } > evidence/topology.txt
cat evidence/topology.txt
worker-stop:
sh scripts/worker.sh stop
worker-start:
sh scripts/worker.sh start
broker-restart:
sh scripts/broker.sh restart
suite-preflight:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --preflight
level3:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 3 --output evidence/level3.json
pipeline-evidence:
$(PYTHON) scripts/pipeline_evidence.py --base-url "$(BASE_URL)" > evidence/pipeline-stand.txt; \
status=$$?; cat evidence/pipeline-stand.txt; exit $$status
@@ -40,9 +40,9 @@
## 7. Сценарии и обвязка
- [ ] 7.1 `scripts/pipeline_suite.py`, `scripts/make_images.py` из `.prep/lab5`; цели Makefile `queues`, `worker-stop`, `worker-start`, `broker-restart`, `suite-preflight`, `level3`, `topology`; ruff для новых `scripts/*.py`
- [ ] 7.2 CI-job под лабу 5: брокер, relay и worker в стенде, сценарии уровня «3»
- [ ] 7.3 Проверка группы: `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` с `make restore`, `make suite-preflight`, `make level3`
- [x] 7.1 `scripts/pipeline_suite.py`, `scripts/make_images.py` из `.prep/lab5`; цели Makefile `queues`, `worker-stop`, `worker-start`, `broker-restart`, `suite-preflight`, `level3`, `topology`; ruff для новых `scripts/*.py`
- [x] 7.2 CI-job под лабу 5: брокер, relay и worker в стенде, сценарии уровня «3»
- [x] 7.3 Проверка группы: `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` с `make restore`, `make suite-preflight`, `make level3`
## 8. Отчёт
+2
View File
@@ -83,6 +83,8 @@ ignore = [
[tool.ruff.lint.per-file-ignores]
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "T20", "TID251"]
"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"]
"scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"]
"scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"]
"src/messenger/*/infrastructure/**" = ["TID251", "ARG002"]
"src/messenger/*/domain/errors.py" = ["N818"]
"src/messenger/*/application/errors.py" = ["N818"]
+3 -1
View File
@@ -36,8 +36,10 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)"
echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)"
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)"
echo "packages (uv.lock):"
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow; do
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
faststream aio-pika aiormq dishka-faststream; do
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
done
echo "process users (service uid command):"
+4 -3
View File
@@ -23,9 +23,9 @@ def compose(*args, stdin=None):
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
def upload(c, actor, chat, expected=201):
def upload(c, actor, chat, expected=202):
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
schema = "Attachment" if expected == 201 else None
schema = "Attachment" if expected == 202 else None
return c.call(
"POST",
path,
@@ -74,6 +74,7 @@ def failover_upload(c, args):
alice, bob, _, chat = c.bootstrap()
sent = upload(c, alice, chat)
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
c.wait_ready(bob, att)
started = time.monotonic()
hook = subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
@@ -109,7 +110,7 @@ def failover_upload(c, args):
def s3_outage(c, args):
alice, bob, _, chat = c.bootstrap()
att = upload(c, alice, chat)["json"]
att = c.wait_ready(bob, upload(c, alice, chat)["json"])
check(
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
)
+354
View File
@@ -0,0 +1,354 @@
#!/usr/bin/env python3
import argparse
import json
import struct
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MAX_BYTES = 10485760
MAX_PIXELS = 40000000
REJECTED_OR_FAILED = {"upload": [202, 415], "final": "failed", "error_code": "invalid_image"}
GIF = (
b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00"
b",\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;"
)
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
def png(width, height, rows, color_type=2, depth=8, *, extra=(), level=6):
packer = zlib.compressobj(level)
body = b"".join(packer.compress(b"\x00" + row) for row in rows) + packer.flush()
header = struct.pack(">IIBBBBB", width, height, depth, color_type, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ b"".join(extra)
+ chunk(b"IDAT", body)
+ chunk(b"IEND", b"")
)
def ramp(value, size):
return value * 255 // max(1, size - 1)
def gradient(width, height, channels):
for y in range(height):
yield bytes(
channel
for x in range(width)
for channel in (ramp(x, width), ramp(y, height), 160, ramp(x + y, width + height))[
:channels
]
)
def blank(width, height, channels, depth=8):
row = bytes(width * channels * depth // 8)
for _ in range(height):
yield row
def padded(data, total):
pad = total - len(data) - 12
return data[:33] + chunk(b"paDd", bytes(pad)) + data[33:]
class Bits:
def __init__(self):
self.out = bytearray()
self.acc = 0
self.size = 0
def put(self, value, length):
self.acc = (self.acc << length) | value
self.size += length
while self.size >= 8:
self.size -= 8
byte = (self.acc >> self.size) & 0xFF
self.out += b"\xff\x00" if byte == 0xFF else bytes((byte,))
self.acc &= (1 << self.size) - 1
def zeros(self, count):
head = min(count, (8 - self.size) % 8)
self.put(0, head)
count -= head
self.out += bytes(count // 8)
self.put(0, count % 8)
def flush(self):
if self.size:
self.put((1 << (8 - self.size)) - 1, 8 - self.size)
return bytes(self.out)
def segment(marker, payload):
return b"\xff" + bytes((marker,)) + struct.pack(">H", len(payload) + 2) + payload
def ycbcr(red, green, blue):
return (
0.299 * red + 0.587 * green + 0.114 * blue,
128 - 0.168736 * red - 0.331264 * green + 0.5 * blue,
128 + 0.5 * red - 0.418688 * green - 0.081312 * blue,
)
def exif(orientation):
entry = struct.pack(">HHIHH", 0x0112, 3, 1, orientation, 0)
tiff = b"MM\x00\x2a\x00\x00\x00\x08" + struct.pack(">H", 1) + entry + struct.pack(">I", 0)
return segment(0xE1, b"Exif\x00\x00" + tiff)
def jpeg(width, height, mode="rgb", orientation=None, uniform=False):
components = {"gray": 1, "rgb": 3, "cmyk": 4}[mode]
head = b"\xff\xd8" + (
exif(orientation)
if orientation
else segment(0xE0, b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00")
)
if mode == "cmyk":
head += segment(0xEE, b"Adobe\x00\x64\x00\x00\x00\x00\x00")
head += segment(0xDB, b"\x00" + bytes([1] * 64))
head += segment(
0xC0,
struct.pack(">BHHB", 8, height, width, components)
+ b"".join(bytes((i + 1, 0x11, 0)) for i in range(components)),
)
head += segment(0xC4, b"\x00" + bytes([0, 0, 0, 12] + [0] * 12) + bytes(range(12)))
head += segment(0xC4, b"\x10" + bytes([1] + [0] * 15) + b"\x00")
head += segment(
0xDA,
bytes((components,))
+ b"".join(bytes((i + 1, 0)) for i in range(components))
+ b"\x00\x3f\x00",
)
columns, rows = (width + 7) // 8, (height + 7) // 8
bits, previous = Bits(), [0] * components
for block in range(1 if uniform else columns * rows):
bx, by = block % columns, block // columns
red, green, blue = ramp(bx, columns), ramp(by, rows), 160
values = {"gray": (red,), "rgb": ycbcr(red, green, blue), "cmyk": (red, green, blue, 64)}[
mode
]
for index, value in enumerate(values):
dc = round(8 * (value - 128))
diff, previous[index] = dc - previous[index], dc
category = abs(diff).bit_length()
bits.put(category, 4)
if category:
bits.put(diff if diff > 0 else diff + (1 << category) - 1, category)
bits.put(0, 1)
if uniform:
bits.zeros((columns * rows - 1) * components * 5)
return head + bits.flush() + b"\xff\xd9"
def fit(width, height, box, upscale=False):
if not upscale and max(width, height) <= box:
return [width, height]
if width >= height:
return [box, max(1, height * box // width)]
return [max(1, width * box // height), box]
def ready(width, height, orientation=1):
if orientation in (5, 6, 7, 8):
width, height = height, width
return {
"upload": [202],
"final": "ready",
"variants": {
"thumbnail": fit(width, height, 256),
"crop": [128, 128],
"optimized": fit(width, height, 1280),
},
}
def catalog(heavy):
small = png(64, 64, gradient(64, 64, 3))
truncated_png = png(320, 200, gradient(320, 200, 3))
idat = truncated_png.index(b"IDAT")
corrupt = bytearray(truncated_png)
corrupt[idat + 40 : idat + 48] = b"\x00" * 8
palette = bytes(channel for index in range(256) for channel in (index, 255 - index, 128))
alpha = bytes(index for index in range(256))
items = [
(
"png-rgb-320x200.png",
"image/png",
lambda: png(320, 200, gradient(320, 200, 3)),
ready(320, 200),
),
(
"png-rgba-300x300.png",
"image/png",
lambda: png(300, 300, gradient(300, 300, 4), 6),
ready(300, 300),
),
(
"png-palette-trns-200x100.png",
"image/png",
lambda: png(
200,
100,
(bytes((x + y) % 256 for x in range(200)) for y in range(100)),
3,
extra=(chunk(b"PLTE", palette), chunk(b"tRNS", alpha)),
),
ready(200, 100),
),
(
"png-gray-17x1000.png",
"image/png",
lambda: png(17, 1000, gradient(17, 1000, 1), 0),
ready(17, 1000),
),
(
"png-gray-alpha-640x480.png",
"image/png",
lambda: png(640, 480, gradient(640, 480, 2), 4),
ready(640, 480),
),
(
"png-gray16-300x200.png",
"image/png",
lambda: png(300, 200, gradient(600, 200, 1), 0, 16),
ready(300, 200),
),
(
"png-rgba16-64x64.png",
"image/png",
lambda: png(64, 64, gradient(512, 64, 1), 6, 16),
ready(64, 64),
),
("png-1x1.png", "image/png", lambda: png(1, 1, gradient(1, 1, 3)), ready(1, 1)),
(
"png-wide-3000x2.png",
"image/png",
lambda: png(3000, 2, gradient(3000, 2, 3)),
ready(3000, 2),
),
(
"png-small-50x30.png",
"image/png",
lambda: png(50, 30, gradient(50, 30, 3)),
ready(50, 30),
),
("jpeg-rgb-1000x750.jpg", "image/jpeg", lambda: jpeg(1000, 750), ready(1000, 750)),
("jpeg-gray-801x601.jpg", "image/jpeg", lambda: jpeg(801, 601, "gray"), ready(801, 601)),
("jpeg-cmyk-400x300.jpg", "image/jpeg", lambda: jpeg(400, 300, "cmyk"), ready(400, 300)),
(
"jpeg-exif6-400x200.jpg",
"image/jpeg",
lambda: jpeg(400, 200, orientation=6),
ready(400, 200, 6),
),
(
"jpeg-exif3-300x200.jpg",
"image/jpeg",
lambda: jpeg(300, 200, orientation=3),
ready(300, 200, 3),
),
("jpeg-rgb-4000x3000.jpg", "image/jpeg", lambda: jpeg(4000, 3000), ready(4000, 3000)),
("png-exactly-10mib.png", "image/png", lambda: padded(small, MAX_BYTES), ready(64, 64)),
(
"png-over-10mib.png",
"image/png",
lambda: padded(small, MAX_BYTES + 1),
{"upload": [413]},
),
(
"png-over-pixels-8001x5000.png",
"image/png",
lambda: png(8001, 5000, blank(8001, 5000, 3), level=9),
{"upload": [413]},
),
(
"jpeg-over-pixels-10000x5000.jpg",
"image/jpeg",
lambda: jpeg(10000, 5000, uniform=True),
{"upload": [413]},
),
("png-truncated.png", "image/png", lambda: truncated_png[: idat + 600], REJECTED_OR_FAILED),
("png-corrupt-idat.png", "image/png", lambda: bytes(corrupt), REJECTED_OR_FAILED),
("jpeg-truncated.jpg", "image/jpeg", lambda: jpeg(1000, 750)[:3000], REJECTED_OR_FAILED),
(
"png-zero-width.png",
"image/png",
zero_width,
{"upload": [400, 415]},
),
(
"not-an-image.png",
"image/png",
lambda: b"plain text, not an image\n" * 8,
{"upload": [415]},
),
("gif-declared-as-png.png", "image/png", lambda: GIF, {"upload": [415]}),
]
if heavy:
items.append(
(
"png-at-pixel-limit-8000x5000.png",
"image/png",
lambda: png(8000, 5000, blank(8000, 5000, 3), level=9),
ready(8000, 5000),
)
)
return items
def zero_width():
header = struct.pack(">IIBBBBB", 0, 10, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(b""))
+ chunk(b"IEND", b"")
)
def build(out, heavy=False, only=None):
out.mkdir(parents=True, exist_ok=True)
manifest = []
for name, content_type, make, expect in catalog(heavy):
if only and name not in only:
continue
data = make()
(out / name).write_bytes(data)
manifest.append(
{"file": name, "content_type": content_type, "size_bytes": len(data), "expect": expect}
)
(out / "manifest.json").write_text(
json.dumps(
{"rules": {"max_bytes": MAX_BYTES, "max_pixels": MAX_PIXELS}, "images": manifest},
indent=2,
)
+ "\n"
)
return manifest
def main():
parser = argparse.ArgumentParser(
description="Generate lab5 test images and manifest.json with expected outcomes"
)
parser.add_argument("--out", default=str(ROOT / "build/lab5-images"))
parser.add_argument(
"--heavy", action="store_true", help="also write a 40 Mpx image at the pixel limit"
)
parser.add_argument("--only", default="", help="comma separated file names")
args = parser.parse_args()
manifest = build(Path(args.out), args.heavy, {x for x in args.only.split(",") if x})
for item in manifest:
print(f"{item['file']:36} {item['size_bytes']:>10} {item['expect']['upload']}")
if __name__ == "__main__":
main()
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check
COMPOSE = os.environ.get("COMPOSE", "docker compose").split()
PEEK = (
'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/'
'rabbitmq_ops_password)" exec rabbitmqadmin -V messenger '
"--table-style ascii get messages -q attachments.process -a ack_requeue_true"
)
QUEUES = "rabbitmqctl -q list_queues -p messenger name messages_ready messages_unacknowledged"
def compose(*argv):
done = subprocess.run(
[*COMPOSE, *argv], cwd=ROOT, capture_output=True, text=True, timeout=180, check=False
)
check(done.returncode == 0, f"{' '.join(argv[:3])}: {done.stderr.strip()[-300:]}")
return done.stdout
def ready_messages():
for line in compose("exec", "-T", "broker", *QUEUES.split()).splitlines():
name, ready, _ = [*line.split("\t"), "", ""][:3]
if name == "attachments.process":
return int(ready)
return 0
def say(line):
print(line, flush=True)
def stop(service):
began = time.monotonic()
compose("stop", "-t", "10", service)
seconds = time.monotonic() - began
container = compose("ps", "-aq", service).split()[0]
code = subprocess.run(
["docker", "inspect", "-f", "{{.State.ExitCode}}", container],
capture_output=True,
text=True,
check=True,
).stdout.strip()
say(f"$ docker compose stop -t 10 {service} -> {seconds:.1f} s, exit code {code}")
check(seconds <= 10 and code == "0", f"{service} did not stop cleanly on SIGTERM")
def start(service):
compose("up", "-d", "--no-deps", "--wait", "--wait-timeout", "90", service)
say(f"$ docker compose up -d --no-deps --wait {service} -> healthy")
def main():
parser = argparse.ArgumentParser(description="Lab5 stand evidence: queue, message, SIGTERM")
parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
args = parser.parse_args()
client = Client(args.base_url)
alice, bob, _, chat = client.bootstrap()
say(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}")
stop("worker")
attachment, _ = client.attachment(alice, chat)
say(f"upload -> 202 status={attachment['status']} id={attachment['id']}")
path = f"/api/v1/attachments/{attachment['id']}"
for variant in ("original", "thumbnail"):
client.call("GET", f"{path}/content?variant={variant}", 409, actor=bob)
say(f"GET content?variant={variant} while worker is stopped -> 409")
deadline = time.monotonic() + 30
while ready_messages() < 1:
check(time.monotonic() < deadline, "the job did not reach the broker")
time.sleep(0.5)
say("$ rabbitmqctl list_queues -p messenger name messages_ready messages_unacknowledged")
say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip())
say("$ rabbitmqadmin get messages -q attachments.process -a ack_requeue_true")
peeked = compose("exec", "-T", "broker", "sh", "-c", PEEK)
say(peeked.rstrip())
body = json.loads(peeked[peeked.index('{"') : peeked.index("}", peeked.index('{"')) + 1])
say(f"payload keys: {sorted(body)}")
check(body["attachment_id"] == attachment["id"], "the queued job is for another attachment")
stop("relay")
start("relay")
start("worker")
ready = client.wait_ready(bob, attachment)
names = [(v["name"], v["width"], v["height"]) for v in ready["variants"]]
say(f"after worker start -> status={ready['status']} variants={names}")
deadline = time.monotonic() + 30
while ready_messages():
check(time.monotonic() < deadline, "the queue did not drain")
time.sleep(0.5)
say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip())
say("PASS: pipeline stand evidence")
if __name__ == "__main__":
main()
+945
View File
@@ -0,0 +1,945 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
import os
import shlex
import struct
import subprocess
import sys
import time
import urllib.request
import uuid
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
import make_images
from client import Client, Failure, check
from image_probe import jpeg_size
FIXTURE = ROOT / "tests/fixtures/sample.png"
EXCHANGE, ROUTING_KEY = "attachments", "attachment.process.v1"
QUEUE, DLQ = "attachments.process", "attachments.process.dlq"
RABBITMQADMIN = (
'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/'
'rabbitmq_ops_password)" exec rabbitmqadmin --non-interactive -V messenger "$@"'
)
PSQL = 'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"'
ERRORS = (Failure, OSError, ValueError, KeyError, subprocess.TimeoutExpired)
SCENARIOS = (
("smoke_pipeline", 3, "upload, ready, variants match metadata"),
("worker_down_broker_restart", 3, "worker down, API and broker restart"),
("duplicate_after_ready", 3, "redelivery of a finished job"),
("poison_image", 3, "permanent error without retries"),
("s3_outage_recovers", 3, "transient error with retry"),
("retry_exhausted", 3, "transient error until JOB_MAX_ATTEMPTS"),
("image_matrix", 4, "limits and variants"),
("crash_commit_publish", 4, "after COMMIT, before publish"),
("crash_publish_confirm", 4, "after publish, before confirm"),
("crash_after_s3_before_commit", 4, "after S3 write, before COMMIT"),
("crash_after_commit_before_ack", 4, "after COMMIT, before ack"),
("lease_expiry_fencing", 4, "stalled worker, lease expiry"),
("replay_failed", 4, "operator replay"),
("malformed_message_dead_lettered", 4, "message that is not a job goes to the DLQ"),
("crash_before_side_effect", 5, "after claim, before side effects"),
("worker_kill_mid_task", 5, "SIGKILL during processing"),
("duplicate_in_flight", 5, "two copies of one job in flight"),
("reorder_attempts", 5, "attempt 2 delivered before attempt 1"),
("reorder_versions", 5, "v2 delivered before v1"),
("stale_after_new_version", 5, "v1 job after v2 is published"),
)
QUERIES = """
attachment:
SELECT status, published_version, error_code, staging_key FROM attachments WHERE id = :'id'
jobs:
SELECT job_id, pipeline_version, status, attempt FROM jobs
WHERE attachment_id = :'id' ORDER BY pipeline_version
outbox:
SELECT o.payload, o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id)
WHERE j.attachment_id = :'id' ORDER BY o.id
variants:
SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants
WHERE attachment_id = :'id' ORDER BY pipeline_version, name
events:
SELECT kind, attempt FROM job_events WHERE attachment_id = :'id' ORDER BY id
nonterminal:
SELECT id, status FROM attachments
WHERE id = ANY(:'ids'::uuid[]) AND status NOT IN ('ready', 'failed')
known:
SELECT id FROM attachments WHERE id = ANY(:'ids'::uuid[])
duplicate_effects:
SELECT attachment_id, pipeline_version, name, count(*) AS copies FROM attachment_variants
WHERE attachment_id = ANY(:'ids'::uuid[]) GROUP BY 1, 2, 3 HAVING count(*) > 1
attempts:
SELECT job_id, attempt FROM jobs WHERE attachment_id = ANY(:'ids'::uuid[]) AND attempt > :max
pending_outbox:
SELECT o.job_id FROM outbox o JOIN jobs j USING (job_id)
WHERE j.attachment_id = ANY(:'ids'::uuid[]) AND o.published_at IS NULL
unpublished_version:
SELECT a.id, a.published_version, max(j.pipeline_version) AS done FROM attachments a
JOIN jobs j ON j.attachment_id = a.id AND j.status = 'done' WHERE a.id = ANY(:'ids'::uuid[])
GROUP BY a.id, a.published_version HAVING a.published_version <> max(j.pipeline_version)
staging_left:
SELECT id FROM attachments
WHERE id = ANY(:'ids'::uuid[]) AND status = 'ready' AND staging_key IS NOT NULL
object_keys:
SELECT object_key AS key FROM attachment_variants
UNION ALL SELECT original_key FROM attachments WHERE original_key IS NOT NULL
UNION ALL SELECT staging_key FROM attachments WHERE staging_key IS NOT NULL
columns:
SELECT table_name, column_name FROM information_schema.columns
WHERE table_schema = current_schema()
"""
SQL = {
name.strip(): " ".join(body.split())
for name, body in (block.split(":\n", 1) for block in QUERIES.strip().split("\n\n"))
}
REQUIRED_COLUMNS = {
"attachments": "id status published_version error_code staging_key original_key",
"attachment_variants": "attachment_id pipeline_version name sha256 size_bytes object_key",
"jobs": "job_id attachment_id pipeline_version status attempt",
"outbox": "job_id payload published_at",
"job_events": "job_id attachment_id kind attempt",
}
def now():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
def run(cmd, stdin=None, env=None, timeout=180):
done = subprocess.run(
cmd, input=stdin, capture_output=True, text=True, env=env, timeout=timeout, check=False
)
if done.returncode:
raise Failure(f"{' '.join(cmd[:7])}: exit {done.returncode}: {done.stderr.strip()[-400:]}")
return done.stdout
def poll(probe, timeout, what, interval=0.5):
deadline = time.monotonic() + timeout
while True:
value = probe()
if value:
return value
if time.monotonic() > deadline:
raise Failure(f"timeout {timeout}s waiting for {what}")
time.sleep(interval)
def http_ready(base_url):
try:
with urllib.request.urlopen(base_url.rstrip("/") + "/health/ready", timeout=3) as response:
return response.status == 200
except OSError:
return False
def image_size(data, content_type):
if content_type == "image/jpeg":
return jpeg_size(data)
if content_type == "image/png":
return struct.unpack(">II", data[16:24])
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
if data[12:16] == b"VP8 ":
width, height = struct.unpack("<HH", data[26:30])
return width & 0x3FFF, height & 0x3FFF
if data[12:16] == b"VP8L":
bits = int.from_bytes(data[21:25], "little")
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
return int.from_bytes(data[24:27], "little") + 1, int.from_bytes(data[27:30], "little") + 1
class Stack:
def __init__(self, args):
self.args = args
self.base = shlex.split(args.compose)
self.faults = shlex.split(args.compose_faults)
def dc(self, *argv, faults=False, stdin=None, env=None, timeout=180):
command = [*(self.faults if faults else self.base), *argv]
return run(command, stdin, dict(os.environ, **(env or {})), timeout)
def up(self, *services, faults=False, env=None):
wait = ["up", "-d", "--no-deps", "--no-recreate", "--wait", "--wait-timeout", "90"]
if faults or env:
wait.remove("--no-recreate")
self.dc(*wait, *services, faults=faults, env=env)
def launch(self, service, point=""):
variable = "WORKER_FAULT_POINT" if service == self.args.worker else "RELAY_FAULT_POINT"
self.up(service, faults=True, env={variable: point})
def restore(self):
self.dc("up", "-d", "--no-deps", "--wait", self.args.relay, self.args.worker)
def stop(self, *services):
self.dc("stop", "-t", "10", *services)
def kill(self, service):
self.dc("kill", "-s", "KILL", service)
def restart(self, *services):
self.dc("restart", "-t", "20", *services)
self.up(*services)
def state(self, service):
text = self.dc("ps", "-a", "--format", "json", service).strip()
rows = (
json.loads(text) if text.startswith("[") else [json.loads(x) for x in text.split("\n")]
)
check(len(rows) == 1, "expected one container for " + service)
return rows[0]["State"], rows[0].get("ExitCode")
def wait_exit(self, service, timeout=60):
def probe():
state, code = self.state(service)
return (code,) if state in ("exited", "dead") else None
return poll(probe, timeout, service + " exit")[0]
def sql(self, name, **params):
query = f"SELECT coalesce(json_agg(q), '[]'::json) FROM ({SQL[name]}) q;\n"
variables = [x for key, value in params.items() for x in ("-v", f"{key}={value}")]
out = self.dc(
"exec", "-T", self.args.postgres, "sh", "-c", PSQL, "sh", *variables, stdin=query
)
return json.loads(out)
def queues(self):
columns = "name messages_ready messages_unacknowledged --formatter json"
listing = f"exec -T {self.args.broker} rabbitmqctl -q list_queues -p messenger {columns}"
rows = json.loads(self.dc(*listing.split()))
return {q["name"]: (q["messages_ready"], q["messages_unacknowledged"]) for q in rows}
def rabbitmqadmin(self, *argv, stdin=None):
return self.dc(
"exec", "-T", self.args.broker, "sh", "-c", RABBITMQADMIN, "sh", *argv, stdin=stdin
)
def publish(self, payload):
properties = {"delivery_mode": 2, "content_type": "application/json", "type": ROUTING_KEY}
properties["message_id"] = payload["job_id"]
command = f"publish message -e {EXCHANGE} -k {ROUTING_KEY} -f - -p".split()
out = self.rabbitmqadmin(*command, json.dumps(properties), stdin=json.dumps(payload))
check("routed successfully" in out, "broker did not route the injected job")
def admin(self, *argv):
command = shlex.split(self.args.admin) + list(argv)
out = self.dc("run", "--rm", "--no-deps", "-T", self.args.worker, *command)
return json.loads(out.strip().splitlines()[-1])
class Api:
def __init__(self, args):
self.base_url = args.base_url
self.client = Client(args.base_url, args.ca_file)
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
self.accepted, self.traces, self.final = [], {}, {}
def upload(self, data, kind="image/png", name="sample.png", expected=202, final=("ready",)):
path = f"/api/v1/chats/{self.chat['id']}/attachments?filename={quote(name)}"
allowed = [expected] if isinstance(expected, int) else expected
headers = {"Content-Type": kind}
response = self.client.call(
"POST", path, allowed, actor=self.alice, raw=data, headers=headers, schema="Attachment"
)
if response["status"] != 202:
return None
body = response["json"]
snapshot = (
body["status"] == "queued" and body["variants"] == [] and body["error_code"] is None
)
check(snapshot, "202 is not a queued snapshot")
same = body["sha256"] == hashlib.sha256(data).hexdigest() and body["size_bytes"] == len(
data
)
check(same, "202 checksum or size differs from the uploaded bytes")
self.accepted.append(body["id"])
self.final[body["id"]] = set(final)
self.traces[body["id"]] = ["queued"]
return body
def fixture(self, **kwargs):
return self.upload(FIXTURE.read_bytes(), **kwargs)
def get(self, ident):
path = "/api/v1/attachments/" + ident
body = self.client.call("GET", path, actor=self.bob, schema="Attachment")["json"]
self.traces.setdefault(ident, []).append(body["status"])
return body
def wait(self, ident, final=("ready",), timeout=90):
def probe():
body = self.get(ident)
failed = body["status"] == "failed" and "failed" not in final
check(not failed, f"attachment reached failed: {body['error_code']}")
return body if body["status"] in final else None
return poll(probe, timeout, f"{ident} {'/'.join(final)}", 0.3)
def content(self, ident, variant=None, expected=200):
path = f"/api/v1/attachments/{ident}/content" + (f"?variant={variant}" if variant else "")
return self.client.call("GET", path, expected, actor=self.bob)
def message(self, text, attachments=None, expected=201):
if expected == 201:
return self.client.message(self.alice, self.chat, text, attachments=attachments)["json"]
body = {"text": text, "attachment_ids": attachments or []}
headers = {"Idempotency-Key": os.urandom(8).hex()}
path = f"/api/v1/chats/{self.chat['id']}/messages"
return self.client.call(
"POST", path, expected, actor=self.alice, body=body, headers=headers
)
class Suite:
def __init__(self, args):
self.args = args
self.stack = Stack(args)
self.api = None
self.samples = {}
self.dead_letters = 0
def scenarios(self):
return [(name, level, window, getattr(self, name)) for name, level, window in SCENARIOS]
def drained(self, timeout=60):
poll(lambda: self.stack.queues()[QUEUE] == (0, 0), timeout, "main queue drained")
def queued(self, count, timeout=30):
poll(lambda: self.stack.queues()[QUEUE][0] >= count, timeout, f"{count} ready messages")
def record(self, ident):
row = self.stack.sql("attachment", id=ident)[0]
self.samples.setdefault(ident, []).append(row["published_version"])
self.api.traces.setdefault(ident, []).append(row["status"])
return row
def snapshot(self, ident):
body = self.api.get(ident)
public = sorted(body["variants"], key=lambda v: v["name"])
published = self.record(ident)["published_version"]
rows = self.stack.sql("variants", id=ident)
return {"status": body["status"], "public": public, "rows": rows, "published": published}
def payload(self, ident, version=1):
rows = self.stack.sql("outbox", id=ident)
found = [r["payload"] for r in rows if r["payload"]["pipeline_version"] == version]
check(bool(found), f"no outbox payload for v{version}")
return found[-1]
def verify(self, ready, expect=None):
names = sorted(v["name"] for v in ready["variants"])
check(len(names) == len(set(names)), "duplicate variant names in API")
missing = set(self.args.expect_variants.split(",")) - set(names)
check(not missing, f"missing variants {sorted(missing)}")
original = self.api.content(ready["id"])
data = original["bytes"]
same = (
hashlib.sha256(data).hexdigest() == ready["sha256"] and len(data) == ready["size_bytes"]
)
check(same, "original bytes differ from metadata")
check(
original["headers"].get_content_type() == ready["content_type"], "original type differs"
)
for variant in ready["variants"]:
name = variant["name"]
got = self.api.content(ready["id"], name)
data, kind = got["bytes"], got["headers"].get_content_type()
check(kind == variant["content_type"], name + " content type differs")
same = len(data) == variant["size_bytes"]
check(
same and hashlib.sha256(data).hexdigest() == variant["sha256"],
name + " bytes differ",
)
size = (variant["width"], variant["height"])
check(tuple(image_size(data, kind)) == size, name + " real size differs from metadata")
wanted = (expect or {}).get(name)
slack = 1 if name == "optimized" else 0
fits = not wanted or all(abs(a - b) <= slack for a, b in zip(wanted, size))
check(fits, f"{name} is {size}, expected {wanted}")
return names
def crash(self, service, point, observe):
self.drained()
self.stack.launch(service, point)
ident = self.api.fixture()["id"]
code = self.stack.wait_exit(service)
check(code == 137, f"{service} exited with {code}, expected 137 at {point}")
seen = observe(ident)
self.stack.launch(service)
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox")
self.drained()
self.verify(ready)
variants = self.stack.sql("variants", id=ident)
return {
"attachment_id": ident,
"exit_code": code,
"after_crash": seen,
"variants": variants,
}
def smoke_pipeline(self):
att = self.api.fixture()
ready = self.api.wait(att["id"])
return {
"attachment_id": att["id"],
"variants": self.verify(ready, {"thumbnail": [256, 160]}),
}
def worker_down_broker_restart(self):
steps = []
def mark(step, **extra):
steps.append({"at": now(), "step": step, **extra})
self.stack.stop(self.args.worker)
mark("worker stopped")
att = self.api.fixture()
mark("upload 202 queued", attachment_id=att["id"])
self.api.content(att["id"], expected=409)
mark("content 409 while queued")
message = self.api.message("", [att["id"]])
self.api.message("text messages still work")
path = f"/api/v1/chats/{self.api.chat['id']}/messages?limit=5"
self.api.client.call("GET", path, actor=self.api.bob, schema="MessagePage")
mark("message with attachment 201, other messages work", message_id=message["id"])
self.queued(1)
mark("job is in the broker queue", queues=self.stack.queues())
self.stack.restart(*self.args.api, self.args.relay)
poll(lambda: http_ready(self.api.base_url), 90, "API ready")
mark("API replicas and relay restarted")
self.stack.restart(self.args.broker)
self.queued(1)
mark("broker restarted with its volume, job survived", queues=self.stack.queues())
self.stack.launch(self.args.worker)
mark("worker started")
ready = self.api.wait(att["id"])
mark("ready", variants=self.verify(ready))
return {"attachment_id": att["id"], "timeline": steps}
def duplicate_after_ready(self):
ident = self.api.fixture()["id"]
self.api.wait(ident)
self.drained()
before = self.snapshot(ident)
self.stack.publish(self.payload(ident))
self.drained()
time.sleep(1)
check(self.snapshot(ident) == before, "duplicate delivery changed the published result")
return {"attachment_id": ident, "events": self.stack.sql("events", id=ident)}
def poison_image(self):
data = make_images.png(320, 200, make_images.gradient(320, 200, 3))
broken = data[: data.index(b"IDAT") + 600]
att = self.api.upload(broken, expected=[202, 415], final=("failed",))
if att is None:
return {"rejected_by_api": 415}
failed = self.api.wait(att["id"], ("failed",))
check(
failed["error_code"] == "invalid_image", f"unexpected error_code {failed['error_code']}"
)
jobs = self.stack.sql("jobs", id=att["id"])
check(all(j["attempt"] == 1 for j in jobs), "permanent error was retried")
self.api.content(att["id"], expected=409)
self.api.message("", [att["id"]], expected=409)
return {"attachment_id": att["id"], "error_code": failed["error_code"], "jobs": jobs}
def s3_outage_recovers(self):
self.stack.stop(self.args.worker)
ident = self.api.fixture()["id"]
self.queued(1)
self.stack.stop(self.args.s3)
try:
self.stack.launch(self.args.worker)
poll(
lambda: any(j["attempt"] > 1 for j in self.stack.sql("jobs", id=ident)), 60, "retry"
)
status = self.record(ident)["status"]
check(status in ("queued", "processing"), "transient error produced " + status)
finally:
self.stack.up(self.args.s3)
poll(lambda: http_ready(self.api.base_url), 90, "API ready")
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)}
def exhaust(self):
self.drained()
self.stack.launch(self.args.worker, "transient_error")
ident = self.api.fixture(final=("failed", "ready"))["id"]
failed = self.api.wait(ident, ("failed",), timeout=self.args.recovery_timeout)
self.stack.launch(self.args.worker)
jobs = self.stack.sql("jobs", id=ident)
check(failed["error_code"] == "retry_exhausted", f"unexpected {failed['error_code']}")
check(max(j["attempt"] for j in jobs) == self.args.max_attempts, "attempts != max")
return ident, jobs
def retry_exhausted(self):
ident, jobs = self.exhaust()
self.api.final[ident] = {"failed"}
self.api.content(ident, expected=409)
self.api.message("", [ident], expected=409)
return {"attachment_id": ident, "jobs": jobs}
def replay_failed(self):
ident, _ = self.exhaust()
self.api.final[ident] = {"ready"}
first = self.stack.admin("replay", "--attachment-id", ident)
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
self.drained()
before = self.snapshot(ident)
second = self.stack.admin("replay", "--attachment-id", ident)
self.drained()
check(self.snapshot(ident) == before, "replay of a ready attachment changed the result")
events = self.stack.sql("events", id=ident)
check(any("replay" in e["kind"] for e in events), "replay left no audit event")
return {
"attachment_id": ident,
"replay": first,
"replay_of_ready": second,
"events": events,
}
def malformed_message_dead_lettered(self):
before = self.stack.queues()[DLQ][0]
self.stack.publish({"schema_version": 2, "job_id": str(uuid.uuid4())})
poll(lambda: self.stack.queues()[DLQ][0] > before, 30, "dead letter")
self.dead_letters += 1
return {"dead_letters": self.stack.queues()[DLQ][0] - before}
def image_matrix(self):
folder = Path(self.args.images)
results = []
for item in make_images.build(folder, self.args.heavy):
expect, name = item["expect"], item["file"]
ready = expect.get("final") == "ready"
data = (folder / name).read_bytes()
final = ("ready",) if ready else ("failed", "ready")
att = self.api.upload(data, item["content_type"], name, expect["upload"], final)
row = {"file": name, "http": 202 if att else "rejected"}
if att:
done = self.api.wait(att["id"], ("ready", "failed"), self.args.recovery_timeout)
row.update(status=done["status"], error_code=done["error_code"])
if ready:
check(done["status"] == "ready", name + " did not become ready")
row["variants"] = self.verify(done, expect["variants"])
else:
self.api.final[att["id"]] = {"failed"}
code = expect.get("error_code")
check(done["error_code"] == code, f"{name} should fail with {code}")
results.append(row)
return {"images": results}
def crash_commit_publish(self):
def observe(ident):
outbox = self.stack.sql("outbox", id=ident)
check(outbox and not any(r["published"] for r in outbox), "outbox already published")
check(self.stack.queues()[QUEUE] == (0, 0), "job reached the broker before publish")
return {"outbox_published": False, "status": self.record(ident)["status"]}
return self.crash(self.args.relay, "after_commit_before_publish", observe)
def crash_publish_confirm(self):
def observe(ident):
outbox = [r["published"] for r in self.stack.sql("outbox", id=ident)]
return {"outbox_published": outbox, "queues": self.stack.queues()}
return self.crash(self.args.relay, "after_publish_before_confirm", observe)
def crash_after_s3_before_commit(self):
def observe(ident):
check(not self.stack.sql("variants", id=ident), "variants committed before the crash")
return {"status": self.record(ident)["status"], "variant_rows": 0}
return self.crash(self.args.worker, "after_s3_write_before_commit", observe)
def crash_after_commit_before_ack(self):
def observe(ident):
row = self.record(ident)
check(row["status"] == "ready", "result was not committed before the crash point")
variants = self.stack.sql("variants", id=ident)
return {"status": row["status"], "variants": variants, "queues": self.stack.queues()}
result = self.crash(self.args.worker, "after_commit_before_ack", observe)
same = result["variants"] == result["after_crash"]["variants"]
check(same, "redelivery after COMMIT changed variants")
return result
def crash_before_side_effect(self):
def observe(ident):
return {
"status": self.record(ident)["status"],
"jobs": self.stack.sql("jobs", id=ident),
}
return self.crash(self.args.worker, "after_claim_before_side_effect", observe)
def lease_expiry_fencing(self):
self.drained()
self.stack.launch(self.args.worker, "stall_after_claim")
ident = self.api.fixture()["id"]
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
before = self.snapshot(ident)
time.sleep(self.args.stall_seconds + 3)
after = self.snapshot(ident)
self.stack.launch(self.args.worker)
check(before == after, "stalled worker overwrote the result after its lease expired")
jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident)
return {"attachment_id": ident, "jobs": jobs, "events": events}
def worker_kill_mid_task(self):
self.drained()
self.stack.launch(self.args.worker, "stall_after_claim")
ident = self.api.fixture()["id"]
poll(lambda: self.record(ident)["status"] == "processing", 30, "processing")
self.stack.kill(self.args.worker)
code = self.stack.wait_exit(self.args.worker)
self.stack.launch(self.args.worker)
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
self.drained()
self.verify(ready)
return {"attachment_id": ident, "exit_code": code, "jobs": self.stack.sql("jobs", id=ident)}
def duplicate_in_flight(self):
self.stack.stop(self.args.worker)
ident = self.api.fixture()["id"]
self.queued(1)
self.stack.publish(self.payload(ident))
self.queued(2)
self.stack.launch(self.args.worker)
ready = self.api.wait(ident)
self.drained()
self.verify(ready)
variants, events = self.stack.sql("variants", id=ident), self.stack.sql("events", id=ident)
return {"attachment_id": ident, "variants": variants, "events": events}
def reorder_attempts(self):
self.stack.stop(self.args.worker, self.args.relay)
ident = self.api.fixture()["id"]
original = self.payload(ident)
self.stack.publish({**original, "attempt": original["attempt"] + 1})
self.queued(1)
self.stack.launch(self.args.relay)
self.queued(2)
self.stack.launch(self.args.worker)
ready = self.api.wait(ident)
self.drained()
self.verify(ready)
jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident)
return {"attachment_id": ident, "jobs": jobs, "events": events}
def published(self, ident, version):
poll(
lambda: self.record(ident)["published_version"] >= version,
self.args.recovery_timeout,
f"published v{version}",
)
def reprocess(self, ident):
return self.stack.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
def reorder_versions(self):
self.stack.stop(self.args.worker, self.args.relay)
ident = self.api.fixture()["id"]
self.reprocess(ident)
self.stack.publish(self.payload(ident, 2))
self.queued(1)
self.stack.launch(self.args.relay)
self.queued(3)
self.stack.launch(self.args.worker)
self.published(ident, 2)
self.api.wait(ident)
self.drained()
final = self.snapshot(ident)
check(final["published"] == 2, f"published_version rolled back to {final['published']}")
served = {v["name"]: v["sha256"] for v in final["public"]}
stored = {r["name"]: r["sha256"] for r in final["rows"] if r["pipeline_version"] == 2}
check(served == stored, "API serves variants of a version that is not published")
self.verify(self.api.get(ident))
return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)}
def stale_after_new_version(self):
ident = self.api.fixture()["id"]
self.api.wait(ident)
self.reprocess(ident)
self.published(ident, 2)
self.drained()
before = self.snapshot(ident)
self.stack.publish(self.payload(ident, 1))
self.drained()
time.sleep(1)
after = self.snapshot(ident)
check(before == after and after["published"] == 2, "stale v1 job changed the v2 result")
return {"attachment_id": ident, "published_version": after["published"]}
def invariants(self, dead_before):
ids = self.api.accepted
array = "{" + ",".join(ids) + "}"
if not ids:
return [{"id": "accepted_work", "result": "FAIL", "details": "no upload got 202"}]
def empty(name, message, **params):
rows = self.stack.sql(name, ids=array, **params)
check(not rows, f"{message}: {rows[:5]}")
return {"rows": 0}
def terminal():
empty("nonterminal", "non-terminal attachments left")
wrong = [i for i in ids if self.api.get(i)["status"] not in self.api.final[i]]
check(not wrong, "unexpected final status: " + ", ".join(wrong))
return {"accepted": len(ids)}
def no_rollback():
traces = self.api.traces.items()
broken = [
i for i, t in traces if "ready" in t and set(t[t.index("ready") :]) != {"ready"}
]
check(not broken, "ready rolled back: " + ", ".join(broken))
return {"traced": len(self.api.traces)}
def monotonic():
broken = [i for i, values in self.samples.items() if values != sorted(values)]
check(not broken, "published_version went backwards: " + ", ".join(broken))
empty("unpublished_version", "published_version differs from the newest done job")
return {"sampled": len(self.samples)}
def served():
bodies = [self.api.get(i) for i in ids]
ready = [b for b in bodies if b["status"] == "ready"]
for body in ready:
self.verify(body)
return {"verified": len(ready)}
def orphans():
empty("pending_outbox", "unpublished outbox rows")
empty("staging_left", "staging kept after ready")
queues = self.stack.queues()
check(queues[QUEUE] == (0, 0), "messages left in " + QUEUE)
expected = dead_before + self.dead_letters
check(queues[DLQ][0] == expected, f"unexpected dead letters: {queues[DLQ]}")
if not self.args.s3_list:
return {"queues": queues, "s3": "SKIP: --s3-list is not set"}
keys = {r["key"] for r in self.stack.sql("object_keys")}
listing = run(shlex.split(self.args.s3_list)).splitlines()
stray = [k for k in (x.split()[-1] for x in listing if x.strip()) if k not in keys]
check(not stray, f"S3 objects without metadata: {stray[:5]}")
return {"queues": queues, "s3_objects": len(listing)}
def no_loss():
known = {r["id"] for r in self.stack.sql("known", ids=array)}
check(known == set(ids), "accepted attachments are missing in PostgreSQL")
return {"accepted": len(ids)}
claims = [
("terminal", "every 202 reaches its expected terminal state", terminal),
(
"single_effect",
"one variant set per (attachment_id, pipeline_version, variant)",
lambda: empty("duplicate_effects", "duplicate effect rows"),
),
("no_ready_rollback", "ready never returns to queued or processing", no_rollback),
("monotonic_publication", "published_version only moves forward", monotonic),
("bytes_match_metadata", "sha256, size and real dimensions match served bytes", served),
(
"bounded_attempts",
"attempt <= JOB_MAX_ATTEMPTS",
lambda: empty("attempts", "attempt above limit", max=self.args.max_attempts),
),
("no_orphans", "no pending outbox, messages, dead letters, staging, objects", orphans),
("no_lost_202", "every 202 is present in PostgreSQL and the API", no_loss),
]
return [self.claim(name, statement, probe) for name, statement, probe in claims]
@staticmethod
def claim(name, statement, probe):
try:
return {"id": name, "statement": statement, "result": "PASS", "details": probe()}
except ERRORS as error:
return {"id": name, "statement": statement, "result": "FAIL", "details": str(error)}
def preflight(self):
def services():
self.detect_api()
present = set(self.stack.dc("config", "--services").split())
wanted = {*self.args.api, self.args.relay, self.args.worker, self.args.broker}
wanted |= {self.args.postgres, self.args.s3}
check(wanted <= present, f"missing services {sorted(wanted - present)}")
return sorted(wanted)
def schema():
columns = {}
for row in self.stack.sql("columns"):
columns.setdefault(row["table_name"], set()).add(row["column_name"])
needed = {t: set(c.split()) for t, c in REQUIRED_COLUMNS.items()}
missing = {t: sorted(c - columns.get(t, set())) for t, c in needed.items()}
check(not any(missing.values()), f"missing columns {missing}")
return sorted(needed)
def broker():
queues = self.stack.queues()
check({QUEUE, DLQ} <= set(queues), "queues are not declared")
check(QUEUE in self.stack.rabbitmqadmin("list", "queues"), "rabbitmqadmin failed")
return queues
def faults():
check("FAULT_POINT" in self.stack.dc("config", faults=True), "no FAULT_POINT in faults")
return self.args.compose_faults
def api():
check(http_ready(self.args.base_url), "GET /health/ready is not 200")
return self.args.base_url
steps = [
("compose services", services),
("api ready", api),
("postgres schema", schema),
("broker topology and ops CLI", broker),
("faults override", faults),
]
return [self.claim(name, name, probe) for name, probe in steps]
def main(self):
report = {
"suite": "lab5-pipeline-idempotency",
"started_at": now(),
"config": self.config(),
}
if self.args.preflight:
report["preflight"] = self.preflight()
return self.finish(report, report["preflight"])
results = []
try:
self.detect_api()
self.api = Api(self.args)
dead_before = self.stack.queues()[DLQ][0]
self.stack.launch(self.args.relay)
self.stack.launch(self.args.worker)
for name, level, window, scenario in self.scenarios():
if self.selected(name, level):
results.append(self.attempt(name, level, window, scenario))
report["invariants"] = self.invariants(dead_before)
except ERRORS as error:
report["error"] = str(error)
finally:
self.stack.up(self.args.s3)
self.stack.restore()
report["scenarios"] = results
return self.finish(report, results + report.get("invariants", []))
def attempt(self, name, level, window, scenario):
began = time.monotonic()
print("RUN " + name, flush=True)
try:
outcome = {"result": "PASS", "observed": scenario()}
except ERRORS as error:
outcome = {"result": "FAIL", "error": str(error)}
self.heal()
print(f"{outcome['result']} {name}", flush=True)
seconds = round(time.monotonic() - began, 1)
return {"name": name, "level": level, "window": window, "seconds": seconds, **outcome}
def heal(self):
try:
self.stack.up(self.args.s3, self.args.broker)
self.stack.launch(self.args.relay)
self.stack.launch(self.args.worker)
except ERRORS as error:
print("heal failed: " + str(error), flush=True)
def detect_api(self):
if not self.args.api:
names = self.stack.dc("config", "--services").split()
api = [x for x in names if x == "api" or (x[:4] == "api-" and x[4:].isdigit())]
self.args.api = api
check(bool(self.args.api), "no API service: expected api or api-N, or pass --api")
def selected(self, name, level):
return name in self.args.only.split(",") if self.args.only else level <= self.args.level
def config(self):
keys = (
"base_url level only expect_variants max_attempts stall_seconds compose compose_faults"
)
return {key: getattr(self.args, key) for key in keys.split()}
def finish(self, report, verdicts):
passed = bool(verdicts) and "error" not in report
report["result"] = (
"PASS" if passed and all(v["result"] == "PASS" for v in verdicts) else "FAIL"
)
report["finished_at"] = now()
text = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if self.args.output:
Path(self.args.output).parent.mkdir(parents=True, exist_ok=True)
Path(self.args.output).write_text(text)
else:
print(text)
print(f"{report['result']}: lab5 pipeline suite, report {self.args.output or 'stdout'}")
return 0 if report["result"] == "PASS" else 1
def arguments():
env = os.environ.get
faults = "docker compose -f compose.yaml -f infra/compose.faults.yaml"
parser = argparse.ArgumentParser(description="Lab5 pipeline fault injection with invariants")
parser.add_argument("--base-url", default=env("BASE_URL", "http://localhost:8080"))
parser.add_argument("--ca-file", default=env("CA_FILE"))
parser.add_argument("--compose", default=env("COMPOSE", "docker compose"))
parser.add_argument("--compose-faults", default=env("COMPOSE_FAULTS", faults))
parser.add_argument("--admin", default=env("ADMIN_CLI", "python -m messenger.worker.admin"))
parser.add_argument("--api", nargs="+", default=env("API_SERVICES", "").split())
parser.add_argument("--worker", default="worker")
parser.add_argument("--relay", default="relay")
parser.add_argument("--broker", default="broker")
parser.add_argument("--postgres", default="postgres")
parser.add_argument("--s3", default="s3")
parser.add_argument("--s3-list", default=env("S3_LIST_CMD", ""), help="prints object keys")
parser.add_argument("--level", type=int, choices=[3, 4, 5], default=5)
parser.add_argument("--only", default="", help="comma separated scenario names")
parser.add_argument("--list", action="store_true")
parser.add_argument("--preflight", action="store_true")
parser.add_argument("--expect-variants", default=None)
parser.add_argument("--max-attempts", type=int, default=int(env("JOB_MAX_ATTEMPTS", "5")))
parser.add_argument(
"--stall-seconds", type=float, default=float(env("FAULT_STALL_SECONDS", "15"))
)
parser.add_argument("--recovery-timeout", type=float, default=120)
parser.add_argument("--images", default=str(ROOT / "build/lab5-images"))
parser.add_argument("--heavy", action="store_true")
parser.add_argument("--output", default="")
args = parser.parse_args()
if args.expect_variants is None:
args.expect_variants = "thumbnail" if args.level == 3 else "thumbnail,crop,optimized"
return args
def main():
suite = Suite(arguments())
if suite.args.list:
for name, level, window, _ in suite.scenarios():
print(f"{name:32} level {level} {window}")
return 0
return suite.main()
if __name__ == "__main__":
sys.exit(main())
+13 -9
View File
@@ -37,7 +37,10 @@ HEAD = (
FAULTS = """
CREATE TABLE lab4_fault_seen (id uuid PRIMARY KEY, object_key text, filename text);
CREATE FUNCTION lab4_fault_seen() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.object_key, NEW.filename); RETURN NEW; END $$;
BEGIN
INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.staging_key, NEW.filename);
RETURN NEW;
END $$;
CREATE FUNCTION lab4_fault_commit() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NEW.filename LIKE 'fault-fail-%' THEN
@@ -61,7 +64,7 @@ END $$;
CREATE TRIGGER lab4_fault_seen AFTER INSERT ON attachments FOR EACH ROW
WHEN (NEW.filename LIKE 'fault-%') EXECUTE FUNCTION lab4_fault_seen();
CREATE TRIGGER lab4_fault_commit BEFORE UPDATE ON attachments FOR EACH ROW
WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'ready')
WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'queued')
EXECUTE FUNCTION lab4_fault_commit();
"""
CLEANUP = """
@@ -94,7 +97,7 @@ def bucket_keys() -> set[str]:
"list",
"--recursive",
"--json",
f"root/{BUCKET}/attachments/",
f"root/{BUCKET}/staging/",
)
return {item["key"] for item in json.loads(listing)["items"]}
@@ -221,7 +224,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
created = 0
attachment = await upload(http, alice, chat)
exact = await upload(http, alice, chat, png(16, 16, size=LIMIT))
assert (attachment.status_code, exact.status_code) == (201, 201)
assert (attachment.status_code, exact.status_code) == (202, 202)
assert exact.json()["size_bytes"] == LIMIT
created += 2
mine = attachment.json()["id"]
@@ -319,9 +322,9 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
400,
)
expect(
"variant=thumbnail",
"variant=preview",
await http.get(
f"/api/v1/attachments/{mine}/content?variant=thumbnail", headers=bearer(alice)
f"/api/v1/attachments/{mine}/content?variant=preview", headers=bearer(alice)
),
400,
)
@@ -332,7 +335,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
)
statuses = [entry["status"] for entry in logs(since) if entry.get("event") == "request.handled"]
print(
f"строк вложений в чате: {rows} (= ответов 201); ответов в журнале реплик: "
f"строк вложений в чате: {rows} (= ответов 202); ответов в журнале реплик: "
f"{len(statuses)}, из них 5xx: {sum(code >= 500 for code in statuses)}"
)
assert not [status for status in statuses if status >= 500]
@@ -375,7 +378,7 @@ async def crash(http: AsyncClient, token: str, chat: str) -> tuple[str, str]:
name = f"fault-crash-{uuid4().hex}.png"
sent = asyncio.create_task(upload(http, token, chat, filename=name))
address = await asyncio.to_thread(sql, "SELECT lab4_fault_sleeper()")
assert address, "UPDATE ready не дошёл до триггера"
assert address, "UPDATE queued не дошёл до триггера"
victim = replica_at(address)
kill(victim)
try:
@@ -396,6 +399,7 @@ async def test_crash_between_put_and_commit_is_swept_after_grace(
s = await scene(http)
alice, chat = s["alice"], s["chats"][0]
kept = (await upload(http, alice, chat, filename="kept.png")).json()
await waited(lambda: status_of(kept["id"]) == "ready", 60)
assert (
await message(http, alice, chat, {"text": "", "attachment_ids": [kept["id"]]})
).status_code == 201
@@ -465,4 +469,4 @@ async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, fault
assert (status, body["error"]["code"]) == (503, "unavailable")
assert elapsed <= REQUEST_TIMEOUT + 2
assert seen(name) is None
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 201
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 202