forked from mtusi2026/mtusi2026lab5
Add the level 3 pipeline scenarios and the stand evidence
Bring the stdlib suite and image generator of the lab 5 preparation: make suite-preflight checks services, readiness, schema columns, broker topology and the faults override, and make level3 runs the grade 3 scenarios (smoke through the pipeline, worker down with API, relay and broker restarts, duplicate delivery after ready, poison image, S3 outage with recovery, retries up to JOB_MAX_ATTEMPTS) and checks the invariants into evidence/level3.json. make pipeline-evidence shows the queued job on the broker while the worker is stopped, its properties and contract payload, the 409 before ready and SIGTERM of the relay and the worker; make topology dumps queues, consumers with prefetch, permissions and users. The lab 4 failover and S3 outage scenarios and the upload fault suite now expect 202 and wait for ready, the stack evidence lists the broker and its client packages, and the CI acceptance job runs the pipeline scenarios and the SIGTERM check of the new processes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c55fb1b998
commit
fe2693870e
@@ -19,7 +19,7 @@ jobs:
|
||||
- run: make student-test VENV=.venv
|
||||
|
||||
acceptance:
|
||||
name: Стенд Compose с RustFS за HAProxy, smoke, вложения, отказ реплики, рестарт и отказы зависимостей
|
||||
name: Стенд Compose с RabbitMQ, relay и worker за HAProxy, smoke, pipeline, отказ реплики, рестарт и отказы зависимостей
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -34,6 +34,17 @@ jobs:
|
||||
- name: Публичный smoke через балансировщик
|
||||
run: make test
|
||||
|
||||
- name: Топология брокера, consumer с prefetch и права клиентов
|
||||
run: make topology
|
||||
|
||||
- name: Уровень «3» pipeline — worker остановлен, рестарт API и брокера, дубль, S3, исчерпание попыток
|
||||
run: |
|
||||
make suite-preflight
|
||||
make level3
|
||||
|
||||
- name: Сообщение задания в очереди, 409 до ready и SIGTERM relay и worker
|
||||
run: make pipeline-evidence
|
||||
|
||||
- name: Вложение после остановки принявшей его реплики и при недоступном S3
|
||||
run: |
|
||||
make scenario SCENARIO=failover-upload
|
||||
@@ -68,12 +79,22 @@ jobs:
|
||||
|
||||
- name: Наружу опубликован только балансировщик
|
||||
run: |
|
||||
docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" {print; bad=1} END {exit bad}'
|
||||
docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" && $1 != "broker" {print; bad=1} END {exit bad}'
|
||||
docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$1 == "broker" && $2 !~ /^127\.0\.0\.1:/ {print; bad=1} END {exit bad}'
|
||||
|
||||
- name: Процессы работают не от root
|
||||
run: |
|
||||
docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}'
|
||||
|
||||
- name: Корректная остановка relay и worker по SIGTERM
|
||||
run: |
|
||||
start=$(date +%s)
|
||||
docker compose stop -t 10 relay worker
|
||||
elapsed=$(( $(date +%s) - start ))
|
||||
codes=$(docker inspect -f '{{.State.ExitCode}}' $(docker compose ps -aq relay worker) | sort -u)
|
||||
echo "остановка за ${elapsed} с, коды завершения ${codes}"
|
||||
test "$elapsed" -le 10 && test "$codes" = "0"
|
||||
|
||||
- name: Корректная остановка реплик API по SIGTERM
|
||||
run: |
|
||||
start=$(date +%s)
|
||||
|
||||
@@ -11,7 +11,8 @@ RATE ?= 100
|
||||
API_IMAGE ?= mtusi-messenger:lab5
|
||||
FROM_REV ?=
|
||||
SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||
scripts/upload_memory.py
|
||||
scripts/upload_memory.py scripts/make_images.py scripts/pipeline_suite.py \
|
||||
scripts/pipeline_evidence.py
|
||||
SCENARIO ?= failover-upload
|
||||
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
|
||||
|
||||
@@ -20,7 +21,9 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
|
||||
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
|
||||
persistence-evidence storage-evidence cache-evidence test-faults seed explain \
|
||||
limits detection failover-each rolling-update level4 \
|
||||
scenario s3-check attachments-evidence upload-faults upload-memory
|
||||
scenario s3-check attachments-evidence upload-faults upload-memory \
|
||||
images queues topology worker-stop worker-start broker-restart suite-preflight level3 \
|
||||
pipeline-evidence
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -31,6 +34,7 @@ help:
|
||||
@echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence"
|
||||
@echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда"
|
||||
@echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/"
|
||||
@echo "lab5: queues, topology, worker-stop/worker-start, broker-restart, images, suite-preflight, level3"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -168,3 +172,42 @@ explain: seed
|
||||
echo "\$$ make explain # scripts/sql/explain-history.sql"; \
|
||||
$(PSQL) -f - < scripts/sql/explain-history.sql; } > evidence/explain-history.txt
|
||||
cat evidence/explain-history.txt
|
||||
|
||||
images:
|
||||
$(PYTHON) scripts/make_images.py
|
||||
|
||||
queues:
|
||||
sh scripts/broker.sh queues
|
||||
|
||||
topology:
|
||||
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; \
|
||||
echo "$$ rabbitmqctl list_queues -p messenger name type durable policy messages"; \
|
||||
$(COMPOSE) exec -T broker rabbitmqctl -q list_queues -p messenger name type durable policy messages; \
|
||||
echo "$$ rabbitmqctl list_consumers -p messenger"; \
|
||||
$(COMPOSE) exec -T broker rabbitmqctl -q list_consumers -p messenger queue_name ack_required prefetch_count; \
|
||||
echo "$$ rabbitmqctl list_permissions -p messenger"; \
|
||||
$(COMPOSE) exec -T broker rabbitmqctl -q list_permissions -p messenger; \
|
||||
echo "$$ rabbitmqctl list_users"; \
|
||||
$(COMPOSE) exec -T broker rabbitmqctl -q list_users; \
|
||||
echo "$$ docker compose ps"; \
|
||||
$(COMPOSE) ps --format '{{.Service}}\t{{.Status}}\t{{.Ports}}'; } > evidence/topology.txt
|
||||
cat evidence/topology.txt
|
||||
|
||||
worker-stop:
|
||||
sh scripts/worker.sh stop
|
||||
|
||||
worker-start:
|
||||
sh scripts/worker.sh start
|
||||
|
||||
broker-restart:
|
||||
sh scripts/broker.sh restart
|
||||
|
||||
suite-preflight:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --preflight
|
||||
|
||||
level3:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 3 --output evidence/level3.json
|
||||
|
||||
pipeline-evidence:
|
||||
$(PYTHON) scripts/pipeline_evidence.py --base-url "$(BASE_URL)" > evidence/pipeline-stand.txt; \
|
||||
status=$$?; cat evidence/pipeline-stand.txt; exit $$status
|
||||
|
||||
@@ -40,9 +40,9 @@
|
||||
|
||||
## 7. Сценарии и обвязка
|
||||
|
||||
- [ ] 7.1 `scripts/pipeline_suite.py`, `scripts/make_images.py` из `.prep/lab5`; цели Makefile `queues`, `worker-stop`, `worker-start`, `broker-restart`, `suite-preflight`, `level3`, `topology`; ruff для новых `scripts/*.py`
|
||||
- [ ] 7.2 CI-job под лабу 5: брокер, relay и worker в стенде, сценарии уровня «3»
|
||||
- [ ] 7.3 Проверка группы: `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` с `make restore`, `make suite-preflight`, `make level3`
|
||||
- [x] 7.1 `scripts/pipeline_suite.py`, `scripts/make_images.py` из `.prep/lab5`; цели Makefile `queues`, `worker-stop`, `worker-start`, `broker-restart`, `suite-preflight`, `level3`, `topology`; ruff для новых `scripts/*.py`
|
||||
- [x] 7.2 CI-job под лабу 5: брокер, relay и worker в стенде, сценарии уровня «3»
|
||||
- [x] 7.3 Проверка группы: `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` с `make restore`, `make suite-preflight`, `make level3`
|
||||
|
||||
## 8. Отчёт
|
||||
|
||||
|
||||
@@ -83,6 +83,8 @@ ignore = [
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "T20", "TID251"]
|
||||
"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"]
|
||||
"scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"]
|
||||
"scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"]
|
||||
"src/messenger/*/infrastructure/**" = ["TID251", "ARG002"]
|
||||
"src/messenger/*/domain/errors.py" = ["N818"]
|
||||
"src/messenger/*/application/errors.py" = ["N818"]
|
||||
|
||||
@@ -36,8 +36,10 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)"
|
||||
echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)"
|
||||
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
|
||||
echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)"
|
||||
echo "packages (uv.lock):"
|
||||
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow; do
|
||||
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
|
||||
faststream aio-pika aiormq dishka-faststream; do
|
||||
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
|
||||
done
|
||||
echo "process users (service uid command):"
|
||||
|
||||
@@ -23,9 +23,9 @@ def compose(*args, stdin=None):
|
||||
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def upload(c, actor, chat, expected=201):
|
||||
def upload(c, actor, chat, expected=202):
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
||||
schema = "Attachment" if expected == 201 else None
|
||||
schema = "Attachment" if expected == 202 else None
|
||||
return c.call(
|
||||
"POST",
|
||||
path,
|
||||
@@ -74,6 +74,7 @@ def failover_upload(c, args):
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
sent = upload(c, alice, chat)
|
||||
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
|
||||
c.wait_ready(bob, att)
|
||||
started = time.monotonic()
|
||||
hook = subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")],
|
||||
@@ -109,7 +110,7 @@ def failover_upload(c, args):
|
||||
|
||||
def s3_outage(c, args):
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat)["json"]
|
||||
att = c.wait_ready(bob, upload(c, alice, chat)["json"])
|
||||
check(
|
||||
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import struct
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAX_BYTES = 10485760
|
||||
MAX_PIXELS = 40000000
|
||||
REJECTED_OR_FAILED = {"upload": [202, 415], "final": "failed", "error_code": "invalid_image"}
|
||||
GIF = (
|
||||
b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00"
|
||||
b",\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;"
|
||||
)
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
||||
|
||||
|
||||
def png(width, height, rows, color_type=2, depth=8, *, extra=(), level=6):
|
||||
packer = zlib.compressobj(level)
|
||||
body = b"".join(packer.compress(b"\x00" + row) for row in rows) + packer.flush()
|
||||
header = struct.pack(">IIBBBBB", width, height, depth, color_type, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ b"".join(extra)
|
||||
+ chunk(b"IDAT", body)
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def ramp(value, size):
|
||||
return value * 255 // max(1, size - 1)
|
||||
|
||||
|
||||
def gradient(width, height, channels):
|
||||
for y in range(height):
|
||||
yield bytes(
|
||||
channel
|
||||
for x in range(width)
|
||||
for channel in (ramp(x, width), ramp(y, height), 160, ramp(x + y, width + height))[
|
||||
:channels
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def blank(width, height, channels, depth=8):
|
||||
row = bytes(width * channels * depth // 8)
|
||||
for _ in range(height):
|
||||
yield row
|
||||
|
||||
|
||||
def padded(data, total):
|
||||
pad = total - len(data) - 12
|
||||
return data[:33] + chunk(b"paDd", bytes(pad)) + data[33:]
|
||||
|
||||
|
||||
class Bits:
|
||||
def __init__(self):
|
||||
self.out = bytearray()
|
||||
self.acc = 0
|
||||
self.size = 0
|
||||
|
||||
def put(self, value, length):
|
||||
self.acc = (self.acc << length) | value
|
||||
self.size += length
|
||||
while self.size >= 8:
|
||||
self.size -= 8
|
||||
byte = (self.acc >> self.size) & 0xFF
|
||||
self.out += b"\xff\x00" if byte == 0xFF else bytes((byte,))
|
||||
self.acc &= (1 << self.size) - 1
|
||||
|
||||
def zeros(self, count):
|
||||
head = min(count, (8 - self.size) % 8)
|
||||
self.put(0, head)
|
||||
count -= head
|
||||
self.out += bytes(count // 8)
|
||||
self.put(0, count % 8)
|
||||
|
||||
def flush(self):
|
||||
if self.size:
|
||||
self.put((1 << (8 - self.size)) - 1, 8 - self.size)
|
||||
return bytes(self.out)
|
||||
|
||||
|
||||
def segment(marker, payload):
|
||||
return b"\xff" + bytes((marker,)) + struct.pack(">H", len(payload) + 2) + payload
|
||||
|
||||
|
||||
def ycbcr(red, green, blue):
|
||||
return (
|
||||
0.299 * red + 0.587 * green + 0.114 * blue,
|
||||
128 - 0.168736 * red - 0.331264 * green + 0.5 * blue,
|
||||
128 + 0.5 * red - 0.418688 * green - 0.081312 * blue,
|
||||
)
|
||||
|
||||
|
||||
def exif(orientation):
|
||||
entry = struct.pack(">HHIHH", 0x0112, 3, 1, orientation, 0)
|
||||
tiff = b"MM\x00\x2a\x00\x00\x00\x08" + struct.pack(">H", 1) + entry + struct.pack(">I", 0)
|
||||
return segment(0xE1, b"Exif\x00\x00" + tiff)
|
||||
|
||||
|
||||
def jpeg(width, height, mode="rgb", orientation=None, uniform=False):
|
||||
components = {"gray": 1, "rgb": 3, "cmyk": 4}[mode]
|
||||
head = b"\xff\xd8" + (
|
||||
exif(orientation)
|
||||
if orientation
|
||||
else segment(0xE0, b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00")
|
||||
)
|
||||
if mode == "cmyk":
|
||||
head += segment(0xEE, b"Adobe\x00\x64\x00\x00\x00\x00\x00")
|
||||
head += segment(0xDB, b"\x00" + bytes([1] * 64))
|
||||
head += segment(
|
||||
0xC0,
|
||||
struct.pack(">BHHB", 8, height, width, components)
|
||||
+ b"".join(bytes((i + 1, 0x11, 0)) for i in range(components)),
|
||||
)
|
||||
head += segment(0xC4, b"\x00" + bytes([0, 0, 0, 12] + [0] * 12) + bytes(range(12)))
|
||||
head += segment(0xC4, b"\x10" + bytes([1] + [0] * 15) + b"\x00")
|
||||
head += segment(
|
||||
0xDA,
|
||||
bytes((components,))
|
||||
+ b"".join(bytes((i + 1, 0)) for i in range(components))
|
||||
+ b"\x00\x3f\x00",
|
||||
)
|
||||
columns, rows = (width + 7) // 8, (height + 7) // 8
|
||||
bits, previous = Bits(), [0] * components
|
||||
for block in range(1 if uniform else columns * rows):
|
||||
bx, by = block % columns, block // columns
|
||||
red, green, blue = ramp(bx, columns), ramp(by, rows), 160
|
||||
values = {"gray": (red,), "rgb": ycbcr(red, green, blue), "cmyk": (red, green, blue, 64)}[
|
||||
mode
|
||||
]
|
||||
for index, value in enumerate(values):
|
||||
dc = round(8 * (value - 128))
|
||||
diff, previous[index] = dc - previous[index], dc
|
||||
category = abs(diff).bit_length()
|
||||
bits.put(category, 4)
|
||||
if category:
|
||||
bits.put(diff if diff > 0 else diff + (1 << category) - 1, category)
|
||||
bits.put(0, 1)
|
||||
if uniform:
|
||||
bits.zeros((columns * rows - 1) * components * 5)
|
||||
return head + bits.flush() + b"\xff\xd9"
|
||||
|
||||
|
||||
def fit(width, height, box, upscale=False):
|
||||
if not upscale and max(width, height) <= box:
|
||||
return [width, height]
|
||||
if width >= height:
|
||||
return [box, max(1, height * box // width)]
|
||||
return [max(1, width * box // height), box]
|
||||
|
||||
|
||||
def ready(width, height, orientation=1):
|
||||
if orientation in (5, 6, 7, 8):
|
||||
width, height = height, width
|
||||
return {
|
||||
"upload": [202],
|
||||
"final": "ready",
|
||||
"variants": {
|
||||
"thumbnail": fit(width, height, 256),
|
||||
"crop": [128, 128],
|
||||
"optimized": fit(width, height, 1280),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def catalog(heavy):
|
||||
small = png(64, 64, gradient(64, 64, 3))
|
||||
truncated_png = png(320, 200, gradient(320, 200, 3))
|
||||
idat = truncated_png.index(b"IDAT")
|
||||
corrupt = bytearray(truncated_png)
|
||||
corrupt[idat + 40 : idat + 48] = b"\x00" * 8
|
||||
palette = bytes(channel for index in range(256) for channel in (index, 255 - index, 128))
|
||||
alpha = bytes(index for index in range(256))
|
||||
items = [
|
||||
(
|
||||
"png-rgb-320x200.png",
|
||||
"image/png",
|
||||
lambda: png(320, 200, gradient(320, 200, 3)),
|
||||
ready(320, 200),
|
||||
),
|
||||
(
|
||||
"png-rgba-300x300.png",
|
||||
"image/png",
|
||||
lambda: png(300, 300, gradient(300, 300, 4), 6),
|
||||
ready(300, 300),
|
||||
),
|
||||
(
|
||||
"png-palette-trns-200x100.png",
|
||||
"image/png",
|
||||
lambda: png(
|
||||
200,
|
||||
100,
|
||||
(bytes((x + y) % 256 for x in range(200)) for y in range(100)),
|
||||
3,
|
||||
extra=(chunk(b"PLTE", palette), chunk(b"tRNS", alpha)),
|
||||
),
|
||||
ready(200, 100),
|
||||
),
|
||||
(
|
||||
"png-gray-17x1000.png",
|
||||
"image/png",
|
||||
lambda: png(17, 1000, gradient(17, 1000, 1), 0),
|
||||
ready(17, 1000),
|
||||
),
|
||||
(
|
||||
"png-gray-alpha-640x480.png",
|
||||
"image/png",
|
||||
lambda: png(640, 480, gradient(640, 480, 2), 4),
|
||||
ready(640, 480),
|
||||
),
|
||||
(
|
||||
"png-gray16-300x200.png",
|
||||
"image/png",
|
||||
lambda: png(300, 200, gradient(600, 200, 1), 0, 16),
|
||||
ready(300, 200),
|
||||
),
|
||||
(
|
||||
"png-rgba16-64x64.png",
|
||||
"image/png",
|
||||
lambda: png(64, 64, gradient(512, 64, 1), 6, 16),
|
||||
ready(64, 64),
|
||||
),
|
||||
("png-1x1.png", "image/png", lambda: png(1, 1, gradient(1, 1, 3)), ready(1, 1)),
|
||||
(
|
||||
"png-wide-3000x2.png",
|
||||
"image/png",
|
||||
lambda: png(3000, 2, gradient(3000, 2, 3)),
|
||||
ready(3000, 2),
|
||||
),
|
||||
(
|
||||
"png-small-50x30.png",
|
||||
"image/png",
|
||||
lambda: png(50, 30, gradient(50, 30, 3)),
|
||||
ready(50, 30),
|
||||
),
|
||||
("jpeg-rgb-1000x750.jpg", "image/jpeg", lambda: jpeg(1000, 750), ready(1000, 750)),
|
||||
("jpeg-gray-801x601.jpg", "image/jpeg", lambda: jpeg(801, 601, "gray"), ready(801, 601)),
|
||||
("jpeg-cmyk-400x300.jpg", "image/jpeg", lambda: jpeg(400, 300, "cmyk"), ready(400, 300)),
|
||||
(
|
||||
"jpeg-exif6-400x200.jpg",
|
||||
"image/jpeg",
|
||||
lambda: jpeg(400, 200, orientation=6),
|
||||
ready(400, 200, 6),
|
||||
),
|
||||
(
|
||||
"jpeg-exif3-300x200.jpg",
|
||||
"image/jpeg",
|
||||
lambda: jpeg(300, 200, orientation=3),
|
||||
ready(300, 200, 3),
|
||||
),
|
||||
("jpeg-rgb-4000x3000.jpg", "image/jpeg", lambda: jpeg(4000, 3000), ready(4000, 3000)),
|
||||
("png-exactly-10mib.png", "image/png", lambda: padded(small, MAX_BYTES), ready(64, 64)),
|
||||
(
|
||||
"png-over-10mib.png",
|
||||
"image/png",
|
||||
lambda: padded(small, MAX_BYTES + 1),
|
||||
{"upload": [413]},
|
||||
),
|
||||
(
|
||||
"png-over-pixels-8001x5000.png",
|
||||
"image/png",
|
||||
lambda: png(8001, 5000, blank(8001, 5000, 3), level=9),
|
||||
{"upload": [413]},
|
||||
),
|
||||
(
|
||||
"jpeg-over-pixels-10000x5000.jpg",
|
||||
"image/jpeg",
|
||||
lambda: jpeg(10000, 5000, uniform=True),
|
||||
{"upload": [413]},
|
||||
),
|
||||
("png-truncated.png", "image/png", lambda: truncated_png[: idat + 600], REJECTED_OR_FAILED),
|
||||
("png-corrupt-idat.png", "image/png", lambda: bytes(corrupt), REJECTED_OR_FAILED),
|
||||
("jpeg-truncated.jpg", "image/jpeg", lambda: jpeg(1000, 750)[:3000], REJECTED_OR_FAILED),
|
||||
(
|
||||
"png-zero-width.png",
|
||||
"image/png",
|
||||
zero_width,
|
||||
{"upload": [400, 415]},
|
||||
),
|
||||
(
|
||||
"not-an-image.png",
|
||||
"image/png",
|
||||
lambda: b"plain text, not an image\n" * 8,
|
||||
{"upload": [415]},
|
||||
),
|
||||
("gif-declared-as-png.png", "image/png", lambda: GIF, {"upload": [415]}),
|
||||
]
|
||||
if heavy:
|
||||
items.append(
|
||||
(
|
||||
"png-at-pixel-limit-8000x5000.png",
|
||||
"image/png",
|
||||
lambda: png(8000, 5000, blank(8000, 5000, 3), level=9),
|
||||
ready(8000, 5000),
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def zero_width():
|
||||
header = struct.pack(">IIBBBBB", 0, 10, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(b""))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def build(out, heavy=False, only=None):
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
manifest = []
|
||||
for name, content_type, make, expect in catalog(heavy):
|
||||
if only and name not in only:
|
||||
continue
|
||||
data = make()
|
||||
(out / name).write_bytes(data)
|
||||
manifest.append(
|
||||
{"file": name, "content_type": content_type, "size_bytes": len(data), "expect": expect}
|
||||
)
|
||||
(out / "manifest.json").write_text(
|
||||
json.dumps(
|
||||
{"rules": {"max_bytes": MAX_BYTES, "max_pixels": MAX_PIXELS}, "images": manifest},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
return manifest
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate lab5 test images and manifest.json with expected outcomes"
|
||||
)
|
||||
parser.add_argument("--out", default=str(ROOT / "build/lab5-images"))
|
||||
parser.add_argument(
|
||||
"--heavy", action="store_true", help="also write a 40 Mpx image at the pixel limit"
|
||||
)
|
||||
parser.add_argument("--only", default="", help="comma separated file names")
|
||||
args = parser.parse_args()
|
||||
manifest = build(Path(args.out), args.heavy, {x for x in args.only.split(",") if x})
|
||||
for item in manifest:
|
||||
print(f"{item['file']:36} {item['size_bytes']:>10} {item['expect']['upload']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, check
|
||||
|
||||
COMPOSE = os.environ.get("COMPOSE", "docker compose").split()
|
||||
PEEK = (
|
||||
'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/'
|
||||
'rabbitmq_ops_password)" exec rabbitmqadmin -V messenger '
|
||||
"--table-style ascii get messages -q attachments.process -a ack_requeue_true"
|
||||
)
|
||||
QUEUES = "rabbitmqctl -q list_queues -p messenger name messages_ready messages_unacknowledged"
|
||||
|
||||
|
||||
def compose(*argv):
|
||||
done = subprocess.run(
|
||||
[*COMPOSE, *argv], cwd=ROOT, capture_output=True, text=True, timeout=180, check=False
|
||||
)
|
||||
check(done.returncode == 0, f"{' '.join(argv[:3])}: {done.stderr.strip()[-300:]}")
|
||||
return done.stdout
|
||||
|
||||
|
||||
def ready_messages():
|
||||
for line in compose("exec", "-T", "broker", *QUEUES.split()).splitlines():
|
||||
name, ready, _ = [*line.split("\t"), "", ""][:3]
|
||||
if name == "attachments.process":
|
||||
return int(ready)
|
||||
return 0
|
||||
|
||||
|
||||
def say(line):
|
||||
print(line, flush=True)
|
||||
|
||||
|
||||
def stop(service):
|
||||
began = time.monotonic()
|
||||
compose("stop", "-t", "10", service)
|
||||
seconds = time.monotonic() - began
|
||||
container = compose("ps", "-aq", service).split()[0]
|
||||
code = subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.State.ExitCode}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
say(f"$ docker compose stop -t 10 {service} -> {seconds:.1f} s, exit code {code}")
|
||||
check(seconds <= 10 and code == "0", f"{service} did not stop cleanly on SIGTERM")
|
||||
|
||||
|
||||
def start(service):
|
||||
compose("up", "-d", "--no-deps", "--wait", "--wait-timeout", "90", service)
|
||||
say(f"$ docker compose up -d --no-deps --wait {service} -> healthy")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Lab5 stand evidence: queue, message, SIGTERM")
|
||||
parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
say(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}")
|
||||
stop("worker")
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
say(f"upload -> 202 status={attachment['status']} id={attachment['id']}")
|
||||
path = f"/api/v1/attachments/{attachment['id']}"
|
||||
for variant in ("original", "thumbnail"):
|
||||
client.call("GET", f"{path}/content?variant={variant}", 409, actor=bob)
|
||||
say(f"GET content?variant={variant} while worker is stopped -> 409")
|
||||
deadline = time.monotonic() + 30
|
||||
while ready_messages() < 1:
|
||||
check(time.monotonic() < deadline, "the job did not reach the broker")
|
||||
time.sleep(0.5)
|
||||
say("$ rabbitmqctl list_queues -p messenger name messages_ready messages_unacknowledged")
|
||||
say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip())
|
||||
say("$ rabbitmqadmin get messages -q attachments.process -a ack_requeue_true")
|
||||
peeked = compose("exec", "-T", "broker", "sh", "-c", PEEK)
|
||||
say(peeked.rstrip())
|
||||
body = json.loads(peeked[peeked.index('{"') : peeked.index("}", peeked.index('{"')) + 1])
|
||||
say(f"payload keys: {sorted(body)}")
|
||||
check(body["attachment_id"] == attachment["id"], "the queued job is for another attachment")
|
||||
stop("relay")
|
||||
start("relay")
|
||||
start("worker")
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
names = [(v["name"], v["width"], v["height"]) for v in ready["variants"]]
|
||||
say(f"after worker start -> status={ready['status']} variants={names}")
|
||||
deadline = time.monotonic() + 30
|
||||
while ready_messages():
|
||||
check(time.monotonic() < deadline, "the queue did not drain")
|
||||
time.sleep(0.5)
|
||||
say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip())
|
||||
say("PASS: pipeline stand evidence")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,945 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
import make_images
|
||||
from client import Client, Failure, check
|
||||
from image_probe import jpeg_size
|
||||
|
||||
FIXTURE = ROOT / "tests/fixtures/sample.png"
|
||||
EXCHANGE, ROUTING_KEY = "attachments", "attachment.process.v1"
|
||||
QUEUE, DLQ = "attachments.process", "attachments.process.dlq"
|
||||
RABBITMQADMIN = (
|
||||
'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/'
|
||||
'rabbitmq_ops_password)" exec rabbitmqadmin --non-interactive -V messenger "$@"'
|
||||
)
|
||||
PSQL = 'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"'
|
||||
ERRORS = (Failure, OSError, ValueError, KeyError, subprocess.TimeoutExpired)
|
||||
SCENARIOS = (
|
||||
("smoke_pipeline", 3, "upload, ready, variants match metadata"),
|
||||
("worker_down_broker_restart", 3, "worker down, API and broker restart"),
|
||||
("duplicate_after_ready", 3, "redelivery of a finished job"),
|
||||
("poison_image", 3, "permanent error without retries"),
|
||||
("s3_outage_recovers", 3, "transient error with retry"),
|
||||
("retry_exhausted", 3, "transient error until JOB_MAX_ATTEMPTS"),
|
||||
("image_matrix", 4, "limits and variants"),
|
||||
("crash_commit_publish", 4, "after COMMIT, before publish"),
|
||||
("crash_publish_confirm", 4, "after publish, before confirm"),
|
||||
("crash_after_s3_before_commit", 4, "after S3 write, before COMMIT"),
|
||||
("crash_after_commit_before_ack", 4, "after COMMIT, before ack"),
|
||||
("lease_expiry_fencing", 4, "stalled worker, lease expiry"),
|
||||
("replay_failed", 4, "operator replay"),
|
||||
("malformed_message_dead_lettered", 4, "message that is not a job goes to the DLQ"),
|
||||
("crash_before_side_effect", 5, "after claim, before side effects"),
|
||||
("worker_kill_mid_task", 5, "SIGKILL during processing"),
|
||||
("duplicate_in_flight", 5, "two copies of one job in flight"),
|
||||
("reorder_attempts", 5, "attempt 2 delivered before attempt 1"),
|
||||
("reorder_versions", 5, "v2 delivered before v1"),
|
||||
("stale_after_new_version", 5, "v1 job after v2 is published"),
|
||||
)
|
||||
QUERIES = """
|
||||
attachment:
|
||||
SELECT status, published_version, error_code, staging_key FROM attachments WHERE id = :'id'
|
||||
|
||||
jobs:
|
||||
SELECT job_id, pipeline_version, status, attempt FROM jobs
|
||||
WHERE attachment_id = :'id' ORDER BY pipeline_version
|
||||
|
||||
outbox:
|
||||
SELECT o.payload, o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id)
|
||||
WHERE j.attachment_id = :'id' ORDER BY o.id
|
||||
|
||||
variants:
|
||||
SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants
|
||||
WHERE attachment_id = :'id' ORDER BY pipeline_version, name
|
||||
|
||||
events:
|
||||
SELECT kind, attempt FROM job_events WHERE attachment_id = :'id' ORDER BY id
|
||||
|
||||
nonterminal:
|
||||
SELECT id, status FROM attachments
|
||||
WHERE id = ANY(:'ids'::uuid[]) AND status NOT IN ('ready', 'failed')
|
||||
|
||||
known:
|
||||
SELECT id FROM attachments WHERE id = ANY(:'ids'::uuid[])
|
||||
|
||||
duplicate_effects:
|
||||
SELECT attachment_id, pipeline_version, name, count(*) AS copies FROM attachment_variants
|
||||
WHERE attachment_id = ANY(:'ids'::uuid[]) GROUP BY 1, 2, 3 HAVING count(*) > 1
|
||||
|
||||
attempts:
|
||||
SELECT job_id, attempt FROM jobs WHERE attachment_id = ANY(:'ids'::uuid[]) AND attempt > :max
|
||||
|
||||
pending_outbox:
|
||||
SELECT o.job_id FROM outbox o JOIN jobs j USING (job_id)
|
||||
WHERE j.attachment_id = ANY(:'ids'::uuid[]) AND o.published_at IS NULL
|
||||
|
||||
unpublished_version:
|
||||
SELECT a.id, a.published_version, max(j.pipeline_version) AS done FROM attachments a
|
||||
JOIN jobs j ON j.attachment_id = a.id AND j.status = 'done' WHERE a.id = ANY(:'ids'::uuid[])
|
||||
GROUP BY a.id, a.published_version HAVING a.published_version <> max(j.pipeline_version)
|
||||
|
||||
staging_left:
|
||||
SELECT id FROM attachments
|
||||
WHERE id = ANY(:'ids'::uuid[]) AND status = 'ready' AND staging_key IS NOT NULL
|
||||
|
||||
object_keys:
|
||||
SELECT object_key AS key FROM attachment_variants
|
||||
UNION ALL SELECT original_key FROM attachments WHERE original_key IS NOT NULL
|
||||
UNION ALL SELECT staging_key FROM attachments WHERE staging_key IS NOT NULL
|
||||
|
||||
columns:
|
||||
SELECT table_name, column_name FROM information_schema.columns
|
||||
WHERE table_schema = current_schema()
|
||||
"""
|
||||
SQL = {
|
||||
name.strip(): " ".join(body.split())
|
||||
for name, body in (block.split(":\n", 1) for block in QUERIES.strip().split("\n\n"))
|
||||
}
|
||||
REQUIRED_COLUMNS = {
|
||||
"attachments": "id status published_version error_code staging_key original_key",
|
||||
"attachment_variants": "attachment_id pipeline_version name sha256 size_bytes object_key",
|
||||
"jobs": "job_id attachment_id pipeline_version status attempt",
|
||||
"outbox": "job_id payload published_at",
|
||||
"job_events": "job_id attachment_id kind attempt",
|
||||
}
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
|
||||
def run(cmd, stdin=None, env=None, timeout=180):
|
||||
done = subprocess.run(
|
||||
cmd, input=stdin, capture_output=True, text=True, env=env, timeout=timeout, check=False
|
||||
)
|
||||
if done.returncode:
|
||||
raise Failure(f"{' '.join(cmd[:7])}: exit {done.returncode}: {done.stderr.strip()[-400:]}")
|
||||
return done.stdout
|
||||
|
||||
|
||||
def poll(probe, timeout, what, interval=0.5):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
value = probe()
|
||||
if value:
|
||||
return value
|
||||
if time.monotonic() > deadline:
|
||||
raise Failure(f"timeout {timeout}s waiting for {what}")
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
def http_ready(base_url):
|
||||
try:
|
||||
with urllib.request.urlopen(base_url.rstrip("/") + "/health/ready", timeout=3) as response:
|
||||
return response.status == 200
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def image_size(data, content_type):
|
||||
if content_type == "image/jpeg":
|
||||
return jpeg_size(data)
|
||||
if content_type == "image/png":
|
||||
return struct.unpack(">II", data[16:24])
|
||||
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
|
||||
if data[12:16] == b"VP8 ":
|
||||
width, height = struct.unpack("<HH", data[26:30])
|
||||
return width & 0x3FFF, height & 0x3FFF
|
||||
if data[12:16] == b"VP8L":
|
||||
bits = int.from_bytes(data[21:25], "little")
|
||||
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
|
||||
return int.from_bytes(data[24:27], "little") + 1, int.from_bytes(data[27:30], "little") + 1
|
||||
|
||||
|
||||
class Stack:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.base = shlex.split(args.compose)
|
||||
self.faults = shlex.split(args.compose_faults)
|
||||
|
||||
def dc(self, *argv, faults=False, stdin=None, env=None, timeout=180):
|
||||
command = [*(self.faults if faults else self.base), *argv]
|
||||
return run(command, stdin, dict(os.environ, **(env or {})), timeout)
|
||||
|
||||
def up(self, *services, faults=False, env=None):
|
||||
wait = ["up", "-d", "--no-deps", "--no-recreate", "--wait", "--wait-timeout", "90"]
|
||||
if faults or env:
|
||||
wait.remove("--no-recreate")
|
||||
self.dc(*wait, *services, faults=faults, env=env)
|
||||
|
||||
def launch(self, service, point=""):
|
||||
variable = "WORKER_FAULT_POINT" if service == self.args.worker else "RELAY_FAULT_POINT"
|
||||
self.up(service, faults=True, env={variable: point})
|
||||
|
||||
def restore(self):
|
||||
self.dc("up", "-d", "--no-deps", "--wait", self.args.relay, self.args.worker)
|
||||
|
||||
def stop(self, *services):
|
||||
self.dc("stop", "-t", "10", *services)
|
||||
|
||||
def kill(self, service):
|
||||
self.dc("kill", "-s", "KILL", service)
|
||||
|
||||
def restart(self, *services):
|
||||
self.dc("restart", "-t", "20", *services)
|
||||
self.up(*services)
|
||||
|
||||
def state(self, service):
|
||||
text = self.dc("ps", "-a", "--format", "json", service).strip()
|
||||
rows = (
|
||||
json.loads(text) if text.startswith("[") else [json.loads(x) for x in text.split("\n")]
|
||||
)
|
||||
check(len(rows) == 1, "expected one container for " + service)
|
||||
return rows[0]["State"], rows[0].get("ExitCode")
|
||||
|
||||
def wait_exit(self, service, timeout=60):
|
||||
def probe():
|
||||
state, code = self.state(service)
|
||||
return (code,) if state in ("exited", "dead") else None
|
||||
|
||||
return poll(probe, timeout, service + " exit")[0]
|
||||
|
||||
def sql(self, name, **params):
|
||||
query = f"SELECT coalesce(json_agg(q), '[]'::json) FROM ({SQL[name]}) q;\n"
|
||||
variables = [x for key, value in params.items() for x in ("-v", f"{key}={value}")]
|
||||
out = self.dc(
|
||||
"exec", "-T", self.args.postgres, "sh", "-c", PSQL, "sh", *variables, stdin=query
|
||||
)
|
||||
return json.loads(out)
|
||||
|
||||
def queues(self):
|
||||
columns = "name messages_ready messages_unacknowledged --formatter json"
|
||||
listing = f"exec -T {self.args.broker} rabbitmqctl -q list_queues -p messenger {columns}"
|
||||
rows = json.loads(self.dc(*listing.split()))
|
||||
return {q["name"]: (q["messages_ready"], q["messages_unacknowledged"]) for q in rows}
|
||||
|
||||
def rabbitmqadmin(self, *argv, stdin=None):
|
||||
return self.dc(
|
||||
"exec", "-T", self.args.broker, "sh", "-c", RABBITMQADMIN, "sh", *argv, stdin=stdin
|
||||
)
|
||||
|
||||
def publish(self, payload):
|
||||
properties = {"delivery_mode": 2, "content_type": "application/json", "type": ROUTING_KEY}
|
||||
properties["message_id"] = payload["job_id"]
|
||||
command = f"publish message -e {EXCHANGE} -k {ROUTING_KEY} -f - -p".split()
|
||||
out = self.rabbitmqadmin(*command, json.dumps(properties), stdin=json.dumps(payload))
|
||||
check("routed successfully" in out, "broker did not route the injected job")
|
||||
|
||||
def admin(self, *argv):
|
||||
command = shlex.split(self.args.admin) + list(argv)
|
||||
out = self.dc("run", "--rm", "--no-deps", "-T", self.args.worker, *command)
|
||||
return json.loads(out.strip().splitlines()[-1])
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, args):
|
||||
self.base_url = args.base_url
|
||||
self.client = Client(args.base_url, args.ca_file)
|
||||
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
|
||||
self.accepted, self.traces, self.final = [], {}, {}
|
||||
|
||||
def upload(self, data, kind="image/png", name="sample.png", expected=202, final=("ready",)):
|
||||
path = f"/api/v1/chats/{self.chat['id']}/attachments?filename={quote(name)}"
|
||||
allowed = [expected] if isinstance(expected, int) else expected
|
||||
headers = {"Content-Type": kind}
|
||||
response = self.client.call(
|
||||
"POST", path, allowed, actor=self.alice, raw=data, headers=headers, schema="Attachment"
|
||||
)
|
||||
if response["status"] != 202:
|
||||
return None
|
||||
body = response["json"]
|
||||
snapshot = (
|
||||
body["status"] == "queued" and body["variants"] == [] and body["error_code"] is None
|
||||
)
|
||||
check(snapshot, "202 is not a queued snapshot")
|
||||
same = body["sha256"] == hashlib.sha256(data).hexdigest() and body["size_bytes"] == len(
|
||||
data
|
||||
)
|
||||
check(same, "202 checksum or size differs from the uploaded bytes")
|
||||
self.accepted.append(body["id"])
|
||||
self.final[body["id"]] = set(final)
|
||||
self.traces[body["id"]] = ["queued"]
|
||||
return body
|
||||
|
||||
def fixture(self, **kwargs):
|
||||
return self.upload(FIXTURE.read_bytes(), **kwargs)
|
||||
|
||||
def get(self, ident):
|
||||
path = "/api/v1/attachments/" + ident
|
||||
body = self.client.call("GET", path, actor=self.bob, schema="Attachment")["json"]
|
||||
self.traces.setdefault(ident, []).append(body["status"])
|
||||
return body
|
||||
|
||||
def wait(self, ident, final=("ready",), timeout=90):
|
||||
def probe():
|
||||
body = self.get(ident)
|
||||
failed = body["status"] == "failed" and "failed" not in final
|
||||
check(not failed, f"attachment reached failed: {body['error_code']}")
|
||||
return body if body["status"] in final else None
|
||||
|
||||
return poll(probe, timeout, f"{ident} {'/'.join(final)}", 0.3)
|
||||
|
||||
def content(self, ident, variant=None, expected=200):
|
||||
path = f"/api/v1/attachments/{ident}/content" + (f"?variant={variant}" if variant else "")
|
||||
return self.client.call("GET", path, expected, actor=self.bob)
|
||||
|
||||
def message(self, text, attachments=None, expected=201):
|
||||
if expected == 201:
|
||||
return self.client.message(self.alice, self.chat, text, attachments=attachments)["json"]
|
||||
body = {"text": text, "attachment_ids": attachments or []}
|
||||
headers = {"Idempotency-Key": os.urandom(8).hex()}
|
||||
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
||||
return self.client.call(
|
||||
"POST", path, expected, actor=self.alice, body=body, headers=headers
|
||||
)
|
||||
|
||||
|
||||
class Suite:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.stack = Stack(args)
|
||||
self.api = None
|
||||
self.samples = {}
|
||||
self.dead_letters = 0
|
||||
|
||||
def scenarios(self):
|
||||
return [(name, level, window, getattr(self, name)) for name, level, window in SCENARIOS]
|
||||
|
||||
def drained(self, timeout=60):
|
||||
poll(lambda: self.stack.queues()[QUEUE] == (0, 0), timeout, "main queue drained")
|
||||
|
||||
def queued(self, count, timeout=30):
|
||||
poll(lambda: self.stack.queues()[QUEUE][0] >= count, timeout, f"{count} ready messages")
|
||||
|
||||
def record(self, ident):
|
||||
row = self.stack.sql("attachment", id=ident)[0]
|
||||
self.samples.setdefault(ident, []).append(row["published_version"])
|
||||
self.api.traces.setdefault(ident, []).append(row["status"])
|
||||
return row
|
||||
|
||||
def snapshot(self, ident):
|
||||
body = self.api.get(ident)
|
||||
public = sorted(body["variants"], key=lambda v: v["name"])
|
||||
published = self.record(ident)["published_version"]
|
||||
rows = self.stack.sql("variants", id=ident)
|
||||
return {"status": body["status"], "public": public, "rows": rows, "published": published}
|
||||
|
||||
def payload(self, ident, version=1):
|
||||
rows = self.stack.sql("outbox", id=ident)
|
||||
found = [r["payload"] for r in rows if r["payload"]["pipeline_version"] == version]
|
||||
check(bool(found), f"no outbox payload for v{version}")
|
||||
return found[-1]
|
||||
|
||||
def verify(self, ready, expect=None):
|
||||
names = sorted(v["name"] for v in ready["variants"])
|
||||
check(len(names) == len(set(names)), "duplicate variant names in API")
|
||||
missing = set(self.args.expect_variants.split(",")) - set(names)
|
||||
check(not missing, f"missing variants {sorted(missing)}")
|
||||
original = self.api.content(ready["id"])
|
||||
data = original["bytes"]
|
||||
same = (
|
||||
hashlib.sha256(data).hexdigest() == ready["sha256"] and len(data) == ready["size_bytes"]
|
||||
)
|
||||
check(same, "original bytes differ from metadata")
|
||||
check(
|
||||
original["headers"].get_content_type() == ready["content_type"], "original type differs"
|
||||
)
|
||||
for variant in ready["variants"]:
|
||||
name = variant["name"]
|
||||
got = self.api.content(ready["id"], name)
|
||||
data, kind = got["bytes"], got["headers"].get_content_type()
|
||||
check(kind == variant["content_type"], name + " content type differs")
|
||||
same = len(data) == variant["size_bytes"]
|
||||
check(
|
||||
same and hashlib.sha256(data).hexdigest() == variant["sha256"],
|
||||
name + " bytes differ",
|
||||
)
|
||||
size = (variant["width"], variant["height"])
|
||||
check(tuple(image_size(data, kind)) == size, name + " real size differs from metadata")
|
||||
wanted = (expect or {}).get(name)
|
||||
slack = 1 if name == "optimized" else 0
|
||||
fits = not wanted or all(abs(a - b) <= slack for a, b in zip(wanted, size))
|
||||
check(fits, f"{name} is {size}, expected {wanted}")
|
||||
return names
|
||||
|
||||
def crash(self, service, point, observe):
|
||||
self.drained()
|
||||
self.stack.launch(service, point)
|
||||
ident = self.api.fixture()["id"]
|
||||
code = self.stack.wait_exit(service)
|
||||
check(code == 137, f"{service} exited with {code}, expected 137 at {point}")
|
||||
seen = observe(ident)
|
||||
self.stack.launch(service)
|
||||
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
|
||||
poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox")
|
||||
self.drained()
|
||||
self.verify(ready)
|
||||
variants = self.stack.sql("variants", id=ident)
|
||||
return {
|
||||
"attachment_id": ident,
|
||||
"exit_code": code,
|
||||
"after_crash": seen,
|
||||
"variants": variants,
|
||||
}
|
||||
|
||||
def smoke_pipeline(self):
|
||||
att = self.api.fixture()
|
||||
ready = self.api.wait(att["id"])
|
||||
return {
|
||||
"attachment_id": att["id"],
|
||||
"variants": self.verify(ready, {"thumbnail": [256, 160]}),
|
||||
}
|
||||
|
||||
def worker_down_broker_restart(self):
|
||||
steps = []
|
||||
|
||||
def mark(step, **extra):
|
||||
steps.append({"at": now(), "step": step, **extra})
|
||||
|
||||
self.stack.stop(self.args.worker)
|
||||
mark("worker stopped")
|
||||
att = self.api.fixture()
|
||||
mark("upload 202 queued", attachment_id=att["id"])
|
||||
self.api.content(att["id"], expected=409)
|
||||
mark("content 409 while queued")
|
||||
message = self.api.message("", [att["id"]])
|
||||
self.api.message("text messages still work")
|
||||
path = f"/api/v1/chats/{self.api.chat['id']}/messages?limit=5"
|
||||
self.api.client.call("GET", path, actor=self.api.bob, schema="MessagePage")
|
||||
mark("message with attachment 201, other messages work", message_id=message["id"])
|
||||
self.queued(1)
|
||||
mark("job is in the broker queue", queues=self.stack.queues())
|
||||
self.stack.restart(*self.args.api, self.args.relay)
|
||||
poll(lambda: http_ready(self.api.base_url), 90, "API ready")
|
||||
mark("API replicas and relay restarted")
|
||||
self.stack.restart(self.args.broker)
|
||||
self.queued(1)
|
||||
mark("broker restarted with its volume, job survived", queues=self.stack.queues())
|
||||
self.stack.launch(self.args.worker)
|
||||
mark("worker started")
|
||||
ready = self.api.wait(att["id"])
|
||||
mark("ready", variants=self.verify(ready))
|
||||
return {"attachment_id": att["id"], "timeline": steps}
|
||||
|
||||
def duplicate_after_ready(self):
|
||||
ident = self.api.fixture()["id"]
|
||||
self.api.wait(ident)
|
||||
self.drained()
|
||||
before = self.snapshot(ident)
|
||||
self.stack.publish(self.payload(ident))
|
||||
self.drained()
|
||||
time.sleep(1)
|
||||
check(self.snapshot(ident) == before, "duplicate delivery changed the published result")
|
||||
return {"attachment_id": ident, "events": self.stack.sql("events", id=ident)}
|
||||
|
||||
def poison_image(self):
|
||||
data = make_images.png(320, 200, make_images.gradient(320, 200, 3))
|
||||
broken = data[: data.index(b"IDAT") + 600]
|
||||
att = self.api.upload(broken, expected=[202, 415], final=("failed",))
|
||||
if att is None:
|
||||
return {"rejected_by_api": 415}
|
||||
failed = self.api.wait(att["id"], ("failed",))
|
||||
check(
|
||||
failed["error_code"] == "invalid_image", f"unexpected error_code {failed['error_code']}"
|
||||
)
|
||||
jobs = self.stack.sql("jobs", id=att["id"])
|
||||
check(all(j["attempt"] == 1 for j in jobs), "permanent error was retried")
|
||||
self.api.content(att["id"], expected=409)
|
||||
self.api.message("", [att["id"]], expected=409)
|
||||
return {"attachment_id": att["id"], "error_code": failed["error_code"], "jobs": jobs}
|
||||
|
||||
def s3_outage_recovers(self):
|
||||
self.stack.stop(self.args.worker)
|
||||
ident = self.api.fixture()["id"]
|
||||
self.queued(1)
|
||||
self.stack.stop(self.args.s3)
|
||||
try:
|
||||
self.stack.launch(self.args.worker)
|
||||
poll(
|
||||
lambda: any(j["attempt"] > 1 for j in self.stack.sql("jobs", id=ident)), 60, "retry"
|
||||
)
|
||||
status = self.record(ident)["status"]
|
||||
check(status in ("queued", "processing"), "transient error produced " + status)
|
||||
finally:
|
||||
self.stack.up(self.args.s3)
|
||||
poll(lambda: http_ready(self.api.base_url), 90, "API ready")
|
||||
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
|
||||
return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)}
|
||||
|
||||
def exhaust(self):
|
||||
self.drained()
|
||||
self.stack.launch(self.args.worker, "transient_error")
|
||||
ident = self.api.fixture(final=("failed", "ready"))["id"]
|
||||
failed = self.api.wait(ident, ("failed",), timeout=self.args.recovery_timeout)
|
||||
self.stack.launch(self.args.worker)
|
||||
jobs = self.stack.sql("jobs", id=ident)
|
||||
check(failed["error_code"] == "retry_exhausted", f"unexpected {failed['error_code']}")
|
||||
check(max(j["attempt"] for j in jobs) == self.args.max_attempts, "attempts != max")
|
||||
return ident, jobs
|
||||
|
||||
def retry_exhausted(self):
|
||||
ident, jobs = self.exhaust()
|
||||
self.api.final[ident] = {"failed"}
|
||||
self.api.content(ident, expected=409)
|
||||
self.api.message("", [ident], expected=409)
|
||||
return {"attachment_id": ident, "jobs": jobs}
|
||||
|
||||
def replay_failed(self):
|
||||
ident, _ = self.exhaust()
|
||||
self.api.final[ident] = {"ready"}
|
||||
first = self.stack.admin("replay", "--attachment-id", ident)
|
||||
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
|
||||
self.drained()
|
||||
before = self.snapshot(ident)
|
||||
second = self.stack.admin("replay", "--attachment-id", ident)
|
||||
self.drained()
|
||||
check(self.snapshot(ident) == before, "replay of a ready attachment changed the result")
|
||||
events = self.stack.sql("events", id=ident)
|
||||
check(any("replay" in e["kind"] for e in events), "replay left no audit event")
|
||||
return {
|
||||
"attachment_id": ident,
|
||||
"replay": first,
|
||||
"replay_of_ready": second,
|
||||
"events": events,
|
||||
}
|
||||
|
||||
def malformed_message_dead_lettered(self):
|
||||
before = self.stack.queues()[DLQ][0]
|
||||
self.stack.publish({"schema_version": 2, "job_id": str(uuid.uuid4())})
|
||||
poll(lambda: self.stack.queues()[DLQ][0] > before, 30, "dead letter")
|
||||
self.dead_letters += 1
|
||||
return {"dead_letters": self.stack.queues()[DLQ][0] - before}
|
||||
|
||||
def image_matrix(self):
|
||||
folder = Path(self.args.images)
|
||||
results = []
|
||||
for item in make_images.build(folder, self.args.heavy):
|
||||
expect, name = item["expect"], item["file"]
|
||||
ready = expect.get("final") == "ready"
|
||||
data = (folder / name).read_bytes()
|
||||
final = ("ready",) if ready else ("failed", "ready")
|
||||
att = self.api.upload(data, item["content_type"], name, expect["upload"], final)
|
||||
row = {"file": name, "http": 202 if att else "rejected"}
|
||||
if att:
|
||||
done = self.api.wait(att["id"], ("ready", "failed"), self.args.recovery_timeout)
|
||||
row.update(status=done["status"], error_code=done["error_code"])
|
||||
if ready:
|
||||
check(done["status"] == "ready", name + " did not become ready")
|
||||
row["variants"] = self.verify(done, expect["variants"])
|
||||
else:
|
||||
self.api.final[att["id"]] = {"failed"}
|
||||
code = expect.get("error_code")
|
||||
check(done["error_code"] == code, f"{name} should fail with {code}")
|
||||
results.append(row)
|
||||
return {"images": results}
|
||||
|
||||
def crash_commit_publish(self):
|
||||
def observe(ident):
|
||||
outbox = self.stack.sql("outbox", id=ident)
|
||||
check(outbox and not any(r["published"] for r in outbox), "outbox already published")
|
||||
check(self.stack.queues()[QUEUE] == (0, 0), "job reached the broker before publish")
|
||||
return {"outbox_published": False, "status": self.record(ident)["status"]}
|
||||
|
||||
return self.crash(self.args.relay, "after_commit_before_publish", observe)
|
||||
|
||||
def crash_publish_confirm(self):
|
||||
def observe(ident):
|
||||
outbox = [r["published"] for r in self.stack.sql("outbox", id=ident)]
|
||||
return {"outbox_published": outbox, "queues": self.stack.queues()}
|
||||
|
||||
return self.crash(self.args.relay, "after_publish_before_confirm", observe)
|
||||
|
||||
def crash_after_s3_before_commit(self):
|
||||
def observe(ident):
|
||||
check(not self.stack.sql("variants", id=ident), "variants committed before the crash")
|
||||
return {"status": self.record(ident)["status"], "variant_rows": 0}
|
||||
|
||||
return self.crash(self.args.worker, "after_s3_write_before_commit", observe)
|
||||
|
||||
def crash_after_commit_before_ack(self):
|
||||
def observe(ident):
|
||||
row = self.record(ident)
|
||||
check(row["status"] == "ready", "result was not committed before the crash point")
|
||||
variants = self.stack.sql("variants", id=ident)
|
||||
return {"status": row["status"], "variants": variants, "queues": self.stack.queues()}
|
||||
|
||||
result = self.crash(self.args.worker, "after_commit_before_ack", observe)
|
||||
same = result["variants"] == result["after_crash"]["variants"]
|
||||
check(same, "redelivery after COMMIT changed variants")
|
||||
return result
|
||||
|
||||
def crash_before_side_effect(self):
|
||||
def observe(ident):
|
||||
return {
|
||||
"status": self.record(ident)["status"],
|
||||
"jobs": self.stack.sql("jobs", id=ident),
|
||||
}
|
||||
|
||||
return self.crash(self.args.worker, "after_claim_before_side_effect", observe)
|
||||
|
||||
def lease_expiry_fencing(self):
|
||||
self.drained()
|
||||
self.stack.launch(self.args.worker, "stall_after_claim")
|
||||
ident = self.api.fixture()["id"]
|
||||
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
|
||||
before = self.snapshot(ident)
|
||||
time.sleep(self.args.stall_seconds + 3)
|
||||
after = self.snapshot(ident)
|
||||
self.stack.launch(self.args.worker)
|
||||
check(before == after, "stalled worker overwrote the result after its lease expired")
|
||||
jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident)
|
||||
return {"attachment_id": ident, "jobs": jobs, "events": events}
|
||||
|
||||
def worker_kill_mid_task(self):
|
||||
self.drained()
|
||||
self.stack.launch(self.args.worker, "stall_after_claim")
|
||||
ident = self.api.fixture()["id"]
|
||||
poll(lambda: self.record(ident)["status"] == "processing", 30, "processing")
|
||||
self.stack.kill(self.args.worker)
|
||||
code = self.stack.wait_exit(self.args.worker)
|
||||
self.stack.launch(self.args.worker)
|
||||
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
|
||||
self.drained()
|
||||
self.verify(ready)
|
||||
return {"attachment_id": ident, "exit_code": code, "jobs": self.stack.sql("jobs", id=ident)}
|
||||
|
||||
def duplicate_in_flight(self):
|
||||
self.stack.stop(self.args.worker)
|
||||
ident = self.api.fixture()["id"]
|
||||
self.queued(1)
|
||||
self.stack.publish(self.payload(ident))
|
||||
self.queued(2)
|
||||
self.stack.launch(self.args.worker)
|
||||
ready = self.api.wait(ident)
|
||||
self.drained()
|
||||
self.verify(ready)
|
||||
variants, events = self.stack.sql("variants", id=ident), self.stack.sql("events", id=ident)
|
||||
return {"attachment_id": ident, "variants": variants, "events": events}
|
||||
|
||||
def reorder_attempts(self):
|
||||
self.stack.stop(self.args.worker, self.args.relay)
|
||||
ident = self.api.fixture()["id"]
|
||||
original = self.payload(ident)
|
||||
self.stack.publish({**original, "attempt": original["attempt"] + 1})
|
||||
self.queued(1)
|
||||
self.stack.launch(self.args.relay)
|
||||
self.queued(2)
|
||||
self.stack.launch(self.args.worker)
|
||||
ready = self.api.wait(ident)
|
||||
self.drained()
|
||||
self.verify(ready)
|
||||
jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident)
|
||||
return {"attachment_id": ident, "jobs": jobs, "events": events}
|
||||
|
||||
def published(self, ident, version):
|
||||
poll(
|
||||
lambda: self.record(ident)["published_version"] >= version,
|
||||
self.args.recovery_timeout,
|
||||
f"published v{version}",
|
||||
)
|
||||
|
||||
def reprocess(self, ident):
|
||||
return self.stack.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
|
||||
|
||||
def reorder_versions(self):
|
||||
self.stack.stop(self.args.worker, self.args.relay)
|
||||
ident = self.api.fixture()["id"]
|
||||
self.reprocess(ident)
|
||||
self.stack.publish(self.payload(ident, 2))
|
||||
self.queued(1)
|
||||
self.stack.launch(self.args.relay)
|
||||
self.queued(3)
|
||||
self.stack.launch(self.args.worker)
|
||||
self.published(ident, 2)
|
||||
self.api.wait(ident)
|
||||
self.drained()
|
||||
final = self.snapshot(ident)
|
||||
check(final["published"] == 2, f"published_version rolled back to {final['published']}")
|
||||
served = {v["name"]: v["sha256"] for v in final["public"]}
|
||||
stored = {r["name"]: r["sha256"] for r in final["rows"] if r["pipeline_version"] == 2}
|
||||
check(served == stored, "API serves variants of a version that is not published")
|
||||
self.verify(self.api.get(ident))
|
||||
return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)}
|
||||
|
||||
def stale_after_new_version(self):
|
||||
ident = self.api.fixture()["id"]
|
||||
self.api.wait(ident)
|
||||
self.reprocess(ident)
|
||||
self.published(ident, 2)
|
||||
self.drained()
|
||||
before = self.snapshot(ident)
|
||||
self.stack.publish(self.payload(ident, 1))
|
||||
self.drained()
|
||||
time.sleep(1)
|
||||
after = self.snapshot(ident)
|
||||
check(before == after and after["published"] == 2, "stale v1 job changed the v2 result")
|
||||
return {"attachment_id": ident, "published_version": after["published"]}
|
||||
|
||||
def invariants(self, dead_before):
|
||||
ids = self.api.accepted
|
||||
array = "{" + ",".join(ids) + "}"
|
||||
if not ids:
|
||||
return [{"id": "accepted_work", "result": "FAIL", "details": "no upload got 202"}]
|
||||
|
||||
def empty(name, message, **params):
|
||||
rows = self.stack.sql(name, ids=array, **params)
|
||||
check(not rows, f"{message}: {rows[:5]}")
|
||||
return {"rows": 0}
|
||||
|
||||
def terminal():
|
||||
empty("nonterminal", "non-terminal attachments left")
|
||||
wrong = [i for i in ids if self.api.get(i)["status"] not in self.api.final[i]]
|
||||
check(not wrong, "unexpected final status: " + ", ".join(wrong))
|
||||
return {"accepted": len(ids)}
|
||||
|
||||
def no_rollback():
|
||||
traces = self.api.traces.items()
|
||||
broken = [
|
||||
i for i, t in traces if "ready" in t and set(t[t.index("ready") :]) != {"ready"}
|
||||
]
|
||||
check(not broken, "ready rolled back: " + ", ".join(broken))
|
||||
return {"traced": len(self.api.traces)}
|
||||
|
||||
def monotonic():
|
||||
broken = [i for i, values in self.samples.items() if values != sorted(values)]
|
||||
check(not broken, "published_version went backwards: " + ", ".join(broken))
|
||||
empty("unpublished_version", "published_version differs from the newest done job")
|
||||
return {"sampled": len(self.samples)}
|
||||
|
||||
def served():
|
||||
bodies = [self.api.get(i) for i in ids]
|
||||
ready = [b for b in bodies if b["status"] == "ready"]
|
||||
for body in ready:
|
||||
self.verify(body)
|
||||
return {"verified": len(ready)}
|
||||
|
||||
def orphans():
|
||||
empty("pending_outbox", "unpublished outbox rows")
|
||||
empty("staging_left", "staging kept after ready")
|
||||
queues = self.stack.queues()
|
||||
check(queues[QUEUE] == (0, 0), "messages left in " + QUEUE)
|
||||
expected = dead_before + self.dead_letters
|
||||
check(queues[DLQ][0] == expected, f"unexpected dead letters: {queues[DLQ]}")
|
||||
if not self.args.s3_list:
|
||||
return {"queues": queues, "s3": "SKIP: --s3-list is not set"}
|
||||
keys = {r["key"] for r in self.stack.sql("object_keys")}
|
||||
listing = run(shlex.split(self.args.s3_list)).splitlines()
|
||||
stray = [k for k in (x.split()[-1] for x in listing if x.strip()) if k not in keys]
|
||||
check(not stray, f"S3 objects without metadata: {stray[:5]}")
|
||||
return {"queues": queues, "s3_objects": len(listing)}
|
||||
|
||||
def no_loss():
|
||||
known = {r["id"] for r in self.stack.sql("known", ids=array)}
|
||||
check(known == set(ids), "accepted attachments are missing in PostgreSQL")
|
||||
return {"accepted": len(ids)}
|
||||
|
||||
claims = [
|
||||
("terminal", "every 202 reaches its expected terminal state", terminal),
|
||||
(
|
||||
"single_effect",
|
||||
"one variant set per (attachment_id, pipeline_version, variant)",
|
||||
lambda: empty("duplicate_effects", "duplicate effect rows"),
|
||||
),
|
||||
("no_ready_rollback", "ready never returns to queued or processing", no_rollback),
|
||||
("monotonic_publication", "published_version only moves forward", monotonic),
|
||||
("bytes_match_metadata", "sha256, size and real dimensions match served bytes", served),
|
||||
(
|
||||
"bounded_attempts",
|
||||
"attempt <= JOB_MAX_ATTEMPTS",
|
||||
lambda: empty("attempts", "attempt above limit", max=self.args.max_attempts),
|
||||
),
|
||||
("no_orphans", "no pending outbox, messages, dead letters, staging, objects", orphans),
|
||||
("no_lost_202", "every 202 is present in PostgreSQL and the API", no_loss),
|
||||
]
|
||||
return [self.claim(name, statement, probe) for name, statement, probe in claims]
|
||||
|
||||
@staticmethod
|
||||
def claim(name, statement, probe):
|
||||
try:
|
||||
return {"id": name, "statement": statement, "result": "PASS", "details": probe()}
|
||||
except ERRORS as error:
|
||||
return {"id": name, "statement": statement, "result": "FAIL", "details": str(error)}
|
||||
|
||||
def preflight(self):
|
||||
def services():
|
||||
self.detect_api()
|
||||
present = set(self.stack.dc("config", "--services").split())
|
||||
wanted = {*self.args.api, self.args.relay, self.args.worker, self.args.broker}
|
||||
wanted |= {self.args.postgres, self.args.s3}
|
||||
check(wanted <= present, f"missing services {sorted(wanted - present)}")
|
||||
return sorted(wanted)
|
||||
|
||||
def schema():
|
||||
columns = {}
|
||||
for row in self.stack.sql("columns"):
|
||||
columns.setdefault(row["table_name"], set()).add(row["column_name"])
|
||||
needed = {t: set(c.split()) for t, c in REQUIRED_COLUMNS.items()}
|
||||
missing = {t: sorted(c - columns.get(t, set())) for t, c in needed.items()}
|
||||
check(not any(missing.values()), f"missing columns {missing}")
|
||||
return sorted(needed)
|
||||
|
||||
def broker():
|
||||
queues = self.stack.queues()
|
||||
check({QUEUE, DLQ} <= set(queues), "queues are not declared")
|
||||
check(QUEUE in self.stack.rabbitmqadmin("list", "queues"), "rabbitmqadmin failed")
|
||||
return queues
|
||||
|
||||
def faults():
|
||||
check("FAULT_POINT" in self.stack.dc("config", faults=True), "no FAULT_POINT in faults")
|
||||
return self.args.compose_faults
|
||||
|
||||
def api():
|
||||
check(http_ready(self.args.base_url), "GET /health/ready is not 200")
|
||||
return self.args.base_url
|
||||
|
||||
steps = [
|
||||
("compose services", services),
|
||||
("api ready", api),
|
||||
("postgres schema", schema),
|
||||
("broker topology and ops CLI", broker),
|
||||
("faults override", faults),
|
||||
]
|
||||
return [self.claim(name, name, probe) for name, probe in steps]
|
||||
|
||||
def main(self):
|
||||
report = {
|
||||
"suite": "lab5-pipeline-idempotency",
|
||||
"started_at": now(),
|
||||
"config": self.config(),
|
||||
}
|
||||
if self.args.preflight:
|
||||
report["preflight"] = self.preflight()
|
||||
return self.finish(report, report["preflight"])
|
||||
results = []
|
||||
try:
|
||||
self.detect_api()
|
||||
self.api = Api(self.args)
|
||||
dead_before = self.stack.queues()[DLQ][0]
|
||||
self.stack.launch(self.args.relay)
|
||||
self.stack.launch(self.args.worker)
|
||||
for name, level, window, scenario in self.scenarios():
|
||||
if self.selected(name, level):
|
||||
results.append(self.attempt(name, level, window, scenario))
|
||||
report["invariants"] = self.invariants(dead_before)
|
||||
except ERRORS as error:
|
||||
report["error"] = str(error)
|
||||
finally:
|
||||
self.stack.up(self.args.s3)
|
||||
self.stack.restore()
|
||||
report["scenarios"] = results
|
||||
return self.finish(report, results + report.get("invariants", []))
|
||||
|
||||
def attempt(self, name, level, window, scenario):
|
||||
began = time.monotonic()
|
||||
print("RUN " + name, flush=True)
|
||||
try:
|
||||
outcome = {"result": "PASS", "observed": scenario()}
|
||||
except ERRORS as error:
|
||||
outcome = {"result": "FAIL", "error": str(error)}
|
||||
self.heal()
|
||||
print(f"{outcome['result']} {name}", flush=True)
|
||||
seconds = round(time.monotonic() - began, 1)
|
||||
return {"name": name, "level": level, "window": window, "seconds": seconds, **outcome}
|
||||
|
||||
def heal(self):
|
||||
try:
|
||||
self.stack.up(self.args.s3, self.args.broker)
|
||||
self.stack.launch(self.args.relay)
|
||||
self.stack.launch(self.args.worker)
|
||||
except ERRORS as error:
|
||||
print("heal failed: " + str(error), flush=True)
|
||||
|
||||
def detect_api(self):
|
||||
if not self.args.api:
|
||||
names = self.stack.dc("config", "--services").split()
|
||||
api = [x for x in names if x == "api" or (x[:4] == "api-" and x[4:].isdigit())]
|
||||
self.args.api = api
|
||||
check(bool(self.args.api), "no API service: expected api or api-N, or pass --api")
|
||||
|
||||
def selected(self, name, level):
|
||||
return name in self.args.only.split(",") if self.args.only else level <= self.args.level
|
||||
|
||||
def config(self):
|
||||
keys = (
|
||||
"base_url level only expect_variants max_attempts stall_seconds compose compose_faults"
|
||||
)
|
||||
return {key: getattr(self.args, key) for key in keys.split()}
|
||||
|
||||
def finish(self, report, verdicts):
|
||||
passed = bool(verdicts) and "error" not in report
|
||||
report["result"] = (
|
||||
"PASS" if passed and all(v["result"] == "PASS" for v in verdicts) else "FAIL"
|
||||
)
|
||||
report["finished_at"] = now()
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
if self.args.output:
|
||||
Path(self.args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(self.args.output).write_text(text)
|
||||
else:
|
||||
print(text)
|
||||
print(f"{report['result']}: lab5 pipeline suite, report {self.args.output or 'stdout'}")
|
||||
return 0 if report["result"] == "PASS" else 1
|
||||
|
||||
|
||||
def arguments():
|
||||
env = os.environ.get
|
||||
faults = "docker compose -f compose.yaml -f infra/compose.faults.yaml"
|
||||
parser = argparse.ArgumentParser(description="Lab5 pipeline fault injection with invariants")
|
||||
parser.add_argument("--base-url", default=env("BASE_URL", "http://localhost:8080"))
|
||||
parser.add_argument("--ca-file", default=env("CA_FILE"))
|
||||
parser.add_argument("--compose", default=env("COMPOSE", "docker compose"))
|
||||
parser.add_argument("--compose-faults", default=env("COMPOSE_FAULTS", faults))
|
||||
parser.add_argument("--admin", default=env("ADMIN_CLI", "python -m messenger.worker.admin"))
|
||||
parser.add_argument("--api", nargs="+", default=env("API_SERVICES", "").split())
|
||||
parser.add_argument("--worker", default="worker")
|
||||
parser.add_argument("--relay", default="relay")
|
||||
parser.add_argument("--broker", default="broker")
|
||||
parser.add_argument("--postgres", default="postgres")
|
||||
parser.add_argument("--s3", default="s3")
|
||||
parser.add_argument("--s3-list", default=env("S3_LIST_CMD", ""), help="prints object keys")
|
||||
parser.add_argument("--level", type=int, choices=[3, 4, 5], default=5)
|
||||
parser.add_argument("--only", default="", help="comma separated scenario names")
|
||||
parser.add_argument("--list", action="store_true")
|
||||
parser.add_argument("--preflight", action="store_true")
|
||||
parser.add_argument("--expect-variants", default=None)
|
||||
parser.add_argument("--max-attempts", type=int, default=int(env("JOB_MAX_ATTEMPTS", "5")))
|
||||
parser.add_argument(
|
||||
"--stall-seconds", type=float, default=float(env("FAULT_STALL_SECONDS", "15"))
|
||||
)
|
||||
parser.add_argument("--recovery-timeout", type=float, default=120)
|
||||
parser.add_argument("--images", default=str(ROOT / "build/lab5-images"))
|
||||
parser.add_argument("--heavy", action="store_true")
|
||||
parser.add_argument("--output", default="")
|
||||
args = parser.parse_args()
|
||||
if args.expect_variants is None:
|
||||
args.expect_variants = "thumbnail" if args.level == 3 else "thumbnail,crop,optimized"
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
suite = Suite(arguments())
|
||||
if suite.args.list:
|
||||
for name, level, window, _ in suite.scenarios():
|
||||
print(f"{name:32} level {level} {window}")
|
||||
return 0
|
||||
return suite.main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -37,7 +37,10 @@ HEAD = (
|
||||
FAULTS = """
|
||||
CREATE TABLE lab4_fault_seen (id uuid PRIMARY KEY, object_key text, filename text);
|
||||
CREATE FUNCTION lab4_fault_seen() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.object_key, NEW.filename); RETURN NEW; END $$;
|
||||
BEGIN
|
||||
INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.staging_key, NEW.filename);
|
||||
RETURN NEW;
|
||||
END $$;
|
||||
CREATE FUNCTION lab4_fault_commit() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NEW.filename LIKE 'fault-fail-%' THEN
|
||||
@@ -61,7 +64,7 @@ END $$;
|
||||
CREATE TRIGGER lab4_fault_seen AFTER INSERT ON attachments FOR EACH ROW
|
||||
WHEN (NEW.filename LIKE 'fault-%') EXECUTE FUNCTION lab4_fault_seen();
|
||||
CREATE TRIGGER lab4_fault_commit BEFORE UPDATE ON attachments FOR EACH ROW
|
||||
WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'ready')
|
||||
WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'queued')
|
||||
EXECUTE FUNCTION lab4_fault_commit();
|
||||
"""
|
||||
CLEANUP = """
|
||||
@@ -94,7 +97,7 @@ def bucket_keys() -> set[str]:
|
||||
"list",
|
||||
"--recursive",
|
||||
"--json",
|
||||
f"root/{BUCKET}/attachments/",
|
||||
f"root/{BUCKET}/staging/",
|
||||
)
|
||||
return {item["key"] for item in json.loads(listing)["items"]}
|
||||
|
||||
@@ -221,7 +224,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
|
||||
created = 0
|
||||
attachment = await upload(http, alice, chat)
|
||||
exact = await upload(http, alice, chat, png(16, 16, size=LIMIT))
|
||||
assert (attachment.status_code, exact.status_code) == (201, 201)
|
||||
assert (attachment.status_code, exact.status_code) == (202, 202)
|
||||
assert exact.json()["size_bytes"] == LIMIT
|
||||
created += 2
|
||||
mine = attachment.json()["id"]
|
||||
@@ -319,9 +322,9 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
|
||||
400,
|
||||
)
|
||||
expect(
|
||||
"variant=thumbnail",
|
||||
"variant=preview",
|
||||
await http.get(
|
||||
f"/api/v1/attachments/{mine}/content?variant=thumbnail", headers=bearer(alice)
|
||||
f"/api/v1/attachments/{mine}/content?variant=preview", headers=bearer(alice)
|
||||
),
|
||||
400,
|
||||
)
|
||||
@@ -332,7 +335,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
|
||||
)
|
||||
statuses = [entry["status"] for entry in logs(since) if entry.get("event") == "request.handled"]
|
||||
print(
|
||||
f"строк вложений в чате: {rows} (= ответов 201); ответов в журнале реплик: "
|
||||
f"строк вложений в чате: {rows} (= ответов 202); ответов в журнале реплик: "
|
||||
f"{len(statuses)}, из них 5xx: {sum(code >= 500 for code in statuses)}"
|
||||
)
|
||||
assert not [status for status in statuses if status >= 500]
|
||||
@@ -375,7 +378,7 @@ async def crash(http: AsyncClient, token: str, chat: str) -> tuple[str, str]:
|
||||
name = f"fault-crash-{uuid4().hex}.png"
|
||||
sent = asyncio.create_task(upload(http, token, chat, filename=name))
|
||||
address = await asyncio.to_thread(sql, "SELECT lab4_fault_sleeper()")
|
||||
assert address, "UPDATE ready не дошёл до триггера"
|
||||
assert address, "UPDATE queued не дошёл до триггера"
|
||||
victim = replica_at(address)
|
||||
kill(victim)
|
||||
try:
|
||||
@@ -396,6 +399,7 @@ async def test_crash_between_put_and_commit_is_swept_after_grace(
|
||||
s = await scene(http)
|
||||
alice, chat = s["alice"], s["chats"][0]
|
||||
kept = (await upload(http, alice, chat, filename="kept.png")).json()
|
||||
await waited(lambda: status_of(kept["id"]) == "ready", 60)
|
||||
assert (
|
||||
await message(http, alice, chat, {"text": "", "attachment_ids": [kept["id"]]})
|
||||
).status_code == 201
|
||||
@@ -465,4 +469,4 @@ async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, fault
|
||||
assert (status, body["error"]["code"]) == (503, "unavailable")
|
||||
assert elapsed <= REQUEST_TIMEOUT + 2
|
||||
assert seen(name) is None
|
||||
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 201
|
||||
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 202
|
||||
|
||||
Reference in New Issue
Block a user