Files
mtusi2026lab5/student-tests/burst.py
T
zemiles 13bec4ab8b ЛР 5 — БПИ2303 — Лазарев — на 4
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher
confirms, manual ack, DLQ), transactional outbox, отдельный worker.

- upload: оригинал в staging S3, затем одной транзакцией queued + jobs +
  outbox -> 202; API к брокеру не подключается
- relay: публикация outbox с confirm, reaper истёкших аренд, redrive
- worker: claim по строке jobs, аренда с heartbeat и fencing по
  lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом,
  thumbnail 256, crop 128, WebP 1280 по детерминированным ключам
- retry с backoff и jitter, постоянные/временные ошибки, реестр failed,
  ops replay (ready не повторяется), DLQ для невалидных сообщений
- janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4
- тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack,
  заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
2026-11-13 16:28:18 +03:00

125 lines
6.1 KiB
Python

#!/usr/bin/env python3
"""Уровень 4: ограниченная конкуренция/prefetch, очистка staging, память worker под всплеском.
python3 student-tests/burst.py --count 24 --output evidence/burst.json
N несжимаемых PNG 1500x1500 (~6.8 MB) загружаются 8 потоками. Во время обработки раз в
секунду снимается память контейнеров worker (docker stats). По job_events считаются:
максимальное число одновременно обрабатываемых заданий (claimed..ready), задержка очереди
(enqueued -> claimed) и время до ready. Ожидание: одновременно <= WORKER_REPLICAS x
WORKER_CONCURRENCY (prefetch=1 на потребителя), staging всех заданий удалён.
Иллюстрация на одной машине, не статистика.
"""
import argparse
import concurrent.futures
import datetime as dt
import json
import statistics
import subprocess
import threading
import time
from lab5lib import Client, Log, check, compose, noise_png_fast, queues, sql, upload, wait_status
def ts(text):
return dt.datetime.fromisoformat(text.replace("Z", "+00:00"))
def mem_sampler(stop, samples):
while not stop.is_set():
ids = compose("ps", "-q", "worker").stdout.split()
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}} {{.CPUPerc}}"]
+ ids, capture_output=True, text=True).stdout
for line in out.splitlines():
name, used, _slash, limit, cpu = line.split()[:5]
samples.append({"name": name, "used": used, "limit": limit, "cpu": cpu})
stop.wait(1)
def mib(text):
num = float("".join(ch for ch in text if ch.isdigit() or ch == "."))
return num / 1024 if text.endswith("KiB") else num * 1024 if text.endswith("GiB") else num
def pct(values, p):
values = sorted(values)
return round(values[min(len(values) - 1, int(round(p / 100 * (len(values) - 1))))], 3)
def main():
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=None)
p.add_argument("--count", type=int, default=24)
p.add_argument("--output")
args = p.parse_args()
log = Log("burst of uploads: bounded worker concurrency")
c = Client(args.base_url, timeout=60)
alice, bob, _eve, chat = c.bootstrap()
consumers = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "-p", "messenger", "list_consumers",
"queue_name", "prefetch_count", "ack_required", "--formatter", "json").stdout
consumers = json.loads(consumers)
log.step("consumers", count=len(consumers), prefetch=sorted({x["prefetch_count"] for x in consumers}))
images = [noise_png_fast(1500, 1500) for _ in range(4)]
log.step("generated images", bytes=[len(x) for x in images])
stop = threading.Event()
samples = []
sampler = threading.Thread(target=mem_sampler, args=(stop, samples), daemon=True)
sampler.start()
t0 = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(8) as pool:
atts = list(pool.map(lambda i: upload(c, alice, chat, images[i % 4], "burst-%d.png" % i),
range(args.count)))
t_up = time.monotonic() - t0
q = queues()["attachments.process.v1"]
log.step("all uploads accepted (202)", count=len(atts), seconds=round(t_up, 2), queue_right_after=q)
for a in atts:
st = wait_status(c, bob, a["id"], {"ready", "failed"}, timeout=180, interval=1)
check(st["status"] == "ready", "burst job failed")
total = time.monotonic() - t0
stop.set()
sampler.join()
ids = ",".join("'%s'" % a["id"] for a in atts)
rows = sql("""SELECT j.attachment_id, e.event, to_char(e.at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')
FROM job_events e JOIN jobs j ON j.id = e.job_id
WHERE j.attachment_id IN (%s) AND e.event IN ('enqueued','claimed','ready') ORDER BY e.id""" % ids)
per = {}
for att_id, ev, at in rows:
per.setdefault(att_id, {})[ev] = ts(at)
intervals = [(v["claimed"], v["ready"]) for v in per.values()]
points = sorted([(a, 1) for a, _ in intervals] + [(b, -1) for _, b in intervals], key=lambda x: (x[0], x[1]))
cur = peak = 0
for _, d in points:
cur += d
peak = max(peak, cur)
lag = [(v["claimed"] - v["enqueued"]).total_seconds() for v in per.values()]
to_ready = [(v["ready"] - v["enqueued"]).total_seconds() for v in per.values()]
work = [(v["ready"] - v["claimed"]).total_seconds() for v in per.values()]
staging_left = sql("SELECT count(*) FROM attachments WHERE id IN (%s) AND staging_key IS NOT NULL" % ids)[0][0]
limit = len(consumers)
by_name = {}
for s in samples:
by_name[s["name"]] = max(by_name.get(s["name"], 0), mib(s["used"]))
result = {
"count": args.count, "image_bytes": len(images[0]), "upload_seconds": round(t_up, 2),
"total_seconds_to_all_ready": round(total, 2),
"throughput_jobs_per_s": round(args.count / total, 2),
"consumers": limit, "prefetch_per_consumer": sorted({x["prefetch_count"] for x in consumers}),
"max_concurrent_processing": peak,
"queue_lag_s": {"p50": pct(lag, 50), "p95": pct(lag, 95), "max": round(max(lag), 3)},
"enqueued_to_ready_s": {"p50": pct(to_ready, 50), "p95": pct(to_ready, 95), "max": round(max(to_ready), 3)},
"processing_s": {"p50": pct(work, 50), "max": round(max(work), 3), "mean": round(statistics.mean(work), 3)},
"worker_peak_mem_mib": {k: round(v, 1) for k, v in by_name.items()},
"worker_mem_limit": samples[0]["limit"] if samples else None,
"staging_left": int(staging_left),
}
check(peak <= limit, "concurrency exceeded consumers x prefetch")
check(result["staging_left"] == 0, "staging not cleaned")
log.step("result", **result)
log.save(args.output, result=result)
print("PASS: %d jobs, max concurrent %d <= %d consumers" % (args.count, peak, limit))
if __name__ == "__main__":
main()