forked from mtusi2026/mtusi2026lab5
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher confirms, manual ack, DLQ), transactional outbox, отдельный worker. - upload: оригинал в staging S3, затем одной транзакцией queued + jobs + outbox -> 202; API к брокеру не подключается - relay: публикация outbox с confirm, reaper истёкших аренд, redrive - worker: claim по строке jobs, аренда с heartbeat и fencing по lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом, thumbnail 256, crop 128, WebP 1280 по детерминированным ключам - retry с backoff и jitter, постоянные/временные ошибки, реестр failed, ops replay (ready не повторяется), DLQ для невалидных сообщений - janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4 - тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack, заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
125 lines
6.1 KiB
Python
125 lines
6.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Уровень 4: ограниченная конкуренция/prefetch, очистка staging, память worker под всплеском.
|
|
|
|
python3 student-tests/burst.py --count 24 --output evidence/burst.json
|
|
|
|
N несжимаемых PNG 1500x1500 (~6.8 MB) загружаются 8 потоками. Во время обработки раз в
|
|
секунду снимается память контейнеров worker (docker stats). По job_events считаются:
|
|
максимальное число одновременно обрабатываемых заданий (claimed..ready), задержка очереди
|
|
(enqueued -> claimed) и время до ready. Ожидание: одновременно <= WORKER_REPLICAS x
|
|
WORKER_CONCURRENCY (prefetch=1 на потребителя), staging всех заданий удалён.
|
|
Иллюстрация на одной машине, не статистика.
|
|
"""
|
|
import argparse
|
|
import concurrent.futures
|
|
import datetime as dt
|
|
import json
|
|
import statistics
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
|
|
from lab5lib import Client, Log, check, compose, noise_png_fast, queues, sql, upload, wait_status
|
|
|
|
|
|
def ts(text):
|
|
return dt.datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
|
|
|
|
def mem_sampler(stop, samples):
|
|
while not stop.is_set():
|
|
ids = compose("ps", "-q", "worker").stdout.split()
|
|
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}} {{.CPUPerc}}"]
|
|
+ ids, capture_output=True, text=True).stdout
|
|
for line in out.splitlines():
|
|
name, used, _slash, limit, cpu = line.split()[:5]
|
|
samples.append({"name": name, "used": used, "limit": limit, "cpu": cpu})
|
|
stop.wait(1)
|
|
|
|
|
|
def mib(text):
|
|
num = float("".join(ch for ch in text if ch.isdigit() or ch == "."))
|
|
return num / 1024 if text.endswith("KiB") else num * 1024 if text.endswith("GiB") else num
|
|
|
|
|
|
def pct(values, p):
|
|
values = sorted(values)
|
|
return round(values[min(len(values) - 1, int(round(p / 100 * (len(values) - 1))))], 3)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--base-url", default=None)
|
|
p.add_argument("--count", type=int, default=24)
|
|
p.add_argument("--output")
|
|
args = p.parse_args()
|
|
log = Log("burst of uploads: bounded worker concurrency")
|
|
c = Client(args.base_url, timeout=60)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
consumers = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "-p", "messenger", "list_consumers",
|
|
"queue_name", "prefetch_count", "ack_required", "--formatter", "json").stdout
|
|
consumers = json.loads(consumers)
|
|
log.step("consumers", count=len(consumers), prefetch=sorted({x["prefetch_count"] for x in consumers}))
|
|
images = [noise_png_fast(1500, 1500) for _ in range(4)]
|
|
log.step("generated images", bytes=[len(x) for x in images])
|
|
stop = threading.Event()
|
|
samples = []
|
|
sampler = threading.Thread(target=mem_sampler, args=(stop, samples), daemon=True)
|
|
sampler.start()
|
|
t0 = time.monotonic()
|
|
with concurrent.futures.ThreadPoolExecutor(8) as pool:
|
|
atts = list(pool.map(lambda i: upload(c, alice, chat, images[i % 4], "burst-%d.png" % i),
|
|
range(args.count)))
|
|
t_up = time.monotonic() - t0
|
|
q = queues()["attachments.process.v1"]
|
|
log.step("all uploads accepted (202)", count=len(atts), seconds=round(t_up, 2), queue_right_after=q)
|
|
for a in atts:
|
|
st = wait_status(c, bob, a["id"], {"ready", "failed"}, timeout=180, interval=1)
|
|
check(st["status"] == "ready", "burst job failed")
|
|
total = time.monotonic() - t0
|
|
stop.set()
|
|
sampler.join()
|
|
ids = ",".join("'%s'" % a["id"] for a in atts)
|
|
rows = sql("""SELECT j.attachment_id, e.event, to_char(e.at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')
|
|
FROM job_events e JOIN jobs j ON j.id = e.job_id
|
|
WHERE j.attachment_id IN (%s) AND e.event IN ('enqueued','claimed','ready') ORDER BY e.id""" % ids)
|
|
per = {}
|
|
for att_id, ev, at in rows:
|
|
per.setdefault(att_id, {})[ev] = ts(at)
|
|
intervals = [(v["claimed"], v["ready"]) for v in per.values()]
|
|
points = sorted([(a, 1) for a, _ in intervals] + [(b, -1) for _, b in intervals], key=lambda x: (x[0], x[1]))
|
|
cur = peak = 0
|
|
for _, d in points:
|
|
cur += d
|
|
peak = max(peak, cur)
|
|
lag = [(v["claimed"] - v["enqueued"]).total_seconds() for v in per.values()]
|
|
to_ready = [(v["ready"] - v["enqueued"]).total_seconds() for v in per.values()]
|
|
work = [(v["ready"] - v["claimed"]).total_seconds() for v in per.values()]
|
|
staging_left = sql("SELECT count(*) FROM attachments WHERE id IN (%s) AND staging_key IS NOT NULL" % ids)[0][0]
|
|
limit = len(consumers)
|
|
by_name = {}
|
|
for s in samples:
|
|
by_name[s["name"]] = max(by_name.get(s["name"], 0), mib(s["used"]))
|
|
result = {
|
|
"count": args.count, "image_bytes": len(images[0]), "upload_seconds": round(t_up, 2),
|
|
"total_seconds_to_all_ready": round(total, 2),
|
|
"throughput_jobs_per_s": round(args.count / total, 2),
|
|
"consumers": limit, "prefetch_per_consumer": sorted({x["prefetch_count"] for x in consumers}),
|
|
"max_concurrent_processing": peak,
|
|
"queue_lag_s": {"p50": pct(lag, 50), "p95": pct(lag, 95), "max": round(max(lag), 3)},
|
|
"enqueued_to_ready_s": {"p50": pct(to_ready, 50), "p95": pct(to_ready, 95), "max": round(max(to_ready), 3)},
|
|
"processing_s": {"p50": pct(work, 50), "max": round(max(work), 3), "mean": round(statistics.mean(work), 3)},
|
|
"worker_peak_mem_mib": {k: round(v, 1) for k, v in by_name.items()},
|
|
"worker_mem_limit": samples[0]["limit"] if samples else None,
|
|
"staging_left": int(staging_left),
|
|
}
|
|
check(peak <= limit, "concurrency exceeded consumers x prefetch")
|
|
check(result["staging_left"] == 0, "staging not cleaned")
|
|
log.step("result", **result)
|
|
log.save(args.output, result=result)
|
|
print("PASS: %d jobs, max concurrent %d <= %d consumers" % (args.count, peak, limit))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|