forked from mtusi2026/mtusi2026lab5
65 lines
3.3 KiB
Python
65 lines
3.3 KiB
Python
"""lab5, уровень 4: ограниченная конкуренция worker и prefetch брокера.
|
|
|
|
make concurrency (= python student-tests/concurrency.py)
|
|
|
|
Точка worker_slow=6: каждая обработка «работает» 6 с. В очередь сразу попадают 6 заданий.
|
|
Раз в 0,5 с смотрим:
|
|
- сколько заданий в processing (PostgreSQL) — не больше WORKER_CONCURRENCY (2);
|
|
- сколько сообщений выдано worker и не подтверждено (RabbitMQ messages_unacknowledged) — не больше
|
|
prefetch (2), остальные лежат в очереди (messages_ready) и ждут: worker не набирает работу впрок,
|
|
и при втором worker они достались бы ему.
|
|
Все 6 вложений в итоге ready.
|
|
"""
|
|
import time
|
|
|
|
import ops
|
|
from filegen import upload_when_ready, wait_status
|
|
from helpers import Client, check, run
|
|
|
|
COUNT = 6
|
|
LIMIT = 2
|
|
|
|
|
|
def main():
|
|
api = Client(timeout=15)
|
|
ops.scale("worker", 1)
|
|
worker = ops.wait_healthy("worker", 1)[0]
|
|
alice, bob, _eve, chat = api.bootstrap()
|
|
try:
|
|
ops.arm("worker", "worker_slow", "6", container=worker)
|
|
ids = [upload_when_ready(api, alice, chat, name=f"conc-{i}.png")["json"]["id"] for i in range(COUNT)]
|
|
quoted = ", ".join(f"'{i}'" for i in ids)
|
|
samples = []
|
|
started = time.monotonic()
|
|
while time.monotonic() - started < 60:
|
|
rows = ops.psql(f"SELECT status, count(*) FROM attachment_jobs WHERE attachment_id IN ({quoted}) "
|
|
"GROUP BY status").splitlines()
|
|
by_status = {status: int(n) for status, n in (row.split("|") for row in rows if row)}
|
|
ready_q, unacked = ops.queues().get("attachments.process", (0, 0))
|
|
samples.append((round(time.monotonic() - started, 1), by_status.get("processing", 0), unacked, ready_q))
|
|
if by_status.get("ready", 0) == COUNT:
|
|
break
|
|
time.sleep(0.5)
|
|
finally:
|
|
ops.disarm("worker", "worker_slow")
|
|
for attachment_id in ids:
|
|
wait_status(api, bob, attachment_id, seconds=60)
|
|
peak_processing = max(s[1] for s in samples)
|
|
peak_unacked = max(s[2] for s in samples)
|
|
waited = max(s[3] for s in samples)
|
|
print("time s | processing (DB) | unacked (broker) | waiting in queue")
|
|
for sample in samples[:: max(1, len(samples) // 12)]:
|
|
print(f"{sample[0]:>6} | {sample[1]:>15} | {sample[2]:>16} | {sample[3]:>16}")
|
|
check(peak_processing <= LIMIT, f"more than {LIMIT} jobs were processed at once: {peak_processing}")
|
|
check(peak_unacked <= LIMIT, f"the worker took more than prefetch={LIMIT} messages: {peak_unacked}")
|
|
check(peak_processing == LIMIT, "the worker did not use its parallelism")
|
|
check(waited >= 1, "the other jobs should wait in the broker, not in the worker")
|
|
print(f"peak: {peak_processing} jobs in processing, {peak_unacked} unacknowledged messages, up to {waited} "
|
|
f"waiting in the queue; all {COUNT} ready after {samples[-1][0]} s")
|
|
print(f"PASS: at most WORKER_CONCURRENCY={LIMIT} jobs run at once and the broker gives the worker at most "
|
|
f"prefetch={LIMIT} messages; the rest wait in the durable queue")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|