diff --git a/.env.example b/.env.example index 119384b5..4f281476 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,8 @@ MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +MAX_PENDING_JOBS=100 diff --git a/Makefile b/Makefile index 5d86d171..da8c6dee 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load track1 test-persistence test-failover help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -21,6 +21,10 @@ test: load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +track1: + @test -n "$(TRACK_ARGS)" || (echo "Set TRACK_ARGS, e.g. --run-name c1-p1 --workers 1 --prefetch 1 --max-pending 20"; exit 2) + .venv/bin/python scripts/track1.py --base-url "$(BASE_URL)" $(TLS_ARGS) $(TRACK_ARGS) + test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index 6c46d248..2aa09501 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 5 - Автор, группа: Matvey Kozyrev, BVT2301 -- Целевая оценка; выбранный трек 5, если нужен: 3 +- Целевая оценка; выбранный трек 5, если нужен: 5, трек 1 «Очередь под перегрузкой» - Commit/PR: TODO после публикации - Использованные источники, библиотеки и помощь ИИ: TODO @@ -22,13 +22,15 @@ PostgreSQL хранит метаданные, jobs и transactional outbox; Seaw | Публичный smoke | make test | TODO | TODO | | Остальные пункты уровня 3 | TODO | TODO | TODO | | Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Трек 5: очередь под перегрузкой | `make track1 TRACK_ARGS="..."` | TODO: заполнить только результатом выполненного прогона | `evidence/track1-*.json` | TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза: увеличение worker concurrency сокращает p95 «202 → ready» до точки насыщения CPU/RAM; prefetch выше concurrency может повысить throughput ценой queue lag и RSS. Проверяются минимум три конфигурации: `1/1`, `2/2`, `4/4` (concurrency/prefetch) и несколько размеров входного PNG. Для каждой конфигурации используются одинаковые CPU/RAM limits Compose, одинаковое число изображений и не менее трёх прогонов после прогрева. + +Команда запуска профиля: `WORKER_CONCURRENCY=N WORKER_PREFETCH=P MAX_PENDING_JOBS=20 docker compose up -d --build`. Команда измерения: `make track1 TRACK_ARGS="--run-name cN-pP-sizeS --workers N --prefetch P --max-pending 20 --images 40 --concurrency 8 --size S"`. Скрипт сохраняет raw JSON с successful throughput, p50/p95 времени принятия и ready, queue depth RabbitMQ и RSS worker. Значения будут внесены после реального прогона; сейчас их нет, так как Docker daemon недоступен. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | diff --git a/compose.yaml b/compose.yaml index 4b4edd5d..b040b762 100644 --- a/compose.yaml +++ b/compose.yaml @@ -20,6 +20,7 @@ x-api: &api UPLOAD_TIMEOUT_SECONDS: "30" UPLOAD_FAULT_AFTER_S3_PUT: ${UPLOAD_FAULT_AFTER_S3_PUT:-0} OUTBOX_FAULT_AFTER_COMMIT: ${OUTBOX_FAULT_AFTER_COMMIT:-0} + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100} secrets: - database_url - cursor_secret @@ -197,6 +198,7 @@ services: S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key BROKER_URL_FILE: /run/secrets/broker_url WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} diff --git a/scripts/track1.py b/scripts/track1.py new file mode 100755 index 00000000..eb3d238c --- /dev/null +++ b/scripts/track1.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Repeatable level-5 queue-overload experiment. + +Run a Compose profile first, for example: + WORKER_CONCURRENCY=2 MAX_PENDING_JOBS=20 docker compose up -d --build + .venv/bin/python scripts/track1.py --run-name c2-p2 --workers 2 --prefetch 2 --images 40 --size 1600 + +The script records client-observed acceptance/ready latency, RabbitMQ queue depth +and Docker RSS samples. It does not print credentials or session tokens. +""" +from concurrent.futures import ThreadPoolExecutor, as_completed +from io import BytesIO +import json +import math +from pathlib import Path +import subprocess +import sys +import threading +import time +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + + +def percentile(values, point): + values = sorted(values) + return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None + + +def docker_text(*args): + try: + return subprocess.run(args, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + timeout=10, check=True).stdout.strip() + except (OSError, subprocess.SubprocessError): + return None + + +def broker_depth(): + text = docker_text("docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "list_queues", "name", "messages_ready", "messages_unacknowledged", "--formatter", "json") + if not text: + return None + try: + rows = json.loads(text) + return next((row for row in rows if row["name"] == "attachment.process.v1"), None) + except (ValueError, KeyError, TypeError): + return None + + +def worker_rss(): + container = docker_text("docker", "compose", "ps", "-q", "worker") + if not container: + return None + text = docker_text("docker", "stats", "--no-stream", "--format", "{{json .}}", container) + if not text: + return None + try: + return json.loads(text).get("MemUsage") + except ValueError: + return None + + +def main(): + def configure(parser): + parser.add_argument("--run-name", required=True) + parser.add_argument("--workers", type=int, required=True, help="documented WORKER_CONCURRENCY setting") + parser.add_argument("--prefetch", type=int, required=True, help="documented RabbitMQ prefetch setting") + parser.add_argument("--max-pending", type=int, required=True, help="documented MAX_PENDING_JOBS setting") + parser.add_argument("--images", type=int, default=40) + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--size", type=int, default=1600, help="square synthetic PNG dimensions") + parser.add_argument("--max-ready-seconds", type=float, default=120) + parser.add_argument("--output", default=None) + args = arguments("Lab 5 track 1 image-pipeline experiment", configure) + check(1 <= args.workers <= 64 and 1 <= args.prefetch <= 1024 and 1 <= args.max_pending <= 100000, "invalid worker/prefetch/max-pending") + check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128 and 1 <= args.size <= 6000, "invalid workload") + + from PIL import Image + image = Image.new("RGB", (args.size, args.size), "#427aa1") + raw = BytesIO(); image.save(raw, "PNG", optimize=True); data = raw.getvalue() + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, _eve, chat = client.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png" + barrier = threading.Barrier(args.concurrency) + queue_samples, rss_samples = [], [] + stop_sampling = threading.Event() + + def sampler(): + while not stop_sampling.wait(1): + queue_samples.append({"seconds": round(time.monotonic() - started, 3), "queue": broker_depth()}) + rss_samples.append({"seconds": round(time.monotonic() - started, 3), "rss": worker_rss()}) + + def upload(index): + local = Client(args.base_url, args.ca_file, timeout=30) + barrier.wait() + started_at = time.monotonic() + try: + response = local.call("POST", path + "-" + str(index), 202, actor=alice, raw=data, + headers={"Content-Type": "image/png"}, schema="Attachment") + return {"accepted": True, "id": response["json"]["id"], "accepted_seconds": time.monotonic() - started_at} + except Failure as error: + return {"accepted": False, "error": str(error), "accepted_seconds": time.monotonic() - started_at} + + started = time.monotonic() + thread = threading.Thread(target=sampler, daemon=True); thread.start() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + results = list(pool.map(upload, range(args.images))) + accepted = [item for item in results if item["accepted"]] + ready_latencies, failed_ready = [], 0 + deadline = time.monotonic() + args.max_ready_seconds + pending = {item["id"]: started + item["accepted_seconds"] for item in accepted} + while pending and time.monotonic() < deadline: + for attachment_id, accepted_at in list(pending.items()): + try: + metadata = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=bob, schema="Attachment")["json"] + except Failure: + continue + if metadata["status"] == "ready": + ready_latencies.append(time.monotonic() - accepted_at) + del pending[attachment_id] + elif metadata["status"] == "failed": + failed_ready += 1; del pending[attachment_id] + time.sleep(.2) + stop_sampling.set(); thread.join(timeout=2) + elapsed = time.monotonic() - started + report = { + "track": "queue-overload", "run_name": args.run_name, + "settings": {"worker_concurrency": args.workers, "prefetch": args.prefetch, "max_pending_jobs": args.max_pending}, + "workload": {"images": args.images, "client_concurrency": args.concurrency, "png_dimensions": [args.size, args.size], "input_bytes": len(data)}, + "results": {"elapsed_seconds": round(elapsed, 3), "accepted": len(accepted), "rejected_or_failed": args.images - len(accepted), "ready": len(ready_latencies), "failed": failed_ready, "timed_out": len(pending), "throughput_ready_per_second": round(len(ready_latencies) / elapsed, 4), "accept_latency_ms": {"p50": percentile([x["accepted_seconds"] for x in accepted], .5), "p95": percentile([x["accepted_seconds"] for x in accepted], .95)}, "time_to_ready_ms": {"p50": percentile(ready_latencies, .5), "p95": percentile(ready_latencies, .95)}}, + "queue_samples": queue_samples, "worker_rss_samples": rss_samples, + "limitations": "Closed-loop client; queue/RSS samples are null when Docker CLI is unavailable. p95 covers ready tasks only.", + } + text = json.dumps(report, ensure_ascii=False, indent=2) + destination = Path(args.output or (ROOT / "evidence" / (args.run_name + ".json"))) + destination.parent.mkdir(parents=True, exist_ok=True); destination.write_text(text + "\n", encoding="utf-8") + print(text) + check(bool(accepted), "no accepted uploads") + + +if __name__ == "__main__": + run(main) diff --git a/src/attachments.py b/src/attachments.py index 790732de..17c0cb82 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -87,6 +87,13 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...), # Disabled by default; models a crash after PUT, before INSERT. if os.environ.get("UPLOAD_FAULT_AFTER_S3_PUT") == "1": raise ApiError(503, "unavailable", "Инъецирован отказ после S3 PUT") + try: + max_pending = int(os.environ.get("MAX_PENDING_JOBS", "100")) + except ValueError as error: + raise ApiError(503, "unavailable", "Некорректный лимит очереди") from error + if max_pending < 1: + raise ApiError(503, "unavailable", "Некорректный лимит очереди") + await store.ensure_attachment_queue_capacity(connection, max_pending) attachment = await store.create_attachment( attachment_id, chat_id, user["id"], filename, content_type, size, digest.hexdigest(), key, job_id, connection=connection, diff --git a/src/storage/postgres.py b/src/storage/postgres.py index e87fec81..f63699db 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -346,6 +346,18 @@ class PostgresStore: raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error return dict(row) + async def ensure_attachment_queue_capacity(self, connection: asyncpg.Connection, maximum: int) -> None: + """Globally serialize admission so replicas cannot over-accept queued work.""" + try: + await connection.execute("SELECT pg_advisory_xact_lock(41005, 1)") + pending = await connection.fetchval( + "SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')" + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if pending >= maximum: + raise ApiError(429, "rate_limited", "Очередь обработки заполнена") + async def attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> dict: try: row = await self.pool.fetchrow( diff --git a/src/worker.py b/src/worker.py index 730f94e9..3e7f2d54 100644 --- a/src/worker.py +++ b/src/worker.py @@ -6,6 +6,7 @@ import json import logging import os import random +from concurrent.futures import ThreadPoolExecutor from uuid import UUID from PIL import Image, ImageOps @@ -87,29 +88,45 @@ async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_second def main() -> None: settings = load_settings() - store = PostgresStore(settings.database_url) - asyncio.run(store.connect()) - s3 = S3Store(S3Settings.load()) connection = _connection() channel = connection.channel() - channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_CONCURRENCY", "2")))) + workers = max(1, int(os.environ.get("WORKER_CONCURRENCY", "2"))) + channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_PREFETCH", str(workers))))) - def consume(ch, method, properties, body): - del properties + async def handle_async(payload): + """Each executor thread owns its event loop and its DB/S3 clients.""" + store = PostgresStore(settings.database_url) + s3 = S3Store(S3Settings.load()) + await store.connect() + try: + await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))) + finally: + s3.client.close() + await store.close() + + async def schedule_retry(attachment_id, delay): + retry_store = PostgresStore(settings.database_url) + await retry_store.connect() + try: + return await retry_store.retry_attachment_job( + attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay + ) + finally: + await retry_store.close() + + def handle(body): try: payload = json.loads(body) if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1: raise ValueError("unexpected broker payload") - asyncio.run(process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))))) + asyncio.run(handle_async(payload)) except Exception: logging.exception("job failed") try: attachment_id = UUID(payload["attachment_id"]) attempt = int(payload.get("attempt", 1)) delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5) - retry = asyncio.run(store.retry_attachment_job( - attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay - )) + retry = asyncio.run(schedule_retry(attachment_id, delay)) except Exception: retry = False if not retry: @@ -117,17 +134,21 @@ def main() -> None: asyncio.run(publish_dlq(payload)) except Exception: logging.exception("could not publish failed job to DLQ") - ch.basic_ack(method.delivery_tag) - else: - ch.basic_ack(method.delivery_tag) + return None + + def consume(ch, method, properties, body): + del properties + future = executor.submit(handle, body) + def acknowledged(_future): + connection.add_callback_threadsafe(lambda: ch.basic_ack(method.delivery_tag)) + future.add_done_callback(acknowledged) try: - channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False) - channel.start_consuming() + with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="attachment-worker") as executor: + channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False) + channel.start_consuming() finally: connection.close() - s3.client.close() - asyncio.run(store.close()) if __name__ == "__main__":