add level 5
This commit is contained in:
@@ -20,6 +20,8 @@ MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
MAX_PENDING_JOBS=100
|
||||
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load track1 test-persistence test-failover
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -21,6 +21,10 @@ test:
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
track1:
|
||||
@test -n "$(TRACK_ARGS)" || (echo "Set TRACK_ARGS, e.g. --run-name c1-p1 --workers 1 --prefetch 1 --max-pending 20"; exit 2)
|
||||
.venv/bin/python scripts/track1.py --base-url "$(BASE_URL)" $(TLS_ARGS) $(TRACK_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: Matvey Kozyrev, BVT2301
|
||||
- Целевая оценка; выбранный трек 5, если нужен: 3
|
||||
- Целевая оценка; выбранный трек 5, если нужен: 5, трек 1 «Очередь под перегрузкой»
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
|
||||
@@ -22,13 +22,15 @@ PostgreSQL хранит метаданные, jobs и transactional outbox; Seaw
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Трек 5: очередь под перегрузкой | `make track1 TRACK_ARGS="..."` | TODO: заполнить только результатом выполненного прогона | `evidence/track1-*.json` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Гипотеза: увеличение worker concurrency сокращает p95 «202 → ready» до точки насыщения CPU/RAM; prefetch выше concurrency может повысить throughput ценой queue lag и RSS. Проверяются минимум три конфигурации: `1/1`, `2/2`, `4/4` (concurrency/prefetch) и несколько размеров входного PNG. Для каждой конфигурации используются одинаковые CPU/RAM limits Compose, одинаковое число изображений и не менее трёх прогонов после прогрева.
|
||||
|
||||
Команда запуска профиля: `WORKER_CONCURRENCY=N WORKER_PREFETCH=P MAX_PENDING_JOBS=20 docker compose up -d --build`. Команда измерения: `make track1 TRACK_ARGS="--run-name cN-pP-sizeS --workers N --prefetch P --max-pending 20 --images 40 --concurrency 8 --size S"`. Скрипт сохраняет raw JSON с successful throughput, p50/p95 времени принятия и ready, queue depth RabbitMQ и RSS worker. Значения будут внесены после реального прогона; сейчас их нет, так как Docker daemon недоступен.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
|
||||
@@ -20,6 +20,7 @@ x-api: &api
|
||||
UPLOAD_TIMEOUT_SECONDS: "30"
|
||||
UPLOAD_FAULT_AFTER_S3_PUT: ${UPLOAD_FAULT_AFTER_S3_PUT:-0}
|
||||
OUTBOX_FAULT_AFTER_COMMIT: ${OUTBOX_FAULT_AFTER_COMMIT:-0}
|
||||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100}
|
||||
secrets:
|
||||
- database_url
|
||||
- cursor_secret
|
||||
@@ -197,6 +198,7 @@ services:
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
|
||||
Executable
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Repeatable level-5 queue-overload experiment.
|
||||
|
||||
Run a Compose profile first, for example:
|
||||
WORKER_CONCURRENCY=2 MAX_PENDING_JOBS=20 docker compose up -d --build
|
||||
.venv/bin/python scripts/track1.py --run-name c2-p2 --workers 2 --prefetch 2 --images 40 --size 1600
|
||||
|
||||
The script records client-observed acceptance/ready latency, RabbitMQ queue depth
|
||||
and Docker RSS samples. It does not print credentials or session tokens.
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from io import BytesIO
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
|
||||
def percentile(values, point):
|
||||
values = sorted(values)
|
||||
return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None
|
||||
|
||||
|
||||
def docker_text(*args):
|
||||
try:
|
||||
return subprocess.run(args, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
||||
timeout=10, check=True).stdout.strip()
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
|
||||
|
||||
def broker_depth():
|
||||
text = docker_text("docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "list_queues", "name", "messages_ready", "messages_unacknowledged", "--formatter", "json")
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
rows = json.loads(text)
|
||||
return next((row for row in rows if row["name"] == "attachment.process.v1"), None)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def worker_rss():
|
||||
container = docker_text("docker", "compose", "ps", "-q", "worker")
|
||||
if not container:
|
||||
return None
|
||||
text = docker_text("docker", "stats", "--no-stream", "--format", "{{json .}}", container)
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return json.loads(text).get("MemUsage")
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument("--run-name", required=True)
|
||||
parser.add_argument("--workers", type=int, required=True, help="documented WORKER_CONCURRENCY setting")
|
||||
parser.add_argument("--prefetch", type=int, required=True, help="documented RabbitMQ prefetch setting")
|
||||
parser.add_argument("--max-pending", type=int, required=True, help="documented MAX_PENDING_JOBS setting")
|
||||
parser.add_argument("--images", type=int, default=40)
|
||||
parser.add_argument("--concurrency", type=int, default=4)
|
||||
parser.add_argument("--size", type=int, default=1600, help="square synthetic PNG dimensions")
|
||||
parser.add_argument("--max-ready-seconds", type=float, default=120)
|
||||
parser.add_argument("--output", default=None)
|
||||
args = arguments("Lab 5 track 1 image-pipeline experiment", configure)
|
||||
check(1 <= args.workers <= 64 and 1 <= args.prefetch <= 1024 and 1 <= args.max_pending <= 100000, "invalid worker/prefetch/max-pending")
|
||||
check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128 and 1 <= args.size <= 6000, "invalid workload")
|
||||
|
||||
from PIL import Image
|
||||
image = Image.new("RGB", (args.size, args.size), "#427aa1")
|
||||
raw = BytesIO(); image.save(raw, "PNG", optimize=True); data = raw.getvalue()
|
||||
client = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png"
|
||||
barrier = threading.Barrier(args.concurrency)
|
||||
queue_samples, rss_samples = [], []
|
||||
stop_sampling = threading.Event()
|
||||
|
||||
def sampler():
|
||||
while not stop_sampling.wait(1):
|
||||
queue_samples.append({"seconds": round(time.monotonic() - started, 3), "queue": broker_depth()})
|
||||
rss_samples.append({"seconds": round(time.monotonic() - started, 3), "rss": worker_rss()})
|
||||
|
||||
def upload(index):
|
||||
local = Client(args.base_url, args.ca_file, timeout=30)
|
||||
barrier.wait()
|
||||
started_at = time.monotonic()
|
||||
try:
|
||||
response = local.call("POST", path + "-" + str(index), 202, actor=alice, raw=data,
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
return {"accepted": True, "id": response["json"]["id"], "accepted_seconds": time.monotonic() - started_at}
|
||||
except Failure as error:
|
||||
return {"accepted": False, "error": str(error), "accepted_seconds": time.monotonic() - started_at}
|
||||
|
||||
started = time.monotonic()
|
||||
thread = threading.Thread(target=sampler, daemon=True); thread.start()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
results = list(pool.map(upload, range(args.images)))
|
||||
accepted = [item for item in results if item["accepted"]]
|
||||
ready_latencies, failed_ready = [], 0
|
||||
deadline = time.monotonic() + args.max_ready_seconds
|
||||
pending = {item["id"]: started + item["accepted_seconds"] for item in accepted}
|
||||
while pending and time.monotonic() < deadline:
|
||||
for attachment_id, accepted_at in list(pending.items()):
|
||||
try:
|
||||
metadata = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=bob, schema="Attachment")["json"]
|
||||
except Failure:
|
||||
continue
|
||||
if metadata["status"] == "ready":
|
||||
ready_latencies.append(time.monotonic() - accepted_at)
|
||||
del pending[attachment_id]
|
||||
elif metadata["status"] == "failed":
|
||||
failed_ready += 1; del pending[attachment_id]
|
||||
time.sleep(.2)
|
||||
stop_sampling.set(); thread.join(timeout=2)
|
||||
elapsed = time.monotonic() - started
|
||||
report = {
|
||||
"track": "queue-overload", "run_name": args.run_name,
|
||||
"settings": {"worker_concurrency": args.workers, "prefetch": args.prefetch, "max_pending_jobs": args.max_pending},
|
||||
"workload": {"images": args.images, "client_concurrency": args.concurrency, "png_dimensions": [args.size, args.size], "input_bytes": len(data)},
|
||||
"results": {"elapsed_seconds": round(elapsed, 3), "accepted": len(accepted), "rejected_or_failed": args.images - len(accepted), "ready": len(ready_latencies), "failed": failed_ready, "timed_out": len(pending), "throughput_ready_per_second": round(len(ready_latencies) / elapsed, 4), "accept_latency_ms": {"p50": percentile([x["accepted_seconds"] for x in accepted], .5), "p95": percentile([x["accepted_seconds"] for x in accepted], .95)}, "time_to_ready_ms": {"p50": percentile(ready_latencies, .5), "p95": percentile(ready_latencies, .95)}},
|
||||
"queue_samples": queue_samples, "worker_rss_samples": rss_samples,
|
||||
"limitations": "Closed-loop client; queue/RSS samples are null when Docker CLI is unavailable. p95 covers ready tasks only.",
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
destination = Path(args.output or (ROOT / "evidence" / (args.run_name + ".json")))
|
||||
destination.parent.mkdir(parents=True, exist_ok=True); destination.write_text(text + "\n", encoding="utf-8")
|
||||
print(text)
|
||||
check(bool(accepted), "no accepted uploads")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -87,6 +87,13 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...),
|
||||
# Disabled by default; models a crash after PUT, before INSERT.
|
||||
if os.environ.get("UPLOAD_FAULT_AFTER_S3_PUT") == "1":
|
||||
raise ApiError(503, "unavailable", "Инъецирован отказ после S3 PUT")
|
||||
try:
|
||||
max_pending = int(os.environ.get("MAX_PENDING_JOBS", "100"))
|
||||
except ValueError as error:
|
||||
raise ApiError(503, "unavailable", "Некорректный лимит очереди") from error
|
||||
if max_pending < 1:
|
||||
raise ApiError(503, "unavailable", "Некорректный лимит очереди")
|
||||
await store.ensure_attachment_queue_capacity(connection, max_pending)
|
||||
attachment = await store.create_attachment(
|
||||
attachment_id, chat_id, user["id"], filename, content_type,
|
||||
size, digest.hexdigest(), key, job_id, connection=connection,
|
||||
|
||||
@@ -346,6 +346,18 @@ class PostgresStore:
|
||||
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
||||
return dict(row)
|
||||
|
||||
async def ensure_attachment_queue_capacity(self, connection: asyncpg.Connection, maximum: int) -> None:
|
||||
"""Globally serialize admission so replicas cannot over-accept queued work."""
|
||||
try:
|
||||
await connection.execute("SELECT pg_advisory_xact_lock(41005, 1)")
|
||||
pending = await connection.fetchval(
|
||||
"SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')"
|
||||
)
|
||||
except DB_ERRORS as error:
|
||||
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
||||
if pending >= maximum:
|
||||
raise ApiError(429, "rate_limited", "Очередь обработки заполнена")
|
||||
|
||||
async def attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> dict:
|
||||
try:
|
||||
row = await self.pool.fetchrow(
|
||||
|
||||
+38
-17
@@ -6,6 +6,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import UUID
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
@@ -87,29 +88,45 @@ async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_second
|
||||
|
||||
def main() -> None:
|
||||
settings = load_settings()
|
||||
store = PostgresStore(settings.database_url)
|
||||
asyncio.run(store.connect())
|
||||
s3 = S3Store(S3Settings.load())
|
||||
connection = _connection()
|
||||
channel = connection.channel()
|
||||
channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_CONCURRENCY", "2"))))
|
||||
workers = max(1, int(os.environ.get("WORKER_CONCURRENCY", "2")))
|
||||
channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_PREFETCH", str(workers)))))
|
||||
|
||||
def consume(ch, method, properties, body):
|
||||
del properties
|
||||
async def handle_async(payload):
|
||||
"""Each executor thread owns its event loop and its DB/S3 clients."""
|
||||
store = PostgresStore(settings.database_url)
|
||||
s3 = S3Store(S3Settings.load())
|
||||
await store.connect()
|
||||
try:
|
||||
await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))))
|
||||
finally:
|
||||
s3.client.close()
|
||||
await store.close()
|
||||
|
||||
async def schedule_retry(attachment_id, delay):
|
||||
retry_store = PostgresStore(settings.database_url)
|
||||
await retry_store.connect()
|
||||
try:
|
||||
return await retry_store.retry_attachment_job(
|
||||
attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay
|
||||
)
|
||||
finally:
|
||||
await retry_store.close()
|
||||
|
||||
def handle(body):
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1:
|
||||
raise ValueError("unexpected broker payload")
|
||||
asyncio.run(process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))))
|
||||
asyncio.run(handle_async(payload))
|
||||
except Exception:
|
||||
logging.exception("job failed")
|
||||
try:
|
||||
attachment_id = UUID(payload["attachment_id"])
|
||||
attempt = int(payload.get("attempt", 1))
|
||||
delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5)
|
||||
retry = asyncio.run(store.retry_attachment_job(
|
||||
attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay
|
||||
))
|
||||
retry = asyncio.run(schedule_retry(attachment_id, delay))
|
||||
except Exception:
|
||||
retry = False
|
||||
if not retry:
|
||||
@@ -117,17 +134,21 @@ def main() -> None:
|
||||
asyncio.run(publish_dlq(payload))
|
||||
except Exception:
|
||||
logging.exception("could not publish failed job to DLQ")
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
else:
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
return None
|
||||
|
||||
def consume(ch, method, properties, body):
|
||||
del properties
|
||||
future = executor.submit(handle, body)
|
||||
def acknowledged(_future):
|
||||
connection.add_callback_threadsafe(lambda: ch.basic_ack(method.delivery_tag))
|
||||
future.add_done_callback(acknowledged)
|
||||
|
||||
try:
|
||||
channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False)
|
||||
channel.start_consuming()
|
||||
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="attachment-worker") as executor:
|
||||
channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False)
|
||||
channel.start_consuming()
|
||||
finally:
|
||||
connection.close()
|
||||
s3.client.close()
|
||||
asyncio.run(store.close())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user