From 9049c404edd1b9a2dad17d0eb15d1cdb7ae59707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=94=D0=B0=D0=B4?= =?UTF-8?q?=D0=B0=D0=B9=D0=BA=D0=B8=D0=BD?= Date: Mon, 9 Nov 2026 21:59:21 +0300 Subject: [PATCH 1/2] Force LF line endings --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..9fe3034b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.sh text eol=lf +*.py text eol=lf \ No newline at end of file -- 2.54.0 From fb8a43920e74e3712a5ca3a7fcb8933ce9d85bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=94=D0=B0=D0=B4?= =?UTF-8?q?=D0=B0=D0=B9=D0=BA=D0=B8=D0=BD?= Date: Mon, 9 Nov 2026 23:39:53 +0300 Subject: [PATCH 2/2] Complete Lab 5: implementation, fault scenarios, and Track 1 report --- Dockerfile | 11 ++ REPORT.md | 133 +++++++++++++-- app/main.py | 299 +++++++++++++++++++++++++++++++++ app/migrate.py | 18 ++ app/worker.py | 131 +++++++++++++++ compose.yaml | 131 +++++++++++++++ evidence/backpressure.json | 11 ++ evidence/broker-restart.json | 9 + evidence/fault-redelivery.json | 15 ++ evidence/pipeline-load-c1.json | 29 ++++ evidence/pipeline-load-c2.json | 28 +++ evidence/pipeline-load-c4.json | 28 +++ evidence/retry-replay.json | 11 ++ evidence/worker-outage.json | 13 ++ infra/bootstrap-rustfs.sh | 17 ++ infra/haproxy.cfg | 28 +++ infra/rustfs-policy.json | 1 + migrations/001_initial.sql | 36 ++++ migrations/002_idempotency.sql | 6 + requirements.lock | 26 +++ scripts/bootstrap.py | 22 +++ scripts/replay_failed.py | 18 ++ scripts/restart.bat | 1 + scripts/restart.sh | 6 + scripts/stop-one.bat | 1 + scripts/stop-one.sh | 16 ++ tests/backpressure.py | 31 ++++ tests/broker_restart.py | 37 ++++ tests/fault_redelivery.py | 18 ++ tests/pipeline_load.py | 68 ++++++++ tests/retry_replay.py | 41 +++++ tests/worker_outage.py | 30 ++++ 32 files changed, 1252 insertions(+), 18 deletions(-) create mode 100644 Dockerfile create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/worker.py create mode 100644 compose.yaml create mode 100644 evidence/backpressure.json create mode 100644 evidence/broker-restart.json create mode 100644 evidence/fault-redelivery.json create mode 100644 evidence/pipeline-load-c1.json create mode 100644 evidence/pipeline-load-c2.json create mode 100644 evidence/pipeline-load-c4.json create mode 100644 evidence/retry-replay.json create mode 100644 evidence/worker-outage.json create mode 100644 infra/bootstrap-rustfs.sh create mode 100644 infra/haproxy.cfg create mode 100644 infra/rustfs-policy.json create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 requirements.lock create mode 100644 scripts/bootstrap.py create mode 100644 scripts/replay_failed.py create mode 100644 scripts/restart.bat create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.bat create mode 100644 scripts/stop-one.sh create mode 100644 tests/backpressure.py create mode 100644 tests/broker_restart.py create mode 100644 tests/fault_redelivery.py create mode 100644 tests/pipeline_load.py create mode 100644 tests/retry_replay.py create mode 100644 tests/worker_outage.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..25fd1868 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.12.12-slim +WORKDIR /srv +COPY requirements.lock . +RUN pip install --no-cache-dir --timeout 120 --retries 5 -r requirements.lock +COPY app app +COPY migrations migrations +COPY scripts scripts +RUN useradd --uid 10001 --create-home api +USER api +EXPOSE 8080 +CMD ["gunicorn", "--bind", "0.0.0.0:8080", "--workers", "1", "--threads", "2", "--timeout", "35", "--graceful-timeout", "5", "--keep-alive", "1", "app.main:app"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..777cf8e2 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,134 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Дадайкин Андрей, БВТ2302 +- Целевая оценка: 5; трек: 1 — очередь под перегрузкой +- Commit/PR: локальная работа, не опубликована +- Использованные источники, библиотеки и помощь ИИ: README и контракты курса; реализация продолжает lab4; Flask, psycopg, Argon2id, boto3, Pillow, Redis Streams, PostgreSQL, RustFS, HAProxy. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine/Desktop с Compose v2 и Python 3.9+ на хосте для команд проверок. Compose собирает приложение из `Dockerfile`; наружу опубликован только HAProxy на `127.0.0.1:8081`. Теги образов зафиксированы в `compose.yaml`; Python-образ приложения — `python:3.12.12-slim`, зависимости приложения — в `requirements.lock`. +```sh +python scripts/bootstrap.py +docker compose config --quiet +docker compose up -d --build --scale api=2 --wait --wait-timeout 180 +python scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://localhost:8081 +``` + +bootstrap.py создаёт локальные secrets и .env с правами доступа, не печатая значения. Миграция выполняется отдельной одноразовой задачей migrate; bootstrap приватного bucket выполняет bucket-bootstrap. Тесты используют http://localhost:8081. Для контрольных отказов и исследования: + +```sh +python tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url http://localhost:8081 +python tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url http://localhost:8081 +python tests/worker_outage.py --base-url http://localhost:8081 +python tests/broker_restart.py --base-url http://localhost:8081 +python tests/retry_replay.py --base-url http://localhost:8081 +FAULT_AFTER_S3_WRITE_ONCE=1 JOB_TIMEOUT_SECONDS=5 docker compose up -d --force-recreate worker +python tests/fault_redelivery.py --base-url http://localhost:8081 +FAULT_AFTER_S3_WRITE_ONCE=0 JOB_TIMEOUT_SECONDS=60 docker compose up -d --force-recreate worker +python tests/backpressure.py --base-url http://localhost:8081 +``` + +Трек нагрузки запускается трижды (concurrency 1, 2, 4): + +```sh +WORKER_CONCURRENCY=1 docker compose up -d --force-recreate worker +python tests/pipeline_load.py --base-url http://localhost:8081 --count 12 --profile mixed --upload-concurrency 6 --worker-concurrency 1 --output evidence/pipeline-load-c1.json + +WORKER_CONCURRENCY=2 docker compose up -d --force-recreate worker +python tests/pipeline_load.py --base-url http://localhost:8081 --count 12 --profile mixed --upload-concurrency 6 --worker-concurrency 2 --output evidence/pipeline-load-c2.json + +WORKER_CONCURRENCY=4 docker compose up -d --force-recreate worker +python tests/pipeline_load.py --base-url http://localhost:8081 --count 12 --profile mixed --upload-concurrency 6 --worker-concurrency 4 --output evidence/pipeline-load-c4.json +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +клиент → HAProxy → API × 2 → PostgreSQL + ├── Redis: кэш профиля + └── RustFS/S3: staging, оригиналы, варианты +PostgreSQL outbox → Redis Streams/AOF → отдельный worker → RustFS/S3 +``` + +Сохранены маршруты и поведение lab4. Upload принимает все HTTP bytes и сначала записывает оригинал в приватный S3 staging-prefix. Затем в одной PostgreSQL-транзакции создаются metadata со статусом queued и запись attachment_jobs; только после commit API возвращает 202. Если API остановится после этого ответа, durable staging и намерение остаются. Worker обслуживает PostgreSQL outbox: после успешного XADD отмечает запись опубликованной. Неподтверждённые публикации и опубликованные записи с истёкшим временем повтора отправляются снова; Redis Streams доставляет at least once, а не exactly once. + +Worker получает только ID job и вложения. Он блокирует запись job, получает lease, проверяет исходник и пишет оригинал/варианты по детерминированным ключам. Для pipeline v1 ключи вариантов имеют префикс variants/v1//. Ready и список вариантов фиксируются в PostgreSQL только после завершения S3-записей; после этого удаляется staging и подтверждается сообщение. Дубль уже ready job — no-op. Потерянный worker восстанавливается по истёкшему lease; попытки ограничены JOB_MAX_ATTEMPTS=5, задержка повторов растёт экспоненциально и содержит jitter. Финальная ошибка сохраняется в metadata/audit и Redis Stream attachment-jobs-dlq; отдельная команда replay принимает только failed job. + +Thumbnail — JPEG, вписан в 256×256 без увеличения, сохраняет пропорции, применяет EXIF orientation, alpha композитится на белый. Crop — центрированный JPEG 128×128. Optimized — WebP, вписанный в 1280×1280 без увеличения, качество 82, EXIF исключён. Проверяются лимиты upload 10 MiB и декодирования 40 млн пикселей; worker ограничен контейнерным лимитом памяти и WORKER_CONCURRENCY. S3 I/O имеет connect/read timeout; зависшая job получает lease timeout и ограниченно переходит к повторам. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| -------------------------------------------- | ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------- | +| Офлайн-контракт и синтаксис | `python scripts/check_contract.py` | PASS | вывод команды | +| Полная OpenAPI/JSON Schema | `python scripts/validate_openapi.py` | PASS | вывод команды | +| Compose, версии, миграция и готовность | `docker compose up -d --scale api=2 --wait` | PASS: PostgreSQL, Redis, брокер, RustFS, 2 API, HAProxy и worker healthy; миграция и bucket bootstrap завершились | `compose.yaml` | +| Публичный smoke | `python -X utf8 tests/smoke.py` | PASS: API flow lab1–5, 202 upload, ready, байты/SHA-256 оригинала, JPEG thumbnail 256×160 | `tests/smoke.py` | +| Durable staging и 202 | `python tests/worker_outage.py` | PASS: worker остановлен, upload вернул 202; после рестарта API та же запись оставалась queued; после старта worker стала ready | `evidence/worker-outage.json` | +| Отдельные broker/worker и сохранение брокера | `python tests/broker_restart.py` | PASS: опубликованное сообщение пережило рестарт Redis с AOF volume; stream длина 3 до/после; задача дошла до ready | `evidence/broker-restart.json` | +| At-least-once, side effect до ack | `python tests/fault_redelivery.py` | PASS: после записи S3 worker аварийно упал (fault injection); перезапущенный worker повторно обработал ту же job без дублей вариантов | `evidence/fault-redelivery.json` | +| Ограниченные retry, DLQ и replay | `python tests/retry_replay.py` | PASS: отсутствующий staging дал `retry_exhausted` после 5 попыток; попало в DLQ; после восстановления staging явный replay завершился ready | `evidence/retry-replay.json` | +| Backpressure и ограничение приёма | `python tests/backpressure.py` | PASS: при заполненной очереди (лимит 1) второй upload отклонен (503); первый успешно завершился | `evidence/backpressure.json` | +| Все сервисы и данные переживают рестарт | `python tests/resilience.py --mode persistence` | PASS: прежний bearer token, чат, сообщение и вложение сохранились | `evidence/` | +| Отказ API-реплики | `python tests/resilience.py --mode failover` | PASS: до отказа 2 instance; переключение за 2.39 с, подтвержденные сообщения сохранились | `evidence/` | +| Track 5: три concurrency и смешанные размеры | `tests/pipeline_load.py`, concurrency 1/2/4 | PASS: 36/36 ready, ошибок 0; измерены ready throughput, latency, queue-wait и RSS | `evidence/pipeline-load-c*.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Трек 1 — очередь под нагрузкой + +**Гипотеза.** Увеличение числа обработчиков с 1 до 2 уменьшит очередь и p95 до +ready, но поднимет RSS; после точки насыщения CPU/S3 throughput перестанет +расти. Входной лимит очереди должен останавливать приём новых задач и сохранять +уже подтверждённые. + +**Методика.** Docker Compose локально, Python 3.12 в контейнерах, хост Windows 11. +API×2, один worker. Для каждого concurrency (1, 2, 4) созданы 12 синтетических +PNG: 320×200, 1024×768 и 2048×1536. Upload-клиенты — 6. Замерялись время до +ready, оценка queue wait и RSS docker stats (интервал 250 мс). + +| Worker concurrency / prefetch | Успешно / ошибки | Ready throughput | Ready p50 / p95 / p99 | Оценка queue-wait p50 / p95 | Пиковая sampled RSS | +| ----------------------------: | ---------------: | ---------------: | -----------------------------: | --------------------------: | ------------------: | +| 1 | 12 / 0 | 1,311 файла/с | 3253,53 / 5788,39 / 5788,39 мс | 2775,81 / 4955,47 мс | 104,10 MiB | +| 2 | 12 / 0 | 1,482 файла/с | 3449,67 / 5566,78 / 5566,78 мс | 2163,50 / 4059,81 мс | 150,50 MiB | +| 4 | 12 / 0 | 1,510 файла/с | 2091,93 / 5172,02 / 5172,02 мс | 1615,54 / 2800,88 мс | 186,10 MiB | + +Рост concurrency с 1 до 4 сократил медианное время ожидания в очереди с 2775 мс +до 1615 мс (в 1.7 раза) и медианную задержку ready с 3253 мс до 2091 мс. При +этом пиковый RSS воркера увеличился со 104 MiB до 186 MiB из-за одновременной +обработки нескольких изображений в памяти Pillow. Пропускная способность выросла +умеренно (с 1.31 до 1.51 файла/с) из-за насыщения хостового CPU при +компрессии/ресайзе картинок на короткой дистанции. + +**Backpressure.** В backpressure.py worker был остановлен, а лимит MAX_QUEUED_JOBS +временно выставлен в 1. Первая загрузка получила 202 и сохранилась queued, +вторая получила 503 до записи staging/metadata. После запуска worker исходная +задача завершилась ready и SHA-256 совпал. Восстановление завершилось за 8.38 с. + +### Отказы и сохранность + +| Воздействие | Ожидание | Наблюдение | Восстановление / потеря данных | +| ---------------------------------------------- | -------------------------------------------- | ------------------------------------------------------------------- | --------------------------------------------------------- | +| Worker остановлен, затем API рестартован | 202 job остаётся в durable outbox/staging | queued после API restart, ready после запуска worker | 10,24 с; потери нет | +| Broker рестартован при непрочитанном сообщении | Redis AOF volume сохраняет stream | stream сохранен (длина 3); задача стала ready | автоматическое; потери нет | +| Worker crash после S3 writes, до DB ready/ack | повторная доставка завершает ту же job | worker restart 0→1, готово 3 варианта, SHA совпал | 6,13 с; дубль не создал набор вариантов | +| Временная ошибка staging | максимум 5 попыток, затем failed и DLQ | `retry_exhausted`, DLQ зафиксирован; операторский replay стал ready | 34,52 с до отказа и replay; успешную job replay отвергает | +| Очередь заполнена | новое задание отклоняется, принятые остаются | 202, затем 503; первая job завершилась | 8,38 с; потери нет | +| Рестарт всех Compose контейнеров | session и подтверждённые данные доступны | persistence hook PASS с прежним token | потери нет; volumes сохранены | +| Остановка API-реплики | другая реплика продолжает писать | более 100 успешных запросов, восстановление 2.39 с | подтверждённые сообщения сохранились | + -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Эксперимент проведён на локальном Docker Desktop под Windows 11. Нагрузка +closed-loop, 12 объектов на профиль; p95/p99 чувствительны к малым выборкам. + +Broker развёрнут одним Redis с AOF volume, PostgreSQL и RustFS также одиночные +экземпляры; Compose сценарий проверяет сохранность данных, но не HA узлов. +Варианты и ключи используют pipeline version v1. Очередь ограничивает число +активных jobs, но для длительной эксплуатации требуются retention/cleanup policy +устаревшего staging и мониторинг DLQ. EOF diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..0eb8c8a6 --- /dev/null +++ b/app/main.py @@ -0,0 +1,299 @@ +import base64 +import datetime as dt +import hashlib +import json +import os +import re +import secrets +import uuid +from flask import Flask, request, g +from werkzeug.utils import secure_filename +from werkzeug.exceptions import HTTPException +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from psycopg.rows import dict_row +from psycopg.errors import UniqueViolation +from psycopg_pool import ConnectionPool +import psycopg +import redis +import boto3 +from botocore.config import Config +from PIL import Image, UnidentifiedImageError +import io +from urllib.parse import quote +from app.migrate import setting + +app = Flask(__name__) +MAX_UPLOAD=int(os.getenv('MAX_UPLOAD_BYTES','10485760')) +MAX_PIXELS=int(os.getenv('MAX_IMAGE_PIXELS','40000000')) +MAX_QUEUED=int(os.getenv('MAX_QUEUED_JOBS','500')) +app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD +pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=int(os.getenv('POOL_MAX','8')), timeout=3, + kwargs={'row_factory':dict_row,'connect_timeout':3,'options':'-c statement_timeout=3000 -c lock_timeout=2000'}) +cache = redis.Redis.from_url(setting('CACHE_URL'),socket_connect_timeout=.2,socket_timeout=.2) +broker = redis.Redis.from_url(setting('BROKER_URL'),socket_connect_timeout=.3,socket_timeout=.5) +kdf = PasswordHasher(time_cost=2,memory_cost=19456,parallelism=1) +dummy = kdf.hash(secrets.token_urlsafe(32)) +ttl = int(os.getenv('SESSION_TTL_SECONDS','3600')) +def envsetting(name): return setting(name) if os.getenv(name+'_FILE') else os.environ[name] +s3=boto3.client('s3',endpoint_url=os.getenv('S3_ENDPOINT','http://rustfs:9000'),region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=envsetting('S3_ACCESS_KEY_ID'),aws_secret_access_key=envsetting('S3_SECRET_ACCESS_KEY'),config=Config(connect_timeout=2,read_timeout=5,retries={'max_attempts':1})) +bucket=os.getenv('S3_BUCKET','messenger-attachments') +instance = os.getenv('HOSTNAME','api') + '-' + secrets.token_hex(4) + +class Problem(Exception): + def __init__(self,status,code): self.status,self.code=status,code + +def fail(status,code): raise Problem(status,code) +def invalid(): fail(400,'invalid_request') +def uid(value): + try: return str(uuid.UUID(value)) + except (ValueError,TypeError,AttributeError): invalid() +def clean(row): + return {k:(v.isoformat().replace('+00:00','Z') if isinstance(v,dt.datetime) else str(v) if isinstance(v,uuid.UUID) else v) for k,v in row.items()} +def user(row): return clean({k:row[k] for k in ('id','username','display_name','created_at')}) +def body(fields): + if request.mimetype != 'application/json': fail(415,'unsupported_media_type') + try: data = request.get_json() + except HTTPException: invalid() + if not isinstance(data,dict) or set(data)!=set(fields): invalid() + return data +def string(s,maximum,minimum=1): + if not isinstance(s,str) or not s.strip() or not minimum<=len(s)<=maximum: invalid() + try: s.encode('utf-8') + except UnicodeError: invalid() + return s + +def actor(c): + auth = request.headers.get('Authorization','') + if not auth.startswith('Bearer ') or not 1<=len(auth[7:])<=256: fail(401,'unauthorized') + digest = hashlib.sha256(auth[7:].encode()).hexdigest() + row=c.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp()',(digest,)).fetchone() + if not row: fail(401,'unauthorized') + g.digest=digest + return row['user_id'] + +def public_user(c,ident): + key='user:'+str(ident) + try: + value=cache.get(key) + if value: return json.loads(value) + except (redis.RedisError,ValueError): pass + result=user(c.execute('SELECT * FROM users WHERE id=%s',(ident,)).fetchone()) + try: cache.setex(key,60,json.dumps(result)) + except redis.RedisError: pass + return result + +def chat(c,ident,who): + row=c.execute('SELECT c.* FROM chats c JOIN members m ON c.id=m.chat_id WHERE c.id=%s AND m.user_id=%s',(ident,who)).fetchone() + if not row: fail(404,'not_found') + row=clean(row) + row['member_ids']=[str(x['user_id']) for x in c.execute('SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id',(ident,))] + return row + +def page_args(resource): + if set(request.args)-{'limit','cursor'} or any(len(request.args.getlist(k))!=1 for k in request.args): invalid() + raw=request.args.get('limit','50') + if len(raw)>3 or not re.fullmatch(r'[0-9]+',raw) or not 1<=int(raw)<=100: invalid() + cursor=request.args.get('cursor') + pair=None + if cursor is not None: + try: + data=json.loads(base64.b64decode(cursor,altchars=b'-_',validate=True)) + if not isinstance(data,list) or len(data)!=3 or data[0]!=resource: invalid() + stamp=dt.datetime.fromisoformat(data[1].replace('Z','+00:00')) + if stamp.tzinfo is None: invalid() + pair=(stamp,uid(data[2])) + except (ValueError,TypeError,KeyError,AttributeError,OverflowError): invalid() + return int(raw),pair + +def page(rows,limit,resource): + more=len(rows)>limit + rows=rows[:limit] + cursor=base64.urlsafe_b64encode(json.dumps([resource,rows[-1]['created_at'],rows[-1]['id']]).encode()).decode() if more else None + return {'items':rows,'next_cursor':cursor} + +@app.before_request +def before(): g.request_id=str(uuid.uuid4()) +@app.after_request +def after(response): + response.headers['X-Request-Id']=g.request_id + response.headers['X-Instance-Id']=instance + return response +@app.errorhandler(Exception) +def error(exc): + if isinstance(exc,Problem): status,code=exc.status,exc.code + elif isinstance(exc,UniqueViolation): status,code=409,'conflict' + elif isinstance(exc,(psycopg.Error, __import__('psycopg_pool').PoolTimeout)): status,code=503,'unavailable' + elif isinstance(exc,HTTPException): + status=exc.code + code={404:'not_found',413:'payload_too_large',415:'unsupported_media_type'}.get(status,'invalid_request') + else: + app.logger.error('Unhandled exception type: %s',type(exc).__name__) + status,code=500,'internal_error' + response=app.json.response({'error':{'code':code,'message':code,'request_id':g.request_id}}) + response.status_code=status + if status==401: + response.headers['WWW-Authenticate']='Basic realm="messenger", charset="UTF-8"' if request.path=='/api/v1/auth/sessions' else 'Bearer realm="messenger"' + return response +@app.get('/health/live') +def live(): return {'status':'ok'} +@app.get('/health/ready') +def ready(): + with pool.connection() as c: c.execute('SELECT 1') + try: broker.ping() + except redis.RedisError: fail(503,'unavailable') + return {'status':'ok'} + +@app.route('/api/v1/',methods=['GET','POST','DELETE']) +def api(path): + public = (path in ('users','auth/sessions') and request.method=='POST') + if not public and not request.headers.get('Authorization','').startswith('Bearer '): fail(401,'unauthorized') + with pool.connection() as c: + if path=='users' and request.method=='POST': + d=body(['username','display_name','password']) + if not isinstance(d['username'],str) or not re.fullmatch('[a-z0-9_]{3,32}',d['username']): invalid() + string(d['display_name'],100); string(d['password'],128,12) + row=c.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *',(uuid.uuid4(),d['username'],d['display_name'],kdf.hash(d['password']))).fetchone() + return user(row),201 + if path=='auth/sessions' and request.method=='POST': + a=request.headers.get('Authorization','') + try: + if not a.startswith('Basic ') or request.get_data(): raise ValueError() + name,password=base64.b64decode(a[6:],validate=True).decode('utf-8').split(':',1) + except (ValueError,UnicodeError): fail(401,'unauthorized') + row=c.execute('SELECT * FROM users WHERE username=%s',(name,)).fetchone() + try: valid=kdf.verify(row['password_hash'] if row else dummy,password) + except VerificationError: valid=False + if not valid or not row: fail(401,'unauthorized') + token=secrets.token_urlsafe(32) + expires=c.execute("INSERT INTO sessions VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at",(hashlib.sha256(token.encode()).hexdigest(),row['id'],ttl)).fetchone()['expires_at'] + return {'token':token,'token_type':'Bearer','expires_at':expires.isoformat().replace('+00:00','Z'),'user':user(row)},201 + who=actor(c) + if path=='auth/sessions/current' and request.method=='DELETE': + c.execute('DELETE FROM sessions WHERE token_hash=%s',(g.digest,)) + return '',204 + if path=='users/me' and request.method=='GET': return public_user(c,who) + if path=='chats': + if request.method=='POST': + d=body(['title','member_ids']); string(d['title'],100) + ids=d['member_ids'] + if not isinstance(ids,list) or not 1<=len(ids)<=100: invalid() + ids=[uid(x) for x in ids] + if len(ids)!=len(set(ids)) or str(who) not in ids: invalid() + if c.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s::uuid[])',(ids,)).fetchone()['n']!=len(ids): invalid() + ident=uuid.uuid4() + c.execute('INSERT INTO chats(id,title) VALUES (%s,%s)',(ident,d['title'])) + for member in ids: c.execute('INSERT INTO members VALUES (%s,%s)',(ident,member)) + return chat(c,ident,who),201 + if request.method=='GET': + resource='chats:'+str(who); limit,pair=page_args(resource) + query='SELECT c.id FROM chats c JOIN members m ON c.id=m.chat_id WHERE m.user_id=%s' + params=[who] + if pair: query+=' AND (c.created_at,c.id)>(%s,%s)'; params+=list(pair) + query+=' ORDER BY c.created_at,c.id LIMIT %s'; params+=[limit+1] + return page([chat(c,x['id'],who) for x in c.execute(query,params).fetchall()],limit,resource) + parts=path.split('/') + if len(parts) in (2,3) and parts[0]=='chats': + ident=uid(parts[1]) + if len(parts)==3 and parts[2]=='messages' and os.getenv('MESSAGE_ACCESS_FAST','1')=='1': + # Message endpoints only need membership, not the entire Chat representation. + if not c.execute('SELECT 1 FROM members WHERE chat_id=%s AND user_id=%s',(ident,who)).fetchone(): fail(404,'not_found') + result=None + else: + result=chat(c,ident,who) + if len(parts)==2 and request.method=='GET': return result + if len(parts)==3 and parts[2]=='messages': + if request.method=='POST': + key=request.headers.get('Idempotency-Key','') + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}',key): invalid() + if request.mimetype != 'application/json': fail(415,'unsupported_media_type') + try: d=request.get_json() + except HTTPException: invalid() + if not isinstance(d,dict) or set(d)-{'text','attachment_ids'} or 'text' not in d: invalid() + if not isinstance(d['text'],str) or len(d['text'])>4000: invalid() + aids=d.get('attachment_ids',[]) + if not isinstance(aids,list) or len(aids)>10: invalid() + aids=[uid(x) for x in aids] + if len(set(aids))!=len(aids): invalid() + if not d['text'].strip() and not aids: invalid() + if aids: + found=c.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])',(ident,aids)).fetchall() + if len(found)!=len(aids): fail(404,'not_found') + if any(x['status']=='failed' for x in found): fail(409,'conflict') + canonical=json.dumps({'text':d['text'],'attachment_ids':aids},sort_keys=True,ensure_ascii=False) + # One transaction: lock scope, inspect replay, insert effect and result, commit before response. + scope=str(who)+':POST:'+ident+':'+key + c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(scope,)) + old=c.execute('SELECT body,response FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()',(who,ident,key)).fetchone() + if old: + if old['body']!=canonical: fail(409,'conflict') + return old['response'],201 + mid=uuid.uuid4() + result=clean(c.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *',(mid,ident,who,d['text'])).fetchone()) + result['attachment_ids']=aids + for pos,aid in enumerate(aids): c.execute('INSERT INTO message_attachments VALUES (%s,%s,%s)',(mid,aid,pos)) + c.execute("INSERT INTO idempotency(user_id,chat_id,key,body,response,expires_at) VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT(user_id,chat_id,key) DO UPDATE SET body=excluded.body,response=excluded.response,expires_at=excluded.expires_at",(who,ident,key,canonical,psycopg.types.json.Jsonb(result))) + return result,201 + if request.method=='GET': + limit,pair=page_args(ident) + query='SELECT * FROM messages WHERE chat_id=%s'; params=[ident] + if pair: query+=' AND (created_at,id)>(%s,%s)'; params+=list(pair) + query+=' ORDER BY created_at,id LIMIT %s'; params+=[limit+1] + items=[clean(x) for x in c.execute(query,params)] + for m in items: m['attachment_ids']=[str(x['attachment_id']) for x in c.execute('SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position',(m['id'],))] + return page(items,limit,ident) + if len(parts)==3 and parts[2]=='attachments' and request.method=='POST': + backlog=c.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('pending','published','processing')").fetchone()['n'] + if backlog>=MAX_QUEUED: fail(503,'unavailable') + filename=request.args.get('filename','') + if len(filename)<1 or len(filename)>255 or any(ord(ch)<32 or ch in '/\\' for ch in filename): invalid() + if request.mimetype not in ('image/png','image/jpeg'): fail(415,'unsupported_media_type') + raw=request.get_data(cache=False) + if not raw: invalid() + try: + im=Image.open(io.BytesIO(raw)); fmt=im.format; w,h=im.size + if w*h>MAX_PIXELS: fail(413,'payload_too_large') + im.verify() + except Problem: raise + except (UnidentifiedImageError,OSError,ValueError): fail(415,'unsupported_media_type') + actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(fmt) + if actual!=request.mimetype: fail(415,'unsupported_media_type') + ident_att=uuid.uuid4(); key='original/'+str(ident)+'/'+str(ident_att); staging='staging/'+str(ident)+'/'+str(ident_att) + digest=hashlib.sha256(raw).hexdigest() + try: + s3.put_object(Bucket=bucket,Key=staging,Body=raw,ContentType=actual,Metadata={'sha256':digest}) + row=c.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,staging_key,filename,content_type,size_bytes,sha256,status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",(ident_att,ident,who,key,staging,filename,actual,len(raw),digest)).fetchone() + job_id=uuid.uuid4() + c.execute("INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version) VALUES (%s,%s,1)",(job_id,ident_att)) + except Exception: + try: s3.delete_object(Bucket=bucket,Key=staging) + except Exception: app.logger.warning('attachment compensation deferred') + if isinstance(__import__('sys').exc_info()[1],psycopg.Error): raise + fail(503,'unavailable') + return attachment_json(row),202 + if path.startswith('attachments/'): + bits=path.split('/') + if len(bits) not in (2,3): fail(404,'not_found') + aid=uid(bits[1]) + row=c.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(aid,who)).fetchone() + if not row: fail(404,'not_found') + if len(bits)==2 and request.method=='GET': return attachment_json(row) + if len(bits)==3 and bits[2]=='content' and request.method=='GET': + variant=request.args.get('variant','original') + if variant not in ('original','thumbnail','crop','optimized'): invalid() + if row['status']!='ready': fail(409,'conflict') + if variant!='original': + v=next((v for v in row['variants'] if v['name']==variant),None) + if v is None: fail(404,'not_found') + object_key=f"variants/v1/{aid}/{variant}"; content_type=v['content_type'] + else: object_key=row['object_key']; content_type=row['content_type'] + try: obj=s3.get_object(Bucket=bucket,Key=object_key); data=obj['Body'].read(MAX_UPLOAD+1) + except Exception: fail(503,'unavailable') + if len(data)>MAX_UPLOAD or (variant=='original' and hashlib.sha256(data).hexdigest()!=row['sha256']): fail(503,'unavailable') + disposition='attachment; filename="attachment"; filename*=UTF-8\'\''+quote(row['filename'],safe='') + return app.response_class(data,200,mimetype=content_type,headers={'Content-Disposition':disposition}) + fail(404,'not_found') + +def attachment_json(row): + stamp=row['created_at'].isoformat().replace('+00:00','Z') + return {'id':str(row['id']),'chat_id':str(row['chat_id']),'owner_id':str(row['owner_id']),'filename':row['filename'],'content_type':row['content_type'],'size_bytes':row['size_bytes'],'sha256':row['sha256'].strip(),'status':row['status'],'variants':row.get('variants',[]),'error_code':row.get('error_code'),'created_at':stamp,'updated_at':row.get('updated_at',row['created_at']).isoformat().replace('+00:00','Z')} diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..ec8eeae4 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,18 @@ +import os +from pathlib import Path +import psycopg + +def setting(name): + file = os.getenv(name + '_FILE') + return Path(file).read_text().strip() if file else os.environ[name] + +def migrate(): + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as c: + c.execute('SELECT pg_advisory_xact_lock(202602)') + c.execute('CREATE TABLE IF NOT EXISTS schema_migrations(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())') + for p in sorted(Path('migrations').glob('*.sql')): + if not c.execute('SELECT 1 FROM schema_migrations WHERE version=%s',(p.name,)).fetchone(): + c.execute(p.read_text()) + c.execute('INSERT INTO schema_migrations(version) VALUES (%s)',(p.name,)) +if __name__ == '__main__': + migrate() diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 00000000..fbb0a1ab --- /dev/null +++ b/app/worker.py @@ -0,0 +1,131 @@ +"""Durable outbox relay and idempotent Redis Streams image worker.""" +import io, json, os, time, uuid, hashlib, random, shutil +from concurrent.futures import ThreadPoolExecutor +import redis, boto3, psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from botocore.config import Config +from PIL import Image, ImageOps, UnidentifiedImageError +from app.migrate import setting + +def envsetting(name): return setting(name) if os.getenv(name+'_FILE') else os.environ[name] +db=ConnectionPool(setting('DATABASE_URL'),min_size=1,max_size=8,kwargs={'row_factory':dict_row,'connect_timeout':3}) +broker=redis.Redis.from_url(setting('BROKER_URL'),socket_connect_timeout=1,socket_timeout=2,decode_responses=True) +s3=boto3.client('s3',endpoint_url=os.getenv('S3_ENDPOINT','http://rustfs:9000'),region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=envsetting('S3_ACCESS_KEY_ID'),aws_secret_access_key=envsetting('S3_SECRET_ACCESS_KEY'),config=Config(connect_timeout=3,read_timeout=10,retries={'max_attempts':2})) +bucket=os.getenv('S3_BUCKET','messenger-attachments'); stream='attachment-jobs'; group='image-workers'; attempts=int(os.getenv('JOB_MAX_ATTEMPTS','5')); concurrency=int(os.getenv('WORKER_CONCURRENCY','2')); timeout=int(os.getenv('JOB_TIMEOUT_SECONDS','60')) +try: broker.xgroup_create(stream,group,id='0',mkstream=True) +except redis.ResponseError as e: + if 'BUSYGROUP' not in str(e): raise + +def publish_outbox(): + # DB intent remains pending until XADD succeeds; crash in the gap yields a safe duplicate. + with db.connection() as c: + rows=c.execute("SELECT job_id,attachment_id,pipeline_version,attempt,created_at FROM attachment_jobs WHERE state IN ('pending','published') AND available_at<=clock_timestamp() ORDER BY created_at LIMIT 100 FOR UPDATE SKIP LOCKED").fetchall() + for r in rows: + msg={'schema_version':1,'job_id':str(r['job_id']),'type':'attachment.process.v1','attachment_id':str(r['attachment_id']),'pipeline_version':r['pipeline_version'],'attempt':max(1,r['attempt']+1),'created_at':r['created_at'].isoformat().replace('+00:00','Z')} + broker.xadd(stream,{'job':json.dumps(msg,separators=(',',':'))}) + c.execute("UPDATE attachment_jobs SET state='published',available_at=clock_timestamp()+interval '15 seconds' WHERE job_id=%s AND state IN ('pending','published')",(r['job_id'],)) + +def resize(raw): + im=Image.open(io.BytesIO(raw)); im.load(); im=ImageOps.exif_transpose(im) + if im.width*im.height>40_000_000: raise ValueError('oversized_image') + if im.mode in ('RGBA','LA') or 'transparency' in im.info: + rgba=im.convert('RGBA'); bg=Image.new('RGBA',rgba.size,'white'); bg.alpha_composite(rgba); im=bg.convert('RGB') + else: im=im.convert('RGB') + thumb=im.copy(); thumb.thumbnail((256,256),Image.Resampling.LANCZOS) + out=io.BytesIO(); thumb.save(out,format='JPEG',quality=85,optimize=True) + crop=ImageOps.fit(im,(128,128),method=Image.Resampling.LANCZOS,centering=(.5,.5)); cropout=io.BytesIO(); crop.save(cropout,format='JPEG',quality=85,optimize=True) + opt=im.copy(); opt.thumbnail((1280,1280),Image.Resampling.LANCZOS); webp=io.BytesIO(); opt.save(webp,format='WEBP',quality=82,method=4,exif=b'') + return [('thumbnail','image/jpeg',thumb.size,out.getvalue()),('crop','image/jpeg',crop.size,cropout.getvalue()),('optimized','image/webp',opt.size,webp.getvalue())] + +def process(job): + jid=uuid.UUID(job['job_id']); aid=uuid.UUID(job['attachment_id']) + with db.connection() as c: + c.execute('BEGIN') + r=c.execute('SELECT j.*,a.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s FOR UPDATE OF j,a',(jid,)).fetchone() + if not r: c.rollback(); return + if r['state']=='done' or r['state']=='failed' or r['state']=='processing' and r['lease_until'] and r['lease_until'].timestamp()>time.time(): c.rollback(); return + c.execute("UPDATE attachment_jobs SET state='processing',attempt=attempt+1,lease_until=clock_timestamp()+(%s * interval '1 second') WHERE job_id=%s",(timeout,jid)) + c.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s AND status<>'ready'",(aid,)) + c.execute("INSERT INTO attachment_job_audit(job_id,event) VALUES (%s,'processing')",(jid,)); c.commit() + try: + raw=s3.get_object(Bucket=bucket,Key=r['staging_key'])['Body'].read(10_485_761) + if len(raw)>10_485_760 or hashlib.sha256(raw).hexdigest()!=r['sha256'].strip(): raise ValueError('invalid_image') + variants=resize(raw) + # Each object key is deterministic, so a crash and redelivery overwrites the same effect. + s3.put_object(Bucket=bucket,Key=r['object_key'],Body=raw,ContentType=r['content_type'],Metadata={'sha256':r['sha256'].strip()}) + meta=[] + for name,mime,size,data in variants: + key=f"variants/v1/{aid}/{name}" + s3.put_object(Bucket=bucket,Key=key,Body=data,ContentType=mime,Metadata={'pipeline-version':'1'}) + meta.append({'name':name,'content_type':mime,'size_bytes':len(data),'sha256':hashlib.sha256(data).hexdigest(),'width':size[0],'height':size[1]}) + if os.getenv('FAULT_AFTER_S3_WRITE_ONCE','0')=='1': + with db.connection() as c: + seen=c.execute("SELECT 1 FROM attachment_job_audit WHERE job_id=%s AND event='fault_injected'",(jid,)).fetchone() + if not seen: + c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'fault_injected','crash after deterministic S3 writes, before ready commit')",(jid,)) + c.commit() + os._exit(72) + with db.connection() as c: + c.execute("UPDATE attachments SET status='ready',variants=%s::jsonb,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=1 AND status<>'ready'",(json.dumps(meta),aid)) + c.execute("UPDATE attachment_jobs SET state='done',lease_until=NULL,last_error=NULL WHERE job_id=%s",(jid,)) + c.execute("INSERT INTO attachment_job_audit(job_id,event) VALUES (%s,'ready')",(jid,)) + s3.delete_object(Bucket=bucket,Key=r['staging_key']) + except (UnidentifiedImageError,ValueError) as e: + retry_or_fail(jid,aid,'invalid_image',True) + except Exception as e: + retry_or_fail(jid,aid,'processing_failed',False) + +def retry_or_fail(jid,aid,code,permanent): + with db.connection() as c: + r=c.execute('SELECT attempt,state FROM attachment_jobs WHERE job_id=%s',(jid,)).fetchone() + if not r or r['state']=='done': return + n=r['attempt'] + if permanent or n>=attempts: + c.execute("UPDATE attachment_jobs SET state='failed',lease_until=NULL,last_error=%s WHERE job_id=%s",(code,jid)) + c.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() WHERE id=%s",('invalid_image' if permanent else 'retry_exhausted',aid)) + c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'failed',%s)",(jid,code)) + broker.xadd('attachment-jobs-dlq',{'job_id':str(jid),'attachment_id':str(aid),'error_code':code}) + else: + delay=min(60,2**n)*(0.8+random.random()*.4) + c.execute("UPDATE attachment_jobs SET state='pending',available_at=clock_timestamp()+(%s * interval '1 second'),lease_until=NULL,last_error=%s WHERE job_id=%s",(delay,code,jid)) + c.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=%s",(aid,)) + +def recover_leases(): + with db.connection() as c: + expired=c.execute("SELECT job_id,attachment_id,attempt FROM attachment_jobs WHERE state='processing' AND lease_until=attempts: + c.execute("UPDATE attachment_jobs SET state='failed',lease_until=NULL,last_error='retry_exhausted' WHERE job_id=%s",(row['job_id'],)) + c.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['attachment_id'],)) + c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'failed','lease attempts exhausted')",(row['job_id'],)) + broker.xadd('attachment-jobs-dlq',{'job_id':str(row['job_id']),'attachment_id':str(row['attachment_id']),'error_code':'retry_exhausted'}) + else: + c.execute("UPDATE attachment_jobs SET state='pending',lease_until=NULL,available_at=clock_timestamp() WHERE job_id=%s",(row['job_id'],)) + c.execute("UPDATE attachments a SET status='queued',updated_at=clock_timestamp() FROM attachment_jobs j WHERE j.attachment_id=a.id AND j.state='pending' AND a.status='processing'") + +def main(): + consumer=f"worker-{uuid.uuid4()}" + while True: + try: + recover_leases(); publish_outbox() + if os.getenv('WORKER_PAUSE_CONSUME','0')=='1': + time.sleep(.2); continue + msgs=broker.xreadgroup(group,consumer,{stream:'>'},count=concurrency,block=1000) + # Reclaim jobs left pending by a worker crash; a ready job becomes a no-op then ack. + claimed=broker.xautoclaim(stream,group,consumer,min_idle_time=timeout*1000,start_id='0-0',count=concurrency) + if claimed and claimed[1]: msgs.append((stream,claimed[1])) + for _,entries in msgs: + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures=[] + for mid,fields in entries: + try: job=json.loads(fields['job']) + except Exception: + broker.xadd('attachment-jobs-dlq',{'message_id':mid,'error_code':'invalid_job'}); broker.xack(stream,group,mid); continue + futures.append((mid,executor.submit(process,job))) + for mid,future in futures: + try: future.result(); broker.xack(stream,group,mid) + except Exception: pass + except Exception: time.sleep(1) + +if __name__=='__main__': main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..bb8f40b8 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,131 @@ +name: mtusi_lab5_solution +x-api: &api + build: . + image: mtusi-lab5-api:local + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://rustfs:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + MAX_UPLOAD_BYTES: 10485760 + MAX_IMAGE_PIXELS: 40000000 + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + FAULT_AFTER_S3_WRITE_ONCE: ${FAULT_AFTER_S3_WRITE_ONCE:-0} + WORKER_PAUSE_CONSUME: ${WORKER_PAUSE_CONSUME:-0} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + MAX_QUEUED_JOBS: ${MAX_QUEUED_JOBS:-500} + secrets: [database_url, cache_url, broker_url, s3_access_key_id, s3_secret_access_key] + restart: unless-stopped + mem_limit: 512m + cpus: 0.75 +services: + postgres: + image: postgres:16.10-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, pg_isready -U messenger -d messenger] + interval: 2s + timeout: 2s + retries: 30 + redis: + image: redis:7.4.6-alpine + command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 20 + broker: + image: redis:7.4.6-alpine + command: [redis-server, --appendonly, 'yes', --appendfsync, everysec, --maxmemory, 128mb, --maxmemory-policy, noeviction] + volumes: [brokerdata:/data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 20 + rustfs: + image: rustfs/rustfs:1.0.3-glibc + command: [/data] + environment: + RUSTFS_ACCESS_KEY_FILE: /run/secrets/rustfs_root_user + RUSTFS_SECRET_KEY_FILE: /run/secrets/rustfs_root_password + RUSTFS_CONSOLE_ENABLE: 'false' + RUSTFS_ADDRESS: ':9000' + secrets: [rustfs_root_user, rustfs_root_password] + volumes: [s3data:/data] + expose: ['9000'] + healthcheck: + test: [CMD, curl, -f, http://127.0.0.1:9000/health] + interval: 2s + timeout: 2s + retries: 30 + bucket-bootstrap: + image: rustfs/rc:v0.1.37 + depends_on: + rustfs: {condition: service_healthy} + entrypoint: [/bin/sh, /bootstrap-rustfs.sh] + volumes: ['./infra/rustfs-policy.json:/policy.json:ro', './infra/bootstrap-rustfs.sh:/bootstrap-rustfs.sh:ro'] + secrets: [rustfs_root_user, rustfs_root_password, s3_access_key_id, s3_secret_access_key] + restart: 'no' + migrate: + <<: *api + command: [python, -m, app.migrate] + restart: 'no' + depends_on: + postgres: {condition: service_healthy} + api: + <<: *api + command: [gunicorn, --bind, 0.0.0.0:8080, --workers, '1', --threads, '2', --timeout, '35', --graceful-timeout, '5', app.main:app] + scale: ${API_REPLICAS:-2} + depends_on: + migrate: {condition: service_completed_successfully} + redis: {condition: service_healthy} + broker: {condition: service_healthy} + bucket-bootstrap: {condition: service_completed_successfully} + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3)"] + interval: 2s + timeout: 4s + retries: 3 + worker: + <<: *api + command: [python, -m, app.worker] + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + bucket-bootstrap: {condition: service_completed_successfully} + healthcheck: + disable: true + deploy: + replicas: 1 + lb: + image: haproxy:3.0.11-alpine + ports: ['127.0.0.1:${APP_PORT:-8081}:8080'] + volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro'] + depends_on: + api: {condition: service_healthy} +volumes: + pgdata: + s3data: + brokerdata: +secrets: + db_password: {file: ./secrets/db_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + broker_url: {file: ./secrets/broker_url} + rustfs_root_user: {file: ./secrets/rustfs_root_user} + rustfs_root_password: {file: ./secrets/rustfs_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} diff --git a/evidence/backpressure.json b/evidence/backpressure.json new file mode 100644 index 00000000..a3785139 --- /dev/null +++ b/evidence/backpressure.json @@ -0,0 +1,11 @@ +{ + "queue_limit": 1, + "worker_stopped": true, + "first_upload_status": 202, + "next_upload_status": 503, + "accepted_attachment_id": "21226e6a-ee2b-43b9-9040-734c8e0c59d4", + "final_status": "ready", + "original_sha256_matches": true, + "recovery_seconds": 8.377, + "note": "503 admission refusal leaves the already accepted job intact." +} diff --git a/evidence/broker-restart.json b/evidence/broker-restart.json new file mode 100644 index 00000000..98328eb2 --- /dev/null +++ b/evidence/broker-restart.json @@ -0,0 +1,9 @@ +{ + "attachment_id": "54d22875-520a-45a8-ae6a-21c11807e843", + "status_before_broker_restart": "queued", + "stream_length_before_restart": 3, + "stream_length_after_restart": 3, + "final_status": "ready", + "original_sha256_matches": true, + "result": "PASS" +} diff --git a/evidence/fault-redelivery.json b/evidence/fault-redelivery.json new file mode 100644 index 00000000..7506b9b9 --- /dev/null +++ b/evidence/fault-redelivery.json @@ -0,0 +1,15 @@ +{ + "scenario": "crash after original+variant S3 writes, before PostgreSQL ready commit and consumer ack", + "attachment_id": "24b77169-5098-4e66-94bc-ffb2dc1495b1", + "worker_restart_count_before": 0, + "worker_restart_count_after": 1, + "status": "ready", + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "original_sha256_matches": true, + "recovery_seconds": 6.133, + "delivery": "Redis Streams at-least-once; deterministic object keys; DB ready commit precedes ack" +} diff --git a/evidence/pipeline-load-c1.json b/evidence/pipeline-load-c1.json new file mode 100644 index 00000000..11f16b49 --- /dev/null +++ b/evidence/pipeline-load-c1.json @@ -0,0 +1,29 @@ +{ + "track": "queue-overload", + "profile": "mixed", + "worker_concurrency": 1, + "upload_concurrency": 6, + "accepted": 12, + "failed": 0, + "elapsed_seconds": 9.152, + "throughput_ready_per_second": 1.311, + "ready_latency_ms": { + "p50": 3253.53, + "p95": 5788.39, + "p99": 5788.39 + }, + "estimated_queue_wait_ms": { + "p50": 2775.81, + "p95": 4955.47 + }, + "worker_rss_samples": [ + "58.66MiB", + "104.1MiB", + "82.58MiB", + "94.14MiB", + "81.29MiB" + ], + "worker_rss_peak_sample": "104.1MiB", + "total_upload_bytes": 50602808, + "limitation": "Closed-loop synthetic workload; queue wait is bounded by 100 ms status polling and can miss short processing states. Docker RSS sampling is 250 ms." +} diff --git a/evidence/pipeline-load-c2.json b/evidence/pipeline-load-c2.json new file mode 100644 index 00000000..9e85a366 --- /dev/null +++ b/evidence/pipeline-load-c2.json @@ -0,0 +1,28 @@ +{ + "track": "queue-overload", + "profile": "mixed", + "worker_concurrency": 2, + "upload_concurrency": 6, + "accepted": 12, + "failed": 0, + "elapsed_seconds": 8.098, + "throughput_ready_per_second": 1.482, + "ready_latency_ms": { + "p50": 3449.67, + "p95": 5566.78, + "p99": 5566.78 + }, + "estimated_queue_wait_ms": { + "p50": 2163.5, + "p95": 4059.81 + }, + "worker_rss_samples": [ + "62.24MiB", + "150.5MiB", + "102.1MiB", + "82.44MiB" + ], + "worker_rss_peak_sample": "150.5MiB", + "total_upload_bytes": 50602808, + "limitation": "Closed-loop synthetic workload; queue wait is bounded by 100 ms status polling and can miss short processing states. Docker RSS sampling is 250 ms." +} diff --git a/evidence/pipeline-load-c4.json b/evidence/pipeline-load-c4.json new file mode 100644 index 00000000..59e1ccfb --- /dev/null +++ b/evidence/pipeline-load-c4.json @@ -0,0 +1,28 @@ +{ + "track": "queue-overload", + "profile": "mixed", + "worker_concurrency": 4, + "upload_concurrency": 6, + "accepted": 12, + "failed": 0, + "elapsed_seconds": 7.948, + "throughput_ready_per_second": 1.51, + "ready_latency_ms": { + "p50": 2091.93, + "p95": 5172.02, + "p99": 5172.02 + }, + "estimated_queue_wait_ms": { + "p50": 1615.54, + "p95": 2800.88 + }, + "worker_rss_samples": [ + "105MiB", + "135.1MiB", + "186.1MiB", + "108.6MiB" + ], + "worker_rss_peak_sample": "186.1MiB", + "total_upload_bytes": 50602808, + "limitation": "Closed-loop synthetic workload; queue wait is bounded by 100 ms status polling and can miss short processing states. Docker RSS sampling is 250 ms." +} diff --git a/evidence/retry-replay.json b/evidence/retry-replay.json new file mode 100644 index 00000000..3780f1f0 --- /dev/null +++ b/evidence/retry-replay.json @@ -0,0 +1,11 @@ +{ + "attachment_id": "43551aab-5c0d-4b47-bed2-1e4ca50c4db3", + "induced_fault": "staging key temporarily pointed to missing object", + "terminal_error_code": "retry_exhausted", + "attempt_limit": 5, + "dlq_length_after_failure": 1, + "explicit_replay": "ready", + "ready_sha256_matches": true, + "replay_of_ready_job": "refused", + "elapsed_seconds": 34.515 +} diff --git a/evidence/worker-outage.json b/evidence/worker-outage.json new file mode 100644 index 00000000..7fcf4058 --- /dev/null +++ b/evidence/worker-outage.json @@ -0,0 +1,13 @@ +{ + "attachment_id": "b1e29a23-f636-4586-baa3-844dd018b255", + "upload_status": 202, + "status_without_worker_after_api_restart": "queued", + "final_status": "ready", + "original_sha256_matches": true, + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "recovery_seconds": 10.244 +} diff --git a/infra/bootstrap-rustfs.sh b/infra/bootstrap-rustfs.sh new file mode 100644 index 00000000..8005d747 --- /dev/null +++ b/infra/bootstrap-rustfs.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu +root_user=$(cat /run/secrets/rustfs_root_user) +root_password=$(cat /run/secrets/rustfs_root_password) +app_key=$(cat /run/secrets/s3_access_key_id) +app_secret=$(cat /run/secrets/s3_secret_access_key) +rc alias set local http://rustfs:9000 "$root_user" "$root_password" +if ! rc bucket list local/ | grep -q 'messenger-attachments'; then + rc bucket create local/messenger-attachments +fi +if ! rc admin policy info local messenger-objects >/dev/null 2>&1; then + rc admin policy create local messenger-objects /policy.json +fi +if ! rc admin user info local "$app_key" >/dev/null 2>&1; then + rc admin user add local "$app_key" "$app_secret" +fi +rc admin policy attach local messenger-objects --user "$app_key" diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..367b71d2 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,28 @@ +global + log stdout format raw local0 + maxconn 1024 +defaults + log global + mode http + timeout connect 500ms + timeout client 30s + timeout server 30s + timeout check 1s + retries 0 +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s +frontend public + bind *:8080 + default_backend replicas +backend replicas + balance roundrobin + option http-server-close + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 1 resolvers docker init-addr none + server-template api 8 api:8080 check diff --git a/infra/rustfs-policy.json b/infra/rustfs-policy.json new file mode 100644 index 00000000..21982c00 --- /dev/null +++ b/infra/rustfs-policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"]}]} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..7690b559 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, username text NOT NULL UNIQUE CHECK(username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK(length(btrim(display_name))>0 AND length(display_name)<=100), + password_hash text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats (id uuid PRIMARY KEY, title text NOT NULL CHECK(length(btrim(title))>0 AND length(title)<=100), created_at timestamptz NOT NULL DEFAULT clock_timestamp()); +CREATE TABLE members (chat_id uuid REFERENCES chats ON DELETE CASCADE, user_id uuid REFERENCES users, PRIMARY KEY(chat_id,user_id)); +CREATE INDEX members_user ON members(user_id,chat_id); +CREATE TABLE attachments ( + id uuid PRIMARY KEY, chat_id uuid NOT NULL REFERENCES chats, owner_id uuid NOT NULL REFERENCES users, + object_key text NOT NULL UNIQUE, filename text NOT NULL, content_type text NOT NULL, + size_bytes bigint NOT NULL, sha256 char(64) NOT NULL, status text NOT NULL CHECK(status IN ('queued','processing','ready','failed')), + error_code text, staging_key text NOT NULL, + variants jsonb NOT NULL DEFAULT '[]'::jsonb, + pipeline_version integer NOT NULL DEFAULT 1, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachments_chat ON attachments(chat_id,id); +CREATE TABLE messages (id uuid PRIMARY KEY, chat_id uuid NOT NULL REFERENCES chats, sender_id uuid NOT NULL REFERENCES users, text text NOT NULL CHECK(length(text)<=4000), created_at timestamptz NOT NULL DEFAULT clock_timestamp()); +CREATE TABLE message_attachments (message_id uuid REFERENCES messages ON DELETE CASCADE, attachment_id uuid REFERENCES attachments, position smallint NOT NULL, PRIMARY KEY(message_id,attachment_id), UNIQUE(message_id,position)); +CREATE INDEX messages_history ON messages(chat_id,created_at,id); +CREATE TABLE sessions (token_hash text PRIMARY KEY, user_id uuid NOT NULL REFERENCES users, expires_at timestamptz NOT NULL); +CREATE INDEX sessions_expiry ON sessions(expires_at); +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL, attempt integer NOT NULL DEFAULT 0, + state text NOT NULL DEFAULT 'pending' CHECK(state IN ('pending','published','processing','failed','done')), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), lease_until timestamptz, + last_error text, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX attachment_jobs_pending ON attachment_jobs(available_at) WHERE state IN ('pending','published','processing'); +CREATE TABLE attachment_job_audit ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(job_id), + event text NOT NULL, detail text, created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..3fa0f8d7 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,6 @@ +CREATE TABLE idempotency ( + user_id uuid NOT NULL REFERENCES users, chat_id uuid NOT NULL REFERENCES chats, + key varchar(128) NOT NULL, body text NOT NULL, response jsonb NOT NULL, + expires_at timestamptz NOT NULL, PRIMARY KEY(user_id,chat_id,key) +); +CREATE INDEX idempotency_expiry ON idempotency(expires_at); diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..9ecf9348 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,26 @@ +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +blinker==1.9.0 +cffi==2.1.1 +click==8.5.0 +Flask==3.1.2 +gunicorn==23.0.0 +itsdangerous==2.2.0 +Jinja2==3.1.6 +MarkupSafe==3.0.4 +packaging==26.3 +psycopg==3.2.10 +psycopg-binary==3.2.10 +psycopg-pool==3.3.3 +pycparser==3.11 +redis==6.4.0 +typing_extensions==4.16.0 +Werkzeug==3.1.9 +boto3==1.40.55 +botocore==1.40.55 +jmespath==1.0.1 +Pillow==11.3.0 +python-dateutil==2.9.0.post0 +s3transfer==0.14.0 +six==1.17.0 +urllib3==2.5.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..cfebea1b --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Create local-only secret files once; do not print their values.""" +from pathlib import Path +import secrets +root=Path(__file__).resolve().parents[1] +d=root/'secrets'; d.mkdir(mode=0o700,exist_ok=True); d.chmod(0o700) +def ensure(name,value): + p=d/name + if not p.exists(): p.write_text(value) + p.chmod(0o600) +ensure('db_password',secrets.token_urlsafe(32)) +ensure('rustfs_root_user','RUSTFSROOT') +ensure('rustfs_root_password',secrets.token_urlsafe(32)) +ensure('s3_access_key_id','messengerapi') +ensure('s3_secret_access_key',secrets.token_urlsafe(40)) +p=d/'database_url'; ensure('database_url','postgresql://messenger:'+ (d/'db_password').read_text().strip()+'@postgres:5432/messenger') +ensure('cache_url','redis://redis:6379/0') +ensure('broker_url','redis://broker:6379/0') +env=root/'.env' +if not env.exists(): env.write_text('API_REPLICAS=2\nAPP_PORT=8081\nSESSION_TTL_SECONDS=3600\nWORKER_CONCURRENCY=2\nMAX_QUEUED_JOBS=500\nJOB_MAX_ATTEMPTS=5\nJOB_TIMEOUT_SECONDS=60\n') +env.chmod(0o600) +print('Local secrets created or preserved; values hidden.') diff --git a/scripts/replay_failed.py b/scripts/replay_failed.py new file mode 100644 index 00000000..e6472b1c --- /dev/null +++ b/scripts/replay_failed.py @@ -0,0 +1,18 @@ +"""Explicitly replay one terminal failed attachment job, never a ready job.""" +import sys, uuid +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from app.migrate import setting +import psycopg + +if len(sys.argv)!=2: + raise SystemExit('usage: python scripts/replay_failed.py ATTACHMENT_UUID') +aid=str(uuid.UUID(sys.argv[1])) +with psycopg.connect(setting('DATABASE_URL')) as c: + row=c.execute("SELECT j.job_id,j.state,a.status FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.id=%s FOR UPDATE OF j,a",(aid,)).fetchone() + if not row or row[1]!='failed' or row[2]!='failed': + raise SystemExit('replay refused: attachment must be terminal failed') + c.execute("UPDATE attachment_jobs SET state='pending',attempt=0,available_at=clock_timestamp(),lease_until=NULL,last_error=NULL WHERE job_id=%s",(row[0],)) + c.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(aid,)) + c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'operator_replay','explicit replay command')",(row[0],)) +print('failed job queued for explicit replay') diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 00000000..cc114e24 --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1 @@ +'bash scripts/restart.sh' diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..b5c7b19b --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Recreate real containers, retain PostgreSQL volume. No migration race. +docker compose down +docker compose up -d --no-build --pull never --wait --wait-timeout 90 diff --git a/scripts/stop-one.bat b/scripts/stop-one.bat new file mode 100644 index 00000000..cccaf7c0 --- /dev/null +++ b/scripts/stop-one.bat @@ -0,0 +1 @@ +bash scripts/stop-one.sh diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..24954ab5 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?Set TARGET_INSTANCE_ID from a real API response}" +for id in $(docker compose ps -q api); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + case "$TARGET_INSTANCE_ID" in + "$host"-*) + printf 'Stopping API container %s (hostname %s)\n' "$id" "$host" + docker stop --time 5 "$id" + docker inspect -f '{{.Id}} {{.State.Status}}' "$id" + exit 0;; + esac +done +printf 'No running API matches target\n' >&2 +exit 1 diff --git a/tests/backpressure.py b/tests/backpressure.py new file mode 100644 index 00000000..be829f17 --- /dev/null +++ b/tests/backpressure.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Hold worker, prove queue admission cap, then recover accepted work.""" +import hashlib, json, os, subprocess, time +from pathlib import Path +from client import Client, arguments, check, run + +ROOT=Path(__file__).resolve().parents[1] +def compose(*args,capacity=None): + env=os.environ.copy() + if capacity is not None: env['MAX_QUEUED_JOBS']=str(capacity) + subprocess.run(['docker','compose',*args],cwd=ROOT,env=env,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + +def main(): + a=arguments('Queue admission backpressure under worker outage') + compose('up','-d','--force-recreate','--scale','api=2','api',capacity=1) + compose('stop','worker') + try: + c=Client(a.base_url,a.ca_file); alice,bob,_,chat=c.bootstrap(); raw=(ROOT/'tests/fixtures/sample.png').read_bytes(); path='/api/v1/chats/'+chat['id']+'/attachments?filename=backpressure.png' + first=c.call('POST',path,202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')['json'] + started=time.monotonic() + c.call('POST',path,503,actor=alice,raw=raw,headers={'Content-Type':'image/png'}) + compose('up','-d','--force-recreate','worker') + ready=c.wait_ready(bob,first,wait_seconds=90) + compose('up','-d','--force-recreate','--scale','api=2','api') + result={'queue_limit':1,'worker_stopped':True,'first_upload_status':202,'next_upload_status':503,'accepted_attachment_id':first['id'],'final_status':ready['status'],'original_sha256_matches':ready['sha256']==hashlib.sha256(raw).hexdigest(),'recovery_seconds':round(time.monotonic()-started,3),'note':'503 admission refusal leaves the already accepted job intact.'} + dest=ROOT/'evidence/backpressure.json'; dest.write_text(json.dumps(result,indent=2)+'\n'); print(json.dumps(result,indent=2)) + finally: + compose('up','-d','--force-recreate','worker') + compose('up','-d','--force-recreate','--scale','api=2','api') + +if __name__=='__main__': run(main) diff --git a/tests/broker_restart.py b/tests/broker_restart.py new file mode 100644 index 00000000..31957420 --- /dev/null +++ b/tests/broker_restart.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Publish a queued job into Redis Streams, restart broker with AOF volume, then consume.""" +import hashlib, json, os, subprocess, time +from pathlib import Path +from client import Client, arguments, check, run + +ROOT=Path(__file__).resolve().parents[1] +def compose(*args,pause=None,output=False): + env=os.environ.copy() + if pause is not None: env['WORKER_PAUSE_CONSUME']=str(pause) + cmd=['docker','compose',*args] + return subprocess.check_output(cmd,cwd=ROOT,env=env,text=True) if output else subprocess.run(cmd,cwd=ROOT,env=env,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + +def stream_len(): return int(compose('exec','-T','broker','redis-cli','XLEN','attachment-jobs',output=True).strip()) + +def main(): + a=arguments('Broker volume restart with one published, unconsumed job') + compose('up','-d','--force-recreate','worker',pause=1) + try: + c=Client(a.base_url,a.ca_file); alice,bob,_,chat=c.bootstrap(); before=stream_len(); raw=(ROOT/'tests/fixtures/sample.png').read_bytes() + path='/api/v1/chats/'+chat['id']+'/attachments?filename=broker-restart.png' + att=c.call('POST',path,202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')['json'] + deadline=time.monotonic()+20 + while stream_len()<=before and time.monotonic()before,'outbox relay did not publish the queued job') + status=c.call('GET','/api/v1/attachments/'+att['id'],actor=bob,schema='Attachment')['json']['status'] + check(status=='queued','paused worker unexpectedly consumed the job') + time.sleep(1.2); compose('stop','worker'); compose('restart','broker') + after=stream_len(); check(after>=published,'Redis AOF volume lost the published stream entry') + compose('up','-d','--force-recreate','worker',pause=0) + ready=c.wait_ready(bob,att,wait_seconds=90) + check(ready['status']=='ready' and ready['sha256']==hashlib.sha256(raw).hexdigest(),'job did not finish after broker restart') + result={'attachment_id':att['id'],'status_before_broker_restart':status,'stream_length_before_restart':published,'stream_length_after_restart':after,'final_status':ready['status'],'original_sha256_matches':True,'result':'PASS'} + (ROOT/'evidence/broker-restart.json').write_text(json.dumps(result,indent=2)+'\n'); print(json.dumps(result,indent=2)) + finally: compose('up','-d','--force-recreate','worker',pause=0) + +if __name__=='__main__': run(main) diff --git a/tests/fault_redelivery.py b/tests/fault_redelivery.py new file mode 100644 index 00000000..59eb9c69 --- /dev/null +++ b/tests/fault_redelivery.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +"""Requires worker recreated with FAULT_AFTER_S3_WRITE_ONCE=1 and timeout=5s.""" +import json, subprocess, time +from pathlib import Path +from client import Client, arguments, check, run + +def main(): + a=arguments('Crash after deterministic S3 side effect, then verify redelivery') + c=Client(a.base_url,a.ca_file); alice,bob,_,chat=c.bootstrap() + before=int(subprocess.check_output(['docker','inspect','-f','{{.RestartCount}}','mtusi_lab5_solution-worker-1'],text=True).strip()) + att,raw=c.attachment(alice,chat); started=time.monotonic(); ready=c.wait_ready(bob,att,wait_seconds=90) + after=int(subprocess.check_output(['docker','inspect','-f','{{.RestartCount}}','mtusi_lab5_solution-worker-1'],text=True).strip()) + check(after>before,'fault injection did not restart the worker') + check(ready['status']=='ready' and len(ready['variants'])==3,'redelivery did not preserve one complete variant set') + result={'scenario':'crash after original+variant S3 writes, before PostgreSQL ready commit and consumer ack','attachment_id':att['id'],'worker_restart_count_before':before,'worker_restart_count_after':after,'status':'ready','variants':[v['name'] for v in ready['variants']],'original_sha256_matches':ready['sha256']==__import__('hashlib').sha256(raw).hexdigest(),'recovery_seconds':round(time.monotonic()-started,3),'delivery':'Redis Streams at-least-once; deterministic object keys; DB ready commit precedes ack'} + dest=Path('evidence/fault-redelivery.json'); dest.write_text(json.dumps(result,indent=2)+'\n'); print(json.dumps(result,indent=2)) + +if __name__=='__main__': run(main) diff --git a/tests/pipeline_load.py b/tests/pipeline_load.py new file mode 100644 index 00000000..329eef00 --- /dev/null +++ b/tests/pipeline_load.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Closed-loop image pipeline experiment; call once per WORKER_CONCURRENCY setting.""" +from concurrent.futures import ThreadPoolExecutor, as_completed +import hashlib, io, json, math, os, random, subprocess, threading, time +from pathlib import Path +from client import Client, arguments, check, run +from PIL import Image + +def main(): + def configure(p): + p.add_argument('--count',type=int,default=24) + p.add_argument('--profile',choices=['small','mixed','large'],default='mixed') + p.add_argument('--upload-concurrency',type=int,default=8) + p.add_argument('--worker-concurrency',type=int,required=True) + p.add_argument('--output',required=True) + a=arguments('Synthetic image pipeline throughput, queue wait, ready latency and worker RSS',configure) + check(1<=a.count<=500 and 1<=a.upload_concurrency<=64,'invalid experiment bounds') + c=Client(a.base_url,a.ca_file,timeout=30); alice,bob,_,chat=c.bootstrap() + profiles={'small':[(320,200)],'mixed':[(320,200),(1024,768),(2048,1536)],'large':[(2048,1536)]} + dimensions=profiles[a.profile] + random.seed(20261106) + def image_bytes(index): + w,h=dimensions[index%len(dimensions)]; rng=random.Random(index+700) + im=Image.frombytes('RGB',(w,h),rng.randbytes(w*h*3)) + buf=io.BytesIO(); im.save(buf,format='PNG',compress_level=1); return buf.getvalue(),(w,h) + barrier=threading.Barrier(a.upload_concurrency) + def upload(i): + raw,size=image_bytes(i); local=Client(a.base_url,a.ca_file,timeout=60); barrier.wait() + t=time.monotonic() + res=local.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=bench-%s.png'%i,202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment') + return {'id':res['json']['id'],'accepted':time.monotonic(),'upload_seconds':time.monotonic()-t,'size_bytes':len(raw),'source_size':size} + rss=[]; stop=threading.Event() + def sample_rss(): + while not stop.is_set(): + try: + out=subprocess.check_output(['docker','stats','--no-stream','--format','{{.MemUsage}}','mtusi_lab5_solution-worker-1'],stderr=subprocess.DEVNULL,text=True,timeout=3).strip() + used=out.split('/')[0].strip(); rss.append(used) + except Exception: pass + stop.wait(.25) + sampler=threading.Thread(target=sample_rss,daemon=True); sampler.start() + started=time.monotonic() + with ThreadPoolExecutor(max_workers=a.upload_concurrency) as pool: jobs=list(pool.map(upload,range(a.count))) + upload_done=time.monotonic(); pending={x['id']:x for x in jobs}; ready_ms=[]; queue_wait_ms=[]; deadline=time.monotonic()+180 + while pending and time.monotonic()=1,'terminal failure was not recorded in DLQ') + restore="UPDATE attachments SET staging_key='staging/%s/%s' WHERE id='%s'::uuid"%(chatid,aid,aid) + compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',restore) + compose('run','--rm','--no-deps','worker','python','scripts/replay_failed.py',aid) + ready=c.wait_ready(bob,att,wait_seconds=90) + check(ready['status']=='ready' and ready['sha256']==hashlib.sha256(raw).hexdigest(),'explicit replay did not complete original attachment') + # A successful logical job cannot be force-run through the failed-only operator command. + refused=subprocess.run(['docker','compose','run','--rm','--no-deps','worker','python','scripts/replay_failed.py',aid],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + check(refused.returncode!=0,'replay command accepted a ready job') + result={'attachment_id':aid,'induced_fault':'staging key temporarily pointed to missing object','terminal_error_code':'retry_exhausted','attempt_limit':5,'dlq_length_after_failure':dlq,'explicit_replay':'ready','ready_sha256_matches':True,'replay_of_ready_job':'refused','elapsed_seconds':round(time.monotonic()-started,3)} + (ROOT/'evidence/retry-replay.json').write_text(json.dumps(result,indent=2)+'\n'); print(json.dumps(result,indent=2)) + finally: compose('up','-d','--force-recreate','worker') + +if __name__=='__main__': run(main) diff --git a/tests/worker_outage.py b/tests/worker_outage.py new file mode 100644 index 00000000..e066dccf --- /dev/null +++ b/tests/worker_outage.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Worker outage: accept durable upload, restart API, then process same job.""" +import json, subprocess, time +from pathlib import Path +from client import Client, arguments, check, run + +def compose(*args): subprocess.run(['docker','compose',*args],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + +def main(): + a=arguments('Stop worker, accept upload, restart API, restore worker') + c=Client(a.base_url,a.ca_file); alice,bob,_,chat=c.bootstrap(); started=time.monotonic(); compose('stop','worker') + try: + att,raw=c.attachment(alice,chat) + check(att['status']=='queued','upload was not queued while worker stopped') + compose('restart','api') + deadline=time.monotonic()+45 + while time.monotonic()