Implement lab5 async image processing with RabbitMQ and outbox

This commit is contained in:
ferraronp
2026-11-16 16:08:39 +03:00
parent 276dbdad8a
commit b61a1cae9e
43 changed files with 3020 additions and 45 deletions
+18 -20
View File
@@ -1,25 +1,23 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Copy to .env and REPLACE all sample secrets with random unique values.
# For local synthetic data only. Never commit .env.
POSTGRES_DB=messenger
POSTGRES_USER=lab
POSTGRES_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_PASSWORD
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
DB_POOL_MAX=8
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
S3_ADMIN_ACCESS_KEY=adminlab4
S3_ADMIN_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_ADMIN_SECRET
S3_ACCESS_KEY=messengerapi
S3_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_API_SECRET
# RabbitMQ persistent queue (different from session Redis).
BROKER_USER=labworker
BROKER_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_BROKER_PASSWORD
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
JOB_BACKLOG_LIMIT=1000
WORKER_PREFETCH=1
# Only for deliberate crash experiment: once. Keep off for normal operation.
LAB5_FAULT_AFTER_S3=off
+10
View File
@@ -0,0 +1,10 @@
FROM python:3.12.10-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
WORKDIR /srv
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 999 app && useradd -m --uid 999 --gid app app
COPY app ./app
COPY db ./db
USER app
EXPOSE 8080
CMD ["python", "-m", "app"]
+12 -3
View File
@@ -1,10 +1,14 @@
ifeq ($(OS),Windows_NT)
PYTHON ?= python
else
PYTHON ?= python3
endif
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover test-own
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -22,9 +26,14 @@ load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) -c "import pathlib,sys; sys.exit(0 if pathlib.Path(sys.argv[1]).is_file() else 2)" "$(ACTION_SCRIPT)"
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) -c "import pathlib,sys; sys.exit(0 if pathlib.Path(sys.argv[1]).is_file() else 2)" "$(ACTION_SCRIPT)"
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-own:
$(PYTHON) tests/own_unit.py
$(PYTHON) tests/idempotency_parallel.py
$(PYTHON) tests/attachments_negative.py
+106 -22
View File
@@ -1,37 +1,121 @@
# Отчёт — лабораторная 5
# Отчёт — лабораторная 5: очередь и асинхронная обработка изображений
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
**Автор / группа:** Авзалов Артур Марсович / БПИ2301
**Целевая оценка:** 4
**Заявленная оценка:** 4; основной интеграционный набор выполнен на учебном Windows/Docker Desktop стенде (см. результаты и ограничения)
**Использование ИИ:** ChatGPT помогал проектировать и реализовать API, outbox, worker, инфраструктуру, тесты и этот отчёт.
## Что перенесено и изменено
Сохранён каркас Lab4: две stateless FastAPI-реплики за HAProxy, PostgreSQL, Redis-сессии, S3-совместимый SeaweedFS, авторизация по Bearer, идемпотентность сообщений, пагинация, проверки MIME/ACL и миграции. Публичные `contracts/` и `tests/smoke.py` взяты из **оригинального skeleton Lab5**, не из Lab4.
Изменение Lab5: загрузка `POST /api/v1/chats/{id}/attachments?filename=...` отвечает `202` со снимком `status=queued`. Обещание на момент ответа: файл целиком проверен и находится в устойчивом S3 staging, а метаданные вложения, job и запись transactional outbox находятся в **одном коммите PostgreSQL**. Публикация RabbitMQ и обработка — после HTTP-ответа, в отдельных процессах.
## Архитектура
```text
HAProxy
/ \
api1 api2 GET status / content / messages
\ /
PostgreSQL (attachments, jobs, transactional outbox)
| ^
outbox dispatcher worker (RabbitMQ consumer)
| ^
RabbitMQ (durable queue, persistent messages, manual ack)
|
worker child (Pillow transforms, time budget)
|
SeaweedFS (private S3 bucket)
staging/ -> attachments/ + recovery/ + variants/thumbnail,crop,optimized
```
**Единственный источник истины по статусу** — PostgreSQL. В сообщении RabbitMQ только `job_id`, `attachment_id`, `pipeline_version`, `attempt`, дата и тип; байтов, клиентского filename, токенов и S3 credentials там нет. S3 staging не исчезает при перезапуске API/worker/broker. RabbitMQ имеет named volume и durable queue. Pubisher confirm фиксирует приём брокером; consumer ack вызывается после устойчивой фиксации результата в БД либо очередной попытки/failed. Доставка **at least once**, но не exactly-once.
### Разрыв между COMMIT и publish
`_admit_upload()` загружает исходник в `staging/{id}/original`, затем в одной транзакции создаёт `attachments(status=queued)`, `processing_jobs` и `job_outbox`. `dispatcher` читает ещё не опубликованные outbox-записи, публикует persistent RabbitMQ message с publisher confirms и только после подтверждения выставляет `published_at`. Если dispatcher падает до publish или после publish до обновления outbox — он повторяет отправку. Повторы безопасны благодаря `processing_jobs` и `attachment_variants` с уникальностью `(attachment_id,pipeline_version,name)`.
### Worker, таймауты и retry
Worker имеет ограниченный `prefetch` (по умолчанию 1), одну обработку за раз на процесс, и отдельный subprocess для каждой картинки с таймаутом `JOB_TIMEOUT_SECONDS=60`. Изображения ограничены 10 MiB и 40 млн пикселей; Docker memory limit worker — 768 MiB, CPU 1.0. Для большого формата subprocess может завершиться неуспешно: фиксируется ограниченный retry либо failed, но API не зависает синхронно.
Worker захватывает job через строковую блокировку в PostgreSQL и создаёт lease на 105 с при таймауте 60 с. При падении lease истекает, dispatcher повторно ставит job в очередь. При временной ошибке: retry с экспоненциальным backoff + jitter, по умолчанию до пяти попыток. Повреждённый файл — постоянная ошибка `invalid_image`. По исчерпании — `retry_exhausted`. Для failed доступен **только операторский** `python -m app.jobs replay --id ...`, с аудитом в `job_replay_audit`. Job с `status=ready` не перезапускается по старому сообщению.
### Варианты изображений
- `original` — неизменённые исходные bytes, первичный ключ `attachments/...`, recovery-копия `recovery/...`.
- `thumbnail` — JPEG, вписанный в 256×256 без увеличения; например, fixture 320×200 → 256×160; quality 85.
- `crop` — JPEG 128×128, центрированный квадрат с допущенным увеличением; quality 85.
- `optimized` — WebP, вписанный в 1280×1280 без увеличения; quality 78, method 4.
EXIF orientation применяется через `ImageOps.exif_transpose`; прозрачность накладывается на белый фон; метаданные не передаются при кодировании новых изображений. Контент queued/processing/failed отвечает `409`, готовые варианты — только через авторизованный API и проверку участника чата.
### Безопасная очистка staging
После `ready` worker удаляет `staging/` только после успешного commit. Если он умер раньше очистки, dispatcher повторяет безопасную уборку. Не привязанные к БД staging-объекты и staging failed-работ удерживаются не менее 86400 с; по истечении могут быть очищены, после чего replay требует нового staging и будет отклонён. S3 и PostgreSQL **не имеют общей транзакции**; очистка построена консервативно.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
1. Создать `.env` из `.env.example` и заменить учебные пароли БД, RabbitMQ и S3 на разные случайные значения. `.env` никогда не включать в Git.
2. Остановить Lab4, если он занимает `127.0.0.1:8080`, без удаления volumes.
3. В PowerShell из корня Lab5:
## Архитектура и решения
```powershell
$env:PYTHONUTF8 = '1'
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait
docker compose --env-file .env ps
python scripts/check_contract.py
python tests/smoke.py
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Все зависимости создаются только этим Compose: PostgreSQL 16.10, Redis 7.4.5, RabbitMQ 4.1.8, SeaweedFS 4.53, HAProxy 3.1.7, Python 3.12.10. Для Windows также нужны Docker Desktop, PowerShell, Python 3.9+.
**Команды тестов и их результаты приведены ниже в разделе доказательств критериев.**
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
**Фактически проверено пользователем 14–15 ноября 2026 года на Windows + Docker Desktop.** Команды и результаты ниже взяты из вывода реального запуска; отсутствующие результаты не обозначены PASS. Исправлены две обнаруженные проблемы: лишний `%s` в SQL `publish_result()` и устаревшее ожидание `404` thumbnail/`201` upload в собственных тестах из Lab4.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий | Команда / опыт | Результат |
| --- | --- | --- |
| Инфраструктура | `docker compose up -d --build --wait`; `docker compose ps` | **PASS**: API1/2, RabbitMQ, PostgreSQL, Redis, SeaweedFS, LB, worker, dispatcher поднялись; health основных сервисов зелёный |
| Структура/контракт | `python scripts/check_contract.py` | **PASS**, offline-only |
| Модели/limit/cursor | `python tests/own_unit.py` | **PASS** |
| API и вариантные файлы | `python tests/smoke.py` после SQL фикса | **PASS**: 202→ready |
| Негативные MIME, размеры, ACL, варианты | `python tests/attachments_negative.py` после актуализации ожиданий под Lab5 | **PASS** |
| Message idempotency | `python tests/idempotency_parallel.py` | **PASS**: 2 API, 201 replay и 409 для иного тела |
| Pillow, isolated | `python -m app.pipeline_selftest` внутри worker-контейнера | **PASS**: PNG→JPEG thumb 256×160, crop 128×128, WebP |
| Остановленный worker / рестарт API и RabbitMQ / ready | `python tests/pipeline_resilience.py --scenario paused` | **PASS**: данные, оригинал, 3 варианта доступны после восстановления |
| PostgreSQL COMMIT → падение dispatcher до publish | `... --scenario outbox-crash` | **PASS**: exit 78, восстановление из outbox |
| Worker после S3 effects, до DB commit/ack | `... --scenario worker-crash` | **PASS**: exit 77, восстановление после lease, 3 варианта |
| 10 одновременных загрузок → все ready | `python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json` | **PASS**: 10/10 accepted, 10/10 ready, 0 failed; повторный прогон: приём 2.391 s / 4.182 RPS, p50 2218.5 ms, max 2313 ms; все ready через 29.781 s, по 111817 bytes |
| Долговечность *уже опубликованного* RabbitMQ сообщения | `python tests/pipeline_extended.py --scenario broker-persistence` | **PASS**: подтверждённая публикация, ready=1 до и после рестарта брокера; dispatcher остановлен |
| Permanent failed / manual replay / аудит / запрет replay ready | `python tests/pipeline_extended.py --scenario failed-replay` | **PASS** (15.11): `invalid_image` без retry/вариантов → исправление метаданных → operator replay → `ready`; audit=1, replay `ready` отклонён |
| Очистка S3 staging | `python tests/pipeline_extended.py --scenario staging-cleanup` | **PASS**: `ready`, `staging_key=NULL`, staging-объект отсутствует |
| Использование памяти контейнеров под нагрузкой | 45-секундная серия `docker stats`, 15 отсчётов на контейнер | **ЗАМЕРЕНО**: наблюдаемый максимум памяти API1 163.10 MiB, API2 157.10 MiB, worker 128.90 MiB; это не peak RSS |
| Persistence Lab2/3 на Lab5 | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | **PASS** (15.11): ранее подтверждённые данные, байты вложения и действующая сессия сохранились после выполнения restart hook |
| Failover одной FastAPI-реплики | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | **PASS** (15.11): 2 экземпляра до отказа, 6 успешных записей, 1 неудачная попытка; первое успешное восстановление за 6.172 s; данные и сессия сохранены |
| Восстановление 2 API и повторный smoke | `docker compose ... up -d --no-deps --wait api1 api2`; `python tests/smoke.py`; `docker compose ps` | **PASS** (15.11): обе API healthy, smoke PASS, инфраструктура запущена |
## Эксперименты и отказы
`4.182 RPS` — скорость **приёма HTTP 202** во втором прогоне, а не производительность worker; `29.781 s` — время завершения всех 10 задач. Дополнительные прогоны 14–15.11: `broker-persistence`, `staging-cleanup`, `failed-replay`, `persistence` и `failover` — PASS после исправления тестового клиента. Первый прогон `failed-replay` падал из-за ошибки теста, затем перепроверен успешно. Новые проверки `pipeline_extended.py` работают только при `LAB5_ALLOW_FAILURE_TEST=yes`, не стирают volumes и не запускаются автоматически; их нельзя объявлять прошедшими без реального вывода.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### Границы проверки restart/failover
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
`tests/resilience.py` **подавляет stdout/stderr hook** и выводит `Hook scope needs operator evidence`. Прочитанные `scripts/restart.cmd`/`scripts/stop-one.cmd` действительно содержат команды остановки и запуска соответствующих контейнеров; однако сам вывод `PASS` подтверждает **наблюдаемый эффект теста**, а не сам по себе независимую запись событий Docker. Для защиты удобно сохранить `docker compose ps`, показать сами `.cmd` и при необходимости провести повторный прогон с дополнительными логами Docker. Failover не бесшовный: **1 неудачная попытка записи**, восстановление в тесте около **6.172 s**. Сценарий с одним Docker-хостом не доказывает устойчивость к потере хоста.
## Ограничения и следующие шаги
Полный фактический результат последнего прогона: [`evidence/verified_results_2026-11-15.md`](evidence/verified_results_2026-11-15.md). Приведённые результаты — пользовательские логи на Windows; архив v6 подготовлен без повторного запуска Docker в среде сборки.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
## Ограничения
- **Одиночный broker, S3, PostgreSQL, Redis и HAProxy** являются точками отказа. RabbitMQ volume выдерживает restart при сохранении диска, но не потерю хоста.
- Время обработки зависит от Pillow и S3. При нагрузке очередь растёт; учебный admission cap проверяется простым счётчиком, а не атомарным глобальным семафором.
- Доставка RabbitMQ at least once, поэтому возможны дубликаты публикаций и обработок, но БД не фиксирует второй набор вариантов.
- Worker lease обеспечивает восстановление после смерти; в этом кандидате нет периодического продления lease во время обработки. Время lease сделано больше budget subprocess; на сильно перегруженном узле возможны повторные попытки.
- Пакет создан на основе **предыдущего кандидата Lab4**, а не автоматически синхронизирован с последним Git-коммитом пользователя. Перед MR сравнить отличия с собственной Lab4.
- Набор тестов на текущем учебном стенде пройден, но не заменяет независимую проверку преподавателем и стресс-/долгосрочное тестирование. Однозначных доказательств устойчивости к отказу физического хоста или exact-once нет.
## Полезные различия для защиты
**Publisher confirm** — брокер подтвердил приём persistent сообщения. **Consumer ack** — worker завершил обработку и клиент брокера подтвердил удаление сообщения из очереди. **202** — уже сохранены staging bytes + metadata/job/outbox, но не варианты. **Idempotency key эффекта** — `(attachment_id,pipeline_version,variant)`. **Lease** — ограниченный срок права обработчика на работу; истёкший lease подбирает dispatcher. **Exactly-once** не обещаем: одна логическая операция может доставляться и исполняться несколько раз, итоговая запись идемпотентна.
+1
View File
@@ -0,0 +1 @@
"""Lab 3 messenger API."""
+4
View File
@@ -0,0 +1,4 @@
import os
import uvicorn
if __name__ == "__main__":
uvicorn.run("app.main:app", host="0.0.0.0", port=int(os.getenv("APP_PORT", "8080")), workers=1, timeout_graceful_shutdown=8)
+193
View File
@@ -0,0 +1,193 @@
"""Messenger HTTP handlers; state lives only in PostgreSQL/Redis."""
import hashlib
import json
import re
import uuid
import psycopg
from argon2 import PasswordHasher
from fastapi import APIRouter, Request
from .auth import current_user, hasher, login, logout, public_user
from .core import APIError, invalid, iso, make_cursor, parse_limit, parse_uuid, read_cursor
from .models import CreateChat, CreateMessage, CreateUser
from .storage import db
router = APIRouter()
KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII)
def actor(req: Request) -> dict:
return getattr(req.state, "actor", None) or current_user(req)
def chat_detail(conn, chat: dict) -> dict:
members = conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position", (chat["id"],)).fetchall()
return {"id": str(chat["id"]), "title": bytes(chat["title"]).decode("utf-8"), "created_at": iso(chat["created_at"]), "member_ids": [str(x["user_id"]) for x in members]}
def message_detail(conn, row: dict) -> dict:
linked = conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (row["id"],)).fetchall()
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": bytes(row["text"]).decode("utf-8"), "created_at": iso(row["created_at"]), "attachment_ids": [str(x["attachment_id"]) for x in linked]}
def require_chat(conn, chat_id: str, actor_id: str) -> dict:
row = conn.execute("SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id WHERE c.id=%s AND cm.user_id=%s", (chat_id, actor_id)).fetchone()
if row is None:
raise APIError(404, "not_found", "Object not found")
return row
@router.get("/health/live")
def live():
return {"status": "ok"}
@router.get("/health/ready")
def ready():
from .storage import ready as storage_ready
if not storage_ready():
raise APIError(503, "unavailable", "Service not ready")
return {"status": "ok"}
@router.post("/api/v1/users", status_code=201)
def register(body: CreateUser):
hashed = hasher.hash(body.password)
ident = str(uuid.uuid4())
try:
with db() as conn:
row = conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (ident, body.username, body.display_name.encode("utf-8"), hashed)).fetchone()
return public_user(row)
except psycopg.errors.UniqueViolation:
raise APIError(409, "conflict", "Username already exists") from None
@router.post("/api/v1/auth/sessions", status_code=201)
def sessions(req: Request):
return login(req)
@router.delete("/api/v1/auth/sessions/current", status_code=204)
def sessions_delete(req: Request):
logout(req)
return None
@router.get("/api/v1/users/me")
def users_me(req: Request):
return actor(req)
@router.post("/api/v1/chats", status_code=201)
def chats_create(req: Request, body: CreateChat):
ident = actor(req)["id"]
if ident not in body.member_ids:
raise invalid("Current user must belong to the chat")
chat_id = str(uuid.uuid4())
try:
with db() as conn:
with conn.transaction():
# Validate all members; FK constraints are the final authority for races.
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s::uuid[])", (body.member_ids,)).fetchall()
if len(found) != len(body.member_ids):
raise invalid("Unknown member")
row = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (chat_id, body.title.encode("utf-8"))).fetchone()
for pos, member in enumerate(body.member_ids):
conn.execute("INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)", (chat_id, member, pos))
result = chat_detail(conn, row)
return result
except psycopg.errors.ForeignKeyViolation:
raise invalid("Unknown member") from None
@router.get("/api/v1/chats")
def chats_list(req: Request, limit: str | None = None, cursor: str | None = None):
who = actor(req)["id"]
n = parse_limit(limit)
mark = read_cursor(cursor, "chats", who)
with db() as conn:
if mark:
existing = conn.execute("SELECT 1 FROM chats c JOIN chat_members cm ON c.id=cm.chat_id WHERE c.id=%s AND c.created_at=%s AND cm.user_id=%s", (mark[1], mark[0], who)).fetchone()
if not existing:
raise invalid("Invalid cursor")
rows = conn.execute("""
SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id
WHERE cm.user_id=%s AND (%s::timestamptz IS NULL OR (c.created_at,c.id)>(%s::timestamptz,%s::uuid))
ORDER BY c.created_at,c.id LIMIT %s
""", (who, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
items = [chat_detail(conn, row) for row in rows[:n]]
more = len(rows) > n
last = rows[n-1] if more else None
return {"items": items, "next_cursor": make_cursor("chats", who, last["created_at"], str(last["id"])) if last else None}
@router.get("/api/v1/chats/{chat_id}")
def chats_get(req: Request, chat_id: str):
who = actor(req)["id"]
ident = parse_uuid(chat_id)
with db() as conn:
row = require_chat(conn, ident, who)
return chat_detail(conn, row)
@router.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def messages_create(req: Request, chat_id: str, body: CreateMessage):
who = actor(req)["id"]
ident = parse_uuid(chat_id)
key = req.headers.get("Idempotency-Key")
if not key or not KEY_RE.fullmatch(key):
raise invalid("Invalid Idempotency-Key")
# Lab3 has only text; canonical JSON avoids unstable formatting/field ordering.
fingerprint = hashlib.sha256(json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
lock_scope = f"{who}:POST:/api/v1/chats/{ident}/messages:{key}"
with db() as conn:
with conn.transaction():
require_chat(conn, ident, who)
# Transaction-scoped advisory lock is GLOBAL across API replicas.
conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (lock_scope,))
previous = conn.execute("""
SELECT idem.body_sha256, m.* FROM message_idempotency idem
JOIN messages m ON m.id=idem.message_id
WHERE idem.actor_id=%s AND idem.chat_id=%s AND idem.method='POST' AND idem.idem_key=%s
""", (who, ident, key)).fetchone()
if previous:
if previous["body_sha256"] != fingerprint:
raise APIError(409, "conflict", "Idempotency key used with another body")
return message_detail(conn, previous)
# 404 for foreign/nonexistent IDs; check BEFORE any message insert.
if body.attachment_ids:
owned = conn.execute("SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])", (ident, body.attachment_ids)).fetchall()
if len(owned) != len(body.attachment_ids):
raise APIError(404, "not_found", "Attachment not found")
if any(item["status"] == "failed" for item in owned):
raise APIError(409, "conflict", "Failed attachment cannot be linked")
message_id = str(uuid.uuid4())
row = conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (message_id, ident, who, body.text.encode("utf-8"))).fetchone()
for pos, attachment_id in enumerate(body.attachment_ids):
conn.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, attachment_id, pos))
conn.execute("INSERT INTO message_idempotency(actor_id,chat_id,method,idem_key,body_sha256,message_id) VALUES (%s,%s,'POST',%s,%s,%s)", (who, ident, key, fingerprint, message_id))
return message_detail(conn, row)
@router.get("/api/v1/chats/{chat_id}/messages")
def messages_list(req: Request, chat_id: str, limit: str | None = None, cursor: str | None = None):
who = actor(req)["id"]
ident = parse_uuid(chat_id)
n = parse_limit(limit)
mark = read_cursor(cursor, "messages", ident)
with db() as conn:
require_chat(conn, ident, who)
if mark:
existing = conn.execute("SELECT 1 FROM messages WHERE id=%s AND chat_id=%s AND created_at=%s", (mark[1], ident, mark[0])).fetchone()
if not existing:
raise invalid("Invalid cursor")
rows = conn.execute("""
SELECT * FROM messages WHERE chat_id=%s
AND (%s::timestamptz IS NULL OR (created_at,id)>(%s::timestamptz,%s::uuid))
ORDER BY created_at,id LIMIT %s
""", (ident, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
with db() as conn:
items = [message_detail(conn, row) for row in rows[:n]]
last = rows[n-1] if len(rows)>n else None
return {"items": items, "next_cursor": make_cursor("messages", ident, last["created_at"], str(last["id"])) if last else None}
+238
View File
@@ -0,0 +1,238 @@
"""Lab5 attachments: synchronous durable admission, asynchronous image pipeline.
202 means verified bytes were staged to private S3 and metadata + a job + an
outbox event were committed in ONE PostgreSQL transaction. No broker availability
is required for admission; the dispatcher replays unsent intents.
"""
import hashlib
import os
import tempfile
import threading
import uuid
from contextlib import closing
import boto3
from boto3.s3.transfer import TransferConfig
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError
from fastapi import APIRouter, Query, Request
from fastapi.responses import StreamingResponse
from PIL import Image, ImageFile, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from .auth import current_user
from .core import APIError, invalid, iso, parse_uuid
from .storage import db
router = APIRouter()
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
SPOOL_MEMORY_BYTES = 1024 * 1024
S3_BUFFER_BYTES = 256 * 1024
DECODE_LIMIT = threading.BoundedSemaphore(1)
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
ImageFile.LOAD_TRUNCATED_IMAGES = False
def s3_client():
return boto3.client(
"s3", endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
region_name="us-east-1", use_ssl=False,
config=Config(s3={"addressing_style": "path"}, connect_timeout=3,
read_timeout=10, retries={"max_attempts": 1}),
)
def s3_error(exc):
return APIError(503, "unavailable", "Object storage unavailable")
def format_attachment(row, variants=()):
return {
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]), "filename": row["filename"],
"content_type": row["content_type"], "size_bytes": row["size_bytes"],
"sha256": row["sha256"], "status": row["status"],
"variants": [{k: item[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
for item in variants] if row["status"] == "ready" else [],
"error_code": row["error_code"] if row["status"] == "failed" else None,
"created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]),
}
def require_member(conn, chat_id, user_id):
if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone():
raise APIError(404, "not_found", "Object not found")
def attachment_row(conn, attachment_id, user_id):
row = conn.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON cm.chat_id=a.chat_id
WHERE a.id=%s AND cm.user_id=%s
""", (attachment_id, user_id)).fetchone()
if not row:
raise APIError(404, "not_found", "Object not found")
return row
def validate_filename(filename: str):
if not (1 <= len(filename) <= 255) or any(c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename):
raise invalid("Invalid filename")
def validate_image(fileobj, mime: str):
"""Validate MIME, magic, dimensions and complete decode; worker does the resize."""
try:
fileobj.seek(0)
with Image.open(fileobj) as im:
if im.format not in ("PNG", "JPEG"):
raise APIError(415, "unsupported_media_type", "Unsupported image encoding")
actual = "image/png" if im.format == "PNG" else "image/jpeg"
if actual != mime:
raise APIError(415, "unsupported_media_type", "Content-Type does not match image")
if im.width * im.height > MAX_PIXELS:
raise APIError(413, "payload_too_large", "Image dimensions too large")
im.verify()
fileobj.seek(0)
with DECODE_LIMIT:
with Image.open(fileobj) as im:
im.load()
except APIError:
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise APIError(415, "unsupported_media_type", "Invalid image") from None
finally:
fileobj.seek(0)
def _admit_upload(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest):
from .jobs import job_payload
stage_key = f"staging/{attachment_id}/original"
key = f"attachments/{attachment_id}/original"
backup = f"recovery/{attachment_id}/original"
job_id = str(uuid.uuid4())
outbox_id = str(uuid.uuid4())
client = s3_client()
try:
spool.seek(0)
client.upload_fileobj(spool, BUCKET, stage_key,
ExtraArgs={"ContentType": mime},
Config=TransferConfig(use_threads=False, multipart_threshold=16*1024*1024,
io_chunksize=S3_BUFFER_BYTES))
except (BotoCoreError, ClientError, EndpointConnectionError, OSError) as exc:
raise s3_error(exc) from exc
# Atomic DB metadata + intent. If DB commit is ambiguous, do not delete the
# staged object: an accepted task may already exist; operator cleanup handles it.
with db() as conn:
with conn.transaction():
require_member(conn, chat_id, user_id)
# Small educational admission bound; pre-existing queued tasks remain safe.
pending = conn.execute("SELECT count(*) AS c FROM processing_jobs WHERE status IN ('queued','processing')").fetchone()["c"]
if pending >= int(os.getenv("JOB_BACKLOG_LIMIT", "1000")):
raise APIError(429, "rate_limited", "Processing queue is full")
row = conn.execute("""
INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename,
content_type,size_bytes,sha256,status,staging_key,pipeline_version)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,1) RETURNING *
""", (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest, stage_key)).fetchone()
conn.execute("""
INSERT INTO processing_jobs(id,attachment_id,pipeline_version,status,attempt)
VALUES (%s,%s,1,'queued',1)
""", (job_id, attachment_id))
payload = job_payload(job_id, attachment_id, 1, 1)
conn.execute("""
INSERT INTO job_outbox(id,job_id,payload) VALUES (%s,%s,%s::jsonb)
""", (outbox_id, job_id, __import__('json').dumps(payload)))
return format_attachment(row) # snapshot = queued, even if worker is very fast
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)):
from .api import actor, require_chat
who = actor(req)["id"]
chat_id = parse_uuid(chat_id)
validate_filename(filename)
mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if mime not in ("image/png", "image/jpeg"):
raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
with db() as conn:
require_chat(conn, chat_id, who)
announced = req.headers.get("content-length")
if announced:
try:
if int(announced) > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
if int(announced) < 0:
raise invalid("Invalid Content-Length")
except ValueError:
raise invalid("Invalid Content-Length") from None
size = 0
digest = hashlib.sha256()
with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool:
async for chunk in req.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
digest.update(chunk)
spool.write(chunk)
if not size:
raise APIError(415, "unsupported_media_type", "Empty image")
await run_in_threadpool(validate_image, spool, mime)
return await run_in_threadpool(_admit_upload, spool, attachment_id=str(uuid.uuid4()),
chat_id=chat_id, user_id=who, filename=filename,
mime=mime, size=size, digest=digest.hexdigest())
@router.get("/api/v1/attachments/{attachment_id}")
def get_attachment(req: Request, attachment_id: str):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
variants = conn.execute("""
SELECT name,content_type,size_bytes,width,height,sha256
FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s
ORDER BY CASE name WHEN 'thumbnail' THEN 0 WHEN 'crop' THEN 1 ELSE 2 END
""", (row["id"], row["pipeline_version"])).fetchall() if row["status"] == "ready" else []
return format_attachment(row, variants)
@router.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(req: Request, attachment_id: str, variant: str = "original"):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
if row["status"] != "ready":
raise APIError(409, "conflict", "Attachment is not ready")
if variant == "original":
key, mime, size = row["object_key"], row["content_type"], row["size_bytes"]
else:
v = conn.execute("""
SELECT * FROM attachment_variants
WHERE attachment_id=%s AND pipeline_version=%s AND name=%s
""", (row["id"], row["pipeline_version"], variant)).fetchone()
if not v:
raise APIError(404, "not_found", "Object not found")
key, mime, size = v["object_key"], v["content_type"], v["size_bytes"]
client = s3_client()
try:
response = client.get_object(Bucket=BUCKET, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"):
raise APIError(503, "unavailable", "Stored object missing") from exc
raise s3_error(exc) from exc
except BotoCoreError as exc:
raise s3_error(exc) from exc
def iterator():
with closing(response["Body"]) as body:
while chunk := body.read(S3_BUFFER_BYTES):
yield chunk
return StreamingResponse(iterator(), media_type=mime, headers={"Content-Length": str(size)})
+104
View File
@@ -0,0 +1,104 @@
"""HTTP Basic only on login; bearer sessions shared by all API replicas."""
import base64
import binascii
import hashlib
import re
import secrets
import uuid
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError, VerificationError
from fastapi import Request
from redis.exceptions import RedisError
from .core import APIError, SESSION_TTL, iso
from .storage import cache, db
hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
PASSWORD_RE = re.compile(r"^[a-z0-9_]{3,32}$")
def public_user(row: dict):
return {"id": str(row["id"]), "username": row["username"], "display_name": bytes(row["display_name"]).decode("utf-8"), "created_at": iso(row["created_at"])}
def unauthorized() -> APIError:
return APIError(401, "unauthorized", "Unauthorized")
def challenge(req: Request) -> str:
return 'Basic realm="messenger", charset="UTF-8"' if req.url.path == "/api/v1/auth/sessions" else 'Bearer realm="messenger"'
def bearer_token(req: Request) -> str:
value = req.headers.get("Authorization", "")
if not value.startswith("Bearer ") or len(value) <= 7 or " " in value[7:] or len(value) > 512:
raise unauthorized()
try:
value[7:].encode("ascii")
except UnicodeEncodeError:
raise unauthorized() from None
return value[7:]
def session_key(token: str) -> str:
return "session:" + hashlib.sha256(token.encode("ascii")).hexdigest()
def current_user(req: Request) -> dict:
token = bearer_token(req)
try:
raw = cache().get(session_key(token))
except (RedisError, UnicodeError) as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
if not raw:
raise unauthorized()
try:
ident = str(uuid.UUID(raw))
except (ValueError, TypeError):
try:
cache().delete(session_key(token))
except RedisError as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
raise unauthorized() from None
with db() as conn:
row = conn.execute("SELECT * FROM users WHERE id = %s", (ident,)).fetchone()
if row is None:
raise unauthorized()
return public_user(row)
def login(req: Request):
auth = req.headers.get("Authorization", "")
if not auth.startswith("Basic "):
raise unauthorized()
try:
decoded = base64.b64decode(auth[6:].encode("ascii"), validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
if not PASSWORD_RE.fullmatch(username):
raise ValueError()
except (ValueError, UnicodeError, binascii.Error):
raise unauthorized() from None
with db() as conn:
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
if not row:
raise unauthorized()
try:
hasher.verify(row["password_hash"], password)
except (VerifyMismatchError, VerificationError):
raise unauthorized() from None
token = secrets.token_urlsafe(32)
try:
cache().set(session_key(token), str(row["id"]), ex=SESSION_TTL)
except RedisError as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
return {"token": token, "token_type": "Bearer", "expires_at": iso(datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)), "user": public_user(row)}
def logout(req: Request):
current_user(req)
try:
cache().delete(session_key(bearer_token(req)))
except RedisError as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
+79
View File
@@ -0,0 +1,79 @@
"""Configuration, HTTP errors and shared representation helpers."""
import base64
import json
import os
import uuid
from datetime import datetime, timezone
from fastapi import Request
INSTANCE_ID = os.getenv("API_INSTANCE_ID", uuid.uuid4().hex[:12])
SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
IDEMPOTENCY_TTL = max(86400, int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")))
class APIError(Exception):
def __init__(self, status: int, code: str, message: str):
super().__init__(message)
self.status = status
self.code = code
self.message = message
def iso(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def invalid(message: str = "Invalid request") -> APIError:
return APIError(400, "invalid_request", message)
def parse_uuid(value: str) -> str:
try:
return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError):
raise invalid("Invalid UUID") from None
def parse_limit(raw: str | None) -> int:
if raw is None:
return 50
try:
if not raw.isascii() or not raw.isdecimal():
raise ValueError()
value = int(raw)
except (ValueError, AttributeError):
raise invalid("Invalid limit") from None
if not 1 <= value <= 100:
raise invalid("Invalid limit")
return value
def make_cursor(kind: str, scope: str, created: datetime, ident: str) -> str:
b = json.dumps({"kind": kind, "scope": scope, "created_at": iso(created), "id": ident}, separators=(",", ":")).encode()
return base64.urlsafe_b64encode(b).decode().rstrip("=")
def read_cursor(raw: str | None, kind: str, scope: str):
if raw is None:
return None
try:
if not raw or len(raw) > 2048 or any(ch not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" for ch in raw):
raise ValueError()
data = json.loads(base64.b64decode(raw + "=" * (-len(raw) % 4), altchars=b"-_", validate=True))
if not isinstance(data, dict) or data.get("kind") != kind or data.get("scope") != scope:
raise ValueError()
created = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
if created.tzinfo is None:
raise ValueError()
ident = str(uuid.UUID(data["id"]))
return created, ident
except (ValueError, TypeError, KeyError, UnicodeError, json.JSONDecodeError):
raise invalid("Invalid cursor") from None
def request_id(req: Request) -> str:
return getattr(req.state, "request_id", uuid.uuid4().hex)
def error_body(req: Request, code: str, message: str):
return {"error": {"code": code, "message": message, "request_id": request_id(req)}}
+115
View File
@@ -0,0 +1,115 @@
"""Sandboxed-per-task image work. Process is killed after JOB_TIMEOUT_SECONDS.
Outputs only JSON variant metadata. Original bytes are copied from durable staging,
never transformed. Pillow reopens/exif-transposes pixels for derivatives, stripping
metadata on re-encode. No API tokens or arbitrary user paths enter this command.
"""
import hashlib
import io
import json
import os
import sys
from PIL import Image, ImageOps, UnidentifiedImageError
from botocore.exceptions import ClientError, BotoCoreError
from .attachments import BUCKET, MAX_BYTES, MAX_PIXELS, s3_client
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
def rgb_image(src):
orient = ImageOps.exif_transpose(src)
orient.load()
if orient.width * orient.height > MAX_PIXELS:
raise ValueError("too many decoded pixels")
# Compose transparency on white, strip EXIF/ICC and do not enlarge.
rgba = orient.convert("RGBA")
white = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
white.alpha_composite(rgba)
return white.convert("RGB")
def transform(raw):
with Image.open(io.BytesIO(raw)) as image:
if image.format not in ("PNG", "JPEG"):
raise ValueError("invalid encoding")
rgb = rgb_image(image)
t = rgb.copy()
t.thumbnail((256, 256), resample=Image.Resampling.LANCZOS)
# Center crop fit scales up small images when needed.
crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
optim = rgb.copy()
optim.thumbnail((1280, 1280), resample=Image.Resampling.LANCZOS)
return [
("thumbnail", t, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
("crop", crop, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
("optimized", optim, "WEBP", "image/webp", {"quality": 78, "method": 4}),
]
def run(attachment_id, pipeline_version):
import psycopg
from psycopg.rows import dict_row
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5) as conn:
row = conn.execute("""
SELECT * FROM attachments WHERE id=%s AND pipeline_version=%s
""", (attachment_id, pipeline_version)).fetchone()
if not row or not row["staging_key"]:
raise ValueError("staging metadata not found")
client = s3_client()
body = client.get_object(Bucket=BUCKET, Key=row["staging_key"])["Body"]
try:
raw = body.read(MAX_BYTES + 1)
finally:
body.close()
if not raw or len(raw) > MAX_BYTES or len(raw) != row["size_bytes"]:
raise ValueError("staging size mismatch")
if hashlib.sha256(raw).hexdigest() != row["sha256"]:
raise ValueError("staging checksum mismatch")
# Recompute every result if a prior run was interrupted. Keys are deterministic
# for (attachment_id,pipeline_version,variant), so duplicates don't multiply objects.
output = []
for name, image, fmt, mime, options in transform(raw):
buf = io.BytesIO()
image.save(buf, format=fmt, **options)
encoded = buf.getvalue()
key = f"variants/{attachment_id}/v{pipeline_version}/{name}"
client.put_object(Bucket=BUCKET, Key=key, Body=encoded, ContentType=mime)
output.append({"name": name, "content_type": mime,
"size_bytes": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(),
"width": image.width, "height": image.height, "object_key": key})
# Only after derivatives exist, publish unchanged primary and recovery copy.
client.copy_object(Bucket=BUCKET, Key=row["object_key"],
CopySource={"Bucket": BUCKET, "Key": row["staging_key"]},
MetadataDirective="REPLACE", ContentType=row["content_type"])
client.copy_object(Bucket=BUCKET, Key=row["backup_key"],
CopySource={"Bucket": BUCKET, "Key": row["object_key"]},
MetadataDirective="REPLACE", ContentType=row["content_type"])
return output
def main():
import uuid
if len(sys.argv) != 3:
raise SystemExit(2)
try:
ident = str(uuid.UUID(sys.argv[1]))
version = int(sys.argv[2])
if version < 1:
raise ValueError("invalid version")
out = run(ident, version)
print(json.dumps({"variants": out}, ensure_ascii=False))
except (ValueError, UnidentifiedImageError, Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
print("Permanent image processing failure", file=sys.stderr)
raise SystemExit(20) from None
except (BotoCoreError, ClientError, OSError):
print("Transient object storage failure", file=sys.stderr)
raise SystemExit(30) from None
if __name__ == "__main__":
main()
+386
View File
@@ -0,0 +1,386 @@
"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases.
There is no exactly-once transport guarantee. We guarantee eventual recovery of
committed intent and idempotent DB results under at-least-once redelivery.
Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`.
"""
import argparse
from contextlib import contextmanager
import json
import logging
import os
import random
import subprocess
import sys
import time
import uuid
from datetime import datetime, timezone, timedelta
import pika
import psycopg
from pika.exceptions import AMQPError
from psycopg.rows import dict_row
log = logging.getLogger("lab5.jobs")
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
QUEUE = "attachment-processing.v1"
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
LEASE_SECONDS = JOB_TIMEOUT + 45
def connect_db():
return psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5)
def job_payload(job_id, attachment_id, version, attempt):
return {
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
"attachment_id": str(attachment_id), "pipeline_version": version,
"attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(),
}
@contextmanager
def connection():
params = pika.ConnectionParameters(
host=os.getenv("BROKER_HOST", "broker"), port=5672,
credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]),
heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120,
socket_timeout=5, connection_attempts=3, retry_delay=1)
mq = pika.BlockingConnection(params)
try:
yield mq
finally:
try:
if mq.is_open:
mq.close()
except AMQPError:
pass
def setup_channel(conn):
ch = conn.channel()
ch.queue_declare(queue=QUEUE, durable=True)
return ch
def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None):
payload = job_payload(job_id, attachment_id, version, attempt)
db.execute("""
INSERT INTO job_outbox(id,job_id,payload,available_at)
VALUES (%s,%s,%s::jsonb,COALESCE(%s,now()))
""", (str(uuid.uuid4()), job_id, json.dumps(payload), available_at))
def cleanup_abandoned_staging():
"""Best effort cleanup of retained stages; never delete current processing work."""
from .attachments import BUCKET, s3_client
client = s3_client()
retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
with connect_db() as db:
rows = db.execute("""
SELECT id,staging_key FROM attachments
WHERE staging_key IS NOT NULL AND
((status='ready' AND updated_at < now()-interval '300 seconds') OR
(status='failed' AND updated_at < now()-(%s*interval '1 second')))
LIMIT 50
""", (retention,)).fetchall()
for row in rows:
with connect_db() as db:
with db.transaction():
# Row-level lock serializes against operator replay.
current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone()
if not current or current["staging_key"] != row["staging_key"]:
continue
age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds()
if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention):
continue
client.delete_object(Bucket=BUCKET, Key=row["staging_key"])
db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"]))
# Also prune staging uploads whose DB COMMIT failed, after a long retention.
from botocore.exceptions import ClientError
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"):
for obj in page.get("Contents", []):
if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention:
continue
key = obj["Key"]
with connect_db() as db:
with db.transaction():
referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
if not referenced:
client.delete_object(Bucket=BUCKET, Key=key)
def rescue_expired_jobs(conn):
"""Requeue leased jobs whose worker died (durably, no manual task recreation)."""
with conn.transaction():
rows = conn.execute("""
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt
FROM processing_jobs j
WHERE j.status='processing' AND j.leased_until < now()
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if row["attempt"] >= MAX_ATTEMPTS:
conn.execute("""UPDATE processing_jobs
SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now()
WHERE id=%s""", (row["id"],))
conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now()
WHERE id=%s""", (row["attachment_id"],))
else:
attempt = row["attempt"] + 1
conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,
lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now()
WHERE id=%s""", (attempt, row["id"]))
conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],))
queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt)
log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt)
def dispatch_loop():
"""Publish outbox rows with publisher confirm, mark only AFTER confirm."""
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.confirm_delivery()
last_clean = time.monotonic()
while True:
# Periodic lease recovery works even if original message was consumed.
with connect_db() as conn:
rescue_expired_jobs(conn)
with conn.transaction():
rows = conn.execute("""
SELECT id,payload FROM job_outbox
WHERE published_at IS NULL AND available_at<=now()
ORDER BY available_at,created_at
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once":
# Persist one-time fault before hard exit; outbox still
# unmarked and dispatcher will publish on restart.
with connect_db() as faultdb:
inserted = faultdb.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish')
ON CONFLICT DO NOTHING RETURNING job_id
""", (row["payload"]["job_id"],)).fetchone()
if inserted:
os._exit(78)
raw = json.dumps(row["payload"], separators=(",", ":"))
# Mandatory + confirm; an ambiguous publish will be retried.
ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"),
mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2))
conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],))
if time.monotonic() - last_clean > 60:
try:
cleanup_abandoned_staging()
except Exception:
log.warning("staging janitor failed; retry later")
last_clean = time.monotonic()
if not rows:
time.sleep(1)
except (AMQPError, OSError, psycopg.Error) as exc:
log.warning("dispatcher reconnect after %s", type(exc).__name__)
time.sleep(2)
def claim_job(db, message):
job_id = message["job_id"]
with db.transaction():
job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone()
if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]:
return "ignore", None
if job["status"] == "processing":
return "busy", None
if job["next_run_at"] > datetime.now(timezone.utc):
return "busy", None
lease = str(uuid.uuid4())
db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s,
leased_until=now()+(%s*interval '1 second'),updated_at=now()
WHERE id=%s""", (lease, LEASE_SECONDS, job_id))
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],))
return "claimed", {"job": job, "lease": lease}
def publish_result(db, *, job, lease, variants):
"""Final result is committed atomically; stale leases/versions cannot publish."""
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]:
log.warning("stale worker result ignored %s", job["id"])
return False
for v in variants:
db.execute("""
INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET
object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,
size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256,
width=EXCLUDED.width,height=EXCLUDED.height
""", (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"],
v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"]))
db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now()
WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"]))
db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL,
error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],))
return True
def fail_job(db, job, lease, *, permanent=False, code="processing_failed"):
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease:
return
if permanent or job["attempt"] >= MAX_ATTEMPTS:
code = code if permanent else "retry_exhausted"
db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL,
leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"]))
db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"]))
else:
attempt = job["attempt"] + 1
delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0)
run_at = datetime.now(timezone.utc) + timedelta(seconds=delay)
db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s,
lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"]))
db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],))
queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at)
log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay)
def cleanup_staging(job):
from .attachments import BUCKET, s3_client
from botocore.exceptions import BotoCoreError, ClientError
try:
with connect_db() as conn:
row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone()
if row and row["status"] == "ready" and row["staging_key"]:
s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"])
with connect_db() as conn:
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s",
(job["attachment_id"], row["staging_key"]))
except (BotoCoreError, ClientError, psycopg.Error, OSError):
log.warning("staging cleanup deferred for %s", job["id"])
def process_message(raw):
try:
msg = json.loads(raw)
assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1"
uuid.UUID(msg["job_id"])
uuid.UUID(msg["attachment_id"])
assert type(msg["attempt"]) is int and msg["attempt"] >= 1
except (ValueError, KeyError, AssertionError, TypeError):
log.error("invalid internal broker message discarded")
return "ack" # no secrets printed, does not touch DB
with connect_db() as db:
status, state = claim_job(db, msg)
if status == "ignore":
return "ack"
if status == "busy":
return "requeue"
job, lease = state["job"], state["lease"]
try:
cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])]
proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False)
if proc.returncode != 0:
with connect_db() as db:
fail_job(db, job, lease, permanent=(proc.returncode == 20),
code="invalid_image" if proc.returncode == 20 else "processing_failed")
return "ack" # next attempt is already safely in DB outbox
result = json.loads(proc.stdout)
variants = result["variants"]
if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]:
raise ValueError("incomplete variants")
if os.getenv("LAB5_FAULT_AFTER_S3") == "once":
# One-time operator-only injected hard death, after all S3 writes and
# before DB result commit/ack; SQL PK prevents endless failure loop.
with connect_db() as db:
with db.transaction():
inserted = db.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3')
ON CONFLICT DO NOTHING RETURNING job_id
""", (job["id"],)).fetchone()
if inserted:
os._exit(77)
with connect_db() as db:
ok = publish_result(db, job=job, lease=lease, variants=variants)
if ok:
cleanup_staging(job)
return "ack"
except subprocess.TimeoutExpired:
with connect_db() as db:
fail_job(db, job, lease, code="processing_timeout")
return "ack"
except Exception:
log.exception("job processing failure (internal details not sent to API)")
# Leave lease for dispatcher recovery if even DB unavailable.
try:
with connect_db() as db:
fail_job(db, job, lease)
return "ack"
except psycopg.Error:
return "requeue"
def worker_loop():
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1"))))
def callback(channel, method, props, body):
outcome = process_message(body)
if outcome == "ack":
channel.basic_ack(method.delivery_tag)
else:
channel.basic_nack(method.delivery_tag, requeue=True)
time.sleep(2) # prevent tight redelivery spin on an active lease
ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False)
log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT)
ch.start_consuming()
except (AMQPError, OSError, psycopg.Error) as exc:
log.warning("worker reconnect after %s", type(exc).__name__)
time.sleep(2)
def replay_failed(ident):
with connect_db() as db:
with db.transaction():
row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone()
if not row or row["status"] != "failed":
raise RuntimeError("Only failed jobs may be explicitly replayed")
attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone()
if not attachment or not attachment["staging_key"]:
raise RuntimeError("Retained staging is required for replay")
db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(),
lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],))
db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,))
queue_outbox(db, row["id"], ident, row["pipeline_version"], 1)
db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"]))
print("Replayed failed job", ident)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"])
parser.add_argument("--id", help="UUID for explicit operator replay/status")
args = parser.parse_args()
if args.mode == "dispatch":
dispatch_loop()
elif args.mode == "worker":
worker_loop()
elif args.mode in ("replay", "status"):
if not args.id:
parser.error("--id is required")
ident = str(uuid.UUID(args.id))
if args.mode == "replay":
replay_failed(ident)
else:
with connect_db() as db:
print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str))
if __name__ == "__main__":
main()
+84
View File
@@ -0,0 +1,84 @@
"""FastAPI process: one stateless replica. No auto-migration on startup."""
import logging
import uuid
from contextlib import asynccontextmanager
import psycopg
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.concurrency import run_in_threadpool
from redis.exceptions import RedisError
from .api import router
from .attachments import router as attachment_router
from .auth import challenge, current_user
from .core import APIError, INSTANCE_ID, error_body, invalid, request_id
from .storage import start, stop
logging.basicConfig(level="INFO")
logger = logging.getLogger("lab5")
@asynccontextmanager
async def lifespan(app: FastAPI):
start()
try:
yield
finally:
stop()
app = FastAPI(title="MTUCI Messenger Lab 5", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan)
app.include_router(router)
app.include_router(attachment_router)
PROTECTED = ("/api/v1/chats", "/api/v1/attachments", "/api/v1/users/me", "/api/v1/auth/sessions/current")
JSON_POST = ("/api/v1/users", "/api/v1/chats")
@app.middleware("http")
async def request_context(req: Request, call_next):
req.state.request_id = uuid.uuid4().hex
try:
path = req.url.path
# Authenticate before parsing the request body or resource-specific fields.
if any(path == p or path.startswith(p + "/") for p in PROTECTED):
req.state.actor = await run_in_threadpool(current_user, req)
if req.method == "POST" and (path in JSON_POST or (path.startswith("/api/v1/chats/") and path.endswith("/messages"))):
if req.headers.get("content-type", "").split(";", 1)[0].strip().lower() != "application/json":
raise APIError(415, "unsupported_media_type", "Content-Type must be application/json")
if req.method == "POST" and path == "/api/v1/auth/sessions" and await req.body():
raise invalid("Login body is not allowed")
response = await call_next(req)
except APIError as exc:
response = api_error(req, exc)
except (RedisError, psycopg.OperationalError, psycopg.InterfaceError):
response = api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
except Exception:
logger.exception("unexpected request failure id=%s", request_id(req))
response = api_error(req, APIError(500, "internal_error", "Internal server error"))
response.headers["X-Request-Id"] = request_id(req)
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
def api_error(req: Request, exc: APIError) -> JSONResponse:
headers = {"WWW-Authenticate": challenge(req)} if exc.status == 401 else {}
return JSONResponse(status_code=exc.status, content=error_body(req, exc.code, exc.message), headers=headers)
@app.exception_handler(APIError)
async def api_error_handler(req: Request, exc: APIError):
return api_error(req, exc)
@app.exception_handler(RequestValidationError)
async def validation_error_handler(req: Request, exc: RequestValidationError):
return api_error(req, invalid())
@app.exception_handler(Exception)
async def unexpected_error_handler(req: Request, exc: Exception):
logger.exception("handler failure id=%s", request_id(req))
if isinstance(exc, (psycopg.OperationalError, psycopg.InterfaceError, RedisError)):
return api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
return api_error(req, APIError(500, "internal_error", "Internal server error"))
+23
View File
@@ -0,0 +1,23 @@
"""Run once before API instances; transactional and advisory-lock coordinated migrations."""
import os
from pathlib import Path
import psycopg
MIGRATIONS = Path(__file__).resolve().parents[1] / "db" / "migrations"
def main():
with psycopg.connect(os.environ["DATABASE_URL"], autocommit=False) as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(62132103)")
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())")
for path in sorted(MIGRATIONS.glob("[0-9]*.sql")):
applied = conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone()
if applied:
continue
conn.execute(path.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (path.name,))
print("Applied", path.name, flush=True)
print("Migrations up to date", flush=True)
if __name__ == "__main__":
main()
+77
View File
@@ -0,0 +1,77 @@
"""HTTP request models; all strings use Unicode code point limits."""
import re
import uuid
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
class CreateUser(StrictModel):
username: str
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_valid(cls, value: str) -> str:
if not USERNAME_RE.fullmatch(value):
raise ValueError("Invalid username")
return value
@field_validator("display_name", "password")
@classmethod
def non_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("Blank value")
value.encode("utf-8", errors="strict")
return value
class CreateChat(StrictModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def non_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("Blank title")
value.encode("utf-8", errors="strict")
return value
@field_validator("member_ids")
@classmethod
def uuid_members(cls, value: list[str]) -> list[str]:
for member in value:
if not isinstance(member, str) or str(uuid.UUID(member)) != member:
raise ValueError("Invalid member UUID")
if len(value) != len(set(value)):
raise ValueError("Duplicate member")
return value
class CreateMessage(StrictModel):
text: str = Field(min_length=0, max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@field_validator("text")
@classmethod
def non_blank(cls, value: str) -> str:
value.encode("utf-8", errors="strict")
return value
@field_validator("attachment_ids")
@classmethod
def valid_attachments(cls, value: list[str]) -> list[str]:
if len(value) != len(set(value)):
raise ValueError("Duplicate attachment IDs")
for item in value:
if not isinstance(item, str) or str(uuid.UUID(item)) != item:
raise ValueError("Invalid attachment UUID")
return value
@model_validator(mode="after")
def nonempty_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Empty message")
return self
+25
View File
@@ -0,0 +1,25 @@
"""Offline image conversion tests inside the application image; no S3 or DB."""
import io
from PIL import Image
from .image_worker import transform
def main():
source = Image.new('RGB',(320,200), (255,0,0))
buf=io.BytesIO();source.save(buf,'PNG')
result=transform(buf.getvalue())
items={name:(img,fmt,mime,opts) for name,img,fmt,mime,opts in result}
assert set(items)=={'thumbnail','crop','optimized'}
assert items['thumbnail'][0].size==(256,160)
assert items['crop'][0].size==(128,128)
assert items['optimized'][0].size==(320,200)
for name,(im,fmt,mime,opts) in items.items():
data=io.BytesIO();im.save(data,format=fmt,**opts)
with Image.open(io.BytesIO(data.getvalue())) as decoded:
assert decoded.size==im.size
assert decoded.format==fmt
assert 'exif' not in decoded.info
print('PASS: PNG 320x200 -> JPEG thumb 256x160, crop 128x128, optimized WebP, metadata stripped')
if __name__=='__main__':
main()
+132
View File
@@ -0,0 +1,132 @@
"""Lab4 grade-5 track 2: repeatable DB/S3 reconciliation.
Dry-run is default. Apply never drops metadata when BOTH object copies are gone:
without a trustworthy copy or external backup, deletion would destroy evidence.
"""
import argparse
import json
import sys
import time
from datetime import datetime, timezone
import psycopg
from psycopg.rows import dict_row
from botocore.exceptions import BotoCoreError, ClientError
from .attachments import BUCKET, s3_client
def inventory(client):
result = {}
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET):
for obj in page.get("Contents", []):
# Only reconciliation-owned prefixes; never mutate arbitrary S3 keys.
key = obj["Key"]
if key.startswith(("attachments/", "recovery/")):
result[key] = obj["LastModified"]
return result
def get_db_rows(conn):
return conn.execute("SELECT id, object_key, backup_key, status FROM attachments").fetchall()
def current_reference(conn, key):
return conn.execute(
"SELECT 1 FROM attachments WHERE object_key=%s OR backup_key=%s", (key, key)
).fetchone() is not None
def reconcile(*, apply=False, grace_seconds=300, now=None, client=None, connect=None):
if grace_seconds < 300 and apply:
raise ValueError("Apply requires >=300 s grace period to protect in-flight uploads")
started = time.monotonic()
now = now or datetime.now(timezone.utc)
client = client or s3_client()
connect = connect or (lambda: psycopg.connect(__import__('os').environ['DATABASE_URL'], row_factory=dict_row))
objects = inventory(client)
stats = {
"mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds,
"s3_keys_scanned": len(objects), "db_rows_scanned": 0,
"orphan_keys": [], "orphan_keys_young": [],
"missing_primary_repairable": [], "missing_backup_repairable": [],
"missing_both_unrecoverable": [], "actions": [], "errors": [],
}
with connect() as conn:
rows = get_db_rows(conn)
stats["db_rows_scanned"] = len(rows)
references = {key for row in rows for key in (row["object_key"], row["backup_key"])}
for row in rows:
if row["status"] != "ready":
continue # queued/processing have durable staging, not final variants yet
primary, backup = row["object_key"], row["backup_key"]
have_primary, have_backup = primary in objects, backup in objects
rid = str(row["id"])
if have_primary and have_backup:
continue
if not have_primary and not have_backup:
stats["missing_both_unrecoverable"].append(rid)
continue
dest, source, typ = (primary, backup, "missing_primary_repairable") if not have_primary else (backup, primary, "missing_backup_repairable")
stats[typ].append(rid)
if apply:
# Defensive recheck. Object may have reappeared after inventory.
try:
client.head_object(Bucket=BUCKET, Key=dest)
stats["actions"].append({"action": "already-restored", "key": dest})
continue
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
if code not in ("404", "NotFound", "NoSuchKey"):
raise
try:
client.copy_object(Bucket=BUCKET, Key=dest,
CopySource={"Bucket": BUCKET, "Key": source})
stats["actions"].append({"action": "restore", "key": dest, "from": source})
except (BotoCoreError, ClientError) as exc:
stats["errors"].append({"action": "restore", "key": dest, "error_type": type(exc).__name__})
for key, modified in objects.items():
if key in references:
continue
age = (now - modified).total_seconds()
if age < grace_seconds:
stats["orphan_keys_young"].append(key)
continue
stats["orphan_keys"].append(key)
if apply:
# Query again immediately before destructive action; never delete
# an object now referenced by a committed metadata row.
if current_reference(conn, key):
stats["actions"].append({"action": "skip-now-referenced", "key": key})
continue
try:
client.delete_object(Bucket=BUCKET, Key=key)
stats["actions"].append({"action": "delete-old-orphan", "key": key})
except (BotoCoreError, ClientError) as exc:
stats["errors"].append({"action": "delete-old-orphan", "key": key, "error_type": type(exc).__name__})
stats["scan_elapsed_seconds"] = round(time.monotonic() - started, 3)
return stats
def main():
parser = argparse.ArgumentParser(description="Reconcile S3 attachments vs PostgreSQL (dry-run default)")
parser.add_argument("--apply", action="store_true", help="Make safe changes; orphans older than 300 seconds only")
parser.add_argument("--grace-seconds", type=int, default=300)
parser.add_argument("--output", help="Write JSON summary (no credentials)")
args = parser.parse_args()
try:
result = reconcile(apply=args.apply, grace_seconds=args.grace_seconds)
except Exception as exc:
# No exception string: botocore errors may contain signed credentials/URLs.
print(json.dumps({"status":"failed", "error_type":type(exc).__name__}), file=sys.stderr)
raise SystemExit(2) from None
rendered = json.dumps(result, ensure_ascii=False, indent=2)
print(rendered)
if args.output:
with open(args.output, "w", encoding="utf-8") as fh:
fh.write(rendered + "\n")
if result["errors"] or result["missing_both_unrecoverable"]:
raise SystemExit(1)
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
"""Operator-only fault injection in a LOCAL TEST stack; never exposed as HTTP."""
import argparse
import json
import os
import uuid
import psycopg
from psycopg.rows import dict_row
from .attachments import BUCKET, s3_client
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["delete-primary", "delete-backup", "create-orphan"])
p.add_argument("--id", help="Attachment UUID for the delete actions")
args = p.parse_args()
if os.environ.get("LAB4_ALLOW_FAULT_INJECTION") != "yes":
raise SystemExit("Fault injection disabled; use LAB4_ALLOW_FAULT_INJECTION=yes in isolated test")
client = s3_client()
if args.action == "create-orphan":
key = f"attachments/{uuid.uuid4()}/original"
client.put_object(Bucket=BUCKET, Key=key, Body=b"synthetic orphan")
print(json.dumps({"fault":"orphan-created", "object_key":key}))
return
if not args.id:
raise SystemExit("--id is required")
ident = str(uuid.UUID(args.id))
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) as conn:
row = conn.execute("SELECT object_key,backup_key FROM attachments WHERE id=%s", (ident,)).fetchone()
if not row:
raise SystemExit("Attachment does not exist")
key = row["object_key"] if args.action == "delete-primary" else row["backup_key"]
client.delete_object(Bucket=BUCKET, Key=key)
print(json.dumps({"fault":args.action, "object_key":key}))
if __name__ == "__main__":
main()
+61
View File
@@ -0,0 +1,61 @@
"""Fast isolated tests for reconciliation safety; run in application image."""
import uuid
from datetime import datetime, timezone, timedelta
from .reconcile import reconcile
from .attachments import BUCKET
NOW = datetime.now(timezone.utc)
class FakeConn:
def __init__(self, rows): self.rows=rows
def __enter__(self): return self
def __exit__(self,*a): return False
def execute(self, sql, args=None):
self.sql=sql
self.args=args
return self
def fetchall(self): return self.rows
def fetchone(self):
key=self.args[0]
return {'id': 1} if any(key in (row['object_key'],row['backup_key']) for row in self.rows) else None
class FakeS3:
def __init__(self,objects): self.objects=objects
def get_paginator(self,op): return self
def paginate(self,**kwargs):
yield {'Contents':[{'Key':k,'LastModified':t} for k,t in self.objects.items()]}
def head_object(self,**kw):
if kw['Key'] not in self.objects:
from botocore.exceptions import ClientError
raise ClientError({'Error':{'Code':'404'}},'HeadObject')
def copy_object(self,**kw):
assert kw['CopySource']['Key'] in self.objects
self.objects[kw['Key']]=NOW
def delete_object(self,**kw):
self.objects.pop(kw['Key'],None)
def test():
i=str(uuid.uuid4())
primary=f'attachments/{i}/original';backup=f'recovery/{i}/original'
young=f'attachments/{uuid.uuid4()}/original'
old=f'attachments/{uuid.uuid4()}/original'
rows=[{'id':i,'object_key':primary,'backup_key':backup,'status':'ready'}]
fake=FakeS3({backup: NOW-timedelta(days=1),young:NOW-timedelta(seconds=10),old:NOW-timedelta(days=1)})
kwargs={'now':NOW,'client':fake,'connect':lambda:FakeConn(rows)}
first=reconcile(apply=False,**kwargs)
assert first['missing_primary_repairable']==[i]
assert old in first['orphan_keys'] and young in first['orphan_keys_young']
assert primary not in fake.objects and old in fake.objects, 'Dry run mutated objects'
second=reconcile(apply=True,**kwargs)
assert primary in fake.objects and old not in fake.objects and young in fake.objects
assert any(x['action']=='restore' for x in second['actions'])
third=reconcile(apply=True,**kwargs)
assert not third['actions'], 'Re-run unexpectedly mutated objects'
assert rows[0]['object_key']==primary, 'Metadata destroyed'
try: reconcile(apply=True, grace_seconds=0, **kwargs)
except ValueError: pass
else: raise AssertionError('Unsafe zero-grace apply accepted')
print('PASS: dry-run, orphan grace, recover primary from backup, apply idempotence')
if __name__ == '__main__': test()
+35
View File
@@ -0,0 +1,35 @@
"""Idempotently create the private S3 bucket with bootstrap-only credentials."""
import os
import time
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError, BotoCoreError
def main():
bucket = os.environ["S3_BUCKET"]
client = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ADMIN_ACCESS_KEY"],
aws_secret_access_key=os.environ["S3_ADMIN_SECRET_KEY"],
region_name="us-east-1",
config=Config(s3={"addressing_style": "path"},
connect_timeout=2, read_timeout=4, retries={"max_attempts": 1}))
for attempt in range(70):
try:
try:
client.head_bucket(Bucket=bucket)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code", "")
if code not in ("404", "NoSuchBucket", "NotFound"):
raise
client.create_bucket(Bucket=bucket)
# PRIVATE: no public bucket policy, ACL or anonymous identity.
print(f"Private bucket {bucket} ready", flush=True)
return
except (ClientError, BotoCoreError, OSError) as exc:
if attempt == 69:
raise RuntimeError("S3 bootstrap failed; inspect S3 container") from exc
time.sleep(1)
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
"""One-off creation of private SeaweedFS S3 identities; secrets stay in Docker volume."""
import json
import os
from pathlib import Path
def main():
bucket = os.environ["S3_BUCKET"]
identities = [
{"name": "bucket-bootstrap", "credentials": [{"accessKey": os.environ["S3_ADMIN_ACCESS_KEY"],
"secretKey": os.environ["S3_ADMIN_SECRET_KEY"]}],
"actions": ["Admin", "Read", "Write", "List"]},
{"name": "messenger-api", "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY"],
"secretKey": os.environ["S3_SECRET_KEY"]}],
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
]
if len({entry["credentials"][0]["accessKey"] for entry in identities}) != 2:
raise ValueError("Admin and API credentials must differ")
path = Path("/s3-config/config.json")
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".tmp")
old_umask = os.umask(0o077)
try:
tmp.write_text(json.dumps({"identities": identities}, separators=(",", ":")), encoding="utf-8")
os.chmod(tmp, 0o644) # readable by the SeaweedFS runtime UID; Docker volume stays private
os.replace(tmp, path)
finally:
os.umask(old_umask)
print("S3 identities configured (no secrets printed)", flush=True)
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
"""Verify bucket is private at S3 boundary (not merely hidden behind HTTP API)."""
import json
import os
import boto3
from botocore import UNSIGNED
from botocore.config import Config
from botocore.exceptions import ClientError
from .attachments import BUCKET, s3_client
def main():
client = s3_client()
client.head_bucket(Bucket=BUCKET) # restricted app identity works
unauth = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
region_name="us-east-1", config=Config(signature_version=UNSIGNED,
s3={"addressing_style":"path"}))
try:
unauth.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status not in (401,403):
raise RuntimeError("Unexpected anonymous S3 response") from None
else:
raise RuntimeError("Private bucket leaked object list to anonymous caller")
listing = client.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
keys = [entry["Key"] for entry in listing.get("Contents", [])]
if keys:
try:
unauth.get_object(Bucket=BUCKET, Key=keys[0])
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status not in (401, 403):
raise RuntimeError("Unexpected anonymous object GET response") from None
else:
raise RuntimeError("Private S3 object downloadable anonymously")
print(json.dumps({"private_bucket": True, "anonymous_listing": "denied",
"anonymous_object_get": "denied" if keys else "not_tested_empty_bucket"}))
if __name__ == "__main__":
main()
+64
View File
@@ -0,0 +1,64 @@
"""Shared external state: psycopg connection pool and Redis session store."""
import os
from contextlib import contextmanager
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from redis import Redis
from redis.exceptions import RedisError
from .core import APIError
pool: ConnectionPool | None = None
redis_client: Redis | None = None
def start() -> None:
global pool, redis_client
url = os.environ["DATABASE_URL"]
size = int(os.getenv("DB_POOL_MAX", "8"))
pool = ConnectionPool(conninfo=url, min_size=1, max_size=size, timeout=3, kwargs={"row_factory": dict_row, "connect_timeout": 5, "options": "-c statement_timeout=5000"}, open=False)
pool.open(wait=True, timeout=30)
redis_client = Redis.from_url(os.environ.get("REDIS_URL", "redis://redis:6379/0"), socket_connect_timeout=2, socket_timeout=2, decode_responses=True)
redis_client.ping()
def stop() -> None:
global pool, redis_client
if pool is not None:
pool.close()
pool = None
if redis_client is not None:
redis_client.close()
redis_client = None
@contextmanager
def db():
if pool is None:
raise APIError(503, "unavailable", "Database unavailable")
try:
with pool.connection() as conn:
yield conn
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout) as exc:
raise APIError(503, "unavailable", "Database unavailable") from exc
def cache() -> Redis:
if redis_client is None:
raise APIError(503, "unavailable", "Session store unavailable")
return redis_client
def ready() -> bool:
try:
with db() as conn:
conn.execute("SELECT 1").fetchone()
cache().ping()
# S3 is needed for the mandatory synchronous upload/download path.
from .attachments import s3_client, BUCKET
s3_client().head_bucket(Bucket=BUCKET)
return True
except Exception:
return False
+251
View File
@@ -0,0 +1,251 @@
services:
postgres:
image: postgres:16.10-bookworm
environment:
POSTGRES_DB: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-lab}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
volumes: ["postgres_data:/var/lib/postgresql/data"]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: 1.0
restart: unless-stopped
networks: [storage]
redis:
image: redis:7.4.5-alpine
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", "60", "1"]
volumes: ["redis_data:/data"]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 256m
cpus: 0.5
restart: unless-stopped
networks: [storage]
broker:
image: rabbitmq:4.1.8-management-alpine
hostname: lab5-broker
environment:
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-labworker}
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env}
volumes: ["rabbitmq_data:/var/lib/rabbitmq"]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 25
start_period: 15s
mem_limit: 512m
cpus: 0.75
restart: unless-stopped
networks: [storage]
s3-policy:
build: .
user: "0:0"
command: ["python", "-m", "app.s3_policy"]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY}
S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
volumes: ["s3_config:/s3-config"]
restart: "no"
networks: [storage]
s3:
image: chrislusf/seaweedfs:4.53
command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.port=8333", "-s3.iam=false", "-s3.config=/s3-config/config.json"]
volumes:
- s3_data:/data
- s3_config:/s3-config:ro
- ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro
depends_on:
s3-policy:
condition: service_completed_successfully
expose: ["8333"]
mem_limit: 640m
cpus: 1.0
restart: unless-stopped
networks: [storage]
s3-bootstrap:
build: .
command: ["python", "-m", "app.s3_bootstrap"]
environment:
S3_ENDPOINT: http://s3:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY}
S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY}
depends_on:
s3:
condition: service_started
restart: "no"
networks: [storage]
migrate:
build: .
command: ["python", "-m", "app.migrate"]
environment:
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
depends_on:
postgres:
condition: service_healthy
restart: "no"
networks: [storage]
reconcile:
profiles: [maintenance]
build: .
command: ["python", "-m", "app.reconcile"]
environment:
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
depends_on:
postgres:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
restart: "no"
networks: [storage]
dispatcher:
build: .
command: ["python", "-m", "app.jobs", "dispatch"]
environment: &job_env
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
BROKER_HOST: broker
BROKER_USER: ${BROKER_USER:-labworker}
BROKER_PASSWORD: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
depends_on:
broker:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
mem_limit: 192m
cpus: 0.5
restart: unless-stopped
networks: [storage]
worker:
build: .
command: ["python", "-m", "app.jobs", "worker"]
environment:
<<: *job_env
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
LAB5_FAULT_AFTER_S3: ${LAB5_FAULT_AFTER_S3:-off}
depends_on:
broker:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
mem_limit: 768m
cpus: 1.0
stop_grace_period: 75s
restart: unless-stopped
networks: [storage]
api1:
build: .
environment: &api_env
APP_PORT: "8080"
API_INSTANCE_ID: api1
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
JOB_BACKLOG_LIMIT: ${JOB_BACKLOG_LIMIT:-1000}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
depends_on: &api_depends
postgres:
condition: service_healthy
redis:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
expose: ["8080"]
healthcheck: &api_health
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 2s
timeout: 3s
retries: 15
start_period: 10s
mem_limit: 384m
cpus: 0.75
stop_grace_period: 12s
restart: unless-stopped
networks: [frontend, storage]
api2:
build: .
environment:
<<: *api_env
API_INSTANCE_ID: api2
depends_on: *api_depends
expose: ["8080"]
healthcheck: *api_health
mem_limit: 384m
cpus: 0.75
stop_grace_period: 12s
restart: unless-stopped
networks: [frontend, storage]
lb:
image: haproxy:3.1.7-alpine
volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"]
ports: ["127.0.0.1:8080:8080"]
depends_on:
api1:
condition: service_healthy
api2:
condition: service_healthy
healthcheck:
test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
interval: 10s
timeout: 3s
retries: 3
mem_limit: 128m
cpus: 0.5
restart: unless-stopped
networks: [frontend]
networks:
frontend:
storage:
internal: true
volumes:
postgres_data:
redis_data:
s3_data:
s3_config:
rabbitmq_data:
+33
View File
@@ -0,0 +1,33 @@
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username VARCHAR(32) NOT NULL UNIQUE,
display_name BYTEA NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_display_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400)
);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title BYTEA NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400)
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id),
user_id UUID NOT NULL REFERENCES users(id),
position INTEGER NOT NULL,
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX IF NOT EXISTS idx_members_user ON chat_members (user_id, chat_id);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
sender_id UUID NOT NULL,
text BYTEA NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id),
CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000)
);
CREATE INDEX IF NOT EXISTS idx_messages_chat_history ON messages (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_chats_history ON chats (created_at, id);
+12
View File
@@ -0,0 +1,12 @@
-- Same transaction creates message and idempotency receipt. Unique key arbitrates concurrent replicas.
CREATE TABLE IF NOT EXISTS message_idempotency (
actor_id UUID NOT NULL REFERENCES users(id),
chat_id UUID NOT NULL REFERENCES chats(id),
method VARCHAR(8) NOT NULL DEFAULT 'POST',
idem_key VARCHAR(128) NOT NULL,
body_sha256 CHAR(64) NOT NULL,
message_id UUID NOT NULL REFERENCES messages(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (actor_id, chat_id, method, idem_key)
);
CREATE INDEX IF NOT EXISTS idx_idempotency_age ON message_idempotency (created_at);
+25
View File
@@ -0,0 +1,25 @@
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_bytes;
ALTER TABLE messages ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000);
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
owner_id UUID NOT NULL REFERENCES users(id),
object_key TEXT NOT NULL UNIQUE,
backup_key TEXT NOT NULL UNIQUE,
filename TEXT NOT NULL,
content_type VARCHAR(32) NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
size_bytes BIGINT NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
sha256 CHAR(64) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id);
CREATE TABLE IF NOT EXISTS message_attachments (
message_id UUID NOT NULL REFERENCES messages(id),
attachment_id UUID NOT NULL REFERENCES attachments(id),
position INTEGER NOT NULL CHECK(position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id,position)
);
CREATE INDEX IF NOT EXISTS idx_message_attachments_attachment ON message_attachments(attachment_id);
+58
View File
@@ -0,0 +1,58 @@
-- Lab5: stage durably, then transact job and outbox atomically.
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'ready'
CHECK (status IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code TEXT;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INTEGER NOT NULL DEFAULT 1;
CREATE UNIQUE INDEX IF NOT EXISTS idx_attachments_staging ON attachments(staging_key) WHERE staging_key IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(status);
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id UUID NOT NULL REFERENCES attachments(id),
pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1),
name TEXT NOT NULL CHECK (name IN ('thumbnail','crop','optimized')),
object_key TEXT NOT NULL UNIQUE,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL CHECK (size_bytes > 0),
sha256 CHAR(64) NOT NULL,
width INTEGER NOT NULL CHECK (width > 0),
height INTEGER NOT NULL CHECK (height > 0),
PRIMARY KEY (attachment_id,pipeline_version,name)
);
CREATE TABLE IF NOT EXISTS processing_jobs (
id UUID PRIMARY KEY,
attachment_id UUID NOT NULL UNIQUE REFERENCES attachments(id),
pipeline_version INTEGER NOT NULL DEFAULT 1,
status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued','processing','ready','failed')),
attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1),
next_run_at TIMESTAMPTZ NOT NULL DEFAULT now(),
lease_token UUID,
leased_until TIMESTAMPTZ,
error_code TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_jobs_leases ON processing_jobs(status, leased_until);
CREATE TABLE IF NOT EXISTS job_outbox (
id UUID PRIMARY KEY,
job_id UUID NOT NULL REFERENCES processing_jobs(id),
payload JSONB NOT NULL,
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
published_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_outbox_due ON job_outbox(available_at, created_at) WHERE published_at IS NULL;
CREATE TABLE IF NOT EXISTS job_replay_audit (
id UUID PRIMARY KEY,
job_id UUID NOT NULL REFERENCES processing_jobs(id),
replayed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
action TEXT NOT NULL DEFAULT 'operator_replay'
);
CREATE TABLE IF NOT EXISTS job_fault_once (
job_id UUID NOT NULL REFERENCES processing_jobs(id),
fault TEXT NOT NULL,
PRIMARY KEY (job_id,fault)
);
+13
View File
@@ -0,0 +1,13 @@
{
"concurrency": 10,
"accepted_uploads": 10,
"ready_attachments": 10,
"failed_requests": 0,
"acceptance_elapsed_seconds": 2.391,
"acceptance_rps": 4.182,
"upload_latency_p50_ms": 2218.5,
"upload_latency_max_ms": 2313.0,
"all_ready_elapsed_seconds": 29.781,
"bytes_per_file": 111817,
"note": "HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats."
}
+66
View File
@@ -0,0 +1,66 @@
# Lab5 — итоговая сводка подтверждённых запусков (14–15.11.2026)
Источник: фактические выводы Windows PowerShell / Docker Desktop, присланные автором 14–15 ноября 2026 года. Архив v6 обновляет документацию; новые контейнерные тесты при упаковке **не запускались**. Для повторяемости команды приведены в [`../LAB5_TEST_COMMANDS.md`](../LAB5_TEST_COMMANDS.md).
## Контракт и функциональность
| Команда / проверка | Фактически наблюдалось |
| --- | --- |
| `docker compose up -d --build --wait`, `docker compose ps` | Основная инфраструктура запустилась; API1/API2/broker/lb/DB/Redis/S3 healthy там, где определён healthcheck |
| `python scripts/check_contract.py` | PASS (только структура/синтаксис, не интеграционный прогон) |
| `python tests/own_unit.py` | PASS: модели, cursor scoping, limits, Unicode U+0000 |
| `python tests/smoke.py` | PASS после SQL-исправления и повторно после восстановления обеих API |
| `python tests/attachments_negative.py` | PASS: image/MIME/size/filename/ACL/variants/cross-chat после исправления ожиданий Lab5 |
| `python tests/idempotency_parallel.py` | PASS: 2 реплики, одно сообщение, replay `201`, конфликт `409` |
| `python -m app.pipeline_selftest` в worker | PASS: PNG 320×200 → JPEG 256×160, crop 128×128, WebP, stripped metadata |
## Очередь, worker и восстановление
| Сценарий | Фактически наблюдалось |
| --- | --- |
| `pipeline_resilience.py --scenario paused` | PASS: durable `queued → ready`; оригинал и 3 варианта, без дублей |
| `pipeline_resilience.py --scenario outbox-crash` | PASS: exit 78 после outbox commit, до RabbitMQ publish; восстановление |
| `pipeline_resilience.py --scenario worker-crash` | PASS: exit 77 после S3 effects, до result/ack; восстановление |
| `pipeline_extended.py --scenario broker-persistence` | PASS: publisher confirmed; `ready=1` до рестарта и после, dispatcher остановлен, нет republish из outbox |
| `pipeline_extended.py --scenario staging-cleanup` | PASS: `ready`, DB `staging_key=NULL`, staging object отсутствует в S3 |
| `pipeline_extended.py --scenario failed-replay` | PASS (15.11): permanent `invalid_image`, retry не запущен, variants отсутствуют; затем исправление метаданных, replay → `ready`, `audit=1`, replay готового задания отклонён |
Первый запуск `failed-replay` упал с `TypeError` из-за отсутствующего `schema='Attachment'` при вызове тестового клиента. После исправления повторный тест **PASS**. Это исправление теста, а не подтверждённый дефект API.
## Параллельность и ресурсы
Повторный запуск `python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json`:
- **PASS**: HTTP `202` 10/10, `ready` 10/10, 0 ошибок, 3 варианта у каждого.
- Acceptance: **2.391 s** и **4.182 requests/s**; это throughput приёма HTTP, **не** скорость обработки worker.
- HTTP upload latency: p50 **2218.5 ms**, max **2313 ms**.
- Время завершения всех 10 задач от старта теста: **29.781 s** (не p95 и не среднее время одного задания).
- Размер каждого файла: **111817 bytes**.
- Параллельная 45-секундная выборка `docker stats`: по 15 отсчётов на контейнер. Наблюдавшийся максимум памяти: API1 **163.10 MiB**, API2 **157.10 MiB**, worker **128.90 MiB**. Это **не истинный peak RSS**.
## Persistence, failover и финальная проверка (15.11)
```text
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 6, "failed_attempts": 1, "first_recovery_seconds": 6.172}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
api1: healthy; api2: healthy
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
```
- Persistence: `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd` — **PASS** сохранности ранее подтверждённых данных, вложения и существующей сессии после возврата сервиса.
- Failover: `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` — **PASS** сохранности и работы оставшейся реплики. **Не zero-downtime:** один неудачный запрос и около **6.172 s** до первого успеха.
- После сценариев `docker compose up -d --no-deps --wait api1 api2`, оба API healthy, повторный `smoke.py` — **PASS**, остальные runtime-контейнеры подняты.
**Оговорка про hook:** в тесте stdout/stderr скриптов подавлен. Исходники `.cmd` показывают их область действия: `restart.cmd` останавливает/поднимает названные Compose-сервисы, `stop-one.cmd` останавливает одну целевую API. Поэтому статус PASS означает успешную проверку клиентского поведения при выполненном hook; для демонстрации масштаба самого hook приложить его исходник и при необходимости независимый лог Docker.
## Что не следует утверждать
- Нет доказательства отказоустойчивости при потере физического хоста: учебный Compose работает на одном Docker-хосте.
- Система использует **at-least-once**, не exactly-once delivery; идемпотентность реализована на уровне устойчивого состояния/эффектов.
- Нагрузка 10 файлов не измеряет устойчивую производительность при тысячах заданий; sampled memory — не точный максимум.
- Оценку назначает преподаватель, а не тесты; это набор технических evidence для защиты уровня 4.
+46
View File
@@ -0,0 +1,46 @@
"time","container","memory_mib","cpu_percent"
"2026-11-14T20:57:07.6236562Z","mtusi2026lab5-api1-1","150,6","39,55"
"2026-11-14T20:57:07.7200386Z","mtusi2026lab5-api2-1","142,7","1,61"
"2026-11-14T20:57:07.7211119Z","mtusi2026lab5-worker-1","67,54","0,01"
"2026-11-14T20:57:10.5614383Z","mtusi2026lab5-api1-1","150,4","2,6"
"2026-11-14T20:57:10.5624387Z","mtusi2026lab5-api2-1","155,7","35,95"
"2026-11-14T20:57:10.5634403Z","mtusi2026lab5-worker-1","67,54","0"
"2026-11-14T20:57:13.6800147Z","mtusi2026lab5-api1-1","163,1","34,99"
"2026-11-14T20:57:13.6805548Z","mtusi2026lab5-api2-1","146,3","9,56"
"2026-11-14T20:57:13.6805548Z","mtusi2026lab5-worker-1","67,54","0"
"2026-11-14T20:57:16.7854336Z","mtusi2026lab5-api1-1","150,5","29,25"
"2026-11-14T20:57:16.7860143Z","mtusi2026lab5-api2-1","143,3","3,53"
"2026-11-14T20:57:16.7860143Z","mtusi2026lab5-worker-1","67,54","0"
"2026-11-14T20:57:19.8297024Z","mtusi2026lab5-api1-1","150,4","1,84"
"2026-11-14T20:57:19.8302414Z","mtusi2026lab5-api2-1","143,3","6,74"
"2026-11-14T20:57:19.8302414Z","mtusi2026lab5-worker-1","67,54","0"
"2026-11-14T20:57:22.9337889Z","mtusi2026lab5-api1-1","150,5","3,43"
"2026-11-14T20:57:22.9343583Z","mtusi2026lab5-api2-1","143,5","28,06"
"2026-11-14T20:57:22.9343583Z","mtusi2026lab5-worker-1","67,54","0"
"2026-11-14T20:57:26.0226361Z","mtusi2026lab5-api1-1","150,5","1,22"
"2026-11-14T20:57:26.0226361Z","mtusi2026lab5-api2-1","143,5","17,2"
"2026-11-14T20:57:26.0226361Z","mtusi2026lab5-worker-1","67,54","0"
"2026-11-14T20:57:29.1343125Z","mtusi2026lab5-api1-1","151,8","22,45"
"2026-11-14T20:57:29.1343125Z","mtusi2026lab5-api2-1","144,4","1,13"
"2026-11-14T20:57:29.1348670Z","mtusi2026lab5-worker-1","84,8","54,15"
"2026-11-14T20:57:32.2450510Z","mtusi2026lab5-api1-1","151,7","32,87"
"2026-11-14T20:57:32.2455994Z","mtusi2026lab5-api2-1","144,4","1,24"
"2026-11-14T20:57:32.2463896Z","mtusi2026lab5-worker-1","91,36","93,69"
"2026-11-14T20:57:35.3731553Z","mtusi2026lab5-api1-1","151,6","2,21"
"2026-11-14T20:57:35.3744212Z","mtusi2026lab5-api2-1","144,4","1,75"
"2026-11-14T20:57:35.3750375Z","mtusi2026lab5-worker-1","128,9","95,95"
"2026-11-14T20:57:38.4925792Z","mtusi2026lab5-api1-1","151,5","1,51"
"2026-11-14T20:57:38.4925792Z","mtusi2026lab5-api2-1","144,3","47,55"
"2026-11-14T20:57:38.4925792Z","mtusi2026lab5-worker-1","118,6","98,38"
"2026-11-14T20:57:41.5595716Z","mtusi2026lab5-api1-1","151,7","73,27"
"2026-11-14T20:57:41.5605761Z","mtusi2026lab5-api2-1","144,3","47,12"
"2026-11-14T20:57:41.5615747Z","mtusi2026lab5-worker-1","85,18","91,04"
"2026-11-14T20:57:44.6992845Z","mtusi2026lab5-api1-1","151,5","25,29"
"2026-11-14T20:57:44.6998215Z","mtusi2026lab5-api2-1","144,3","1,55"
"2026-11-14T20:57:44.6998215Z","mtusi2026lab5-worker-1","81,51","92,75"
"2026-11-14T20:57:47.8226967Z","mtusi2026lab5-api1-1","151,5","32,64"
"2026-11-14T20:57:47.8226967Z","mtusi2026lab5-api2-1","144,6","1,55"
"2026-11-14T20:57:47.8233538Z","mtusi2026lab5-worker-1","68,29","0"
"2026-11-14T20:57:50.8657247Z","mtusi2026lab5-api1-1","151,5","1,54"
"2026-11-14T20:57:50.8662684Z","mtusi2026lab5-api2-1","157,1","27,07"
"2026-11-14T20:57:50.8662684Z","mtusi2026lab5-worker-1","68,29","0,01"
1 time container memory_mib cpu_percent
2 2026-11-14T20:57:07.6236562Z mtusi2026lab5-api1-1 150,6 39,55
3 2026-11-14T20:57:07.7200386Z mtusi2026lab5-api2-1 142,7 1,61
4 2026-11-14T20:57:07.7211119Z mtusi2026lab5-worker-1 67,54 0,01
5 2026-11-14T20:57:10.5614383Z mtusi2026lab5-api1-1 150,4 2,6
6 2026-11-14T20:57:10.5624387Z mtusi2026lab5-api2-1 155,7 35,95
7 2026-11-14T20:57:10.5634403Z mtusi2026lab5-worker-1 67,54 0
8 2026-11-14T20:57:13.6800147Z mtusi2026lab5-api1-1 163,1 34,99
9 2026-11-14T20:57:13.6805548Z mtusi2026lab5-api2-1 146,3 9,56
10 2026-11-14T20:57:13.6805548Z mtusi2026lab5-worker-1 67,54 0
11 2026-11-14T20:57:16.7854336Z mtusi2026lab5-api1-1 150,5 29,25
12 2026-11-14T20:57:16.7860143Z mtusi2026lab5-api2-1 143,3 3,53
13 2026-11-14T20:57:16.7860143Z mtusi2026lab5-worker-1 67,54 0
14 2026-11-14T20:57:19.8297024Z mtusi2026lab5-api1-1 150,4 1,84
15 2026-11-14T20:57:19.8302414Z mtusi2026lab5-api2-1 143,3 6,74
16 2026-11-14T20:57:19.8302414Z mtusi2026lab5-worker-1 67,54 0
17 2026-11-14T20:57:22.9337889Z mtusi2026lab5-api1-1 150,5 3,43
18 2026-11-14T20:57:22.9343583Z mtusi2026lab5-api2-1 143,5 28,06
19 2026-11-14T20:57:22.9343583Z mtusi2026lab5-worker-1 67,54 0
20 2026-11-14T20:57:26.0226361Z mtusi2026lab5-api1-1 150,5 1,22
21 2026-11-14T20:57:26.0226361Z mtusi2026lab5-api2-1 143,5 17,2
22 2026-11-14T20:57:26.0226361Z mtusi2026lab5-worker-1 67,54 0
23 2026-11-14T20:57:29.1343125Z mtusi2026lab5-api1-1 151,8 22,45
24 2026-11-14T20:57:29.1343125Z mtusi2026lab5-api2-1 144,4 1,13
25 2026-11-14T20:57:29.1348670Z mtusi2026lab5-worker-1 84,8 54,15
26 2026-11-14T20:57:32.2450510Z mtusi2026lab5-api1-1 151,7 32,87
27 2026-11-14T20:57:32.2455994Z mtusi2026lab5-api2-1 144,4 1,24
28 2026-11-14T20:57:32.2463896Z mtusi2026lab5-worker-1 91,36 93,69
29 2026-11-14T20:57:35.3731553Z mtusi2026lab5-api1-1 151,6 2,21
30 2026-11-14T20:57:35.3744212Z mtusi2026lab5-api2-1 144,4 1,75
31 2026-11-14T20:57:35.3750375Z mtusi2026lab5-worker-1 128,9 95,95
32 2026-11-14T20:57:38.4925792Z mtusi2026lab5-api1-1 151,5 1,51
33 2026-11-14T20:57:38.4925792Z mtusi2026lab5-api2-1 144,3 47,55
34 2026-11-14T20:57:38.4925792Z mtusi2026lab5-worker-1 118,6 98,38
35 2026-11-14T20:57:41.5595716Z mtusi2026lab5-api1-1 151,7 73,27
36 2026-11-14T20:57:41.5605761Z mtusi2026lab5-api2-1 144,3 47,12
37 2026-11-14T20:57:41.5615747Z mtusi2026lab5-worker-1 85,18 91,04
38 2026-11-14T20:57:44.6992845Z mtusi2026lab5-api1-1 151,5 25,29
39 2026-11-14T20:57:44.6998215Z mtusi2026lab5-api2-1 144,3 1,55
40 2026-11-14T20:57:44.6998215Z mtusi2026lab5-worker-1 81,51 92,75
41 2026-11-14T20:57:47.8226967Z mtusi2026lab5-api1-1 151,5 32,64
42 2026-11-14T20:57:47.8226967Z mtusi2026lab5-api2-1 144,6 1,55
43 2026-11-14T20:57:47.8233538Z mtusi2026lab5-worker-1 68,29 0
44 2026-11-14T20:57:50.8657247Z mtusi2026lab5-api1-1 151,5 1,54
45 2026-11-14T20:57:50.8662684Z mtusi2026lab5-api2-1 157,1 27,07
46 2026-11-14T20:57:50.8662684Z mtusi2026lab5-worker-1 68,29 0,01
+4
View File
@@ -0,0 +1,4 @@
# Persistent SeaweedFS filer metadata for the single-node lab stack.
[leveldb2]
enabled = true
dir = "/data/filerldb2"
+32
View File
@@ -0,0 +1,32 @@
global
log stdout format raw local0
maxconn 1000
defaults
log global
mode http
option httplog
option dontlognull
timeout connect 2s
timeout client 60s
timeout server 60s
timeout http-request 5s
resolvers docker
nameserver localdns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
frontend messenger
bind *:8080
default_backend api_nodes
backend api_nodes
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server inter 1s fall 2 rise 1 slowstart 2s
server api1 api1:8080 check resolvers docker init-addr none
server api2 api2:8080 check resolvers docker init-addr none
+11
View File
@@ -0,0 +1,11 @@
fastapi==0.115.12
uvicorn==0.34.2
pydantic==2.11.4
psycopg[binary]==3.2.6
psycopg-pool==3.2.6
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.40.0
Pillow==11.3.0
pika==1.3.2
+11
View File
@@ -0,0 +1,11 @@
@echo off
setlocal
cd /d "%~dp0.."
docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker
if errorlevel 1 exit /b 1
python scripts\wait_ready.py
exit /b %ERRORLEVEL%
+10
View File
@@ -0,0 +1,10 @@
@echo off
setlocal
cd /d "%~dp0.."
if "%TARGET_INSTANCE_ID%"=="api1" goto stop
if "%TARGET_INSTANCE_ID%"=="api2" goto stop
echo Unexpected TARGET_INSTANCE_ID 1>&2
exit /b 2
:stop
docker compose --env-file .env stop --timeout 1 %TARGET_INSTANCE_ID%
exit /b %ERRORLEVEL%
+14
View File
@@ -0,0 +1,14 @@
"""Operator-facing readiness wait after container replacement."""
import time
import urllib.request
for _ in range(120):
try:
with urllib.request.urlopen("http://localhost:8080/health/ready", timeout=2) as response:
if response.status == 200:
print("[restart] all services ready")
break
except Exception:
pass
time.sleep(.5)
else:
raise SystemExit("[restart] health timeout")
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env python3
"""Concurrent Lab5 uploads (HTTP 202) and eventual processing completion.
Admission metrics measure upload/enqueue requests, not worker throughput.
Completion is checked separately so an accepted-but-stuck job cannot pass.
"""
import argparse
import concurrent.futures
import json
import statistics
import time
from pathlib import Path
from client import Client, check, run
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--output', default='evidence/attachments-concurrent.json')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--wait-seconds', type=int, default=180)
args = parser.parse_args()
check(args.concurrency > 0, 'concurrency must be positive')
check(args.wait_seconds > 0, 'wait-seconds must be positive')
client = Client(args.base_url)
alice, bob, eve, chat = client.bootstrap()
source = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=parallel.png'
def upload(_):
cli = Client(args.base_url)
response = cli.call('POST', path, 202, actor=alice, raw=source,
headers={'Content-Type': 'image/png'}, schema='Attachment')
att = response['json']
check(att['status'] == 'queued', '202 must return a queued attachment')
check(att['size_bytes'] == len(source), 'size mismatch')
check(att['chat_id'] == chat['id'] and att['owner_id'] == alice['id'],
'incorrect attachment ownership')
return att['id'], response['elapsed']
started = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
results = list(pool.map(upload, range(args.concurrency)))
accepted_seconds = time.monotonic() - started
ids = [att_id for att_id, _ in results]
timings = [duration for _, duration in results]
check(len(set(ids)) == args.concurrency, 'duplicate attachment IDs')
# A 202 response alone only proves acceptance, not successful processing.
pending = set(ids)
deadline = time.monotonic() + args.wait_seconds
while pending and time.monotonic() < deadline:
for att_id in list(pending):
att = client.call('GET', '/api/v1/attachments/' + att_id,
actor=alice, schema='Attachment')['json']
check(att['status'] != 'failed', 'attachment reached failed: ' + att_id)
if att['status'] == 'ready':
variants = [v['name'] for v in att['variants']]
check(len(variants) == 3 and set(variants) == {'thumbnail', 'crop', 'optimized'},
'missing/duplicate image variants: ' + att_id)
pending.remove(att_id)
if pending:
time.sleep(0.5)
check(not pending, 'attachments not ready before deadline: ' + str(len(pending)))
complete_seconds = time.monotonic() - started
data = {
'concurrency': args.concurrency,
'accepted_uploads': len(results),
'ready_attachments': args.concurrency,
'failed_requests': 0,
'acceptance_elapsed_seconds': round(accepted_seconds, 3),
'acceptance_rps': round(len(results) / accepted_seconds, 3),
'upload_latency_p50_ms': round(statistics.median(timings) * 1000, 1),
'upload_latency_max_ms': round(max(timings) * 1000, 1),
'all_ready_elapsed_seconds': round(complete_seconds, 3),
'bytes_per_file': len(source),
'note': 'HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats.',
}
output = Path(args.output)
output.parent.mkdir(exist_ok=True, parents=True)
output.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
print('PASS: %d concurrent HTTP 202 uploads -> ready, 3 variants each' % args.concurrency)
print(json.dumps(data, ensure_ascii=False, indent=2))
if __name__ == '__main__':
run(main)
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Lab5 negative API tests with isolated synthetic users (requires live stack)."""
from urllib.parse import quote
import uuid
from client import Client, arguments, run, check
from pathlib import Path
def main():
args = arguments('Lab5 negative file/ACL/security tests')
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
base = '/api/v1/chats/' + chat['id'] + '/attachments?filename='
data = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
c.call('POST', base+'big.png', 413, actor=alice, raw=data+b'\x00'*(10*1024*1024+1-len(data)), headers={'Content-Type':'image/png'})
c.call('POST', base+'bad.png', 415, actor=alice, raw=b'not-an-image', headers={'Content-Type':'image/png'})
c.call('POST', base+'fake.jpg', 415, actor=alice, raw=data, headers={'Content-Type':'image/jpeg'})
c.call('POST', base+quote('../escape.png',safe=''), 400, actor=alice, raw=data, headers={'Content-Type':'image/png'})
c.call('POST', base+'x.png', 404, actor=eve, raw=data, headers={'Content-Type':'image/png'})
c.call('POST', base+'x.png', 401, raw=data, headers={'Content-Type':'image/png'})
uploaded = c.attachment(alice,chat)[0]
ready = c.wait_ready(alice, uploaded)
unknown = str(uuid.uuid4())
c.call('GET','/api/v1/attachments/'+unknown,404,actor=alice)
c.call('GET','/api/v1/attachments/'+uploaded['id'],404,actor=eve)
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',404,actor=eve)
# A ready Lab5 attachment has a downloadable thumbnail for chat members.
# A non-member still gets 404, and an unknown variant must not resolve.
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',404,actor=eve)
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=nonexistent',404,actor=alice)
check(any(v['name'] == 'thumbnail' for v in ready['variants']), 'missing thumbnail variant')
thumb = c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',200,actor=alice)
check(thumb['headers'].get_content_type() == 'image/jpeg', 'wrong thumbnail MIME')
other_chat = c.call('POST','/api/v1/chats',201,actor=alice,
body={'title':'Другой чат','member_ids':[alice['id'],bob['id']]},schema='Chat')['json']
path = '/api/v1/chats/'+other_chat['id']+'/messages'
c.call('POST',path,404,actor=alice,
body={'text':'cross-chat','attachment_ids':[uploaded['id']]},
headers={'Idempotency-Key':uuid.uuid4().hex})
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',404,actor=alice,
body={'text':'unknown','attachment_ids':[unknown]},
headers={'Idempotency-Key':uuid.uuid4().hex})
ok = c.message(bob,chat,'',attachments=[uploaded['id']])['json']
check(ok['attachment_ids']==[uploaded['id']], 'valid attachment not linked')
print('PASS: lab5 negative image, MIME, size, filename, ACL, variants and cross-chat checks')
if __name__=='__main__':
run(main)
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Lab3 own test: 30 requests using identical actor/chat/key/body across both replicas."""
import concurrent.futures
import sys
import uuid
from client import Client, Failure, arguments, check, run
def main():
args = arguments("Check atomic idempotency across API replicas")
c = Client(args.base_url, args.ca_file)
alice, bob, _, chat = c.bootstrap()
key = uuid.uuid4().hex
body = {"text": "Concurrent replay " + key}
path = f"/api/v1/chats/{chat['id']}/messages"
def worker(_):
# Independent request client; one shared bearer token and idempotency key.
client = Client(args.base_url, args.ca_file)
return client.call("POST", path, expected=201, actor=alice, body=body,
headers={"Idempotency-Key": key}, schema="Message")
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor:
results = list(executor.map(worker, range(30)))
ids = {result["json"]["id"] for result in results}
instances = {result["headers"]["X-Instance-Id"] for result in results}
check(len(ids) == 1, "parallel replay created different messages")
check(len(instances) >= 2, "LB did not serve requests via both API replicas")
check(all(r["json"] == results[0]["json"] for r in results), "replay returned altered result")
c.call("POST", path, 409, actor=alice, body={"text": "different body"}, headers={"Idempotency-Key": key})
c.call("POST", path, 201, actor=bob, body=body, headers={"Idempotency-Key": key}, schema="Message")
history = c.call("GET", path + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"]
check(sum(x["id"] in ids for x in history) == 1, "message duplicated in history")
print("PASS: parallel idempotency across", len(instances), "replicas; one message, replay 201, conflicting body 409")
if __name__ == "__main__":
run(main)
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""Fast non-Docker checks for input contracts and cursor scoping."""
import sys
from pathlib import Path
from datetime import datetime, timezone
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.core import make_cursor, read_cursor, parse_limit, APIError
from app.models import CreateUser, CreateChat, CreateMessage
from pydantic import ValidationError
def reject(fn):
try:
fn()
except (APIError, ValidationError, ValueError):
return
raise AssertionError("Expected invalid input")
def main():
token = str(uuid.uuid4())
dt = datetime.now(timezone.utc)
cursor = make_cursor("messages", token, dt, str(uuid.uuid4()))
assert read_cursor(cursor, "messages", token)[0] == dt
reject(lambda: read_cursor(cursor, "messages", str(uuid.uuid4())))
reject(lambda: read_cursor(cursor, "chats", token))
assert parse_limit(None) == 50
reject(lambda: parse_limit("0"))
reject(lambda: parse_limit("101"))
reject(lambda: CreateUser(username="user_name", display_name=" \t", password="Long-enough-secret"))
reject(lambda: CreateUser(username="user_name", display_name="Test", password=" " * 3))
assert CreateMessage(text="A\u0000B").text == "A\u0000B"
reject(lambda: CreateMessage(text=" \t"))
reject(lambda: CreateChat(title="title", member_ids=[token] * 2))
reject(lambda: CreateChat(title="title", member_ids=[token] * 101))
print("PASS: models, cursor scoping, limits, Unicode U+0000")
if __name__ == "__main__":
main()
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
"""Additional *live* Lab5 fault tests; operator-controlled, synthetic records only.
broker-persistence: require a publisher-confirmed RabbitMQ message to survive
broker restart without a running dispatcher or worker. This differs from the
outbox-restoration tests where dispatcher publishes *after* broker recovery.
failed-replay: safely inject a deterministic checksum mismatch into one fresh
synthetic attachment's DB metadata, verify permanent failure, restore the
original hash, replay via the operator-only CLI, and verify success plus audit.
staging-cleanup: verify the staged S3 object is absent after ready (with a
matching cleared staging_key in the DB).
No volumes are deleted. The test may stop/restart worker/dispatcher/broker;
run only against a disposable LAB5 environment and never concurrently.
"""
import argparse
import hashlib
import os
import re
import subprocess
import time
import uuid
from client import Client, Failure, check, run
from pipeline_resilience import verify_ready, wait_api
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, expected=(0,), timeout=240):
proc = subprocess.run(
['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout,
)
if proc.returncode not in expected:
# Don't print env vars, tokens or command stdout/stderr by default.
raise Failure('docker compose %s failed (exit %d); inspect ps/logs' %
(' '.join(args[:2]), proc.returncode))
return proc.stdout.strip()
def pg(sql):
"""Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here."""
user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip()
dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip()
return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t',
'-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql)
def scalar(sql):
return pg(sql).splitlines()[-1].strip()
def wait_for(label, action, predicate, seconds=30, interval=0.5):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
last = action()
if predicate(last):
return last
except Failure:
pass
time.sleep(interval)
raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100])
def job_query(att_id, fields):
uuid.UUID(att_id)
return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' %
(fields, att_id))
def queue_ready():
out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues',
'name', 'messages_ready', 'messages_unacknowledged')
for line in out.splitlines():
parts = re.split(r'\s+', line.strip())
if parts[0] == 'attachment-processing.v1' and len(parts) >= 3:
return int(parts[1]), int(parts[2])
raise Failure('RabbitMQ durable queue missing')
def make_upload(c):
alice, bob, _eve, chat = c.bootstrap()
att, original = c.attachment(alice, chat)
check(att['status'] == 'queued', 'upload must return 202 queued')
uuid.UUID(att['id'])
return alice, bob, att, original
def wait_staging_cleared(att_id, seconds=20):
return wait_for('staging_key cleared after ready',
lambda: scalar("SELECT COALESCE(staging_key,'<NULL>') FROM attachments WHERE id='%s'" % att_id),
lambda value: value == '<NULL>', seconds=seconds)
def verify_staging_absent(att_id):
# This executes *inside* the worker container with its normal S3 credentials.
# Never expose S3 secrets to the host or write them to evidence files.
code = '''import sys
from botocore.exceptions import ClientError
from app.attachments import s3_client, BUCKET
key='staging/'+sys.argv[1]+'/original'
try:
s3_client().head_object(Bucket=BUCKET,Key=key)
except ClientError as exc:
if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404:
print('ABSENT')
else:
raise
else:
print('STILL_PRESENT')'''
result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id)
check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70])
def broker_persistence(c):
compose('stop', 'worker')
compose('up', '-d', 'dispatcher')
baseline = queue_ready()[0]
alice, _bob, att, original = make_upload(c)
print('202 queued:', att['id'], flush=True)
sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id']
wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30)
wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0],
lambda n: n >= baseline + 1, seconds=15)
compose('stop', 'dispatcher')
before = queue_ready()[0]
check(before >= baseline + 1, 'published message disappeared before restart')
print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True)
compose('restart', 'broker')
compose('up', '-d', '--wait', 'broker')
after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0],
lambda n: n >= before, seconds=40, interval=1)
check(scalar(sql) == 't', 'outbox publish confirmation lost')
print('Broker restarted; ready messages persisted=%d' % after, flush=True)
# CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message
# can deliver this job to the worker.
compose('up', '-d', 'worker')
verify_ready(c, alice, att, original, seconds=180)
check(job_query(att['id'], 'status') == 'ready', 'job not committed ready')
print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish')
def failed_replay(c):
compose('stop', 'worker', 'dispatcher')
alice, _bob, att, original = make_upload(c)
ident = att['id']
original_sha = hashlib.sha256(original).hexdigest()
check(original_sha == att['sha256'], 'fixture hash mismatch')
check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha,
'DB hash unexpected before controlled fault')
fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64
changed = False
try:
updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" %
(fault_sha, ident))
check(ident in updated, 'failed to inject synthetic checksum mismatch')
changed = True
compose('up', '-d', 'dispatcher', 'worker')
def get_state():
return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json']
failed = wait_for('permanent failed status', get_state,
lambda a: a['status'] == 'failed', seconds=90)
check(failed['error_code'] == 'invalid_image',
'expected permanent invalid_image, got ' + str(failed['error_code']))
check(job_query(ident, 'status') == 'failed', 'processing job must fail')
check(job_query(ident, 'attempt') == '1', 'permanent error must not retry')
check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0',
'failed job created variants')
print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True)
# Repair only the generated synthetic row before explicit operator replay.
restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" %
(original_sha, ident))
check(ident in restored, 'synthetic metadata could not be restored')
changed = False
compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident)
verify_ready(c, alice, att, original, seconds=180)
audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident)
check(audit == '1', 'operator replay audit count incorrect: ' + audit)
check(job_query(ident, 'status') == 'ready', 'replayed job not ready')
attempt = job_query(ident, 'attempt')
check(attempt == '1', 'replay did not reset attempt: ' + attempt)
invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs',
'replay', '--id', ident, expected=(1,))
print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected')
finally:
if changed:
# Never leave the synthetic attachment with altered metadata.
pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident))
def staging_cleanup(c):
compose('up', '-d', 'dispatcher', 'worker')
alice, _bob, att, original = make_upload(c)
verify_ready(c, alice, att, original, seconds=180)
wait_staging_cleared(att['id'], seconds=30)
verify_staging_absent(att['id'])
print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--scenario', required=True,
choices=['broker-persistence', 'failed-replay', 'staging-cleanup'])
args = parser.parse_args()
if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes':
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack')
c = Client('http://localhost:8080')
try:
wait_api()
if args.scenario == 'broker-persistence':
broker_persistence(c)
elif args.scenario == 'failed-replay':
failed_replay(c)
else:
staging_cleanup(c)
finally:
# Restore normal services even when an assertion fails. NO volume deletion.
try:
compose('up', '-d', '--wait', 'broker')
compose('up', '-d', 'dispatcher', 'worker')
except Failure:
print('WARNING: inspect docker compose ps and restore dispatcher/worker manually')
if __name__ == '__main__':
run(main)
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env python3
"""Explicit operator evidence for Lab5 queue durability and failure windows.
Runs only on an isolated lab5 Compose stack. Never resets named volumes. The test
preserves user/session secrets strictly in process memory and prints no tokens.
"""
import argparse
import hashlib
import os
import subprocess
import sys
import time
from pathlib import Path
from client import Client, Failure, check, run
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, expected=(0,)):
p = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=240)
if p.returncode not in expected:
raise Failure('docker compose command failed: %s (exit %s); inspect docker compose ps/logs'
% (' '.join(args[:2]), p.returncode))
return p
def wait_api(timeout=60):
import urllib.request
deadline=time.monotonic()+timeout
while time.monotonic()<deadline:
try:
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as r:
if r.status==200:
return
except Exception:
pass
time.sleep(.5)
raise Failure('API did not become ready')
def make_queued(client):
alice,bob,eve,chat=client.bootstrap()
att,original=client.attachment(alice,chat)
check(att['status']=='queued','202 must return queued snapshot')
c=client.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=alice)
attached=client.message(alice,chat,'',attachments=[att['id']])['json']
check(attached['attachment_ids']==[att['id']],'queued attachment cannot be linked')
return alice,bob,att,original
def verify_ready(client,actor,att,original,seconds=180):
r=client.wait_ready(actor,att,wait_seconds=seconds)
check(len(r['variants']) == 3, 'expected exactly three variants, no duplicate entries')
variants={v['name']:v for v in r['variants']}
check(set(variants)=={'thumbnail','crop','optimized'},'missing/extra variants')
check((variants['thumbnail']['width'],variants['thumbnail']['height'])==(256,160), 'thumbnail dimensions')
check((variants['crop']['width'],variants['crop']['height'])==(128,128), 'crop dimensions')
raw=client.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)
check(raw['bytes']==original,'original changed across asynchronous pipeline')
for name in ('thumbnail','crop','optimized'):
v=variants[name]
response=client.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
check(len(response['bytes'])==v['size_bytes'] and hashlib.sha256(response['bytes']).hexdigest()==v['sha256'],
'variant checksum mismatch: '+name)
return r
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--scenario',choices=['paused','outbox-crash','worker-crash'],required=True)
args=parser.parse_args()
if os.getenv('LAB5_ALLOW_FAILURE_TEST')!='yes':
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes in PowerShell to run destructive failure test')
c=Client('http://localhost:8080')
# All tests make a fresh attachment while dispatcher/worker are stopped.
compose('stop','worker','dispatcher')
try:
alice,bob,att,original=make_queued(c)
print('Created durable queued attachment',att['id'],flush=True)
if args.scenario=='paused':
compose('restart','api1','api2','broker')
compose('up','-d','--wait','broker')
wait_api()
elif args.scenario=='outbox-crash':
# Transaction committed, dispatcher crashes before publish.
compose('run','--rm','-e','LAB5_FAULT_BEFORE_PUBLISH=once',
'dispatcher',expected=(78,))
print('Injected dispatcher exit 78 after committed outbox, before publish')
else:
compose('up','-d','dispatcher')
# Fault after all S3 writes and before PostgreSQL result COMMIT and ack.
compose('run','--rm','-e','LAB5_FAULT_AFTER_S3=once',
'worker',expected=(77,))
print('Injected worker exit 77 after S3 effects, before result/ack')
compose('up','-d','dispatcher','worker')
wait_api()
ready=verify_ready(c,bob,att,original)
check(ready['status']=='ready','attachment not ready')
print('PASS:',args.scenario,'durable queued->ready; original/3 variants verified; no duplicates')
finally:
# Restore the normal topology even after an assertion failure.
compose('up','-d','dispatcher','worker')
if __name__=='__main__':
run(main)