ЛР 5 - БПИ2303 - Кимпаев - на 3 #48
+2
-1
@@ -2,6 +2,7 @@
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
HTTP_PORT=8085
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
@@ -19,7 +20,7 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_CONCURRENCY=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
*.sh text eol=lf
|
||||
*.py text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.json text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
@@ -1,18 +1,22 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Кимпаев А.Р. (kimpaev), БПИ2303.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Целевая оценка: **3**. Трек 5 не заявлен.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Из ЛР4 перенесены API, авторизация, сообщения, PostgreSQL, приватный S3, Nginx и миграции. Добавлены durable staging, DB-журнал заданий, RabbitMQ с persistence/confirms, dispatcher и отдельный worker с manual ack, JPEG thumbnail, ограниченным retry и идемпотентным результатом.
|
||||
|
||||
Запуск Windows: `start-windows.cmd`. Linux: `sh scripts/setup.sh && docker compose --env-file .env up -d --build --wait --wait-timeout 180`. API: http://localhost:8085.
|
||||
|
||||
Проверки: `py scripts/check_contract.py`, `py tests/smoke.py --base-url http://localhost:8085`, `py scripts/verify-lab5.py`, `py scripts/verify-previous.py http://localhost:8085` — PASS. В pipeline проверены stop worker → upload → restart API/broker → ready, пять дублей job и failed после пяти попыток. Persistence и два failover прошли. Полные команды, наблюдения и ограничения — [REPORT.md](../REPORT.md), журналы — evidence/.
|
||||
|
||||
Ограничения: один worker, нет WebP/crop, DLQ/replay, timeout/lease зависшего процесса и автоматической очистки staging. Одноузловые DB/S3/broker, локальный HTTP.
|
||||
|
||||
Источники: контракт текущего skeleton и собственная ЛР4; использована помощь ИИ для реализации, проверки и документации.
|
||||
|
||||
- [x] Обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] Эквиваленты make check и make test выполнены; реальный вывод приложен.
|
||||
- [x] Сценарии отказов уровня 3 воспроизведены.
|
||||
- [x] Секреты находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию решения.
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
name: Lab5 application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
integration:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: sh scripts/setup.sh
|
||||
- run: docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
- run: make test PYTHON=python BASE_URL=http://localhost:8085
|
||||
- run: python scripts/verify-lab5.py --base-url http://localhost:8085
|
||||
- name: Diagnostics
|
||||
if: failure()
|
||||
run: docker compose logs --tail=60 api1 api2 worker dispatcher broker
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: pipeline-evidence
|
||||
path: evidence/pipeline.*
|
||||
- if: always()
|
||||
run: docker compose down
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
FROM python:3.12.9-slim
|
||||
WORKDIR /srv
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY app ./app
|
||||
COPY migrations ./migrations
|
||||
EXPOSE 8080
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10"]
|
||||
@@ -1,37 +1,112 @@
|
||||
# Отчёт — лабораторная 5
|
||||
# Отчёт — лабораторная работа №5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор: **Кимпаев А.Р. (kimpaev)**.
|
||||
- Группа: БПИ2303 (из отчёта ЛР4).
|
||||
- Целевая оценка: **3**.
|
||||
- Commit/PR: solution/bpi2303-kimpaev.
|
||||
- Источники: README.md, COURSE.md, contracts/README.md, contracts/attachment-states.md, contracts/jobs.schema.json, публичные tests; собственные исходники ЛР4. Использована помощь ИИ для анализа, переноса кода, реализации, настройки, тестирования и отчёта.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Desktop с Linux containers либо Docker Engine + Compose v2; сеть для загрузки образов/PyPI при первой сборке. Для тестов нужен Python 3.9+. Образы закреплены: Python 3.12.9-slim, PostgreSQL 16.8-alpine, RabbitMQ 4.1.0-alpine, Nginx 1.27.5-alpine; MinIO/mc закреплены digest в compose.yaml. Библиотеки: FastAPI 0.115.12, Uvicorn 0.34.2, psycopg 3.2.6, boto3 1.37.38, Pillow 11.2.1, pika 1.3.2. Хостовый Python при проверке — 3.15.0.
|
||||
|
||||
|
||||
Windows / PowerShell:
|
||||
|
||||
```powershell
|
||||
.\start-windows.cmd
|
||||
# Эквивалентные шаги:
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts/setup.ps1
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
curl.exe --fail http://localhost:8085/health/ready
|
||||
py scripts/check_contract.py
|
||||
py tests/smoke.py --base-url http://localhost:8085
|
||||
py scripts/verify-lab5.py --base-url http://localhost:8085
|
||||
py scripts/verify-previous.py http://localhost:8085
|
||||
```
|
||||
|
||||
Основные проверки можно выполнить одной командой: `powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts/verify-windows.ps1`. Она проверяет структуру, smoke, pipeline и persistence. `verify-previous.py` дополнительно проверяет failover и записывает фактические рестарты контейнеров.
|
||||
|
||||
Linux / Git Bash:
|
||||
|
||||
```sh
|
||||
sh scripts/setup.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
make check
|
||||
make test BASE_URL=http://localhost:8085
|
||||
python3 scripts/verify-lab5.py --base-url http://localhost:8085
|
||||
python3 scripts/verify-previous.py http://localhost:8085
|
||||
# Публичные проверки отдельно:
|
||||
chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
make test-persistence BASE_URL=http://localhost:8085 ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover BASE_URL=http://localhost:8085 ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose up -d --no-deps api1 api2
|
||||
```
|
||||
|
||||
Setup создаёт `.env` и случайные DB/S3/RabbitMQ credentials в игнорируемом каталоге `secrets/`, без печати значений. Повторный запуск сохраняет credentials. Нельзя удалять secrets отдельно от уже инициализированных volumes. Bootstrap создаёт приватный S3 bucket и ограниченную identity приложения. `migrate` автоматически выполняет SQL 001–004 с advisory lock и журналом миграций. Оба одноразовых job должны завершиться с кодом 0; `up --wait` ожидает healthy LB.
|
||||
|
||||
Безопасная остановка: `docker compose --env-file .env down` (volumes сохраняются). Повторный запуск — прежняя команда up. Намеренное **необратимое удаление учебных данных ЛР5**: `docker compose --env-file .env down -v`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
HTTP :8085 → Nginx → api1 / api2 → PostgreSQL (metadata + jobs)
|
||||
└────→ private S3 staging/<UUID>
|
||||
PostgreSQL → dispatcher → RabbitMQ → worker
|
||||
├→ S3 results/<UUID>/v1/original
|
||||
├→ S3 results/<UUID>/v1/thumbnail
|
||||
└→ PostgreSQL ready + variants
|
||||
```
|
||||
|
||||
|
||||
Upload ограничивает 10 MiB и 40 млн пикселей, проверяет filename, MIME по содержимому и декодируемость PNG/JPEG. Проверка исходника унаследована из ЛР4; resize выполняется отдельным worker. **202 queued** выдаётся после устойчивого S3 PUT в staging и COMMIT метаданных + job одной PostgreSQL-транзакцией. Staging расположен в S3 с volume; worker публикует оригинал в отдельный конечный prefix. SHA-256 проверяется перед обработкой. Queued/processing можно прикрепить к сообщению, failed — нельзя (409). Остановленный worker не мешает обычным сообщениям.
|
||||
|
||||
`attachment_jobs` — устойчивый журнал намерений. Dispatcher выбирает незавершённые задания с `FOR UPDATE SKIP LOCKED`, публикует persistent-сообщения (delivery_mode=2), ожидает publisher confirm и повторяет публикацию до терминального результата примерно через 10 секунд. Поэтому разрыв publish/COMMIT может дать дубль, но не теряет принятое задание. Durable RabbitMQ queue использует отдельный volume и стабильный hostname. Сообщения соответствуют jobs.schema.json и содержат только job/attachment ID, schema/type, pipeline_version=1, attempt и created_at; bytes, URL и credentials отсутствуют. Attempt сообщения — снимок; реальный счётчик попыток хранится в БД.
|
||||
|
||||
Worker: manual ack, prefetch=1, один процесс; контейнер ограничен 1 CPU и 768 MiB. PostgreSQL session advisory lock сериализует работу одной job; при смерти соединения lock освобождается. После смерти worker оставшийся processing восстанавливается redelivery/dispatcher. Перед обработкой фиксируется попытка. Детерминированные ключи `(attachment_id, pipeline_version=1, variant)` позволяют повторно записать тот же эффект без создания дополнительных объектов. Дубликат ready/failed — no-op. Ack отправляется только после записи original/thumbnail и COMMIT ready с метаданными.
|
||||
|
||||
Thumbnail: JPEG quality=85; EXIF orientation применяется, alpha компонуется на белом, вписывание в 256×256 без увеличения, округление вниз с минимумом 1 px. EXIF в результате не копируется. Fixture 320×200 даёт настоящий JPEG 256×160. Original сохраняется побайтно. Content queued/processing/failed возвращает 409; чужой attachment — 404; публичный API не раскрывает object keys.
|
||||
|
||||
`JOB_MAX_ATTEMPTS=5` задаёт предел. Временная ошибка возвращает queued, следующая публикация назначается через 5 секунд; при исчерпании — failed/retry_exhausted. Постоянная ошибка изображения — failed/invalid_image. Уже доставленные дубли могут сократить паузу: точный backoff и jitter на уровне 3 не заявляются. Ошибки API не содержат stack trace или credentials. Невалидное служебное сообщение отвергается без requeue.
|
||||
|
||||
Readiness проверяет PostgreSQL и S3; RabbitMQ/worker не критичны для безопасного синхронного приёма благодаря staging + DB-журналу. Liveness проверяет только процесс.
|
||||
|
||||
Настройки: HTTP_PORT, SESSION_TTL_SECONDS и JOB_MAX_ATTEMPTS читаются Compose из `.env`; DATABASE_URL_FILE, BROKER_URL_FILE, S3 endpoint/region/bucket и пути S3 secrets заданы в compose.yaml. API внутри контейнера слушает 0.0.0.0:8080. SESSION_STORE=postgres соответствует реализации; CACHE_URL_FILE не используется. IDEMPOTENCY_TTL_SECONDS=86400 — минимальное обещание, ключи сохраняются дольше. MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS реализованы константами 10485760/40000000. API_REPLICAS/WORKER_CONCURRENCY — справочные поля skeleton, фактически две API и один worker с prefetch=1; менять число нужно в Compose. JOB_TIMEOUT_SECONDS/STAGING_RETENTION_SECONDS не реализованы и не заявляются.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Структура | `py scripts/check_contract.py` | PASS | evidence/check.txt |
|
||||
| Публичный smoke, прежний API, upload 202, original/thumbnail | `py tests/smoke.py --base-url http://localhost:8085` | PASS полного сценария lab5 | evidence/smoke.txt |
|
||||
| Отдельные broker/worker, volumes | `docker compose ps -a`, compose.yaml | Отдельные контейнеры и volumes DB/S3/broker | evidence/runtime.json, compose.yaml |
|
||||
| Worker остановлен, API не делает resize | `py scripts/verify-lab5.py` | 202 queued, content 409; сообщение с вложением и обычное сообщение созданы | evidence/pipeline.json |
|
||||
| API/broker restart без потери принятой работы | Тот же сценарий, dispatcher остановлен перед restart | StartedAt api1/api2/broker изменились; 1 сообщение до и после; тот же attachment ready после start worker | evidence/pipeline.json |
|
||||
| Оригинал и настоящий thumbnail | Тот же сценарий | Original bytes неизменны, JPEG 256×160; ready через 1,923 с после start | evidence/pipeline.json |
|
||||
| Повторная доставка | Пять повторных публикаций той же job | Метаданные/thumbnail неизменны, attempts=1, ровно два конечных объекта | evidence/pipeline.json |
|
||||
| Retry и безопасный failed | У нового тестового вложения задан отсутствующий staging key | Ровно 5 попыток, failed/retry_exhausted; content и прикрепление 409, чужой GET 404 | evidence/pipeline.json |
|
||||
| Persistence прежних данных и сессий | `py scripts/verify-previous.py http://localhost:8085` | Результат публичного persistence и реальные времена рестарта всех сервисов | evidence/persistence.txt, evidence/persistence-runtime.json |
|
||||
| Отказ API-реплики | Та же команда, публичный failover | Два PASS: по 60 успешных записей, 0 ошибок; восстановление за 1,388 / 1,154 с | evidence/failover.txt |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Проверка функциональная, уровень 3: один fixture PNG 320×200, один worker, prefetch=1, max_attempts=5, две API. Docker Engine доступно 8 CPU и 8266620928 байт RAM; лимит worker — 1 CPU / 768 MiB. Нагрузочный трек, RPS, p95 и RSS не заявляются. Версии/лимиты Docker и времена старта — evidence/runtime.json. Все пользователи и файлы синтетические; токены в памяти клиента и не сохраняются.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Воздействие | Ожидаем | Что наблюдали |
|
||||
| --- | --- | --- |
|
||||
| Stop worker → upload | Durable queued, работоспособный HTTP API | 202, content 409, сообщения доступны |
|
||||
| Stop dispatcher → restart API/broker | Сохранность очереди без перепубликации | 1 → 1 сообщение, StartedAt изменились |
|
||||
| Start worker | То же вложение ready | 1,923 с, original без потери, thumbnail доступен |
|
||||
| Пять дублей успешной job | No-op и ack | attempts=1, один набор вариантов |
|
||||
| Недоступный исходник | Ограниченный retry | 5 попыток, failed/retry_exhausted |
|
||||
| Restart всех сервисов с volumes | Сессии, сообщения и файлы сохраняются | Публичный persistence; подробности в журнале |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Заявлен только уровень 3. Crop/WebP, DLQ/replay, timeout/lease живого зависшего worker, backoff с jitter, автоматический cleanup staging, исследования concurrency и versioned pipeline не реализованы. Session lock снимается при смерти соединения, но не ограничивает время живого зависшего процесса.
|
||||
|
||||
Staging сохраняется после ready/failed; retention отсутствует. Сбой до 202 может оставить непривязанный staging object. Для учебного сброса используется down -v; подтверждённая 202-задача уже имеет устойчивый DB-журнал. Данные ЛР4 не импортировались: перенесён код, ЛР5 использует новую БД/bucket.
|
||||
|
||||
DB/S3/broker/LB одноузловые; потеря volumes/диска не покрывается. HTTP допустим только для локального стенда с синтетическими данными. Нет ограничения backlog и production-наблюдаемости. At-least-once допускает повторы; идемпотентность эффекта не означает exactly-once доставку. Дальнейшие улучшения: лимит очереди, timeout/lease, cleanup staging и fault-injection после S3 PUT до COMMIT/ack.
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Persistent, confirmed RabbitMQ messages contain identifiers only."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pika
|
||||
|
||||
QUEUE = 'attachments.v1'
|
||||
|
||||
def connect():
|
||||
url = Path(os.environ['BROKER_URL_FILE']).read_text().strip()
|
||||
params = pika.URLParameters(url)
|
||||
params.heartbeat = 0 # one bounded image at a time; connection failure causes redelivery
|
||||
params.socket_timeout = 5
|
||||
params.blocked_connection_timeout = 10
|
||||
connection = pika.BlockingConnection(params)
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True)
|
||||
channel.confirm_delivery()
|
||||
return connection, channel
|
||||
@@ -0,0 +1,26 @@
|
||||
import hashlib
|
||||
from fastapi import Header, HTTPException
|
||||
from app.storage import db
|
||||
|
||||
BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
|
||||
|
||||
def unauthorized(basic=False):
|
||||
raise HTTPException(status_code=401, detail="Unauthorized", headers=BASIC_CHALLENGE if basic else BEARER_CHALLENGE)
|
||||
|
||||
|
||||
def current_user(authorization: str | None = Header(default=None)):
|
||||
if not authorization or not authorization.startswith("Bearer "):
|
||||
unauthorized()
|
||||
token = authorization[7:]
|
||||
if not token or " " in token:
|
||||
unauthorized()
|
||||
with db() as conn:
|
||||
user = conn.execute("""SELECT u.id,u.username,u.display_name,u.created_at FROM sessions s
|
||||
JOIN users u ON u.id=s.user_id
|
||||
WHERE s.token_hash=%s AND s.expires_at > now()""",
|
||||
(hashlib.sha256(token.encode()).hexdigest(),)).fetchone()
|
||||
if user is None:
|
||||
unauthorized()
|
||||
return user
|
||||
@@ -0,0 +1,46 @@
|
||||
"""DB-backed delivery intent, periodically republished until a terminal state.
|
||||
|
||||
Publishing and DB commit are deliberately repeatable: confirmed publish followed
|
||||
by a crash may duplicate a job, never lose an acknowledged upload.
|
||||
"""
|
||||
import json
|
||||
import time
|
||||
import pika
|
||||
from app.broker import connect, QUEUE
|
||||
from app.storage import db
|
||||
from app.models.time_format import utc_z
|
||||
|
||||
def publish_due(channel):
|
||||
with db() as conn:
|
||||
jobs = conn.execute('''SELECT j.* FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id=j.attachment_id
|
||||
WHERE a.status IN ('queued','processing') AND j.next_publish<=now()
|
||||
ORDER BY j.next_publish LIMIT 20 FOR UPDATE OF j SKIP LOCKED''').fetchall()
|
||||
for job in jobs:
|
||||
body = dict(schema_version=1, job_id=str(job['job_id']),
|
||||
type='attachment.process.v1', attachment_id=str(job['attachment_id']),
|
||||
pipeline_version=job['pipeline_version'], attempt=job['attempts']+1,
|
||||
created_at=utc_z(job['created_at']))
|
||||
channel.basic_publish(exchange='', routing_key=QUEUE, body=json.dumps(body),
|
||||
properties=pika.BasicProperties(delivery_mode=2, content_type='application/json',
|
||||
message_id=str(job['job_id'])), mandatory=True)
|
||||
conn.execute("UPDATE attachment_jobs SET next_publish=now()+interval '10 seconds' WHERE job_id=%s",
|
||||
(job['job_id'],))
|
||||
|
||||
def main():
|
||||
while True:
|
||||
connection = None
|
||||
try:
|
||||
connection, channel = connect()
|
||||
while True:
|
||||
publish_due(channel)
|
||||
connection.process_data_events(time_limit=1)
|
||||
except Exception as exc:
|
||||
print('dispatcher reconnect:', type(exc).__name__, flush=True)
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if connection and connection.is_open:
|
||||
connection.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
"""Replicated API. Schema migration is a separate one-shot Compose service."""
|
||||
import os
|
||||
from uuid import uuid4
|
||||
from fastapi import FastAPI, Request, HTTPException
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from psycopg import OperationalError
|
||||
from app.routers import users, chats, messages, attachments
|
||||
from app.s3 import s3_client,bucket
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from app.storage import db
|
||||
|
||||
INSTANCE_ID = os.environ.get('API_INSTANCE_ID') or str(uuid4())
|
||||
app = FastAPI()
|
||||
|
||||
|
||||
def error_code(status):
|
||||
return {400:'invalid_request',401:'unauthorized',404:'not_found',409:'conflict',413:'payload_too_large',415:'unsupported_media_type',429:'rate_limited',503:'unavailable'}.get(status,'internal_error')
|
||||
|
||||
|
||||
def error_response(request,status,message,headers=None):
|
||||
result=JSONResponse(status_code=status,content={'error':{
|
||||
'code':error_code(status),'message':message,'request_id':request.state.request_id}},headers=headers)
|
||||
result.headers['X-Request-Id']=request.state.request_id
|
||||
result.headers['X-Instance-Id']=INSTANCE_ID
|
||||
return result
|
||||
|
||||
|
||||
@app.middleware('http')
|
||||
async def add_request_id(request:Request,call_next):
|
||||
request.state.request_id=str(uuid4())
|
||||
try:
|
||||
response=await call_next(request)
|
||||
except OperationalError:
|
||||
response=error_response(request,503,'Database unavailable')
|
||||
except Exception:
|
||||
response=error_response(request,500,'Internal server error')
|
||||
response.headers['X-Request-Id']=request.state.request_id
|
||||
response.headers['X-Instance-Id']=INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request,exc):
|
||||
return error_response(request,exc.status_code,str(exc.detail),exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request,exc):
|
||||
return error_response(request,400,'Invalid request')
|
||||
|
||||
app.include_router(users.router)
|
||||
app.include_router(chats.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(attachments.router)
|
||||
|
||||
@app.get('/health/live')
|
||||
def live():
|
||||
return {'status':'ok'}
|
||||
|
||||
@app.get('/health/ready')
|
||||
def ready():
|
||||
with db() as conn:
|
||||
conn.execute('SELECT 1')
|
||||
try:
|
||||
s3_client().head_bucket(Bucket=bucket())
|
||||
except (ClientError,BotoCoreError,OSError):
|
||||
raise HTTPException(503,'Storage unavailable')
|
||||
return {'status':'ok'}
|
||||
@@ -0,0 +1,5 @@
|
||||
"""One-shot schema migration (never run from individual API replicas)."""
|
||||
from app.storage import migrate
|
||||
if __name__ == '__main__':
|
||||
migrate()
|
||||
print('Migrations completed')
|
||||
@@ -0,0 +1,39 @@
|
||||
from datetime import datetime
|
||||
from app.models.time_format import utc_z
|
||||
from uuid import UUID
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, field_serializer
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def title_must_not_be_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("title must not be blank")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def member_ids_must_be_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("member_ids must be unique")
|
||||
return value
|
||||
|
||||
class Chat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: list[UUID]
|
||||
created_at: datetime
|
||||
|
||||
@field_serializer("created_at", when_used="json")
|
||||
def serialize_created_at(self, value: datetime) -> str:
|
||||
return utc_z(value)
|
||||
|
||||
class ChatPage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
items: list[Chat]
|
||||
next_cursor: str | None
|
||||
@@ -0,0 +1,35 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, field_serializer, model_validator
|
||||
from app.models.time_format import utc_z
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[UUID] = Field(default_factory=list,max_length=10)
|
||||
|
||||
@model_validator(mode='after')
|
||||
def valid_content(self):
|
||||
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||
raise ValueError('duplicate attachments')
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError('message must contain text or attachments')
|
||||
return self
|
||||
|
||||
class Message(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: list[UUID]
|
||||
|
||||
@field_serializer('created_at', when_used='json')
|
||||
def serialize_created_at(self, value: datetime) -> str:
|
||||
return utc_z(value)
|
||||
|
||||
class MessagePage(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
items: list[Message]
|
||||
next_cursor: str | None
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Contract requires RFC3339 UTC timestamps with a literal Z suffix."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def utc_z(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
# Database columns are TIMESTAMPTZ, but keep the serializer defensive.
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')
|
||||
@@ -0,0 +1,28 @@
|
||||
from datetime import datetime
|
||||
from app.models.time_format import utc_z
|
||||
from uuid import UUID
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, field_serializer
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def nonempty(cls, value):
|
||||
if not value.strip():
|
||||
raise ValueError("Blank display name")
|
||||
return value
|
||||
|
||||
class User(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
@field_serializer("created_at", when_used="json")
|
||||
def serialize_created_at(self, value: datetime) -> str:
|
||||
return utc_z(value)
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Bounded image uploads, authorized streaming downloads and private object storage."""
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from uuid import UUID, uuid4
|
||||
from io import BytesIO
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from PIL import Image, UnidentifiedImageError, ImageFile
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from app.dependencies import current_user
|
||||
from app.routers.chats import get_chat_row
|
||||
from app.storage import db
|
||||
from app.s3 import s3_client, bucket
|
||||
from app.models.time_format import utc_z
|
||||
from pydantic import BaseModel, ConfigDict, field_serializer
|
||||
from datetime import datetime
|
||||
|
||||
router=APIRouter()
|
||||
MAX_BYTES=10*1024*1024
|
||||
MAX_PIXELS=40_000_000
|
||||
CHUNK=64*1024
|
||||
Image.MAX_IMAGE_PIXELS=MAX_PIXELS
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES=False
|
||||
|
||||
class Attachment(BaseModel):
|
||||
model_config=ConfigDict(extra='forbid')
|
||||
id:UUID
|
||||
chat_id:UUID
|
||||
owner_id:UUID
|
||||
filename:str
|
||||
content_type:str
|
||||
size_bytes:int
|
||||
sha256:str
|
||||
status:str
|
||||
variants:list=[]
|
||||
error_code:str|None=None
|
||||
created_at:datetime
|
||||
updated_at:datetime
|
||||
@field_serializer('created_at','updated_at',when_used='json')
|
||||
def as_utc(self,value:datetime)->str:
|
||||
return utc_z(value)
|
||||
|
||||
def public(row):
|
||||
return {k:row[k] for k in ('id','chat_id','owner_id','filename','content_type','size_bytes',
|
||||
'sha256','status','created_at','updated_at')} | {'variants':row.get('variants',[]),'error_code':row.get('error_code')}
|
||||
|
||||
def check_name(name):
|
||||
if not 1<=len(name)<=255 or any(ch in '/\\' or ord(ch)<32 or 127<=ord(ch)<160 or unicodedata.category(ch)=='Cc' for ch in name):
|
||||
raise HTTPException(400,'Invalid filename')
|
||||
|
||||
def inspect_image(file, declared):
|
||||
file.seek(0)
|
||||
try:
|
||||
with Image.open(file) as img:
|
||||
fmt=img.format
|
||||
w,h=img.size
|
||||
if w<=0 or h<=0 or w*h>MAX_PIXELS:
|
||||
raise HTTPException(413,'Image pixel count exceeded')
|
||||
expected={'PNG':'image/png','JPEG':'image/jpeg'}.get(fmt)
|
||||
if not expected or expected!=declared:
|
||||
raise HTTPException(415,'Image content type mismatch')
|
||||
img.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file) as img:
|
||||
img.load() # reject truncated PNG/JPEG and decompression errors
|
||||
except HTTPException:
|
||||
raise
|
||||
except (UnidentifiedImageError, ValueError, OSError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise HTTPException(415,'Invalid image')
|
||||
finally:
|
||||
file.seek(0)
|
||||
|
||||
def put_original(file, key, content_type):
|
||||
client=s3_client()
|
||||
file.seek(0)
|
||||
client.upload_fileobj(file,bucket(),key,ExtraArgs={'ContentType':content_type},
|
||||
Config=__import__('boto3').s3.transfer.TransferConfig(
|
||||
multipart_threshold=16*1024*1024,use_threads=False))
|
||||
|
||||
def delete_original(key):
|
||||
s3_client().delete_object(Bucket=bucket(),Key=key)
|
||||
|
||||
@router.post('/api/v1/chats/{chat_id}/attachments',response_model=Attachment,status_code=202)
|
||||
async def upload(chat_id:UUID,request:Request, filename:str=Query(...),user=Depends(current_user)):
|
||||
check_name(filename)
|
||||
declared=request.headers.get('content-type','').lower()
|
||||
if declared not in ('image/png','image/jpeg'):
|
||||
raise HTTPException(415,'Only PNG and JPEG are supported')
|
||||
with db() as conn:
|
||||
if get_chat_row(conn,chat_id,user['id']) is None:
|
||||
raise HTTPException(404,'Chat not found')
|
||||
length=request.headers.get('content-length')
|
||||
if length is not None:
|
||||
try:
|
||||
if int(length)>MAX_BYTES:raise HTTPException(413,'Upload too large')
|
||||
except ValueError:
|
||||
raise HTTPException(400,'Invalid content length')
|
||||
size=0
|
||||
digest=hashlib.sha256()
|
||||
# Disk-backed temporary spool; no API replica retains permanent content.
|
||||
with tempfile.SpooledTemporaryFile(max_size=1024*1024,mode='w+b') as file:
|
||||
async for part in request.stream():
|
||||
size+=len(part)
|
||||
if size>MAX_BYTES:
|
||||
raise HTTPException(413,'Upload too large')
|
||||
digest.update(part)
|
||||
file.write(part)
|
||||
if not size:raise HTTPException(415,'Empty image')
|
||||
await run_in_threadpool(inspect_image,file,declared)
|
||||
ident=uuid4()
|
||||
key='staging/'+str(ident)
|
||||
try:
|
||||
await run_in_threadpool(put_original,file,key,declared)
|
||||
except (ClientError,BotoCoreError,OSError):
|
||||
# A partially written object is possible after a transport error.
|
||||
try:await run_in_threadpool(delete_original,key)
|
||||
except Exception:pass # periodic reconciler can remove an orphan after grace
|
||||
raise HTTPException(503,'Storage unavailable')
|
||||
try:
|
||||
with db() as conn:
|
||||
if get_chat_row(conn,chat_id,user['id']) is None:
|
||||
raise HTTPException(404,'Chat not found')
|
||||
row=conn.execute('''INSERT INTO attachments
|
||||
(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *''',
|
||||
(ident,chat_id,user['id'],filename,declared,size,digest.hexdigest(),key)).fetchone()
|
||||
conn.execute('INSERT INTO attachment_jobs(job_id,attachment_id) VALUES (%s,%s)',
|
||||
(uuid4(),ident))
|
||||
except Exception:
|
||||
try:await run_in_threadpool(delete_original,key)
|
||||
except Exception:pass
|
||||
raise
|
||||
return public(row)
|
||||
|
||||
def authorized(ident,user):
|
||||
with db() as conn:
|
||||
row=conn.execute('''SELECT a.* FROM attachments a
|
||||
JOIN chat_members cm ON cm.chat_id=a.chat_id AND cm.user_id=%s
|
||||
WHERE a.id=%s''',(user['id'],ident)).fetchone()
|
||||
if row is None:raise HTTPException(404,'Attachment not found')
|
||||
return row
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}',response_model=Attachment)
|
||||
def get_attachment(attachment_id:UUID,user=Depends(current_user)):
|
||||
return public(authorized(attachment_id,user))
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}/content')
|
||||
def download(attachment_id:UUID,variant:str='original',user=Depends(current_user)):
|
||||
row=authorized(attachment_id,user)
|
||||
if row['status']!='ready':raise HTTPException(409,'Attachment is not ready')
|
||||
content_type=row['content_type']
|
||||
size=row['size_bytes']
|
||||
key=row['object_key']
|
||||
if variant!='original':
|
||||
found=next((v for v in row['variants'] if v['name']==variant),None)
|
||||
if found is None:raise HTTPException(404,'Variant not found')
|
||||
key=f"results/{attachment_id}/v1/{variant}"
|
||||
content_type=found['content_type']
|
||||
size=found['size_bytes']
|
||||
try:
|
||||
obj=s3_client().get_object(Bucket=bucket(),Key=key)
|
||||
except (ClientError,BotoCoreError,OSError):
|
||||
raise HTTPException(503,'Storage unavailable')
|
||||
body=obj['Body']
|
||||
def chunks():
|
||||
try:
|
||||
for chunk in body.iter_chunks(chunk_size=CHUNK):
|
||||
if chunk:yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
return StreamingResponse(chunks(),media_type=content_type,headers={
|
||||
'Content-Length':str(size),
|
||||
'Content-Disposition':'attachment',
|
||||
'X-Content-Type-Options':'nosniff'})
|
||||
@@ -0,0 +1,52 @@
|
||||
from uuid import UUID, uuid4
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from app.dependencies import current_user
|
||||
from app.models.chats import Chat, ChatPage, CreateChat
|
||||
from app.storage import db
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def get_chat_row(conn, chat_id, user_id):
|
||||
return conn.execute("""SELECT c.id,c.title,c.created_at FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id=c.id WHERE c.id=%s AND cm.user_id=%s""",
|
||||
(chat_id,user_id)).fetchone()
|
||||
|
||||
|
||||
def chat_output(conn,row):
|
||||
members=conn.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id',(row['id'],)).fetchall()
|
||||
return {**row, 'member_ids':[m['user_id'] for m in members]}
|
||||
|
||||
|
||||
@router.post('/api/v1/chats', response_model=Chat, status_code=201)
|
||||
def create_chat(payload:CreateChat,user=Depends(current_user)):
|
||||
if user['id'] not in payload.member_ids:
|
||||
raise HTTPException(status_code=400,detail='Creator must be a chat member')
|
||||
with db() as conn:
|
||||
rows=conn.execute('SELECT id FROM users WHERE id = ANY(%s)',(payload.member_ids,)).fetchall()
|
||||
if len(rows)!=len(payload.member_ids):
|
||||
raise HTTPException(status_code=400,detail='Unknown member')
|
||||
chat=conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at',
|
||||
(uuid4(),payload.title)).fetchone()
|
||||
for member in payload.member_ids:
|
||||
conn.execute('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)',(chat['id'],member))
|
||||
return chat_output(conn,chat)
|
||||
|
||||
|
||||
@router.get('/api/v1/chats',response_model=ChatPage)
|
||||
def list_chats(limit:int=Query(default=50,ge=1,le=100),user=Depends(current_user)):
|
||||
with db() as conn:
|
||||
rows=conn.execute("""SELECT c.id,c.title,c.created_at FROM chats c
|
||||
JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s
|
||||
ORDER BY c.created_at,c.id LIMIT %s""",(user['id'],limit+1)).fetchall()
|
||||
items=[chat_output(conn,row) for row in rows[:limit]]
|
||||
return {'items':items,'next_cursor':None}
|
||||
|
||||
|
||||
@router.get('/api/v1/chats/{chat_id}',response_model=Chat)
|
||||
def get_chat(chat_id:UUID,user=Depends(current_user)):
|
||||
with db() as conn:
|
||||
row=get_chat_row(conn,chat_id,user['id'])
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404,detail='Chat not found')
|
||||
return chat_output(conn,row)
|
||||
@@ -0,0 +1,115 @@
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import datetime
|
||||
from uuid import UUID,uuid4
|
||||
from fastapi import APIRouter,Depends,HTTPException,Query,Header
|
||||
from app.dependencies import current_user
|
||||
from app.models.messages import CreateMessage,Message,MessagePage
|
||||
from app.routers.chats import get_chat_row
|
||||
from app.storage import db
|
||||
|
||||
router=APIRouter()
|
||||
|
||||
MESSAGE_SELECT = """SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,
|
||||
COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position)
|
||||
FROM message_attachments ma WHERE ma.message_id=m.id), ARRAY[]::uuid[]) AS attachment_ids
|
||||
FROM messages m"""
|
||||
|
||||
|
||||
|
||||
def encode_cursor(chat_id,row):
|
||||
raw=json.dumps({'chat_id':str(chat_id),'created_at':row['created_at'].isoformat(),'id':str(row['id'])}).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
|
||||
|
||||
|
||||
def decode_cursor(chat_id,cursor):
|
||||
try:
|
||||
raw=base64.b64decode(cursor+'='*(-len(cursor)%4),altchars=b'-_',validate=True)
|
||||
data=json.loads(raw)
|
||||
ts=datetime.fromisoformat(data['created_at'])
|
||||
ident=UUID(data['id'])
|
||||
if data['chat_id']!=str(chat_id) or ts.tzinfo is None:
|
||||
raise ValueError('Wrong cursor')
|
||||
return ts,ident
|
||||
except (ValueError,KeyError,TypeError,UnicodeDecodeError,binascii.Error):
|
||||
raise HTTPException(status_code=400,detail='Invalid cursor')
|
||||
|
||||
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r'^[A-Za-z0-9._:-]{1,128}$', re.ASCII)
|
||||
|
||||
|
||||
def canonical_message_hash(payload):
|
||||
# Canonical semantics, not raw JSON bytes or field order.
|
||||
data = payload.model_dump(mode='json', exclude_none=True)
|
||||
data.setdefault('attachment_ids', [])
|
||||
return hashlib.sha256(json.dumps(data, sort_keys=True, separators=(',', ':'),
|
||||
ensure_ascii=False).encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def advisory_lock_id(user_id,chat_id,key):
|
||||
scope = f'{user_id}:{chat_id}:{key}'.encode('utf-8')
|
||||
return int.from_bytes(hashlib.sha256(scope).digest()[:8], 'big', signed=True)
|
||||
|
||||
|
||||
@router.post('/api/v1/chats/{chat_id}/messages',response_model=Message,status_code=201)
|
||||
def create_message(chat_id:UUID,payload:CreateMessage,user=Depends(current_user),
|
||||
idempotency_key:str|None=Header(default=None,alias='Idempotency-Key')):
|
||||
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||
raise HTTPException(status_code=400,detail='Invalid or missing Idempotency-Key')
|
||||
request_hash=canonical_message_hash(payload)
|
||||
with db() as conn:
|
||||
# Explicit authorization is checked on EVERY replay, not just initial POST.
|
||||
if get_chat_row(conn,chat_id,user['id']) is None:
|
||||
raise HTTPException(status_code=404,detail='Chat not found')
|
||||
# Same key on different processes serializes in PostgreSQL. The message and
|
||||
# key record commit together, so a network timeout cannot create duplicates.
|
||||
conn.execute('SELECT pg_advisory_xact_lock(%s)',
|
||||
(advisory_lock_id(user['id'],chat_id,idempotency_key),))
|
||||
saved=conn.execute("""SELECT request_hash,message_id,created_at FROM message_idempotency
|
||||
WHERE user_id=%s AND chat_id=%s AND idem_key=%s FOR UPDATE""",
|
||||
(user['id'],chat_id,idempotency_key)).fetchone()
|
||||
if saved is not None:
|
||||
# Contract promises a replay window of at least 24 hours; retaining
|
||||
# keys longer is safe and prevents late duplicate deliveries.
|
||||
if saved['request_hash'] != request_hash:
|
||||
raise HTTPException(status_code=409,detail='Idempotency-Key used with another body')
|
||||
return conn.execute(MESSAGE_SELECT + """ WHERE m.id=%s""",(saved['message_id'],)).fetchone()
|
||||
if payload.attachment_ids:
|
||||
authorized=conn.execute("""SELECT id,status FROM attachments
|
||||
WHERE chat_id=%s AND id=ANY(%s) FOR SHARE""",
|
||||
(chat_id,payload.attachment_ids)).fetchall()
|
||||
if len(authorized)!=len(payload.attachment_ids):
|
||||
raise HTTPException(status_code=404,detail='Attachment not found')
|
||||
if any(a['status']=='failed' for a in authorized):
|
||||
raise HTTPException(status_code=409,detail='Attachment failed')
|
||||
row=conn.execute("""INSERT INTO messages(id,chat_id,sender_id,text)
|
||||
VALUES (%s,%s,%s,%s) RETURNING id,chat_id,sender_id,text,created_at""",
|
||||
(uuid4(),chat_id,user['id'],payload.text)).fetchone()
|
||||
for pos,attachment_id in enumerate(payload.attachment_ids):
|
||||
conn.execute("""INSERT INTO message_attachments(message_id,attachment_id,position)
|
||||
VALUES (%s,%s,%s)""",(row['id'],attachment_id,pos))
|
||||
row['attachment_ids']=payload.attachment_ids
|
||||
conn.execute("""INSERT INTO message_idempotency(user_id,chat_id,idem_key,request_hash,message_id)
|
||||
VALUES (%s,%s,%s,%s,%s)""",
|
||||
(user['id'],chat_id,idempotency_key,request_hash,row['id']))
|
||||
return row
|
||||
|
||||
|
||||
@router.get('/api/v1/chats/{chat_id}/messages',response_model=MessagePage)
|
||||
def list_messages(chat_id:UUID,limit:int=Query(default=50,ge=1,le=100),
|
||||
cursor:str|None=Query(default=None,min_length=1),user=Depends(current_user)):
|
||||
with db() as conn:
|
||||
if get_chat_row(conn,chat_id,user['id']) is None:
|
||||
raise HTTPException(status_code=404,detail='Chat not found')
|
||||
if cursor is not None:
|
||||
ts,ident=decode_cursor(chat_id,cursor)
|
||||
rows=conn.execute(MESSAGE_SELECT + """ WHERE m.chat_id=%s AND (m.created_at,m.id)>(%s,%s)
|
||||
ORDER BY m.created_at,m.id LIMIT %s""",(chat_id,ts,ident,limit+1)).fetchall()
|
||||
else:
|
||||
rows=conn.execute(MESSAGE_SELECT + """ WHERE m.chat_id=%s ORDER BY m.created_at,m.id LIMIT %s""",(chat_id,limit+1)).fetchall()
|
||||
items=rows[:limit]
|
||||
next_cursor=encode_cursor(chat_id,items[-1]) if len(rows)>limit else None
|
||||
return {'items':items,'next_cursor':next_cursor}
|
||||
@@ -0,0 +1,78 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Response
|
||||
from psycopg.errors import UniqueViolation
|
||||
from app.dependencies import current_user, unauthorized
|
||||
from app.models.users import CreateUser, User
|
||||
from app.models.time_format import utc_z
|
||||
from app.storage import db
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def password_hash(password):
|
||||
salt = os.urandom(16)
|
||||
derived = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=2**14, r=8, p=1, dklen=32)
|
||||
return 'scrypt$' + salt.hex() + '$' + derived.hex()
|
||||
|
||||
|
||||
def verify_password(password, encoded):
|
||||
try:
|
||||
_, salt_hex, expected_hex = encoded.split('$')
|
||||
actual = hashlib.scrypt(password.encode('utf-8'),salt=bytes.fromhex(salt_hex),n=2**14,r=8,p=1,dklen=32)
|
||||
return hmac.compare_digest(actual, bytes.fromhex(expected_hex))
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
@router.post('/api/v1/users', response_model=User, status_code=201)
|
||||
def register_user(payload: CreateUser):
|
||||
with db() as conn:
|
||||
try:
|
||||
user = conn.execute("""INSERT INTO users(id,username,display_name,password_hash)
|
||||
VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at""",
|
||||
(uuid4(),payload.username,payload.display_name,password_hash(payload.password))).fetchone()
|
||||
except UniqueViolation:
|
||||
raise HTTPException(status_code=409,detail='Username already exists')
|
||||
return user
|
||||
|
||||
|
||||
@router.get('/api/v1/users/me', response_model=User)
|
||||
def get_current_user(user=Depends(current_user)):
|
||||
return user
|
||||
|
||||
|
||||
@router.post('/api/v1/auth/sessions', status_code=201)
|
||||
def create_session(authorization: str | None = Header(default=None)):
|
||||
if not authorization or not authorization.startswith('Basic '):
|
||||
unauthorized(basic=True)
|
||||
try:
|
||||
raw = base64.b64decode(authorization[6:],validate=True).decode('utf-8')
|
||||
username, password = raw.split(':',1)
|
||||
except (ValueError, UnicodeDecodeError, binascii.Error):
|
||||
unauthorized(basic=True)
|
||||
with db() as conn:
|
||||
row = conn.execute('SELECT * FROM users WHERE username=%s',(username,)).fetchone()
|
||||
if row is None or not verify_password(password,row['password_hash']):
|
||||
unauthorized(basic=True)
|
||||
token = secrets.token_urlsafe(48)
|
||||
ttl = int(os.getenv('SESSION_TTL_SECONDS','3600'))
|
||||
expires = datetime.now(timezone.utc)+timedelta(seconds=ttl)
|
||||
conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)',
|
||||
(hashlib.sha256(token.encode()).hexdigest(),row['id'],expires))
|
||||
return {'token':token,'token_type':'Bearer','expires_at':utc_z(expires),
|
||||
'user':User.model_validate({key:row[key] for key in ('id','username','display_name','created_at')})}
|
||||
|
||||
|
||||
@router.delete('/api/v1/auth/sessions/current',status_code=204)
|
||||
def logout(authorization: str | None = Header(default=None),user=Depends(current_user)):
|
||||
token=authorization[7:]
|
||||
with db() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE token_hash=%s',(hashlib.sha256(token.encode()).hexdigest(),))
|
||||
return Response(status_code=204)
|
||||
@@ -0,0 +1,19 @@
|
||||
"""S3 access with least-privilege application credentials (never root credentials)."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
|
||||
def s3_client():
|
||||
return boto3.client(
|
||||
's3', endpoint_url=os.environ['S3_ENDPOINT'],
|
||||
region_name=os.getenv('S3_REGION','us-east-1'),
|
||||
aws_access_key_id=Path(os.environ['S3_ACCESS_KEY_ID_FILE']).read_text().strip(),
|
||||
aws_secret_access_key=Path(os.environ['S3_SECRET_ACCESS_KEY_FILE']).read_text().strip(),
|
||||
config=Config(connect_timeout=2,read_timeout=15,retries={'max_attempts':2},
|
||||
s3={'addressing_style':'path'}))
|
||||
|
||||
|
||||
def bucket():
|
||||
return os.getenv('S3_BUCKET','messenger-attachments')
|
||||
@@ -0,0 +1,31 @@
|
||||
"""PostgreSQL is the source of truth, including bearer sessions."""
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
|
||||
def database_url():
|
||||
secret_file = os.getenv("DATABASE_URL_FILE")
|
||||
if secret_file:
|
||||
return Path(secret_file).read_text().strip()
|
||||
return os.environ["DATABASE_URL"]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def db():
|
||||
with psycopg.connect(database_url(), row_factory=dict_row, connect_timeout=5) as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
def migrate():
|
||||
with db() as conn:
|
||||
# Serialize migration execution even if two operators race to bootstrap.
|
||||
conn.execute("SELECT pg_advisory_xact_lock(83475391)")
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)")
|
||||
for file in sorted((Path(__file__).parent.parent / "migrations").glob("*.sql")):
|
||||
if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (file.name,)).fetchone():
|
||||
continue
|
||||
conn.execute(file.read_text())
|
||||
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (file.name,))
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
"""At-least-once consumer; PostgreSQL session lock serializes each logical effect."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from io import BytesIO
|
||||
from uuid import UUID
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
from psycopg.types.json import Jsonb
|
||||
from app.broker import connect, QUEUE
|
||||
from app.s3 import s3_client, bucket
|
||||
from app.storage import db
|
||||
|
||||
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
|
||||
def render(raw):
|
||||
with Image.open(BytesIO(raw)) as source:
|
||||
if source.width * source.height > 40_000_000:
|
||||
raise ValueError('pixel limit')
|
||||
image = ImageOps.exif_transpose(source).convert('RGBA')
|
||||
scale = min(1, 256/image.width, 256/image.height)
|
||||
dimensions = (max(1, int(image.width*scale)), max(1, int(image.height*scale)))
|
||||
image = image.resize(dimensions, Image.Resampling.LANCZOS)
|
||||
white = Image.new('RGB', dimensions, 'white')
|
||||
white.paste(image, mask=image.getchannel('A'))
|
||||
output = BytesIO()
|
||||
white.save(output, format='JPEG', quality=85)
|
||||
data = output.getvalue()
|
||||
return data, dict(name='thumbnail', content_type='image/jpeg', size_bytes=len(data),
|
||||
width=dimensions[0], height=dimensions[1], sha256=hashlib.sha256(data).hexdigest())
|
||||
|
||||
def process(message):
|
||||
job_id = UUID(message['job_id'])
|
||||
# Stable session advisory lock, released automatically on worker death.
|
||||
lock = int.from_bytes(hashlib.sha256(job_id.bytes).digest()[:8], 'big', signed=True)
|
||||
with db() as conn:
|
||||
if not conn.execute('SELECT pg_try_advisory_lock(%s) AS locked', (lock,)).fetchone()['locked']:
|
||||
return # active owner or DB delivery intent will finish/recover this job
|
||||
conn.commit()
|
||||
row = conn.execute('''SELECT a.*, j.attempts, j.pipeline_version FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s''', (job_id,)).fetchone()
|
||||
if not row or row['status'] in ('ready', 'failed'):
|
||||
return
|
||||
if str(row['id']) != message['attachment_id'] or message['pipeline_version'] != row['pipeline_version']:
|
||||
return
|
||||
if row['attempts'] >= MAX_ATTEMPTS:
|
||||
conn.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s", (row['id'],))
|
||||
return
|
||||
conn.execute('UPDATE attachment_jobs SET attempts=attempts+1 WHERE job_id=%s', (job_id,))
|
||||
conn.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (row['id'],))
|
||||
conn.commit()
|
||||
try:
|
||||
client = s3_client()
|
||||
obj = client.get_object(Bucket=bucket(), Key=row['object_key'])
|
||||
try:
|
||||
raw = obj['Body'].read(10*1024*1024+1)
|
||||
finally:
|
||||
obj['Body'].close()
|
||||
if len(raw) != row['size_bytes'] or hashlib.sha256(raw).hexdigest() != row['sha256']:
|
||||
raise ValueError('checksum mismatch')
|
||||
thumbnail, metadata = render(raw)
|
||||
prefix = f"results/{row['id']}/v1/"
|
||||
client.put_object(Bucket=bucket(), Key=prefix+'original', Body=raw, ContentType=row['content_type'])
|
||||
client.put_object(Bucket=bucket(), Key=prefix+'thumbnail', Body=thumbnail, ContentType='image/jpeg')
|
||||
conn.execute("""UPDATE attachments SET status='ready',object_key=%s,variants=%s,
|
||||
error_code=NULL,updated_at=now() WHERE id=%s""",
|
||||
(prefix+'original', Jsonb([metadata]), row['id']))
|
||||
conn.commit() # consumer ack happens only AFTER this durable result
|
||||
print(f"ready attachment={row['id']} attempt={row['attempts']+1}", flush=True)
|
||||
except Exception as exc:
|
||||
conn.rollback()
|
||||
permanent = isinstance(exc, (ValueError, UnidentifiedImageError, Image.DecompressionBombError))
|
||||
exhausted = row['attempts']+1 >= MAX_ATTEMPTS
|
||||
status = 'failed' if permanent or exhausted else 'queued'
|
||||
code = ('invalid_image' if permanent else 'retry_exhausted') if status=='failed' else None
|
||||
conn.execute('UPDATE attachments SET status=%s,error_code=%s,updated_at=now() WHERE id=%s',
|
||||
(status, code, row['id']))
|
||||
conn.execute("UPDATE attachment_jobs SET next_publish=now()+interval '5 seconds' WHERE job_id=%s", (job_id,))
|
||||
print(f"{status} attachment={row['id']} reason={type(exc).__name__}", flush=True)
|
||||
|
||||
def consume(channel, method, properties, body):
|
||||
try:
|
||||
message = json.loads(body)
|
||||
UUID(message['job_id'])
|
||||
UUID(message['attachment_id'])
|
||||
if message['schema_version'] != 1 or message['type'] != 'attachment.process.v1':
|
||||
raise ValueError('unsupported job')
|
||||
except (ValueError, KeyError, TypeError):
|
||||
channel.basic_reject(method.delivery_tag, requeue=False)
|
||||
return
|
||||
process(message)
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
|
||||
def main():
|
||||
while True:
|
||||
connection = None
|
||||
try:
|
||||
connection, channel = connect()
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
channel.basic_consume(queue=QUEUE, on_message_callback=consume)
|
||||
channel.start_consuming()
|
||||
except Exception as exc:
|
||||
print('worker reconnect:', type(exc).__name__, flush=True)
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if connection and connection.is_open:
|
||||
connection.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
name: mtusi2026lab5
|
||||
|
||||
x-app: &app
|
||||
build: .
|
||||
image: mtusi2026lab5-app:local
|
||||
restart: unless-stopped
|
||||
environment: &app-env
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- broker_url
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-bootstrap: {condition: service_completed_successfully}
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:16.8-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
|
||||
secrets: [db_password]
|
||||
volumes: [db-data:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
s3:
|
||||
image: cgr.dev/chainguard/minio@sha256:fb893e36fc369d41e92f678b17ec27378fa5bf0576aeef7ac15460c13629ab5a
|
||||
restart: unless-stopped
|
||||
command: [server, /data, --console-address, ':9001']
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_access
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_secret
|
||||
secrets: [s3_root_access, s3_root_secret]
|
||||
volumes: [s3-data:/data]
|
||||
s3-bootstrap:
|
||||
image: cgr.dev/chainguard/minio-client@sha256:fe649f71872c299489deb3cb9e057e5d2f76a45c85b7599f7c21616ac80b5e63
|
||||
user: '0:0'
|
||||
entrypoint: [/bin/sh, /etc/bootstrap/bootstrap-s3.sh]
|
||||
depends_on: [s3]
|
||||
secrets: [s3_root_access, s3_root_secret, s3_access_key_id, s3_secret_access_key]
|
||||
volumes:
|
||||
- ./infra/bootstrap-s3.sh:/etc/bootstrap/bootstrap-s3.sh:ro
|
||||
- ./infra/s3-policy.json:/etc/bootstrap/s3-policy.json:ro
|
||||
migrate:
|
||||
<<: *app
|
||||
restart: 'no'
|
||||
command: [python, -m, app.migrate]
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
broker:
|
||||
image: rabbitmq:4.1.0-alpine
|
||||
hostname: broker
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger
|
||||
RABBITMQ_CONFIG_FILE: /run/secrets/rabbitmq
|
||||
secrets:
|
||||
- source: rabbitmq_config
|
||||
target: rabbitmq.conf
|
||||
volumes: [broker-data:/var/lib/rabbitmq]
|
||||
healthcheck:
|
||||
test: [CMD, rabbitmq-diagnostics, -q, ping]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
api1:
|
||||
<<: *app
|
||||
environment:
|
||||
<<: *app-env
|
||||
API_INSTANCE_ID: api1
|
||||
api2:
|
||||
<<: *app
|
||||
environment:
|
||||
<<: *app-env
|
||||
API_INSTANCE_ID: api2
|
||||
dispatcher:
|
||||
<<: *app
|
||||
command: [python, -m, app.dispatcher]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
worker:
|
||||
<<: *app
|
||||
command: [python, -m, app.worker]
|
||||
mem_limit: 768m
|
||||
cpus: 1.0
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-bootstrap: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
lb:
|
||||
image: nginx:1.27.5-alpine
|
||||
restart: unless-stopped
|
||||
ports: ['127.0.0.1:${HTTP_PORT:-8085}:8080']
|
||||
volumes: ['./infra/nginx.conf:/etc/nginx/nginx.conf:ro']
|
||||
depends_on: [api1, api2]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 40
|
||||
|
||||
volumes:
|
||||
db-data:
|
||||
s3-data:
|
||||
broker-data:
|
||||
|
||||
secrets:
|
||||
db_password: {file: ./secrets/db_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
s3_root_access: {file: ./secrets/s3_root_access}
|
||||
s3_root_secret: {file: ./secrets/s3_root_secret}
|
||||
s3_access_key_id: {file: ./secrets/s3_access_key_id}
|
||||
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
|
||||
broker_url: {file: ./secrets/broker_url}
|
||||
rabbitmq_config: {file: ./secrets/rabbitmq_config}
|
||||
@@ -0,0 +1 @@
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,9 @@
|
||||
Run 1
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 60, "failed_attempts": 0, "first_recovery_seconds": 1.388}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
Run 2
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 60, "failed_attempts": 0, "first_recovery_seconds": 1.154}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,96 @@
|
||||
{
|
||||
"before": {
|
||||
"api1": {
|
||||
"id": "612826f4ec33",
|
||||
"started_at": "2026-11-11T16:46:45.590637906Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
},
|
||||
"api2": {
|
||||
"id": "9b599b4254f8",
|
||||
"started_at": "2026-11-11T16:46:45.61923996Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
},
|
||||
"broker": {
|
||||
"id": "20286d777e33",
|
||||
"started_at": "2026-11-11T16:46:46.290586048Z",
|
||||
"image": "rabbitmq:4.1.0-alpine"
|
||||
},
|
||||
"db": {
|
||||
"id": "2b1781c64aa8",
|
||||
"started_at": "2026-11-11T16:39:05.943120027Z",
|
||||
"image": "postgres:16.8-alpine"
|
||||
},
|
||||
"dispatcher": {
|
||||
"id": "0439349883a7",
|
||||
"started_at": "2026-11-11T16:47:01.419374194Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
},
|
||||
"lb": {
|
||||
"id": "480f671d010d",
|
||||
"started_at": "2026-11-11T16:39:17.798819925Z",
|
||||
"image": "nginx:1.27.5-alpine"
|
||||
},
|
||||
"s3": {
|
||||
"id": "812c644b3b10",
|
||||
"started_at": "2026-11-11T16:39:06.338675931Z",
|
||||
"image": "cgr.dev/chainguard/minio@sha256:fb893e36fc369d41e92f678b17ec27378fa5bf0576aeef7ac15460c13629ab5a"
|
||||
},
|
||||
"worker": {
|
||||
"id": "a50091a0b7a8",
|
||||
"started_at": "2026-11-11T16:47:25.220191928Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"api1": {
|
||||
"id": "612826f4ec33",
|
||||
"started_at": "2026-11-11T16:50:07.527403019Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
},
|
||||
"api2": {
|
||||
"id": "9b599b4254f8",
|
||||
"started_at": "2026-11-11T16:50:07.595331041Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
},
|
||||
"broker": {
|
||||
"id": "20286d777e33",
|
||||
"started_at": "2026-11-11T16:50:08.421072775Z",
|
||||
"image": "rabbitmq:4.1.0-alpine"
|
||||
},
|
||||
"db": {
|
||||
"id": "2b1781c64aa8",
|
||||
"started_at": "2026-11-11T16:50:07.211588868Z",
|
||||
"image": "postgres:16.8-alpine"
|
||||
},
|
||||
"dispatcher": {
|
||||
"id": "0439349883a7",
|
||||
"started_at": "2026-11-11T16:50:10.075120403Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
},
|
||||
"lb": {
|
||||
"id": "480f671d010d",
|
||||
"started_at": "2026-11-11T16:50:07.299916262Z",
|
||||
"image": "nginx:1.27.5-alpine"
|
||||
},
|
||||
"s3": {
|
||||
"id": "812c644b3b10",
|
||||
"started_at": "2026-11-11T16:50:07.088841496Z",
|
||||
"image": "cgr.dev/chainguard/minio@sha256:fb893e36fc369d41e92f678b17ec27378fa5bf0576aeef7ac15460c13629ab5a"
|
||||
},
|
||||
"worker": {
|
||||
"id": "a50091a0b7a8",
|
||||
"started_at": "2026-11-11T16:50:10.148806874Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
}
|
||||
},
|
||||
"restarted": {
|
||||
"api1": true,
|
||||
"api2": true,
|
||||
"db": true,
|
||||
"s3": true,
|
||||
"broker": true,
|
||||
"worker": true,
|
||||
"dispatcher": true,
|
||||
"lb": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,46 @@
|
||||
[
|
||||
{
|
||||
"scenario": "stopped_worker_and_api_broker_restart",
|
||||
"attachment_id": "44188ba6-770d-4abc-9f87-c12b6c970953",
|
||||
"message_id": "3cec2bb3-2ead-43a3-8b02-e14e0bb90b9b",
|
||||
"queued_before": 1,
|
||||
"queued_after": 1,
|
||||
"broker_restart_verified": true,
|
||||
"started_before": {
|
||||
"api1": "2026-11-11T16:45:34.793026679Z",
|
||||
"api2": "2026-11-11T16:45:34.851702Z",
|
||||
"broker": "2026-11-11T16:45:35.58734277Z"
|
||||
},
|
||||
"started_after": {
|
||||
"api1": "2026-11-11T16:46:45.590637906Z",
|
||||
"api2": "2026-11-11T16:46:45.61923996Z",
|
||||
"broker": "2026-11-11T16:46:46.290586048Z"
|
||||
},
|
||||
"ready_seconds": 1.923,
|
||||
"thumbnail": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"scenario": "five_duplicate_deliveries",
|
||||
"attachment_id": "44188ba6-770d-4abc-9f87-c12b6c970953",
|
||||
"attempts": 1,
|
||||
"final_objects": [
|
||||
"results/44188ba6-770d-4abc-9f87-c12b6c970953/v1/original",
|
||||
"results/44188ba6-770d-4abc-9f87-c12b6c970953/v1/thumbnail"
|
||||
],
|
||||
"unchanged_metadata": true,
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"scenario": "bounded_retry",
|
||||
"attachment_id": "ae50b181-c8d0-4ac9-bda1-312a1ac7ea18",
|
||||
"attempts": 5,
|
||||
"error_code": "retry_exhausted",
|
||||
"download_status": 409,
|
||||
"attach_status": 409,
|
||||
"result": "PASS"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,47 @@
|
||||
[
|
||||
{
|
||||
"scenario": "stopped_worker_and_api_broker_restart",
|
||||
"attachment_id": "44188ba6-770d-4abc-9f87-c12b6c970953",
|
||||
"message_id": "3cec2bb3-2ead-43a3-8b02-e14e0bb90b9b",
|
||||
"queued_before": 1,
|
||||
"queued_after": 1,
|
||||
"broker_restart_verified": true,
|
||||
"started_before": {
|
||||
"api1": "2026-11-11T16:45:34.793026679Z",
|
||||
"api2": "2026-11-11T16:45:34.851702Z",
|
||||
"broker": "2026-11-11T16:45:35.58734277Z"
|
||||
},
|
||||
"started_after": {
|
||||
"api1": "2026-11-11T16:46:45.590637906Z",
|
||||
"api2": "2026-11-11T16:46:45.61923996Z",
|
||||
"broker": "2026-11-11T16:46:46.290586048Z"
|
||||
},
|
||||
"ready_seconds": 1.923,
|
||||
"thumbnail": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"scenario": "five_duplicate_deliveries",
|
||||
"attachment_id": "44188ba6-770d-4abc-9f87-c12b6c970953",
|
||||
"attempts": 1,
|
||||
"final_objects": [
|
||||
"results/44188ba6-770d-4abc-9f87-c12b6c970953/v1/original",
|
||||
"results/44188ba6-770d-4abc-9f87-c12b6c970953/v1/thumbnail"
|
||||
],
|
||||
"unchanged_metadata": true,
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"scenario": "bounded_retry",
|
||||
"attachment_id": "ae50b181-c8d0-4ac9-bda1-312a1ac7ea18",
|
||||
"attempts": 5,
|
||||
"error_code": "retry_exhausted",
|
||||
"download_status": 409,
|
||||
"attach_status": 409,
|
||||
"result": "PASS"
|
||||
}
|
||||
]
|
||||
PASS: lab5 durable queue, restart, redelivery and retry scenarios
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"python": "3.15.0",
|
||||
"host": "Windows-11-10.0.26200-SP0",
|
||||
"docker": {
|
||||
"Client": {
|
||||
"Version": "29.9.2",
|
||||
"ApiVersion": "1.56",
|
||||
"DefaultAPIVersion": "1.56",
|
||||
"GitCommit": "a82e528",
|
||||
"GoVersion": "go1.26.10",
|
||||
"Os": "windows",
|
||||
"Arch": "amd64",
|
||||
"BuildTime": "Tue Oct 20 19:44:07 2026",
|
||||
"Context": "desktop-linux"
|
||||
},
|
||||
"Server": {
|
||||
"Platform": {
|
||||
"Name": "Docker Desktop 4.97.0 (244584)"
|
||||
},
|
||||
"Version": "29.9.2",
|
||||
"ApiVersion": "1.56",
|
||||
"MinAPIVersion": "1.40",
|
||||
"Os": "linux",
|
||||
"Arch": "amd64",
|
||||
"Components": [
|
||||
{
|
||||
"Name": "Engine",
|
||||
"Version": "29.9.2",
|
||||
"Details": {
|
||||
"ApiVersion": "1.56",
|
||||
"Arch": "amd64",
|
||||
"BuildTime": "Tue Oct 20 18:45:02 2026",
|
||||
"Experimental": "false",
|
||||
"GitCommit": "0367cbe",
|
||||
"GoVersion": "go1.26.10",
|
||||
"KernelVersion": "6.18.58.1-microsoft-standard-WSL2",
|
||||
"MinAPIVersion": "1.40",
|
||||
"Module": "github.com/moby/moby/v2",
|
||||
"ModuleVersion": "v2.0.0+unknown",
|
||||
"Os": "linux"
|
||||
}
|
||||
},
|
||||
{
|
||||
"Name": "containerd",
|
||||
"Version": "v2.4.2",
|
||||
"Details": {
|
||||
"GitCommit": "b7a25261e029d68978bbf290df0a62fc819b3c51"
|
||||
}
|
||||
},
|
||||
{
|
||||
"Name": "runc",
|
||||
"Version": "1.5.2",
|
||||
"Details": {
|
||||
"GitCommit": "v1.5.2-0-g29dd3dc2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"Name": "docker-init",
|
||||
"Version": "0.19.0",
|
||||
"Details": {
|
||||
"GitCommit": "de40ad0"
|
||||
}
|
||||
}
|
||||
],
|
||||
"GitCommit": "0367cbe",
|
||||
"GoVersion": "go1.26.10",
|
||||
"KernelVersion": "6.18.58.1-microsoft-standard-WSL2",
|
||||
"BuildTime": "2026-10-20T18:45:02.000000000+00:00"
|
||||
}
|
||||
},
|
||||
"engine": {
|
||||
"NCPU": 8,
|
||||
"MemTotal": 8266620928,
|
||||
"OSType": "linux",
|
||||
"Architecture": "x86_64",
|
||||
"OperatingSystem": "Docker Desktop"
|
||||
},
|
||||
"containers": {
|
||||
"api1": {
|
||||
"id": "612826f4ec33",
|
||||
"started_at": "2026-11-11T16:51:24.765371526Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
},
|
||||
"api2": {
|
||||
"id": "9b599b4254f8",
|
||||
"started_at": "2026-11-11T16:51:04.198535827Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
},
|
||||
"broker": {
|
||||
"id": "20286d777e33",
|
||||
"started_at": "2026-11-11T16:50:08.421072775Z",
|
||||
"image": "rabbitmq:4.1.0-alpine"
|
||||
},
|
||||
"db": {
|
||||
"id": "2b1781c64aa8",
|
||||
"started_at": "2026-11-11T16:50:07.211588868Z",
|
||||
"image": "postgres:16.8-alpine"
|
||||
},
|
||||
"dispatcher": {
|
||||
"id": "0439349883a7",
|
||||
"started_at": "2026-11-11T16:50:10.075120403Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
},
|
||||
"lb": {
|
||||
"id": "480f671d010d",
|
||||
"started_at": "2026-11-11T16:50:07.299916262Z",
|
||||
"image": "nginx:1.27.5-alpine"
|
||||
},
|
||||
"s3": {
|
||||
"id": "812c644b3b10",
|
||||
"started_at": "2026-11-11T16:50:07.088841496Z",
|
||||
"image": "cgr.dev/chainguard/minio@sha256:fb893e36fc369d41e92f678b17ec27378fa5bf0576aeef7ac15460c13629ab5a"
|
||||
},
|
||||
"worker": {
|
||||
"id": "a50091a0b7a8",
|
||||
"started_at": "2026-11-11T16:50:10.148806874Z",
|
||||
"image": "mtusi2026lab5-app:local"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
read_secret() {
|
||||
tr -d '\r\n' < "/run/secrets/$1"
|
||||
}
|
||||
|
||||
ROOT_ACCESS=$(read_secret s3_root_access)
|
||||
ROOT_SECRET=$(read_secret s3_root_secret)
|
||||
API_ACCESS=$(read_secret s3_access_key_id)
|
||||
API_SECRET=$(read_secret s3_secret_access_key)
|
||||
|
||||
# Bound bootstrap wait: fail with diagnostic rather than stall forever.
|
||||
count=0
|
||||
until output=$(timeout 4 mc alias set local http://s3:9000 "$ROOT_ACCESS" "$ROOT_SECRET" 2>&1) && output=$(timeout 4 mc admin info local 2>&1); do
|
||||
count=$((count + 1))
|
||||
if [ "$count" -ge 12 ]; then
|
||||
echo "ERROR: MinIO did not become reachable after 12 attempts; inspect docker compose logs s3" >&2
|
||||
if [ -n "$output" ]; then printf '%s\n' "$output" >&2; fi
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
mc mb --ignore-existing local/messenger-attachments
|
||||
mc anonymous set none local/messenger-attachments
|
||||
# Repeatable bootstrap: policy may exist already after restarts.
|
||||
mc admin policy create local messenger-app /etc/bootstrap/s3-policy.json
|
||||
mc admin user add local "$API_ACCESS" "$API_SECRET"
|
||||
if ! mc admin policy attach local messenger-app --user "$API_ACCESS"; then
|
||||
mc admin policy entities local messenger-app --user "$API_ACCESS" --json | grep -Fq "\"$API_ACCESS\""
|
||||
fi
|
||||
mc anonymous get local/messenger-attachments
|
||||
mc alias set api http://s3:9000 "$API_ACCESS" "$API_SECRET" >/dev/null
|
||||
mc ls api/messenger-attachments >/dev/null
|
||||
printf 'S3 bootstrap completed (private bucket, limited API identity).\n'
|
||||
@@ -0,0 +1,26 @@
|
||||
worker_processes auto;
|
||||
events { worker_connections 1024; }
|
||||
http {
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
upstream messenger_api {
|
||||
zone messenger_api 64k;
|
||||
server api1:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
server api2:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 11m;
|
||||
location / {
|
||||
proxy_pass http://messenger_api;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 35s;
|
||||
proxy_send_timeout 35s;
|
||||
proxy_next_upstream error timeout http_502 http_503;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/staging/*","arn:aws:s3:::messenger-attachments/results/*"]}]}
|
||||
@@ -0,0 +1,35 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username VARCHAR(32) NOT NULL UNIQUE,
|
||||
display_name VARCHAR(100) NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT username_format CHECK (username ~ '^[a-z0-9_]{3,32}$')
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title VARCHAR(100) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS chat_members_user ON chat_members(user_id, chat_id);
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id),
|
||||
text VARCHAR(4000) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT nonblank_message CHECK (length(btrim(text)) > 0)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS messages_chat_cursor ON messages(chat_id, created_at, id);
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash CHAR(64) PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS sessions_expiry ON sessions(expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Lab3: one atomic idempotency result per user, chat and key.
|
||||
CREATE TABLE IF NOT EXISTS message_idempotency (
|
||||
user_id UUID NOT NULL REFERENCES users(id),
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
idem_key VARCHAR(128) NOT NULL,
|
||||
request_hash CHAR(64) NOT NULL,
|
||||
message_id UUID NOT NULL UNIQUE REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, idem_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS message_idempotency_expiry ON message_idempotency(created_at);
|
||||
@@ -0,0 +1,25 @@
|
||||
-- A file is owned by exactly one chat. Binary bytes live only in private S3.
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id UUID NOT NULL REFERENCES users(id),
|
||||
filename VARCHAR(255) NOT NULL,
|
||||
content_type VARCHAR(32) NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes INTEGER NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
object_key VARCHAR(160) NOT NULL UNIQUE,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'ready' CHECK (status='ready'),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
|
||||
ALTER TABLE messages DROP CONSTRAINT nonblank_message;
|
||||
-- Blank text is permitted only with attachment IDs; enforced by application transaction.
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
position SMALLINT NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, position)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments(attachment_id);
|
||||
@@ -0,0 +1,15 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN variants JSONB NOT NULL DEFAULT '[]';
|
||||
ALTER TABLE attachments ADD COLUMN error_code TEXT;
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL UNIQUE REFERENCES attachments(id),
|
||||
pipeline_version INTEGER NOT NULL DEFAULT 1,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_publish TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX attachment_jobs_due ON attachment_jobs(next_publish);
|
||||
@@ -0,0 +1,6 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.2
|
||||
psycopg[binary]==3.2.6
|
||||
boto3==1.37.38
|
||||
Pillow==11.2.1
|
||||
pika==1.3.2
|
||||
@@ -0,0 +1,5 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env restart db s3 broker api1 api2 dispatcher worker lb
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart db s3 broker api1 api2 dispatcher worker lb
|
||||
@@ -0,0 +1,5 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env up -d --no-deps --wait --wait-timeout 55 api1 api2
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env start api1 api2
|
||||
@@ -0,0 +1,40 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-Location (Join-Path $PSScriptRoot '..')
|
||||
|
||||
if (!(Test-Path -LiteralPath '.env')) {
|
||||
Copy-Item -LiteralPath '.env.example' -Destination '.env'
|
||||
}
|
||||
New-Item -ItemType Directory -Path 'secrets' -Force | Out-Null
|
||||
|
||||
function New-SecretFile([string]$Name, [int]$Length) {
|
||||
$path = Join-Path (Get-Location) "secrets/$Name"
|
||||
if (!(Test-Path -LiteralPath $path)) {
|
||||
$bytes = [byte[]]::new($Length)
|
||||
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
try {
|
||||
$rng.GetBytes($bytes)
|
||||
} finally {
|
||||
$rng.Dispose()
|
||||
}
|
||||
$value = [BitConverter]::ToString($bytes).Replace('-', '').ToLowerInvariant()
|
||||
[IO.File]::WriteAllText($path, $value, [Text.Encoding]::ASCII)
|
||||
}
|
||||
}
|
||||
|
||||
New-SecretFile 'db_password' 24
|
||||
New-SecretFile 's3_root_access' 10
|
||||
New-SecretFile 's3_root_secret' 20
|
||||
New-SecretFile 's3_access_key_id' 10
|
||||
New-SecretFile 's3_secret_access_key' 20
|
||||
New-SecretFile 'broker_password' 24
|
||||
$brokerPassword = (Get-Content -LiteralPath 'secrets/broker_password' -Raw).Trim()
|
||||
[IO.File]::WriteAllText((Join-Path (Get-Location) 'secrets/broker_url'), "amqp://messenger:${brokerPassword}@broker:5672/%2F", [Text.Encoding]::ASCII)
|
||||
[IO.File]::WriteAllText((Join-Path (Get-Location) 'secrets/rabbitmq_config'), "default_user = messenger`ndefault_pass = ${brokerPassword}`n", [Text.Encoding]::ASCII)
|
||||
|
||||
if (!(Test-Path -LiteralPath 'secrets/database_url')) {
|
||||
$password = (Get-Content -LiteralPath 'secrets/db_password' -Raw).TrimEnd([char]10)
|
||||
$encoded = [Uri]::EscapeDataString($password)
|
||||
$url = "postgresql://messenger:${encoded}@db:5432/messenger"
|
||||
[IO.File]::WriteAllText((Join-Path (Get-Location) 'secrets/database_url'), $url, [Text.Encoding]::ASCII)
|
||||
}
|
||||
Write-Host 'Local secrets are ready.'
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ ! -e .env ]; then cp .env.example .env; fi
|
||||
mkdir -p secrets
|
||||
if [ ! -e secrets/db_password ]; then
|
||||
python3 -c 'import secrets;print(secrets.token_urlsafe(36),end="")' > secrets/db_password
|
||||
fi
|
||||
if [ ! -e secrets/database_url ]; then
|
||||
python3 -c 'from pathlib import Path;from urllib.parse import quote; p=Path("secrets/db_password").read_text().strip();print("postgresql://messenger:"+quote(p,safe="")+"@db:5432/messenger",end="")' > secrets/database_url
|
||||
fi
|
||||
python3 - <<'PYGEN'
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
root=Path('secrets')
|
||||
for name,length in [('s3_root_access',10),('s3_root_secret',20),('s3_access_key_id',10),('s3_secret_access_key',20),('broker_password',24)]:
|
||||
file=root/name
|
||||
if not file.exists():
|
||||
file.write_bytes(secrets.token_hex(length).encode('ascii'))
|
||||
password=(root/'broker_password').read_text().strip()
|
||||
(root/'broker_url').write_text('amqp://messenger:'+password+'@broker:5672/%2F')
|
||||
(root/'rabbitmq_config').write_text('default_user = messenger\ndefault_pass = '+password+'\n')
|
||||
PYGEN
|
||||
chmod 600 secrets/* .env
|
||||
printf 'Local secrets prepared (values not displayed).\n'
|
||||
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if /I "%TARGET_INSTANCE_ID%"=="api1" goto stop
|
||||
if /I "%TARGET_INSTANCE_ID%"=="api2" goto stop
|
||||
echo Invalid TARGET_INSTANCE_ID; refusing to stop an arbitrary service 1>&2
|
||||
exit /b 2
|
||||
:stop
|
||||
docker compose --env-file .env stop -t 2 %TARGET_INSTANCE_ID%
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
case "${TARGET_INSTANCE_ID:-}" in
|
||||
api1|api2) exec docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID" ;;
|
||||
*) echo 'Unknown TARGET_INSTANCE_ID; refusing to stop an arbitrary service' >&2; exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Reproducible level-3 failure scenarios; synthetic data, no tokens in output."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT/'tests'))
|
||||
from client import Client, check
|
||||
from image_probe import jpeg_size
|
||||
|
||||
def compose(*args):
|
||||
result = subprocess.run(['docker','compose','--env-file','.env',*args], cwd=ROOT,
|
||||
capture_output=True, text=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError('compose action failed: '+ ' '.join(args[:3]))
|
||||
return result.stdout.strip()
|
||||
|
||||
def internal(code):
|
||||
return compose('exec','-T','dispatcher','python','-c',code)
|
||||
|
||||
def queue_count():
|
||||
output = compose('exec','-T','broker','rabbitmqctl','list_queues','name','messages_ready','--silent')
|
||||
for line in output.splitlines():
|
||||
fields = line.split()
|
||||
if len(fields)==2 and fields[0]=='attachments.v1':
|
||||
return int(fields[1])
|
||||
return 0
|
||||
|
||||
def started_at(service):
|
||||
ident = compose('ps','-q',service)
|
||||
return subprocess.check_output(['docker','inspect','--format','{{.State.StartedAt}}',ident],
|
||||
text=True).strip()
|
||||
|
||||
def eventually(predicate, seconds=65):
|
||||
deadline = time.monotonic()+seconds
|
||||
while time.monotonic()<deadline:
|
||||
try:
|
||||
result = predicate()
|
||||
if result:
|
||||
return result
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise AssertionError('condition did not recover before deadline')
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default='http://localhost:8085')
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url)
|
||||
results = []
|
||||
compose('stop','worker')
|
||||
try:
|
||||
alice,bob,eve,chat = client.bootstrap()
|
||||
attachment, raw = client.attachment(alice,chat)
|
||||
ident = attachment['id']
|
||||
path = '/api/v1/attachments/'+ident
|
||||
check(attachment['status']=='queued', 'upload snapshot must be queued')
|
||||
client.call('GET',path+'/content',409,actor=alice)
|
||||
linked = client.message(alice,chat,'',attachments=[ident])['json']
|
||||
client.message(bob,chat,'Worker is stopped; messages still work')
|
||||
eventually(lambda: queue_count()>0)
|
||||
# Freeze publisher so broker persistence is observed without republishing.
|
||||
compose('stop','dispatcher')
|
||||
queued_before = queue_count()
|
||||
started_before = {name:started_at(name) for name in ('api1','api2','broker')}
|
||||
compose('restart','api1','api2','broker')
|
||||
eventually(lambda: client.call('GET','/health/ready',schema='Health'))
|
||||
queued_after = eventually(lambda: queue_count())
|
||||
started_after = {name:started_at(name) for name in ('api1','api2','broker')}
|
||||
check(all(started_before[name]!=started_after[name] for name in started_before), 'process did not actually restart')
|
||||
check(queued_after==queued_before, 'broker lost durable queued messages')
|
||||
pending = client.call('GET',path,actor=alice,schema='Attachment')['json']
|
||||
check(pending['status']=='queued', 'API performed synchronous processing')
|
||||
compose('start','dispatcher','worker')
|
||||
start = time.monotonic()
|
||||
ready = client.wait_ready(bob,attachment)
|
||||
check(client.call('GET',path+'/content',actor=alice)['bytes']==raw, 'original changed')
|
||||
thumbnail = client.call('GET',path+'/content?variant=thumbnail',actor=bob)['bytes']
|
||||
check(jpeg_size(thumbnail)==(256,160), 'invalid thumbnail')
|
||||
results.append(dict(scenario='stopped_worker_and_api_broker_restart', attachment_id=ident,
|
||||
message_id=linked['id'], queued_before=queued_before, queued_after=queued_after,
|
||||
broker_restart_verified=True, started_before=started_before, started_after=started_after,
|
||||
ready_seconds=round(time.monotonic()-start,3),
|
||||
thumbnail=[256,160], result='PASS'))
|
||||
# Publish the exact same logical job five more times, with confirms.
|
||||
snapshot = internal(f"""
|
||||
import json
|
||||
from app.storage import db
|
||||
from app.broker import connect,QUEUE
|
||||
from app.models.time_format import utc_z
|
||||
import pika
|
||||
with db() as c:
|
||||
j=c.execute('SELECT * FROM attachment_jobs WHERE attachment_id=%s',('{ident}',)).fetchone()
|
||||
body=dict(schema_version=1,job_id=str(j['job_id']),type='attachment.process.v1',attachment_id='{ident}',pipeline_version=1,attempt=1,created_at=utc_z(j['created_at']))
|
||||
connection,ch=connect()
|
||||
for _ in range(5):
|
||||
ch.basic_publish('',QUEUE,json.dumps(body),pika.BasicProperties(delivery_mode=2),mandatory=True)
|
||||
connection.close()
|
||||
print(j['attempts'])
|
||||
""")
|
||||
eventually(lambda: queue_count()==0)
|
||||
# Queue empty can mean delivered but unacked; wait for all acknowledgements.
|
||||
eventually(lambda: 'attachments.v1\t0' in compose('exec','-T','broker','rabbitmqctl','list_queues','name','messages','--silent'))
|
||||
check(client.call('GET',path,actor=alice,schema='Attachment')['json']==ready, 'duplicate changed ready metadata')
|
||||
attempts = internal(f"from app.storage import db\nwith db() as c:\n print(c.execute('SELECT attempts FROM attachment_jobs WHERE attachment_id=%s',('{ident}',)).fetchone()['attempts'])")
|
||||
check(attempts==snapshot, 'duplicate reran successful job')
|
||||
check(client.call('GET',path+'/content?variant=thumbnail',actor=bob)['bytes']==thumbnail,'duplicate changed variant')
|
||||
objects = json.loads(internal(f"from app.s3 import s3_client,bucket\nimport json\nprint(json.dumps([o['Key'] for o in s3_client().list_objects_v2(Bucket=bucket(),Prefix='results/{ident}/')['Contents']]))"))
|
||||
check(len(objects)==2,'duplicate created extra objects')
|
||||
results.append(dict(scenario='five_duplicate_deliveries',attachment_id=ident,attempts=int(attempts),
|
||||
final_objects=objects, unchanged_metadata=True, result='PASS'))
|
||||
# Transient storage failure: controlled, nonexistent staging key of a new test job.
|
||||
compose('stop','worker')
|
||||
failed, _ = client.attachment(alice,chat)
|
||||
fid = failed['id']
|
||||
internal(f"from app.storage import db\nwith db() as c:\n c.execute(\"UPDATE attachments SET object_key=%s WHERE id=%s\",('staging/missing-{fid}','{fid}'))")
|
||||
compose('start','worker')
|
||||
fpath = '/api/v1/attachments/'+fid
|
||||
final = eventually(lambda: (lambda a:a if a['status']=='failed' else None)(client.call('GET',fpath,actor=alice,schema='Attachment')['json']))
|
||||
check(final['error_code']=='retry_exhausted','unsafe or wrong failure code')
|
||||
attempts = int(internal(f"from app.storage import db\nwith db() as c:\n print(c.execute('SELECT attempts FROM attachment_jobs WHERE attachment_id=%s',('{fid}',)).fetchone()['attempts'])"))
|
||||
check(attempts==5,'retry bound not respected')
|
||||
client.call('GET',fpath+'/content',409,actor=alice)
|
||||
client.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
|
||||
body={'text':'','attachment_ids':[fid]},headers={'Idempotency-Key':'failed-'+fid})
|
||||
client.call('GET',fpath,404,actor=eve)
|
||||
results.append(dict(scenario='bounded_retry',attachment_id=fid,attempts=attempts,
|
||||
error_code=final['error_code'], download_status=409, attach_status=409,result='PASS'))
|
||||
finally:
|
||||
compose('start','dispatcher','worker')
|
||||
(ROOT/'evidence'/'pipeline.json').write_text(json.dumps(results,indent=2)+'\n',encoding='utf-8')
|
||||
print(json.dumps(results,indent=2))
|
||||
print('PASS: lab5 durable queue, restart, redelivery and retry scenarios')
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Run unchanged public resilience checks and record actual container restarts."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker',*args],cwd=ROOT,text=True).strip()
|
||||
|
||||
def states():
|
||||
ids = docker('compose','ps','-q').splitlines()
|
||||
rows = json.loads(docker('inspect',*ids))
|
||||
return {row['Config']['Labels']['com.docker.compose.service']:
|
||||
dict(id=row['Id'][:12],started_at=row['State']['StartedAt'],image=row['Config']['Image'])
|
||||
for row in rows}
|
||||
|
||||
def main():
|
||||
base = sys.argv[1] if len(sys.argv)>1 else 'http://localhost:8085'
|
||||
suffix = '.cmd' if os.name=='nt' else '.sh'
|
||||
if os.name!='nt':
|
||||
for name in ('restart','stop-one'):
|
||||
(ROOT/'scripts'/(name+suffix)).chmod(0o755)
|
||||
before = states()
|
||||
command = [sys.executable,'tests/resilience.py','--base-url',base]
|
||||
result = subprocess.run(command+['--mode','persistence','--action-script','scripts/restart'+suffix],
|
||||
cwd=ROOT,capture_output=True,text=True)
|
||||
print(result.stdout,end='')
|
||||
(ROOT/'evidence/persistence.txt').write_text(result.stdout+result.stderr,encoding='utf-8')
|
||||
after = states()
|
||||
services = ('api1','api2','db','s3','broker','worker','dispatcher','lb')
|
||||
changed = {name:before[name]['started_at']!=after[name]['started_at'] for name in services}
|
||||
(ROOT/'evidence/persistence-runtime.json').write_text(json.dumps(dict(before=before,after=after,restarted=changed),indent=2)+'\n',encoding='utf-8')
|
||||
if result.returncode or not all(changed.values()):
|
||||
raise RuntimeError('persistence verification failed; see evidence')
|
||||
failover_results = []
|
||||
try:
|
||||
for number in range(2):
|
||||
result = subprocess.run(command+['--mode','failover','--action-script','scripts/stop-one'+suffix],
|
||||
cwd=ROOT,capture_output=True,text=True)
|
||||
print(result.stdout,end='')
|
||||
failover_results.append(f'Run {number+1}\n'+result.stdout+result.stderr)
|
||||
docker('compose','up','-d','--no-deps','api1','api2')
|
||||
if result.returncode:
|
||||
raise RuntimeError('failover verification failed; see evidence')
|
||||
finally:
|
||||
docker('compose','up','-d','--no-deps','api1','api2')
|
||||
(ROOT/'evidence/failover.txt').write_text('\n'.join(failover_results),encoding='utf-8')
|
||||
runtime = dict(python=platform.python_version(),host=platform.platform(),
|
||||
docker=json.loads(docker('version','--format','{{json .}}')),
|
||||
engine=json.loads(docker('info','--format','{{json .}}')),
|
||||
containers=states())
|
||||
# Keep only non-sensitive hardware/version fields from Docker info.
|
||||
runtime['engine']={key:runtime['engine'][key] for key in ('NCPU','MemTotal','OSType','Architecture','OperatingSystem')}
|
||||
(ROOT/'evidence/runtime.json').write_text(json.dumps(runtime,indent=2)+'\n',encoding='utf-8')
|
||||
print('PASS: persistence and two API failover runs; actual process restarts recorded')
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,11 @@
|
||||
param([string]$BaseUrl = 'http://localhost:8085')
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-Location (Join-Path $PSScriptRoot '..')
|
||||
function Invoke-Check([string[]]$Arguments) {
|
||||
& py @Arguments
|
||||
if ($LASTEXITCODE -ne 0) { throw "Verification failed: $($Arguments[0])" }
|
||||
}
|
||||
Invoke-Check -Arguments @('scripts/check_contract.py')
|
||||
Invoke-Check -Arguments @('tests/smoke.py', '--base-url', $BaseUrl)
|
||||
Invoke-Check -Arguments @('scripts/verify-lab5.py', '--base-url', $BaseUrl)
|
||||
Invoke-Check -Arguments @('tests/resilience.py', '--mode', 'persistence', '--action-script', 'scripts/restart.cmd', '--base-url', $BaseUrl)
|
||||
@@ -0,0 +1,30 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0"
|
||||
where docker >nul 2>&1
|
||||
if errorlevel 1 (
|
||||
echo Docker CLI was not found. Start Docker Desktop with Linux containers.
|
||||
exit /b 1
|
||||
)
|
||||
docker info --format "{{.OSType}}"
|
||||
if errorlevel 1 exit /b 1
|
||||
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\setup.ps1"
|
||||
if errorlevel 1 exit /b 1
|
||||
|
||||
docker compose --env-file .env config --quiet
|
||||
if errorlevel 1 exit /b 1
|
||||
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
if errorlevel 1 goto diagnostics
|
||||
|
||||
curl.exe --fail --silent --show-error --retry 10 --retry-connrefused --retry-delay 1 --max-time 5 http://127.0.0.1:8085/health/ready
|
||||
if errorlevel 1 goto diagnostics
|
||||
echo.
|
||||
docker compose --env-file .env ps -a
|
||||
exit /b %errorlevel%
|
||||
|
||||
:diagnostics
|
||||
docker compose --env-file .env ps -a
|
||||
docker compose --env-file .env logs --tail=60 db s3 s3-bootstrap migrate api1 api2 lb broker dispatcher worker
|
||||
exit /b 1
|
||||
Reference in New Issue
Block a user