forked from mtusi2026/mtusi2026lab5
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4fab41b229 | ||
|
|
8f492ec491 |
+10
-8
@@ -1,24 +1,26 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
# Host publish port of the load balancer; change if 8080 is busy locally.
|
||||
HOST_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
DATABASE_URL=postgresql://messenger:messenger@postgres:5432/messenger
|
||||
# DATABASE_URL_FILE=/run/secrets/database_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=minioadmin
|
||||
S3_SECRET_ACCESS_KEY=minioadmin
|
||||
# S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
# S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
BROKER_URL=amqp://messenger:messenger@broker:5672/
|
||||
# BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
FROM python:3.12.8-slim-bookworm
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY migrations/ ./migrations/
|
||||
COPY src/ ./src/
|
||||
COPY scripts/migrate.py ./scripts/migrate.py
|
||||
|
||||
ENV PYTHONPATH=/app/src
|
||||
ENV APP_HOST=0.0.0.0
|
||||
ENV APP_PORT=8080
|
||||
ENV APP_ENV=lab
|
||||
ENV SESSION_TTL_SECONDS=3600
|
||||
ENV WORKER_CONCURRENCY=2
|
||||
ENV JOB_MAX_ATTEMPTS=5
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["sh", "-c", "exec uvicorn messenger.app:app --host ${APP_HOST} --port ${APP_PORT}"]
|
||||
@@ -1,37 +1,183 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Гадяцкий Л. Е., БПИ2302
|
||||
- Целевая оценка: 3
|
||||
- Commit/PR: solution/BPI2302-Gadiatskiy MR#81: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/81
|
||||
- Использованные источники и библиотеки:
|
||||
- контракт и семантика: `contracts/openapi.json`, `contracts/README.md`, `contracts/attachment-states.md`, `README.md`, `COURSE.md`
|
||||
- перенос из lab4: FastAPI, PostgreSQL/сессии, SQLAlchemy, миграции, Nginx LB, идемпотентность, MinIO/S3, вложения
|
||||
- runtime: FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, SQLAlchemy 2.0.36, psycopg 3.2.4, argon2-cffi 23.1.0, boto3 1.35.76, Pillow 11.0.0, pika 1.3.2
|
||||
- PostgreSQL: `postgres:16-alpine`; LB: `nginx:1.27-alpine`; S3/MinIO: `alpine/minio:RELEASE.2025-10-15T17-29-55Z`; broker: `rabbitmq:3.13.7-alpine`
|
||||
- Помощь ИИ использована при переносе lab4, частичной реализации брокера/worker-а и при заполнении отчёта. Результаты подтверждаются сохранёнными запусками.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
**Prerequisites:** Docker и Docker Compose. Python 3.9+ нужен только для публичных проверок (`make check` / `make test` / `tests/resilience.py`). Runtime приложения — внутри образа.
|
||||
|
||||
**Версии:**
|
||||
- API/worker: базовый образ `python:3.12.8-slim-bookworm` → локальный `mtusi-messenger:lab5`
|
||||
- PostgreSQL: `postgres:16-alpine`
|
||||
- Nginx: `nginx:1.27-alpine`
|
||||
- MinIO: `alpine/minio:RELEASE.2025-10-15T17-29-55Z` (community image; официальный `minio/minio` с Docker Hub недоступен)
|
||||
- RabbitMQ: `rabbitmq:3.13.7-alpine`
|
||||
- зависимости: `requirements.txt`
|
||||
|
||||
**Конфигурация:** секретов в Git нет. Локальный `.env` создаётся из примера и не коммитится:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Соответствие `.env.example`:
|
||||
- `APP_HOST` / `APP_PORT` — bind uvicorn внутри контейнеров API
|
||||
- `HOST_PORT` — порт LB на host (по умолчанию 8080; если занят — например 8085)
|
||||
- `APP_ENV` — среда запуска (`lab`)
|
||||
- `SESSION_TTL_SECONDS` — TTL сессии (по умолчанию 3600; для обычного smoke ≥ 300)
|
||||
- `SESSION_STORE=postgres` — сессии в PostgreSQL (общие для всех реплик)
|
||||
- `DATABASE_URL` — строка подключения API/migrate/worker к PostgreSQL в compose-сети
|
||||
(приложение также поддерживает `DATABASE_URL_FILE`)
|
||||
- `IDEMPOTENCY_TTL_SECONDS` — срок хранения идемпотентных записей (по умолчанию 86400)
|
||||
- `API_REPLICAS=2` — в `compose.yaml` зафиксированы сервисы `api1` и `api2`
|
||||
- `S3_ENDPOINT` / `S3_REGION` / `S3_BUCKET` — доступ API/worker к MinIO в compose-сети
|
||||
- `S3_ACCESS_KEY_ID` / `S3_SECRET_ACCESS_KEY` — учебные credentials MinIO (также `*_FILE`)
|
||||
- `MAX_UPLOAD_BYTES=10485760`, `MAX_IMAGE_PIXELS=40000000` — лимиты загрузки/декодирования
|
||||
- `BROKER_URL` — AMQP URL RabbitMQ (также `BROKER_URL_FILE`)
|
||||
- `WORKER_CONCURRENCY` — prefetch consumer worker (по умолчанию 2)
|
||||
- `JOB_MAX_ATTEMPTS=5` — лимит управляемых попыток обработки
|
||||
- `JOB_TIMEOUT_SECONDS`, `STAGING_RETENTION_SECONDS` — параметры эксплуатации (зафиксированы в `.env`)
|
||||
|
||||
Учебные credentials compose (`messenger` / `messenger`, MinIO `minioadmin` / `minioadmin`, RabbitMQ `messenger` / `messenger`) только для локального синтетического стенда.
|
||||
|
||||
**Сборка и запуск:**
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env up -d --build
|
||||
```
|
||||
|
||||
Миграции `migrations/*.sql` применяет one-shot сервис `migrate` до старта `api1`/`api2`/`worker` (учёт в `schema_migrations`). Приватный bucket `S3_BUCKET` идемпотентно создают API и worker при старте (`ensure_bucket`). Реплики API миграции SQL при старте не выполняют.
|
||||
|
||||
Готовность через LB (подставьте свой `HOST_PORT`):
|
||||
|
||||
```sh
|
||||
curl -i http://127.0.0.1:8080/health/ready
|
||||
```
|
||||
|
||||
Ожидается HTTP 200, тело `{"status":"ok"}`, заголовки `X-Request-Id` и `X-Instance-Id` (`api1` или `api2`). `/health/ready` требует доступности PostgreSQL, S3 (MinIO) и RabbitMQ.
|
||||
|
||||
**Публичный smoke** (при уже запущенном стенде):
|
||||
|
||||
```sh
|
||||
python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
# или: make test BASE_URL=http://127.0.0.1:8080
|
||||
```
|
||||
|
||||
**Проверка сохранности** (включая исходный файл вложения):
|
||||
|
||||
```sh
|
||||
# Unix:
|
||||
chmod +x scripts/restart.sh
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
|
||||
|
||||
# Windows:
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
```
|
||||
|
||||
Hook выполняет `docker compose --env-file .env restart` для `postgres`, `s3`, `broker`, `api1`, `api2`, `worker`, `lb` **без** удаления volumes (`down -v` не используется).
|
||||
|
||||
**Проверка отказа одной API-реплики:**
|
||||
|
||||
```sh
|
||||
# Unix:
|
||||
chmod +x scripts/stop-one.sh
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
|
||||
|
||||
# Windows:
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080
|
||||
```
|
||||
|
||||
`TARGET_INSTANCE_ID` = `X-Instance-Id` (`api1`/`api2`); hook делает `docker compose stop` только этой реплики и оставляет её остановленной. После теста: `docker compose --env-file .env start api1` (или `api2`). Для защиты повторить на второй реплике после восстановления первой.
|
||||
|
||||
**Демо остановки worker (уровень 3):**
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env stop worker
|
||||
# upload → 202 queued; GET /content → 409; сообщение с attachment_ids допускается
|
||||
docker compose --env-file .env restart api1 api2
|
||||
docker compose --env-file .env start worker
|
||||
# опросить GET /attachments/{id} до ready; thumbnail доступен
|
||||
```
|
||||
|
||||
**Остановка:**
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down
|
||||
```
|
||||
|
||||
Намеренное удаление учебных данных (volumes):
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down -v
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
HTTP-клиент
|
||||
→ lb (Nginx)
|
||||
→ api1 / api2 (uvicorn + FastAPI)
|
||||
→ PostgreSQL
|
||||
users, chats, chat_members, messages, message_attachments,
|
||||
attachments (+ staging_object_key), attachment_jobs,
|
||||
attachment_variants, sessions, idempotency_records
|
||||
→ MinIO (S3)
|
||||
staging/{chat_id}/{attachment_id}
|
||||
attachments/{chat_id}/{attachment_id}/original
|
||||
attachments/{chat_id}/{attachment_id}/variants/thumbnail.jpg
|
||||
→ RabbitMQ (durable queue attachment.process)
|
||||
→ worker (отдельный процесс)
|
||||
читает staging → пишет final original + thumbnail → status=ready → ack
|
||||
```
|
||||
|
||||
- Источник истины для метаданных — PostgreSQL; байты — только в S3. В сообщениях брокера только ID/служебные поля по `jobs.schema.json`, без bytes и credentials.
|
||||
- До `202`: оригинал в durable staging (S3), запись `attachments` со `status=queued`, запись `attachment_jobs`, затем publish с publisher confirm. Остановка API после ответа не удаляет принятую работу.
|
||||
- Worker отделён от API: переносит оригинал в конечный S3-prefix, строит JPEG thumbnail (вписан в 256×256, без увеличения; EXIF orientation; alpha на белый). Fixture 320×200 → 256×160.
|
||||
- Состояния `queued`/`processing`/`ready`/`failed` видны в API. `GET .../content` для queued/processing/failed → `409`; отсутствующий variant у ready → `404`.
|
||||
- Сообщение может ссылаться на queued/processing вложение; `failed` → `409`.
|
||||
- Retry ограничен `JOB_MAX_ATTEMPTS` (по умолчанию 5); исчерпание → `failed`/`retry_exhausted`. Повтор уже ready-задачи — no-op с ack; variant `thumbnail` не размножается.
|
||||
- Ack — после устойчивой записи результата; delivery at-least-once (persistent messages, durable queue, manual ack).
|
||||
- Сессии в PostgreSQL; `X-Instance-Id` стабилен на жизнь процесса; `Idempotency-Key` на POST сообщений сохранён из lab3/lab4.
|
||||
- Compose: `lb` + `api1` + `api2` + `worker` + `postgres` + `s3` + `broker` + `migrate`, volumes `pgdata`, `miniodata`, `brokerdata`.
|
||||
|
||||
Перенесено из lab4: auth Basic→Bearer, Argon2id, чаты/сообщения, LB, две реплики, идемпотентность, MinIO, upload/download.
|
||||
Изменено в lab5: upload `201/ready` → `202/queued`; staging + jobs; RabbitMQ; worker; thumbnail variant; link queued вложений.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke (202 → ready, thumbnail 256×160) | `python tests/smoke.py --base-url http://127.0.0.1:8085` | `PASS: lab5 public API scenario...` | `evidence/smoke.txt` |
|
||||
| Стенд: ≥2 API, LB, Postgres, MinIO, broker, worker | `docker compose --env-file .env ps` | `lb`, `api1`, `api2`, `postgres`, `s3`, `broker`, `worker`; migrate Exited 0 | `evidence/compose-ps.txt` |
|
||||
| Persistence: рестарт без удаления volumes | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8085` | `PASS: persistence preserved...` | `evidence/restart.txt` |
|
||||
| Failover: остановка одной API-реплики | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8085` | failover PASS; `first_recovery_seconds≈2.89` | `evidence/failover.txt` |
|
||||
| Worker stop → upload 202 → restart API → start worker → ready | `python evidence/worker_stop_demo.py` | upload queued; content 409; после старта worker — ready + thumbnail 256×160 | `evidence/worker-stop.txt` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Для оценки 3 отдельный нагрузочный трек не выполнялся. Проверены сохранность (БД+S3+broker), отказ API-реплики и остановка worker:
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| `docker compose --env-file .env restart` (postgres+s3+broker+api1+api2+worker+lb), volumes сохранены | `/health/ready` ok; прежний Bearer; тот же оригинал по `/content` | persistence PASS | в бюджете теста; потери данных нет |
|
||||
| `docker compose stop` одной API-реплики | LB обслуживает живую реплику | failover PASS; `first_recovery_seconds≈2.89` | подтверждённые сообщения на месте |
|
||||
| `docker compose stop worker` → upload → restart API → `start worker` | 202 queued сохраняется; после worker — ready/thumbnail | demo PASS; attachment `49100060-...` стал ready | сообщение в durable queue пережило остановку worker; потери нет |
|
||||
|
||||
Локальная проверка автора выполнялась на `HOST_PORT=8085`. На Windows для публичных тестов задайте `PYTHONUTF8=1` (иначе чтение `contracts/openapi.json` может упасть на cp1251).
|
||||
|
||||
Аппаратный профиль: локальный Docker Desktop на Windows; 2 API + worker + Nginx + PostgreSQL + MinIO + RabbitMQ без CPU/RAM limits.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- transactional outbox, DLQ/replay, crop/optimized WebP, lease зависшего processing — не реализованы (уровень 4).
|
||||
- Единственный PostgreSQL, MinIO, RabbitMQ и Nginx — SPOF; отказ host/узла не выдерживается.
|
||||
- При сбое между COMMIT задания и publisher confirm возможна запись queued без сообщения в брокере до ручного вмешательства.
|
||||
- Basic по HTTP допустим только на локальном учебном стенде с синтетическими данными.
|
||||
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
x-api: &api
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
expose:
|
||||
- "8080"
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: lab
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger
|
||||
SESSION_STORE: postgres
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-minioadmin}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-minioadmin}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
|
||||
]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: messenger
|
||||
POSTGRES_DB: messenger
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: alpine/minio:RELEASE.2025-10-15T17-29-55Z
|
||||
user: "0:0"
|
||||
command: ["server", "/data", "--address", ":9000", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-minioadmin}
|
||||
MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-minioadmin}
|
||||
volumes:
|
||||
- miniodata:/data
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"wget",
|
||||
"-qO-",
|
||||
"http://127.0.0.1:9000/minio/health/live",
|
||||
]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
broker:
|
||||
image: rabbitmq:3.13.7-alpine
|
||||
hostname: broker
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger
|
||||
RABBITMQ_DEFAULT_PASS: messenger
|
||||
volumes:
|
||||
- brokerdata:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
environment:
|
||||
DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger
|
||||
command: ["python", "/app/scripts/migrate.py"]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api1:
|
||||
<<: *api
|
||||
hostname: api1
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: lab
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger
|
||||
SESSION_STORE: postgres
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-minioadmin}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-minioadmin}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
INSTANCE_ID: api1
|
||||
|
||||
api2:
|
||||
<<: *api
|
||||
hostname: api2
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: lab
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger
|
||||
SESSION_STORE: postgres
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-minioadmin}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-minioadmin}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
INSTANCE_ID: api2
|
||||
|
||||
worker:
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["python", "-m", "messenger.worker"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-minioadmin}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-minioadmin}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: nginx:1.27-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${HOST_PORT:-8080}:80"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
depends_on:
|
||||
api1:
|
||||
condition: service_healthy
|
||||
api2:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1/health/live"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
miniodata:
|
||||
brokerdata:
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,80 @@
|
||||
import base64, hashlib, json, time, urllib.request, uuid
|
||||
from pathlib import Path
|
||||
BASE='http://127.0.0.1:8085'
|
||||
ROOT=Path('.')
|
||||
content=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
def call(method, path, expected=None, body=None, token=None, raw=None, headers=None):
|
||||
h={'Accept':'application/json'}
|
||||
if token: h['Authorization']='Bearer '+token
|
||||
data=None
|
||||
if body is not None:
|
||||
data=json.dumps(body).encode(); h['Content-Type']='application/json'
|
||||
if raw is not None:
|
||||
data=raw
|
||||
if headers: h.update(headers)
|
||||
req=urllib.request.Request(BASE+path, data=data, headers=h, method=method)
|
||||
try:
|
||||
resp=urllib.request.urlopen(req, timeout=30)
|
||||
except urllib.error.HTTPError as e:
|
||||
resp=e
|
||||
status=resp.code; payload=resp.read(); ctype=resp.headers.get_content_type()
|
||||
js=json.loads(payload) if ctype=='application/json' and payload else None
|
||||
if expected is not None and status!=expected:
|
||||
raise SystemExit(f'{method} {path}: expected {expected} got {status} body={payload[:300]!r}')
|
||||
return status, js, payload, resp.headers
|
||||
|
||||
# stop worker
|
||||
import subprocess
|
||||
subprocess.check_call(['docker','compose','--env-file','.env','stop','worker'], cwd=str(ROOT))
|
||||
print('worker stopped')
|
||||
|
||||
# register+login
|
||||
uname='u_demo_'+uuid.uuid4().hex[:10]
|
||||
pwd='T3st-'+uuid.uuid4().hex
|
||||
_, user, _, _ = call('POST','/api/v1/users',201,body={'username':uname,'display_name':'Demo','password':pwd})
|
||||
cred=base64.b64encode((uname+':'+pwd).encode()).decode()
|
||||
_, sess, _, _ = call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+cred})
|
||||
token=sess['token']
|
||||
_, chat, _, _ = call('POST','/api/v1/chats',201,token=token,body={'title':'demo', 'member_ids':[user['id']]})
|
||||
status, att, _, _ = call('POST',f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",202,token=token,raw=content,headers={'Content-Type':'image/png'})
|
||||
print('upload', status, att['status'], att['id'])
|
||||
assert att['status']=='queued'
|
||||
# content must be 409
|
||||
st,_,_,_ = call('GET',f"/api/v1/attachments/{att['id']}/content",409,token=token)
|
||||
print('content while queued', st)
|
||||
# message with queued attachment
|
||||
call('POST',f"/api/v1/chats/{chat['id']}/messages",201,token=token,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
print('message linked while queued')
|
||||
# restart api
|
||||
subprocess.check_call(['docker','compose','--env-file','.env','restart','api1','api2'], cwd=str(ROOT))
|
||||
# wait ready
|
||||
for _ in range(60):
|
||||
try:
|
||||
st,js,_,_=call('GET','/health/ready')
|
||||
if st==200: break
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise SystemExit('api not ready')
|
||||
print('api restarted and ready')
|
||||
# start worker
|
||||
subprocess.check_call(['docker','compose','--env-file','.env','start','worker'], cwd=str(ROOT))
|
||||
print('worker started')
|
||||
# wait ready attachment
|
||||
deadline=time.time()+60
|
||||
ready=None
|
||||
while time.time()<deadline:
|
||||
_, cur, _, _ = call('GET',f"/api/v1/attachments/{att['id']}",token=token)
|
||||
if cur['status']=='ready':
|
||||
ready=cur; break
|
||||
if cur['status']=='failed':
|
||||
raise SystemExit('failed '+str(cur))
|
||||
time.sleep(0.3)
|
||||
if not ready:
|
||||
raise SystemExit('timeout waiting ready')
|
||||
print('ready', ready['id'], 'variants', ready['variants'])
|
||||
_, _, thumb, hdr = call('GET',f"/api/v1/attachments/{att['id']}/content?variant=thumbnail",200,token=token)
|
||||
print('thumbnail bytes', len(thumb), 'ctype', hdr.get_content_type())
|
||||
print('PASS worker-stop demo')
|
||||
@@ -0,0 +1,41 @@
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
access_log /var/log/nginx/access.log;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
client_max_body_size 11m;
|
||||
|
||||
upstream api_backends {
|
||||
least_conn;
|
||||
server api1:8080 max_fails=1 fail_timeout=3s;
|
||||
server api2:8080 max_fails=1 fail_timeout=3s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
location / {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_send_timeout 60s;
|
||||
proxy_read_timeout 60s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_pass http://api_backends;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX users_username_uidx ON users (username);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats (id),
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id),
|
||||
sender_id UUID NOT NULL REFERENCES users (id),
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
revoked_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_user_idx ON sessions (user_id);
|
||||
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE idempotency_records (
|
||||
scope_key TEXT PRIMARY KEY,
|
||||
request_hash TEXT NOT NULL,
|
||||
message_id UUID NOT NULL REFERENCES messages (id),
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
expires_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_records_expires_idx ON idempotency_records (expires_at);
|
||||
@@ -0,0 +1,25 @@
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id),
|
||||
owner_id UUID NOT NULL REFERENCES users (id),
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL,
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_id_idx ON attachments (chat_id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages (id),
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id),
|
||||
position INT NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,32 @@
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_object_key TEXT;
|
||||
|
||||
UPDATE attachments
|
||||
SET staging_object_key = object_key
|
||||
WHERE staging_object_key IS NULL;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ALTER COLUMN staging_object_key SET NOT NULL;
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id),
|
||||
name TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
width INT NOT NULL,
|
||||
height INT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
PRIMARY KEY (attachment_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL UNIQUE REFERENCES attachments (id),
|
||||
pipeline_version INT NOT NULL,
|
||||
attempt INT NOT NULL,
|
||||
max_attempts INT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_jobs_attachment_id_idx ON attachment_jobs (attachment_id);
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi==0.115.6
|
||||
uvicorn[standard]==0.34.0
|
||||
pydantic==2.10.4
|
||||
SQLAlchemy==2.0.36
|
||||
psycopg[binary]==3.2.4
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.35.76
|
||||
Pillow==11.0.0
|
||||
pika==1.3.2
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env python3
|
||||
from messenger.db import apply_migrations, close_engine, init_engine
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_engine()
|
||||
try:
|
||||
apply_migrations()
|
||||
print("migrations applied")
|
||||
finally:
|
||||
close_engine()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0\.."
|
||||
docker compose --env-file .env restart postgres s3 broker api1 api2 worker lb
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart postgres s3 broker api1 api2 worker lb
|
||||
@@ -0,0 +1,12 @@
|
||||
@echo off
|
||||
cd /d "%~dp0\.."
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo TARGET_INSTANCE_ID is required >&2
|
||||
exit /b 1
|
||||
)
|
||||
if /I not "%TARGET_INSTANCE_ID%"=="api1" if /I not "%TARGET_INSTANCE_ID%"=="api2" (
|
||||
echo unknown TARGET_INSTANCE_ID: %TARGET_INSTANCE_ID% >&2
|
||||
exit /b 1
|
||||
)
|
||||
docker compose --env-file .env stop %TARGET_INSTANCE_ID%
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api1|api2) ;;
|
||||
*)
|
||||
printf '%s\n' "unknown TARGET_INSTANCE_ID: $TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
docker compose --env-file .env stop "$TARGET_INSTANCE_ID"
|
||||
@@ -0,0 +1,326 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, Query, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from .cursors import decode_cursor, encode_cursor
|
||||
from .db import close_engine, init_engine, ping
|
||||
from .errors import AppError, error_response, request_id_of
|
||||
from .images import (
|
||||
max_upload_bytes,
|
||||
sniff_image,
|
||||
validate_filename,
|
||||
validate_image_pixels,
|
||||
)
|
||||
from .models import CreateChat, CreateMessage, CreateUser
|
||||
from .broker import ping_broker
|
||||
from .s3util import close_s3, ensure_bucket, ping_s3
|
||||
from .store import store
|
||||
|
||||
BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
init_engine()
|
||||
ensure_bucket()
|
||||
yield
|
||||
close_s3()
|
||||
close_engine()
|
||||
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger lab5", version="1.5.0", lifespan=lifespan)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def attach_request_and_instance(request: Request, call_next):
|
||||
incoming = request.headers.get("x-request-id")
|
||||
request.state.request_id = incoming.strip() if incoming and incoming.strip() else str(uuid.uuid4())
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(AppError)
|
||||
async def app_error_handler(request: Request, exc: AppError) -> JSONResponse:
|
||||
return error_response(
|
||||
exc.status_code,
|
||||
exc.code,
|
||||
exc.message,
|
||||
request_id_of(request),
|
||||
headers=exc.headers,
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
return error_response(
|
||||
400,
|
||||
"invalid_request",
|
||||
"Request validation failed",
|
||||
request_id_of(request),
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
|
||||
code = "not_found" if exc.status_code == 404 else "invalid_request"
|
||||
headers = None
|
||||
if exc.status_code == 401:
|
||||
code = "unauthorized"
|
||||
headers = BEARER_CHALLENGE
|
||||
elif exc.status_code >= 500:
|
||||
code = "internal_error"
|
||||
message = exc.detail if isinstance(exc.detail, str) else "Request failed"
|
||||
return error_response(
|
||||
exc.status_code,
|
||||
code,
|
||||
message,
|
||||
request_id_of(request),
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unhandled_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
return error_response(500, "internal_error", "Internal server error", request_id_of(request))
|
||||
|
||||
|
||||
def current_user_id(authorization: Optional[str] = Header(default=None)) -> str:
|
||||
if authorization is None or not authorization.strip():
|
||||
raise AppError(401, "unauthorized", "Missing bearer token", headers=BEARER_CHALLENGE)
|
||||
scheme, _, credential = authorization.partition(" ")
|
||||
if scheme.lower() != "bearer" or not credential.strip():
|
||||
raise AppError(401, "unauthorized", "Bearer token required", headers=BEARER_CHALLENGE)
|
||||
return store.resolve_bearer(credential.strip())
|
||||
|
||||
|
||||
def require_chat_member(chat_id: UUID, user_id: str):
|
||||
chat = store.get_chat(str(chat_id))
|
||||
if chat is None or user_id not in chat.member_ids:
|
||||
raise AppError(404, "not_found", "Chat not found")
|
||||
return chat
|
||||
|
||||
|
||||
def parse_basic_credentials(authorization: Optional[str]) -> tuple[str, str]:
|
||||
if authorization is None or not authorization.strip():
|
||||
raise AppError(401, "unauthorized", "Missing basic credentials", headers=BASIC_CHALLENGE)
|
||||
scheme, _, credential = authorization.partition(" ")
|
||||
if scheme.lower() != "basic" or not credential.strip():
|
||||
raise AppError(401, "unauthorized", "Basic credentials required", headers=BASIC_CHALLENGE)
|
||||
try:
|
||||
decoded = base64.b64decode(credential.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError, ValueError) as exc:
|
||||
raise AppError(401, "unauthorized", "Invalid basic credentials", headers=BASIC_CHALLENGE) from exc
|
||||
if ":" not in decoded:
|
||||
raise AppError(401, "unauthorized", "Invalid basic credentials", headers=BASIC_CHALLENGE)
|
||||
username, password = decoded.split(":", 1)
|
||||
if not username or not username.isascii():
|
||||
raise AppError(401, "unauthorized", "Invalid basic credentials", headers=BASIC_CHALLENGE)
|
||||
return username, password
|
||||
|
||||
|
||||
def require_idempotency_key(idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key")) -> str:
|
||||
if idempotency_key is None or not idempotency_key.strip():
|
||||
raise AppError(400, "invalid_request", "Idempotency-Key is required")
|
||||
key = idempotency_key.strip()
|
||||
if not IDEMPOTENCY_KEY_RE.fullmatch(key):
|
||||
raise AppError(400, "invalid_request", "Idempotency-Key format is invalid")
|
||||
return key
|
||||
|
||||
|
||||
async def read_upload_body(request: Request) -> bytes:
|
||||
limit = max_upload_bytes()
|
||||
content_length = request.headers.get("content-length")
|
||||
if content_length is not None:
|
||||
try:
|
||||
declared = int(content_length)
|
||||
except ValueError as exc:
|
||||
raise AppError(400, "invalid_request", "Invalid Content-Length") from exc
|
||||
if declared > limit:
|
||||
raise AppError(413, "payload_too_large", "Upload exceeds size limit")
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
async for chunk in request.stream():
|
||||
if not chunk:
|
||||
continue
|
||||
total += len(chunk)
|
||||
if total > limit:
|
||||
raise AppError(413, "payload_too_large", "Upload exceeds size limit")
|
||||
chunks.append(chunk)
|
||||
data = b"".join(chunks)
|
||||
if not data:
|
||||
raise AppError(400, "invalid_request", "Empty upload body")
|
||||
return data
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def health_live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def health_ready():
|
||||
if not ping():
|
||||
raise AppError(503, "unavailable", "Database is unavailable")
|
||||
if not ping_s3():
|
||||
raise AppError(503, "unavailable", "Object storage is unavailable")
|
||||
if not ping_broker():
|
||||
raise AppError(503, "unavailable", "Broker is unavailable")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
def register_user(body: CreateUser):
|
||||
user = store.create_user(body.username, body.display_name, body.password)
|
||||
return user.to_dict()
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
def create_session(authorization: Optional[str] = Header(default=None)):
|
||||
username, password = parse_basic_credentials(authorization)
|
||||
session = store.create_session(username, password)
|
||||
return session.to_dict()
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def revoke_session(authorization: Optional[str] = Header(default=None)):
|
||||
if authorization is None or not authorization.strip():
|
||||
raise AppError(401, "unauthorized", "Missing bearer token", headers=BEARER_CHALLENGE)
|
||||
scheme, _, credential = authorization.partition(" ")
|
||||
if scheme.lower() != "bearer" or not credential.strip():
|
||||
raise AppError(401, "unauthorized", "Bearer token required", headers=BEARER_CHALLENGE)
|
||||
store.revoke_session(credential.strip())
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def get_me(user_id: str = Depends(current_user_id)):
|
||||
user = store.get_user(user_id)
|
||||
if user is None:
|
||||
raise AppError(401, "unauthorized", "Unknown user", headers=BEARER_CHALLENGE)
|
||||
return user.to_dict()
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
def create_chat(body: CreateChat, user_id: str = Depends(current_user_id)):
|
||||
member_ids = [str(m) for m in body.member_ids]
|
||||
if user_id not in member_ids:
|
||||
raise AppError(400, "invalid_request", "Caller must be included in member_ids")
|
||||
chat = store.create_chat(body.title, member_ids)
|
||||
return chat.to_dict()
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def list_chats(
|
||||
user_id: str = Depends(current_user_id),
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||
):
|
||||
resource = f"chats:{user_id}"
|
||||
after = decode_cursor(cursor, resource) if cursor else None
|
||||
page, last_for_cursor = store.list_chats_for_user(user_id, limit, after)
|
||||
next_cursor = None
|
||||
if last_for_cursor is not None:
|
||||
next_cursor = encode_cursor(last_for_cursor.created_at, last_for_cursor.id, resource)
|
||||
return {"items": [c.to_dict() for c in page], "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def get_chat(chat_id: UUID, user_id: str = Depends(current_user_id)):
|
||||
chat = require_chat_member(chat_id, user_id)
|
||||
return chat.to_dict()
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def create_message(
|
||||
chat_id: UUID,
|
||||
body: CreateMessage,
|
||||
request: Request,
|
||||
user_id: str = Depends(current_user_id),
|
||||
idempotency_key: str = Depends(require_idempotency_key),
|
||||
):
|
||||
require_chat_member(chat_id, user_id)
|
||||
message = store.create_message(
|
||||
str(chat_id),
|
||||
user_id,
|
||||
body.text,
|
||||
[str(item) for item in body.attachment_ids],
|
||||
idempotency_key=idempotency_key,
|
||||
request_path=request.url.path,
|
||||
)
|
||||
return message.to_dict()
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def list_messages(
|
||||
chat_id: UUID,
|
||||
user_id: str = Depends(current_user_id),
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||
):
|
||||
require_chat_member(chat_id, user_id)
|
||||
resource = f"messages:{chat_id}"
|
||||
after = decode_cursor(cursor, resource) if cursor else None
|
||||
page, last_for_cursor = store.list_messages(str(chat_id), limit, after)
|
||||
next_cursor = None
|
||||
if last_for_cursor is not None:
|
||||
next_cursor = encode_cursor(last_for_cursor.created_at, last_for_cursor.id, resource)
|
||||
return {"items": [m.to_dict() for m in page], "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(
|
||||
chat_id: UUID,
|
||||
request: Request,
|
||||
filename: str = Query(...),
|
||||
user_id: str = Depends(current_user_id),
|
||||
):
|
||||
require_chat_member(chat_id, user_id)
|
||||
safe_name = validate_filename(filename)
|
||||
content_type = (request.headers.get("content-type") or "").split(";")[0].strip().lower()
|
||||
data = await read_upload_body(request)
|
||||
detected = sniff_image(data, content_type)
|
||||
validate_image_pixels(data, detected)
|
||||
attachment = store.create_attachment(
|
||||
str(chat_id),
|
||||
user_id,
|
||||
safe_name,
|
||||
detected,
|
||||
data,
|
||||
)
|
||||
return attachment.to_dict()
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
def get_attachment(attachment_id: UUID, user_id: str = Depends(current_user_id)):
|
||||
attachment = store.get_attachment_for_user(str(attachment_id), user_id)
|
||||
return attachment.to_dict()
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download_attachment(
|
||||
attachment_id: UUID,
|
||||
user_id: str = Depends(current_user_id),
|
||||
variant: str = Query(default="original"),
|
||||
):
|
||||
data, content_type = store.download_attachment_for_user(
|
||||
str(attachment_id), user_id, variant=variant
|
||||
)
|
||||
return Response(content=data, media_type=content_type)
|
||||
@@ -0,0 +1,105 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import pika
|
||||
from pika.adapters.blocking_connection import BlockingChannel, BlockingConnection
|
||||
from pika.exceptions import AMQPConnectionError, AMQPError, NackError, UnroutableError
|
||||
|
||||
QUEUE_NAME = "attachment.process"
|
||||
|
||||
|
||||
def _read_secret(env_name: str, file_env_name: str) -> str:
|
||||
path = os.environ.get(file_env_name)
|
||||
if path:
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
value = os.environ.get(env_name)
|
||||
if not value:
|
||||
raise RuntimeError(f"{env_name} or {file_env_name} is required")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def broker_url() -> str:
|
||||
return _read_secret("BROKER_URL", "BROKER_URL_FILE")
|
||||
|
||||
|
||||
def connect() -> BlockingConnection:
|
||||
params = pika.URLParameters(broker_url())
|
||||
params.heartbeat = 30
|
||||
params.blocked_connection_timeout = 30
|
||||
return pika.BlockingConnection(params)
|
||||
|
||||
|
||||
def declare_queue(channel: BlockingChannel) -> None:
|
||||
channel.queue_declare(queue=QUEUE_NAME, durable=True)
|
||||
|
||||
|
||||
def publish_job(payload: dict[str, Any]) -> None:
|
||||
connection = connect()
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.confirm_delivery()
|
||||
declare_queue(channel)
|
||||
body = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=QUEUE_NAME,
|
||||
body=body,
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_type="application/json",
|
||||
content_encoding="utf-8",
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
except (AMQPError, NackError, UnroutableError, OSError) as exc:
|
||||
raise RuntimeError("broker publish failed") from exc
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def ping_broker() -> bool:
|
||||
try:
|
||||
connection = connect()
|
||||
try:
|
||||
channel = connection.channel()
|
||||
declare_queue(channel)
|
||||
return True
|
||||
finally:
|
||||
connection.close()
|
||||
except (AMQPConnectionError, AMQPError, OSError, RuntimeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def consume_jobs(
|
||||
handler: Callable[[dict[str, Any], BlockingChannel, Any], None],
|
||||
*,
|
||||
prefetch: int,
|
||||
should_stop: Optional[Callable[[], bool]] = None,
|
||||
) -> None:
|
||||
connection = connect()
|
||||
channel = connection.channel()
|
||||
declare_queue(channel)
|
||||
channel.basic_qos(prefetch_count=max(1, prefetch))
|
||||
|
||||
def _on_message(ch: BlockingChannel, method: Any, _properties: Any, body: bytes) -> None:
|
||||
try:
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("job payload must be an object")
|
||||
handler(payload, ch, method)
|
||||
except Exception:
|
||||
try:
|
||||
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
channel.basic_consume(queue=QUEUE_NAME, on_message_callback=_on_message, auto_ack=False)
|
||||
while True:
|
||||
if should_stop is not None and should_stop():
|
||||
break
|
||||
connection.process_data_events(time_limit=1)
|
||||
connection.close()
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from typing import Tuple
|
||||
|
||||
from .errors import AppError
|
||||
|
||||
|
||||
def encode_cursor(created_at: str, item_id: str, resource: str) -> str:
|
||||
raw = json.dumps(
|
||||
{"t": created_at, "i": item_id, "r": resource},
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(cursor: str, expected_resource: str) -> Tuple[str, str]:
|
||||
try:
|
||||
padded = cursor + "=" * (-len(cursor) % 4)
|
||||
data = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8"))
|
||||
created_at = data["t"]
|
||||
item_id = data["i"]
|
||||
resource = data["r"]
|
||||
if not isinstance(created_at, str) or not isinstance(item_id, str):
|
||||
raise ValueError("bad cursor fields")
|
||||
if resource != expected_resource:
|
||||
raise AppError(400, "invalid_request", "Cursor does not belong to this resource")
|
||||
return created_at, item_id
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(400, "invalid_request", "Invalid cursor") from exc
|
||||
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Iterator, Optional
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
_engine: Optional[Engine] = None
|
||||
_SessionLocal: Optional[sessionmaker[Session]] = None
|
||||
|
||||
|
||||
def _read_database_url() -> str:
|
||||
url_file = os.environ.get("DATABASE_URL_FILE")
|
||||
if url_file:
|
||||
url = Path(url_file).read_text(encoding="utf-8").strip()
|
||||
else:
|
||||
url = os.environ.get("DATABASE_URL")
|
||||
if not url:
|
||||
raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required")
|
||||
url = url.strip()
|
||||
if url.startswith("postgresql://"):
|
||||
url = "postgresql+psycopg://" + url[len("postgresql://") :]
|
||||
elif url.startswith("postgres://"):
|
||||
url = "postgresql+psycopg://" + url[len("postgres://") :]
|
||||
return url
|
||||
|
||||
|
||||
def init_engine() -> Engine:
|
||||
global _engine, _SessionLocal
|
||||
if _engine is not None:
|
||||
return _engine
|
||||
_engine = create_engine(
|
||||
_read_database_url(),
|
||||
pool_size=5,
|
||||
max_overflow=5,
|
||||
pool_pre_ping=True,
|
||||
pool_timeout=10,
|
||||
)
|
||||
_SessionLocal = sessionmaker(bind=_engine, expire_on_commit=False)
|
||||
return _engine
|
||||
|
||||
|
||||
def get_engine() -> Engine:
|
||||
if _engine is None:
|
||||
raise RuntimeError("database engine is not initialized")
|
||||
return _engine
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_scope() -> Iterator[Session]:
|
||||
if _SessionLocal is None:
|
||||
raise RuntimeError("database engine is not initialized")
|
||||
session = _SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def close_engine() -> None:
|
||||
global _engine, _SessionLocal
|
||||
if _engine is not None:
|
||||
_engine.dispose()
|
||||
_engine = None
|
||||
_SessionLocal = None
|
||||
|
||||
|
||||
def migrations_dir() -> Path:
|
||||
return Path(__file__).resolve().parents[2] / "migrations"
|
||||
|
||||
|
||||
def apply_migrations() -> None:
|
||||
engine = get_engine()
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
applied = {
|
||||
row[0]
|
||||
for row in conn.execute(text("SELECT version FROM schema_migrations")).fetchall()
|
||||
}
|
||||
for path in sorted(migrations_dir().glob("*.sql")):
|
||||
version = path.name
|
||||
if version in applied:
|
||||
continue
|
||||
conn.exec_driver_sql(path.read_text(encoding="utf-8"))
|
||||
conn.execute(
|
||||
text("INSERT INTO schema_migrations (version) VALUES (:version)"),
|
||||
{"version": version},
|
||||
)
|
||||
|
||||
|
||||
def ping() -> bool:
|
||||
try:
|
||||
engine = get_engine()
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
|
||||
class AppError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
headers: Optional[dict] = None,
|
||||
) -> None:
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
def request_id_of(request: Request) -> str:
|
||||
return getattr(request.state, "request_id", "unknown")
|
||||
|
||||
|
||||
def error_response(
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
request_id: str,
|
||||
headers: Optional[dict] = None,
|
||||
) -> JSONResponse:
|
||||
payload = {
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": request_id,
|
||||
}
|
||||
}
|
||||
response_headers = {"X-Request-Id": request_id}
|
||||
if headers:
|
||||
response_headers.update(headers)
|
||||
return JSONResponse(status_code=status_code, content=payload, headers=response_headers)
|
||||
@@ -0,0 +1,68 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
from typing import Tuple
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from .errors import AppError
|
||||
|
||||
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||
|
||||
|
||||
def max_upload_bytes() -> int:
|
||||
return max(1, int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")))
|
||||
|
||||
|
||||
def max_image_pixels() -> int:
|
||||
return max(1, int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")))
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> str:
|
||||
if len(filename) < 1 or len(filename) > 255:
|
||||
raise AppError(400, "invalid_request", "filename length is invalid")
|
||||
for ch in filename:
|
||||
code = ord(ch)
|
||||
if ch in "/\\" or code == 0 or code < 32:
|
||||
raise AppError(400, "invalid_request", "filename contains forbidden characters")
|
||||
return filename
|
||||
|
||||
|
||||
def sniff_image(data: bytes, declared_content_type: str) -> str:
|
||||
if declared_content_type not in {"image/png", "image/jpeg"}:
|
||||
raise AppError(415, "unsupported_media_type", "Unsupported media type")
|
||||
if data.startswith(PNG_MAGIC):
|
||||
detected = "image/png"
|
||||
elif data.startswith(JPEG_MAGIC):
|
||||
detected = "image/jpeg"
|
||||
else:
|
||||
raise AppError(415, "unsupported_media_type", "Unsupported or corrupted image")
|
||||
if detected != declared_content_type:
|
||||
raise AppError(415, "unsupported_media_type", "Content-Type does not match body")
|
||||
return detected
|
||||
|
||||
|
||||
def validate_image_pixels(data: bytes, content_type: str) -> Tuple[int, int]:
|
||||
Image.MAX_IMAGE_PIXELS = max_image_pixels()
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as img:
|
||||
fmt = (img.format or "").upper()
|
||||
if content_type == "image/png" and fmt != "PNG":
|
||||
raise AppError(415, "unsupported_media_type", "Invalid PNG image")
|
||||
if content_type == "image/jpeg" and fmt not in {"JPEG", "MPO"}:
|
||||
raise AppError(415, "unsupported_media_type", "Invalid JPEG image")
|
||||
width, height = img.size
|
||||
if width < 1 or height < 1:
|
||||
raise AppError(415, "unsupported_media_type", "Invalid image dimensions")
|
||||
if width * height > max_image_pixels():
|
||||
raise AppError(413, "payload_too_large", "Image exceeds pixel limit")
|
||||
img.load()
|
||||
return width, height
|
||||
except AppError:
|
||||
raise
|
||||
except Image.DecompressionBombError as exc:
|
||||
raise AppError(413, "payload_too_large", "Image exceeds pixel limit") from exc
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
|
||||
raise AppError(415, "unsupported_media_type", "Unsupported or corrupted image") from exc
|
||||
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import List
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
|
||||
|
||||
def _not_blank(value: str, field_name: str) -> str:
|
||||
if value.strip() == "":
|
||||
raise ValueError(f"{field_name} must not be blank")
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
username: str
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def validate_username(cls, value: str) -> str:
|
||||
if not USERNAME_RE.fullmatch(value):
|
||||
raise ValueError("username must match [a-z0-9_]{3,32}")
|
||||
return value
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def validate_display_name(cls, value: str) -> str:
|
||||
return _not_blank(value, "display_name")
|
||||
|
||||
@field_validator("password")
|
||||
@classmethod
|
||||
def validate_password(cls, value: str) -> str:
|
||||
if len(value) < 12 or len(value) > 128:
|
||||
raise ValueError("password length must be 12-128")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: List[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def validate_title(cls, value: str) -> str:
|
||||
return _not_blank(value, "title")
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def validate_members(cls, value: List[UUID]) -> List[UUID]:
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("member_ids must be unique")
|
||||
return value
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: List[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def validate_attachment_ids(cls, value: List[UUID]) -> List[UUID]:
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("attachment_ids must be unique")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_text_or_attachments(self) -> "CreateMessage":
|
||||
if self.text.strip() == "" and not self.attachment_ids:
|
||||
raise ValueError("text must not be blank without attachments")
|
||||
return self
|
||||
@@ -0,0 +1,120 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import boto3
|
||||
from botocore.client import BaseClient, Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from .errors import AppError
|
||||
|
||||
_client: Optional[BaseClient] = None
|
||||
|
||||
|
||||
def _read_secret(env_name: str, file_env_name: str) -> str:
|
||||
path = os.environ.get(file_env_name)
|
||||
if path:
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
value = os.environ.get(env_name)
|
||||
if not value:
|
||||
raise RuntimeError(f"{env_name} or {file_env_name} is required")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def s3_endpoint() -> str:
|
||||
return os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/")
|
||||
|
||||
|
||||
def s3_bucket() -> str:
|
||||
return os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
|
||||
|
||||
def s3_region() -> str:
|
||||
return os.environ.get("S3_REGION", "us-east-1")
|
||||
|
||||
|
||||
def get_s3_client() -> BaseClient:
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=s3_endpoint(),
|
||||
region_name=s3_region(),
|
||||
aws_access_key_id=_read_secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE"),
|
||||
aws_secret_access_key=_read_secret(
|
||||
"S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE"
|
||||
),
|
||||
config=Config(s3={"addressing_style": "path"}),
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
def ensure_bucket() -> None:
|
||||
client = get_s3_client()
|
||||
bucket = s3_bucket()
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
except ClientError:
|
||||
try:
|
||||
client.create_bucket(Bucket=bucket)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||||
raise
|
||||
try:
|
||||
client.put_public_access_block(
|
||||
Bucket=bucket,
|
||||
PublicAccessBlockConfiguration={
|
||||
"BlockPublicAcls": True,
|
||||
"IgnorePublicAcls": True,
|
||||
"BlockPublicPolicy": True,
|
||||
"RestrictPublicBuckets": True,
|
||||
},
|
||||
)
|
||||
except ClientError:
|
||||
pass
|
||||
|
||||
|
||||
def put_object(object_key: str, data: bytes, content_type: str) -> None:
|
||||
try:
|
||||
get_s3_client().put_object(
|
||||
Bucket=s3_bucket(),
|
||||
Key=object_key,
|
||||
Body=data,
|
||||
ContentType=content_type,
|
||||
)
|
||||
except ClientError as exc:
|
||||
raise AppError(503, "unavailable", "Object storage is unavailable") from exc
|
||||
|
||||
|
||||
def get_object_bytes(object_key: str) -> bytes:
|
||||
try:
|
||||
response = get_s3_client().get_object(Bucket=s3_bucket(), Key=object_key)
|
||||
return response["Body"].read()
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code in {"NoSuchKey", "404", "NotFound"}:
|
||||
raise AppError(404, "not_found", "Attachment not found") from exc
|
||||
raise AppError(503, "unavailable", "Object storage is unavailable") from exc
|
||||
|
||||
|
||||
def delete_object(object_key: str) -> None:
|
||||
try:
|
||||
get_s3_client().delete_object(Bucket=s3_bucket(), Key=object_key)
|
||||
except ClientError:
|
||||
pass
|
||||
|
||||
|
||||
def ping_s3() -> bool:
|
||||
try:
|
||||
get_s3_client().head_bucket(Bucket=s3_bucket())
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def close_s3() -> None:
|
||||
global _client
|
||||
_client = None
|
||||
@@ -0,0 +1,629 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Optional, Tuple
|
||||
from uuid import UUID
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
from sqlalchemy import delete, func, select, tuple_
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from .broker import publish_job
|
||||
from .db import session_scope
|
||||
from .errors import AppError
|
||||
from .s3util import get_object_bytes, put_object
|
||||
from .tables import (
|
||||
AttachmentJobRow,
|
||||
AttachmentRow,
|
||||
AttachmentVariantRow,
|
||||
ChatMemberRow,
|
||||
ChatRow,
|
||||
IdempotencyRow,
|
||||
MessageAttachmentRow,
|
||||
MessageRow,
|
||||
SessionRow,
|
||||
UserRow,
|
||||
)
|
||||
|
||||
password_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def format_ts(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def parse_ts(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def session_ttl_seconds() -> int:
|
||||
raw = os.environ.get("SESSION_TTL_SECONDS", "3600")
|
||||
return max(1, int(raw))
|
||||
|
||||
|
||||
def idempotency_ttl_seconds() -> int:
|
||||
raw = os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")
|
||||
return max(1, int(raw))
|
||||
|
||||
|
||||
def job_max_attempts() -> int:
|
||||
return max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5")))
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
return hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def request_body_hash(text: str, attachment_ids: List[str]) -> str:
|
||||
normalized = json.dumps(
|
||||
{"text": text, "attachment_ids": attachment_ids},
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def idempotency_scope(user_id: str, method: str, path: str, key: str) -> str:
|
||||
return f"{user_id}|{method.upper()}|{path}|{key}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class User:
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"username": self.username,
|
||||
"display_name": self.display_name,
|
||||
"created_at": self.created_at,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chat:
|
||||
id: str
|
||||
title: str
|
||||
member_ids: List[str]
|
||||
created_at: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"title": self.title,
|
||||
"member_ids": list(self.member_ids),
|
||||
"created_at": self.created_at,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: List[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"chat_id": self.chat_id,
|
||||
"sender_id": self.sender_id,
|
||||
"text": self.text,
|
||||
"created_at": self.created_at,
|
||||
"attachment_ids": list(self.attachment_ids),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Attachment:
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str
|
||||
variants: List[dict]
|
||||
error_code: Optional[str]
|
||||
created_at: str
|
||||
updated_at: str
|
||||
object_key: str = ""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"chat_id": self.chat_id,
|
||||
"owner_id": self.owner_id,
|
||||
"filename": self.filename,
|
||||
"content_type": self.content_type,
|
||||
"size_bytes": self.size_bytes,
|
||||
"sha256": self.sha256,
|
||||
"status": self.status,
|
||||
"variants": list(self.variants),
|
||||
"error_code": self.error_code,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Session:
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: User
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"token": self.token,
|
||||
"token_type": self.token_type,
|
||||
"expires_at": self.expires_at,
|
||||
"user": self.user.to_dict(),
|
||||
}
|
||||
|
||||
|
||||
def _user_from_row(row: UserRow) -> User:
|
||||
return User(
|
||||
id=str(row.id),
|
||||
username=row.username,
|
||||
display_name=row.display_name,
|
||||
created_at=format_ts(row.created_at),
|
||||
)
|
||||
|
||||
|
||||
def _chat_from_row(row: ChatRow, member_ids: List[str]) -> Chat:
|
||||
return Chat(
|
||||
id=str(row.id),
|
||||
title=row.title,
|
||||
member_ids=member_ids,
|
||||
created_at=format_ts(row.created_at),
|
||||
)
|
||||
|
||||
|
||||
def _variants_for(session, attachment_id: UUID) -> List[dict]:
|
||||
rows = session.scalars(
|
||||
select(AttachmentVariantRow)
|
||||
.where(AttachmentVariantRow.attachment_id == attachment_id)
|
||||
.order_by(AttachmentVariantRow.name.asc())
|
||||
).all()
|
||||
return [
|
||||
{
|
||||
"name": row.name,
|
||||
"content_type": row.content_type,
|
||||
"size_bytes": int(row.size_bytes),
|
||||
"width": int(row.width),
|
||||
"height": int(row.height),
|
||||
"sha256": row.sha256,
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def _attachment_from_row(session, row: AttachmentRow) -> Attachment:
|
||||
return Attachment(
|
||||
id=str(row.id),
|
||||
chat_id=str(row.chat_id),
|
||||
owner_id=str(row.owner_id),
|
||||
filename=row.filename,
|
||||
content_type=row.content_type,
|
||||
size_bytes=int(row.size_bytes),
|
||||
sha256=row.sha256,
|
||||
status=row.status,
|
||||
variants=_variants_for(session, row.id),
|
||||
error_code=row.error_code,
|
||||
created_at=format_ts(row.created_at),
|
||||
updated_at=format_ts(row.updated_at),
|
||||
object_key=row.object_key,
|
||||
)
|
||||
|
||||
|
||||
def _job_payload(job: AttachmentJobRow) -> dict:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job.job_id),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(job.attachment_id),
|
||||
"pipeline_version": int(job.pipeline_version),
|
||||
"attempt": int(job.attempt),
|
||||
"created_at": format_ts(job.created_at),
|
||||
}
|
||||
|
||||
|
||||
def _message_attachment_ids(session, message_id: UUID) -> List[str]:
|
||||
rows = session.scalars(
|
||||
select(MessageAttachmentRow.attachment_id)
|
||||
.where(MessageAttachmentRow.message_id == message_id)
|
||||
.order_by(MessageAttachmentRow.position.asc())
|
||||
).all()
|
||||
return [str(attachment_id) for attachment_id in rows]
|
||||
|
||||
|
||||
def _message_from_row(session, row: MessageRow) -> Message:
|
||||
return Message(
|
||||
id=str(row.id),
|
||||
chat_id=str(row.chat_id),
|
||||
sender_id=str(row.sender_id),
|
||||
text=row.text,
|
||||
created_at=format_ts(row.created_at),
|
||||
attachment_ids=_message_attachment_ids(session, row.id),
|
||||
)
|
||||
|
||||
|
||||
def _member_ids(session, chat_id: UUID) -> List[str]:
|
||||
rows = session.scalars(
|
||||
select(ChatMemberRow.user_id)
|
||||
.where(ChatMemberRow.chat_id == chat_id)
|
||||
.order_by(ChatMemberRow.user_id)
|
||||
).all()
|
||||
return [str(user_id) for user_id in rows]
|
||||
|
||||
|
||||
def _aware(value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value
|
||||
|
||||
|
||||
def _is_chat_member(session, chat_id: UUID, user_id: str) -> bool:
|
||||
row = session.get(ChatMemberRow, {"chat_id": chat_id, "user_id": UUID(user_id)})
|
||||
return row is not None
|
||||
|
||||
|
||||
class Store:
|
||||
def create_user(self, username: str, display_name: str, password: str) -> User:
|
||||
row = UserRow(
|
||||
id=uuid.uuid4(),
|
||||
username=username,
|
||||
display_name=display_name,
|
||||
password_hash=password_hasher.hash(password),
|
||||
created_at=utc_now(),
|
||||
)
|
||||
try:
|
||||
with session_scope() as session:
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return _user_from_row(row)
|
||||
except IntegrityError as exc:
|
||||
raise AppError(409, "conflict", "Username is already taken") from exc
|
||||
|
||||
def get_user(self, user_id: str) -> Optional[User]:
|
||||
with session_scope() as session:
|
||||
row = session.get(UserRow, UUID(user_id))
|
||||
return _user_from_row(row) if row else None
|
||||
|
||||
def create_session(self, username: str, password: str) -> Session:
|
||||
with session_scope() as session:
|
||||
row = session.scalar(select(UserRow).where(UserRow.username == username))
|
||||
if row is None:
|
||||
raise AppError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Invalid credentials",
|
||||
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
|
||||
)
|
||||
try:
|
||||
password_hasher.verify(row.password_hash, password)
|
||||
except VerifyMismatchError as exc:
|
||||
raise AppError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Invalid credentials",
|
||||
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
|
||||
) from exc
|
||||
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = utc_now() + timedelta(seconds=session_ttl_seconds())
|
||||
session.add(
|
||||
SessionRow(
|
||||
token_hash=hash_token(token),
|
||||
user_id=row.id,
|
||||
expires_at=expires_at,
|
||||
created_at=utc_now(),
|
||||
)
|
||||
)
|
||||
user = _user_from_row(row)
|
||||
return Session(
|
||||
token=token,
|
||||
token_type="Bearer",
|
||||
expires_at=format_ts(expires_at),
|
||||
user=user,
|
||||
)
|
||||
|
||||
def resolve_bearer(self, token: str) -> str:
|
||||
with session_scope() as session:
|
||||
row = session.get(SessionRow, hash_token(token))
|
||||
challenge = {"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
if row is None:
|
||||
raise AppError(401, "unauthorized", "Invalid session", headers=challenge)
|
||||
if row.revoked_at is not None:
|
||||
raise AppError(401, "unauthorized", "Session revoked", headers=challenge)
|
||||
if _aware(row.expires_at) <= utc_now():
|
||||
raise AppError(401, "unauthorized", "Session expired", headers=challenge)
|
||||
return str(row.user_id)
|
||||
|
||||
def revoke_session(self, token: str) -> None:
|
||||
with session_scope() as session:
|
||||
row = session.get(SessionRow, hash_token(token), with_for_update=True)
|
||||
challenge = {"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
if row is None:
|
||||
raise AppError(401, "unauthorized", "Invalid session", headers=challenge)
|
||||
if row.revoked_at is not None:
|
||||
raise AppError(401, "unauthorized", "Session revoked", headers=challenge)
|
||||
if _aware(row.expires_at) <= utc_now():
|
||||
raise AppError(401, "unauthorized", "Session expired", headers=challenge)
|
||||
row.revoked_at = utc_now()
|
||||
|
||||
def create_chat(self, title: str, member_ids: List[str]) -> Chat:
|
||||
chat_id = uuid.uuid4()
|
||||
member_uuids = [UUID(m) for m in member_ids]
|
||||
with session_scope() as session:
|
||||
existing = session.scalar(
|
||||
select(func.count()).select_from(UserRow).where(UserRow.id.in_(member_uuids))
|
||||
)
|
||||
if existing != len(member_uuids):
|
||||
raise AppError(400, "invalid_request", "Unknown member in member_ids")
|
||||
chat = ChatRow(id=chat_id, title=title, created_at=utc_now())
|
||||
session.add(chat)
|
||||
for member_id in member_uuids:
|
||||
session.add(ChatMemberRow(chat_id=chat_id, user_id=member_id))
|
||||
session.flush()
|
||||
return _chat_from_row(chat, list(member_ids))
|
||||
|
||||
def get_chat(self, chat_id: str) -> Optional[Chat]:
|
||||
with session_scope() as session:
|
||||
row = session.get(ChatRow, UUID(chat_id))
|
||||
if row is None:
|
||||
return None
|
||||
return _chat_from_row(row, _member_ids(session, row.id))
|
||||
|
||||
def list_chats_for_user(
|
||||
self,
|
||||
user_id: str,
|
||||
limit: int,
|
||||
after: Optional[Tuple[str, str]] = None,
|
||||
) -> Tuple[List[Chat], Optional[Chat]]:
|
||||
with session_scope() as session:
|
||||
stmt = (
|
||||
select(ChatRow)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||
.where(ChatMemberRow.user_id == UUID(user_id))
|
||||
)
|
||||
if after is not None:
|
||||
stmt = stmt.where(
|
||||
tuple_(ChatRow.created_at, ChatRow.id)
|
||||
> (parse_ts(after[0]), UUID(after[1]))
|
||||
)
|
||||
stmt = stmt.order_by(ChatRow.created_at.asc(), ChatRow.id.asc()).limit(limit + 1)
|
||||
rows = list(session.scalars(stmt).all())
|
||||
chats = [_chat_from_row(row, _member_ids(session, row.id)) for row in rows]
|
||||
page = chats[:limit]
|
||||
has_more = len(chats) > limit
|
||||
last = page[-1] if page and has_more else None
|
||||
return page, last
|
||||
|
||||
def create_attachment(
|
||||
self,
|
||||
chat_id: str,
|
||||
owner_id: str,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
data: bytes,
|
||||
) -> Attachment:
|
||||
attachment_id = uuid.uuid4()
|
||||
job_id = uuid.uuid4()
|
||||
staging_key = f"staging/{chat_id}/{attachment_id}"
|
||||
final_key = f"attachments/{chat_id}/{attachment_id}/original"
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
now = utc_now()
|
||||
max_attempts = job_max_attempts()
|
||||
put_object(staging_key, data, content_type)
|
||||
with session_scope() as session:
|
||||
row = AttachmentRow(
|
||||
id=attachment_id,
|
||||
chat_id=UUID(chat_id),
|
||||
owner_id=UUID(owner_id),
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
size_bytes=len(data),
|
||||
sha256=digest,
|
||||
object_key=final_key,
|
||||
staging_object_key=staging_key,
|
||||
status="queued",
|
||||
error_code=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
job = AttachmentJobRow(
|
||||
job_id=job_id,
|
||||
attachment_id=attachment_id,
|
||||
pipeline_version=1,
|
||||
attempt=1,
|
||||
max_attempts=max_attempts,
|
||||
created_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
session.add(job)
|
||||
session.flush()
|
||||
payload = _job_payload(job)
|
||||
attachment = _attachment_from_row(session, row)
|
||||
try:
|
||||
publish_job(payload)
|
||||
except RuntimeError as exc:
|
||||
raise AppError(503, "unavailable", "Broker is unavailable") from exc
|
||||
return attachment
|
||||
|
||||
def get_attachment_for_user(self, attachment_id: str, user_id: str) -> Attachment:
|
||||
with session_scope() as session:
|
||||
row = session.get(AttachmentRow, UUID(attachment_id))
|
||||
if row is None or not _is_chat_member(session, row.chat_id, user_id):
|
||||
raise AppError(404, "not_found", "Attachment not found")
|
||||
return _attachment_from_row(session, row)
|
||||
|
||||
def download_attachment_for_user(
|
||||
self, attachment_id: str, user_id: str, variant: str = "original"
|
||||
) -> Tuple[bytes, str]:
|
||||
allowed = {"original", "thumbnail", "crop", "optimized"}
|
||||
if variant not in allowed:
|
||||
raise AppError(400, "invalid_request", "Unsupported variant")
|
||||
with session_scope() as session:
|
||||
row = session.get(AttachmentRow, UUID(attachment_id))
|
||||
if row is None or not _is_chat_member(session, row.chat_id, user_id):
|
||||
raise AppError(404, "not_found", "Attachment not found")
|
||||
if row.status in {"queued", "processing"}:
|
||||
raise AppError(409, "conflict", "Attachment is not ready")
|
||||
if row.status == "failed":
|
||||
raise AppError(409, "conflict", "Attachment processing failed")
|
||||
if row.status != "ready":
|
||||
raise AppError(409, "conflict", "Attachment is not ready")
|
||||
if variant == "original":
|
||||
object_key = row.object_key
|
||||
content_type = row.content_type
|
||||
else:
|
||||
variant_row = session.get(
|
||||
AttachmentVariantRow, {"attachment_id": row.id, "name": variant}
|
||||
)
|
||||
if variant_row is None:
|
||||
raise AppError(404, "not_found", "Attachment variant not found")
|
||||
object_key = variant_row.object_key
|
||||
content_type = variant_row.content_type
|
||||
return get_object_bytes(object_key), content_type
|
||||
|
||||
def create_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
attachment_ids: List[str],
|
||||
*,
|
||||
idempotency_key: str,
|
||||
request_path: str,
|
||||
) -> Message:
|
||||
scope = idempotency_scope(sender_id, "POST", request_path, idempotency_key)
|
||||
body_hash = request_body_hash(text, attachment_ids)
|
||||
|
||||
for _ in range(5):
|
||||
try:
|
||||
with session_scope() as session:
|
||||
existing = session.get(IdempotencyRow, scope, with_for_update=True)
|
||||
if existing is not None:
|
||||
if _aware(existing.expires_at) > utc_now():
|
||||
if existing.request_hash != body_hash:
|
||||
raise AppError(
|
||||
409,
|
||||
"conflict",
|
||||
"Idempotency-Key reused with a different request body",
|
||||
)
|
||||
message = session.get(MessageRow, existing.message_id)
|
||||
if message is None:
|
||||
raise AppError(500, "internal_error", "Idempotent message missing")
|
||||
return _message_from_row(session, message)
|
||||
session.execute(
|
||||
delete(IdempotencyRow).where(IdempotencyRow.scope_key == scope)
|
||||
)
|
||||
session.flush()
|
||||
|
||||
attachment_uuids = [UUID(item) for item in attachment_ids]
|
||||
if attachment_uuids:
|
||||
rows = list(
|
||||
session.scalars(
|
||||
select(AttachmentRow).where(AttachmentRow.id.in_(attachment_uuids))
|
||||
).all()
|
||||
)
|
||||
by_id = {row.id: row for row in rows}
|
||||
for attachment_id in attachment_uuids:
|
||||
row = by_id.get(attachment_id)
|
||||
if row is None or str(row.chat_id) != chat_id:
|
||||
raise AppError(404, "not_found", "Attachment not found")
|
||||
if row.status == "failed":
|
||||
raise AppError(409, "conflict", "Failed attachment cannot be linked")
|
||||
|
||||
message = MessageRow(
|
||||
id=uuid.uuid4(),
|
||||
chat_id=UUID(chat_id),
|
||||
sender_id=UUID(sender_id),
|
||||
text=text,
|
||||
created_at=utc_now(),
|
||||
)
|
||||
session.add(message)
|
||||
session.flush()
|
||||
for position, attachment_id in enumerate(attachment_uuids):
|
||||
session.add(
|
||||
MessageAttachmentRow(
|
||||
message_id=message.id,
|
||||
attachment_id=attachment_id,
|
||||
position=position,
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
IdempotencyRow(
|
||||
scope_key=scope,
|
||||
request_hash=body_hash,
|
||||
message_id=message.id,
|
||||
created_at=utc_now(),
|
||||
expires_at=utc_now() + timedelta(seconds=idempotency_ttl_seconds()),
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
return _message_from_row(session, message)
|
||||
except IntegrityError:
|
||||
continue
|
||||
|
||||
with session_scope() as session:
|
||||
existing = session.get(IdempotencyRow, scope)
|
||||
if existing is not None and _aware(existing.expires_at) > utc_now():
|
||||
if existing.request_hash != body_hash:
|
||||
raise AppError(
|
||||
409,
|
||||
"conflict",
|
||||
"Idempotency-Key reused with a different request body",
|
||||
)
|
||||
message = session.get(MessageRow, existing.message_id)
|
||||
if message is not None:
|
||||
return _message_from_row(session, message)
|
||||
raise AppError(503, "unavailable", "Could not complete idempotent write")
|
||||
|
||||
def list_messages(
|
||||
self,
|
||||
chat_id: str,
|
||||
limit: int,
|
||||
after: Optional[Tuple[str, str]] = None,
|
||||
) -> Tuple[List[Message], Optional[Message]]:
|
||||
with session_scope() as session:
|
||||
stmt = select(MessageRow).where(MessageRow.chat_id == UUID(chat_id))
|
||||
if after is not None:
|
||||
stmt = stmt.where(
|
||||
tuple_(MessageRow.created_at, MessageRow.id)
|
||||
> (parse_ts(after[0]), UUID(after[1]))
|
||||
)
|
||||
stmt = stmt.order_by(MessageRow.created_at.asc(), MessageRow.id.asc()).limit(
|
||||
limit + 1
|
||||
)
|
||||
rows = list(session.scalars(stmt).all())
|
||||
messages = [_message_from_row(session, row) for row in rows]
|
||||
page = messages[:limit]
|
||||
has_more = len(messages) > limit
|
||||
last = page[-1] if page and has_more else None
|
||||
return page, last
|
||||
|
||||
|
||||
store = Store()
|
||||
@@ -0,0 +1,131 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, Uuid
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
|
||||
username: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
display_name: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class ChatRow(Base):
|
||||
__tablename__ = "chats"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
|
||||
title: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
members: Mapped[List["ChatMemberRow"]] = relationship(back_populates="chat")
|
||||
|
||||
|
||||
class ChatMemberRow(Base):
|
||||
__tablename__ = "chat_members"
|
||||
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
Uuid, ForeignKey("chats.id"), primary_key=True
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
Uuid, ForeignKey("users.id"), primary_key=True
|
||||
)
|
||||
chat: Mapped[ChatRow] = relationship(back_populates="members")
|
||||
|
||||
|
||||
class MessageRow(Base):
|
||||
__tablename__ = "messages"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("chats.id"), nullable=False)
|
||||
sender_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("users.id"), nullable=False)
|
||||
text: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class MessageAttachmentRow(Base):
|
||||
__tablename__ = "message_attachments"
|
||||
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(
|
||||
Uuid, ForeignKey("messages.id"), primary_key=True
|
||||
)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
Uuid, ForeignKey("attachments.id"), primary_key=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
|
||||
|
||||
class AttachmentRow(Base):
|
||||
__tablename__ = "attachments"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("chats.id"), nullable=False)
|
||||
owner_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("users.id"), nullable=False)
|
||||
filename: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
content_type: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
object_key: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
staging_object_key: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
status: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
error_code: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class AttachmentVariantRow(Base):
|
||||
__tablename__ = "attachment_variants"
|
||||
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
Uuid, ForeignKey("attachments.id"), primary_key=True
|
||||
)
|
||||
name: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||
content_type: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
width: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
height: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
object_key: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
|
||||
|
||||
class AttachmentJobRow(Base):
|
||||
__tablename__ = "attachment_jobs"
|
||||
|
||||
job_id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
Uuid, ForeignKey("attachments.id"), nullable=False, unique=True
|
||||
)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class SessionRow(Base):
|
||||
__tablename__ = "sessions"
|
||||
|
||||
token_hash: Mapped[str] = mapped_column(String, primary_key=True)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("users.id"), nullable=False)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
revoked_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class IdempotencyRow(Base):
|
||||
__tablename__ = "idempotency_records"
|
||||
|
||||
scope_key: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||
request_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("messages.id"), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from typing import Tuple
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
from .images import max_image_pixels
|
||||
|
||||
|
||||
def make_thumbnail_jpeg(data: bytes) -> Tuple[bytes, int, int]:
|
||||
Image.MAX_IMAGE_PIXELS = max_image_pixels()
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as img:
|
||||
img = ImageOps.exif_transpose(img)
|
||||
if img.mode in {"RGBA", "LA"} or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.split()[-1])
|
||||
img = background
|
||||
else:
|
||||
img = img.convert("RGB")
|
||||
|
||||
width, height = img.size
|
||||
if width < 1 or height < 1:
|
||||
raise ValueError("invalid image size")
|
||||
|
||||
if width > 256 or height > 256:
|
||||
scale = min(256 / width, 256 / height)
|
||||
new_w = max(1, int(width * scale))
|
||||
new_h = max(1, int(height * scale))
|
||||
img = img.resize((new_w, new_h), Image.Resampling.LANCZOS)
|
||||
else:
|
||||
new_w, new_h = width, height
|
||||
|
||||
out = io.BytesIO()
|
||||
img.save(out, format="JPEG", quality=85, optimize=True)
|
||||
return out.getvalue(), new_w, new_h
|
||||
except Image.DecompressionBombError as exc:
|
||||
raise ValueError("image exceeds pixel limit") from exc
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
|
||||
raise ValueError("invalid image") from exc
|
||||
@@ -0,0 +1,241 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from .broker import consume_jobs, publish_job
|
||||
from .db import close_engine, init_engine, session_scope
|
||||
from .s3util import (
|
||||
close_s3,
|
||||
delete_object,
|
||||
ensure_bucket,
|
||||
get_object_bytes,
|
||||
put_object,
|
||||
)
|
||||
from .tables import AttachmentJobRow, AttachmentRow, AttachmentVariantRow
|
||||
from .thumbnails import make_thumbnail_jpeg
|
||||
|
||||
logger = logging.getLogger("messenger.worker")
|
||||
_stop = False
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def format_ts(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def worker_concurrency() -> int:
|
||||
return max(1, int(os.environ.get("WORKER_CONCURRENCY", "2")))
|
||||
|
||||
|
||||
def _job_payload(job: AttachmentJobRow) -> dict:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job.job_id),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(job.attachment_id),
|
||||
"pipeline_version": int(job.pipeline_version),
|
||||
"attempt": int(job.attempt),
|
||||
"created_at": format_ts(job.created_at),
|
||||
}
|
||||
|
||||
|
||||
def _mark_failed(attachment_id: UUID, error_code: str) -> None:
|
||||
with session_scope() as session:
|
||||
row = session.get(AttachmentRow, attachment_id, with_for_update=True)
|
||||
if row is None or row.status == "ready":
|
||||
return
|
||||
row.status = "failed"
|
||||
row.error_code = error_code
|
||||
row.updated_at = utc_now()
|
||||
|
||||
|
||||
def _schedule_retry(attachment_id: UUID, error_code: str) -> bool:
|
||||
with session_scope() as session:
|
||||
row = session.get(AttachmentRow, attachment_id, with_for_update=True)
|
||||
job = session.scalar(
|
||||
select(AttachmentJobRow).where(AttachmentJobRow.attachment_id == attachment_id)
|
||||
)
|
||||
if row is None or job is None or row.status == "ready":
|
||||
return False
|
||||
if job.attempt >= job.max_attempts:
|
||||
row.status = "failed"
|
||||
row.error_code = "retry_exhausted"
|
||||
row.updated_at = utc_now()
|
||||
return False
|
||||
job.attempt += 1
|
||||
row.status = "queued"
|
||||
row.error_code = error_code
|
||||
row.updated_at = utc_now()
|
||||
payload = _job_payload(job)
|
||||
try:
|
||||
publish_job(payload)
|
||||
except RuntimeError:
|
||||
logger.exception("failed to republish job for %s", attachment_id)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def process_attachment(attachment_id: UUID, pipeline_version: int) -> str:
|
||||
with session_scope() as session:
|
||||
row = session.get(AttachmentRow, attachment_id, with_for_update=True)
|
||||
if row is None:
|
||||
return "ack"
|
||||
if row.status == "ready":
|
||||
existing = session.get(
|
||||
AttachmentVariantRow, {"attachment_id": row.id, "name": "thumbnail"}
|
||||
)
|
||||
if existing is not None:
|
||||
return "ack"
|
||||
if row.status == "failed":
|
||||
return "ack"
|
||||
|
||||
row.status = "processing"
|
||||
row.error_code = None
|
||||
row.updated_at = utc_now()
|
||||
staging_key = row.staging_object_key
|
||||
final_key = row.object_key
|
||||
content_type = row.content_type
|
||||
chat_id = str(row.chat_id)
|
||||
attachment_uuid = row.id
|
||||
|
||||
try:
|
||||
original = get_object_bytes(staging_key)
|
||||
except Exception:
|
||||
if not _schedule_retry(attachment_id, "processing_failed"):
|
||||
_mark_failed(attachment_id, "processing_failed")
|
||||
return "ack"
|
||||
|
||||
try:
|
||||
thumb_bytes, width, height = make_thumbnail_jpeg(original)
|
||||
except ValueError:
|
||||
_mark_failed(attachment_id, "invalid_image")
|
||||
return "ack"
|
||||
except Exception:
|
||||
if not _schedule_retry(attachment_id, "processing_failed"):
|
||||
_mark_failed(attachment_id, "processing_failed")
|
||||
return "ack"
|
||||
|
||||
thumb_key = f"attachments/{chat_id}/{attachment_uuid}/variants/thumbnail.jpg"
|
||||
thumb_digest = hashlib.sha256(thumb_bytes).hexdigest()
|
||||
|
||||
try:
|
||||
put_object(final_key, original, content_type)
|
||||
put_object(thumb_key, thumb_bytes, "image/jpeg")
|
||||
except Exception:
|
||||
if not _schedule_retry(attachment_id, "processing_failed"):
|
||||
_mark_failed(attachment_id, "processing_failed")
|
||||
return "ack"
|
||||
|
||||
try:
|
||||
with session_scope() as session:
|
||||
row = session.get(AttachmentRow, attachment_id, with_for_update=True)
|
||||
if row is None:
|
||||
return "ack"
|
||||
if row.status == "ready":
|
||||
existing = session.get(
|
||||
AttachmentVariantRow, {"attachment_id": row.id, "name": "thumbnail"}
|
||||
)
|
||||
if existing is not None:
|
||||
return "ack"
|
||||
|
||||
existing = session.get(
|
||||
AttachmentVariantRow, {"attachment_id": row.id, "name": "thumbnail"}
|
||||
)
|
||||
if existing is None:
|
||||
session.add(
|
||||
AttachmentVariantRow(
|
||||
attachment_id=row.id,
|
||||
name="thumbnail",
|
||||
content_type="image/jpeg",
|
||||
size_bytes=len(thumb_bytes),
|
||||
width=width,
|
||||
height=height,
|
||||
sha256=thumb_digest,
|
||||
object_key=thumb_key,
|
||||
)
|
||||
)
|
||||
else:
|
||||
existing.content_type = "image/jpeg"
|
||||
existing.size_bytes = len(thumb_bytes)
|
||||
existing.width = width
|
||||
existing.height = height
|
||||
existing.sha256 = thumb_digest
|
||||
existing.object_key = thumb_key
|
||||
|
||||
row.object_key = final_key
|
||||
row.status = "ready"
|
||||
row.error_code = None
|
||||
row.updated_at = utc_now()
|
||||
session.flush()
|
||||
except IntegrityError:
|
||||
return "ack"
|
||||
except Exception:
|
||||
if not _schedule_retry(attachment_id, "processing_failed"):
|
||||
_mark_failed(attachment_id, "processing_failed")
|
||||
return "ack"
|
||||
|
||||
if staging_key != final_key:
|
||||
delete_object(staging_key)
|
||||
return "ack"
|
||||
|
||||
|
||||
def handle_message(payload: dict[str, Any], channel: Any, method: Any) -> None:
|
||||
try:
|
||||
if payload.get("type") != "attachment.process.v1":
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
return
|
||||
attachment_id = UUID(str(payload["attachment_id"]))
|
||||
pipeline_version = int(payload.get("pipeline_version", 1))
|
||||
process_attachment(attachment_id, pipeline_version)
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
except Exception:
|
||||
logger.exception("job handler failed")
|
||||
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
|
||||
|
||||
|
||||
def _on_signal(signum: int, _frame: Any) -> None:
|
||||
global _stop
|
||||
logger.info("received signal %s, stopping", signum)
|
||||
_stop = True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
signal.signal(signal.SIGINT, _on_signal)
|
||||
signal.signal(signal.SIGTERM, _on_signal)
|
||||
|
||||
init_engine()
|
||||
ensure_bucket()
|
||||
prefetch = worker_concurrency()
|
||||
logger.info("worker started prefetch=%s", prefetch)
|
||||
try:
|
||||
while not _stop:
|
||||
try:
|
||||
consume_jobs(handle_message, prefetch=prefetch, should_stop=lambda: _stop)
|
||||
except Exception:
|
||||
logger.exception("broker consumer interrupted")
|
||||
if _stop:
|
||||
break
|
||||
time.sleep(2)
|
||||
finally:
|
||||
close_s3()
|
||||
close_engine()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user