forked from mtusi2026/mtusi2026lab4
lab4
This commit is contained in:
+15
-13
@@ -1,20 +1,22 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_CONCURRENT_UPLOADS=16
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
API_CPU_LIMIT=1.0
|
||||
API_MEMORY_LIMIT=512M
|
||||
POSTGRES_CPU_LIMIT=1.0
|
||||
POSTGRES_MEMORY_LIMIT=1G
|
||||
MINIO_CPU_LIMIT=1.0
|
||||
MINIO_MEMORY_LIMIT=1G
|
||||
REDIS_CPU_LIMIT=0.5
|
||||
REDIS_MEMORY_LIMIT=256M
|
||||
BOOTSTRAP_CPU_LIMIT=0.25
|
||||
BOOTSTRAP_MEMORY_LIMIT=128M
|
||||
MIGRATE_CPU_LIMIT=0.5
|
||||
MIGRATE_MEMORY_LIMIT=256M
|
||||
NGINX_CPU_LIMIT=0.5
|
||||
NGINX_MEMORY_LIMIT=256M
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
## Лабораторная 4
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Журавлев Алексей, БПИ2301
|
||||
Целевая оценка (3/4/5): 4
|
||||
Трек 5, если заявлен: не заявлен
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: MinIO-вложения (upload/download), метаданные PostgreSQL, приватный bucket, две API-реплики, проверки и reconciliation. Перенесена основа lab3: API, авторизация, PostgreSQL/Redis, миграции и Nginx.
|
||||
Команды запуска из чистого clone: `.\scripts\init-local-secrets.ps1`; `docker compose --env-file .env up -d --build` (подробности в REPORT.md).
|
||||
Результаты проверок и ссылка на REPORT.md: smoke, проверки вложений/приватности, S3 outage, persistence и failover прошли; [REPORT.md](../REPORT.md).
|
||||
Ограничения / известные ошибки: одноузловые MinIO/PostgreSQL/Redis; память измерена с периодом около 1 с, возможны непромеренные краткие пики.
|
||||
Источники и использование ИИ: материалы и контракты lab3/lab4; использована помощь ИИ при разработке и проверке.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; результаты приведены в REPORT.md.
|
||||
- [x] Сценарии отказов/безопасности уровня 4 воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
FROM python:3.12.11-slim-bookworm
|
||||
|
||||
ARG BUILD_REVISION=local
|
||||
LABEL org.opencontainers.image.revision=$BUILD_REVISION
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY app.py auth.py db.py sessions.py s3store.py ./
|
||||
COPY migrations ./migrations
|
||||
COPY scripts/migrate.py scripts/reconcile_attachments.py ./scripts/
|
||||
COPY contracts/openapi.json ./contracts/openapi.json
|
||||
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser \
|
||||
&& chown -R 10001:10001 /app
|
||||
USER 10001:10001
|
||||
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" || exit 1
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "app.py"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover test-attachments test-s3-outage test-private-bucket reconcile-attachments
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -22,9 +22,19 @@ load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachments:
|
||||
$(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-s3-outage:
|
||||
$(PYTHON) student-tests/test_s3_outage.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-private-bucket:
|
||||
$(PYTHON) student-tests/check_private_bucket.py
|
||||
|
||||
reconcile-attachments:
|
||||
docker compose exec -T api python scripts/reconcile_attachments.py $(RECONCILE_ARGS)
|
||||
|
||||
@@ -1,37 +1,130 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Журавлев Алексей, БПИ2301.
|
||||
- Целевая оценка: 4; выбран MinIO, S3-совместимый сервер.
|
||||
- Использованные источники и помощь: README/контракты лабораторных 3 и 4; собственная реализация; помощь ИИ при проверке.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Проверено на Windows 11, Docker Engine 29.8.2, Docker Compose v5.5.1,
|
||||
Python 3.14.6 на хосте. Контейнер API собирается на Python 3.12.11.
|
||||
Образы Postgres 16.8, Redis 7.4.2, Nginx 1.27.5 и MinIO
|
||||
`RELEASE.2025-09-07T16-13-09Z` закреплены версиями; MinIO также закреплён
|
||||
digest в `compose.yaml`.
|
||||
|
||||
PowerShell из корня проекта:
|
||||
|
||||
```powershell
|
||||
.\scripts\init-local-secrets.ps1
|
||||
# Если порт 8080 уже занят, изменить APP_PORT в локальном .env на 8081.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
docker compose --env-file .env ps
|
||||
py -3 -m venv .venv
|
||||
.\.venv\Scripts\python.exe -m pip install -r requirements-dev.txt
|
||||
make check PYTHON='py -3 -X utf8'
|
||||
make validate PYTHON='.\.venv\Scripts\python.exe -X utf8'
|
||||
make test PYTHON='py -3 -X utf8' BASE_URL=http://localhost:8081
|
||||
make test-attachments PYTHON='py -3 -X utf8' BASE_URL=http://localhost:8081
|
||||
make test-private-bucket PYTHON='py -3 -X utf8'
|
||||
make test-s3-outage PYTHON='py -3 -X utf8' BASE_URL=http://localhost:8081
|
||||
```
|
||||
|
||||
Скрипт создаёт локальные `.env` и `secrets/`; значения не выводятся и
|
||||
исключены из Git. В этой среде порт `8080` занят стендом lab3, поэтому
|
||||
используется `8081`. При отсутствии конфликта можно оставить порт `8080` и
|
||||
передать соответствующий `BASE_URL`.
|
||||
Для restart/failover checks используйте `make test-persistence` с
|
||||
`ACTION_SCRIPT=scripts/restart.bat` и `make test-failover` с
|
||||
`ACTION_SCRIPT=scripts/stop-one.bat`. Failover hook намеренно оставляет
|
||||
реплику остановленной; после проверки восстановите две реплики:
|
||||
`docker compose --env-file .env up -d --scale api=2`.
|
||||
|
||||
Остановка без удаления учебных данных: `docker compose --env-file .env down`.
|
||||
Команда `docker compose --env-file .env down -v` намеренно удаляет volumes
|
||||
PostgreSQL, Redis и MinIO и нужна только при осознанном сбросе тестового стенда.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Клиент -> Nginx -> API x2 -> PostgreSQL (сессии вложений/сообщений)
|
||||
-> Redis (сессии входа)
|
||||
-> приватный MinIO bucket (байты вложений)
|
||||
```
|
||||
|
||||
Из lab3 перенесены базовые API, авторизация, PostgreSQL/Redis, миграции,
|
||||
идемпотентность сообщений, Dockerfile и Nginx. Метаданные вложений и связь
|
||||
с сообщениями добавлены миграцией `003`; исходные байты хранятся только в
|
||||
MinIO под случайным server-generated key. PostgreSQL определяет статус,
|
||||
владельца и доступ; скачивание проходит через авторизованный API.
|
||||
|
||||
Загрузка ограничена 10 MiB и 40 млн пикселей. PNG/JPEG проверяются по
|
||||
содержимому, MIME и имени файла. Тело читается чанками во временный
|
||||
`SpooledTemporaryFile` с порогом 1 MiB; параллелизм настраивается
|
||||
`MAX_CONCURRENT_UPLOADS` (по умолчанию 16, максимум 64). Есть таймауты S3.
|
||||
Запись сначала резервируется в PostgreSQL как `processing`, затем пишется
|
||||
объект, после чего metadata становится `ready`. Ошибки компенсируются
|
||||
удалением объекта и переводом записи в `failed`.
|
||||
|
||||
У MinIO приватный bucket `messenger-attachments`, отдельный API user и
|
||||
версионированная policy: `Get/Put/DeleteObject` только для `objects/*`,
|
||||
`ListBucket` только с префиксом `objects/`, а `GetBucketLocation` разрешён
|
||||
отдельно для readiness. Bootstrap повторно запускается без удаления volumes
|
||||
и заменяет предыдущую policy. API, PostgreSQL, Redis и MinIO не имеют
|
||||
опубликованных host ports; доступен только Nginx. MinIO console не опубликована.
|
||||
|
||||
Для старых `processing`/неиспользуемых объектов есть reconciler с dry-run
|
||||
по умолчанию и обязательным grace period не менее 900 секунд:
|
||||
|
||||
```powershell
|
||||
docker compose --env-file .env exec -T api python scripts/reconcile_attachments.py
|
||||
docker compose --env-file .env exec -T api python scripts/reconcile_attachments.py --apply
|
||||
```
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Контракт и OpenAPI | `make check`, `make validate` | Оба PASS; offline structure и OpenAPI 3.1/JSON Schema валидны | stdout проверок |
|
||||
| Публичный smoke lab4 | `make test ... BASE_URL=http://localhost:8081` | PASS: публичный сквозной сценарий, включая вложение и контроль доступа | stdout `tests/smoke.py` |
|
||||
| Вложения и ошибки | `make test-attachments ...` | PASS: >10 MiB и >40 MP отклонены; damaged image/MIME spoof отклонены; опасное имя и чужие чаты/ID не дают 500 | stdout `student-tests/test_attachments.py` |
|
||||
| Загрузка и скачивание | Smoke + attachment tests | Метаданные `ready`; скачанный оригинал совпадает побайтно; доступ проверяется членством в чате | публичный smoke |
|
||||
| Отказ одной API-реплики | `make test-failover ... ACTION_SCRIPT=scripts/stop-one.bat` | Наблюдались 2 реплики; поочерёдные тестовые запуски останавливали `api-1` и `api-2`; подтверждённые записи и вложение читались после отказа | stdout `tests/resilience.py`; Compose status |
|
||||
| Restart всего стенда | `make test-persistence ... ACTION_SCRIPT=scripts/restart.bat` | PASS: данные, оригинал вложения и существующая сессия доступны после restart API, PostgreSQL, Redis, MinIO и Nginx без удаления volumes | stdout `tests/resilience.py`; `scripts/restart.bat` |
|
||||
| Отказ MinIO | `make test-s3-outage ...` | Upload при отказе вернул 503; сервис восстановился с сохранёнными volumes | stdout `student-tests/test_s3_outage.py` |
|
||||
| Приватность и права bucket | `make test-private-bucket` | PASS: анонимный list получает 403; порт S3 не опубликован | stdout `student-tests/check_private_bucket.py` |
|
||||
| Bootstrap и ограниченная policy | Два повторных bootstrap; проверка `mc admin policy info` | PASS: policy создана/повторно применена, старая policy удалена; bucket и volumes сохранены | Compose bootstrap logs и policy в MinIO |
|
||||
| Reconciliation | Dry-run, затем `--apply` на искусственно состаренной записи | Dry-run обнаружил одну stale `processing` запись; apply пометил её `failed`. Финальный dry-run: 0 действий, 0 missing/orphan | stdout `reconcile_attachments.py` |
|
||||
| Потоковая обработка и память | 10 параллельных медленных загрузок PNG 2,562,448 байт | PASS за 2.315 с; суммарная sampled RAM API baseline 128,198,901 B, peak 141,882,817 B, delta 13,683,916 B | [attachment-concurrency.json](./evidence/attachment-concurrency.json) |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Failover сценарии на двух репликах восстановили успешные записи примерно за
|
||||
0.68–0.87 с; переходные неопределённые POST повторялись с тем же
|
||||
idempotency key. После тестов обе реплики восстановлены. Выбран только уровень
|
||||
4; исследовательский трек уровня 5 не заявляется.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| Воздействие | Ожидание | Наблюдение | Восстановление / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| 10 одновременных медленных PNG upload | Ограниченный буфер RAM, успех всех валидных файлов | 10/10; файл 2.56 MB; sampled peak delta около 13.7 MB для двух API-контейнеров | 2.315 с; ошибок/потерь нет |
|
||||
| Остановка API-реплики | LB продолжает обслуживать запросы другой репликой; данные не дублируются | В двух запусках проверены `api-1` и `api-2`; финальные повторы дали 102/102 и 120/120 успешных записей, время до первого успеха 0.804/0.788 с | Данные, существующий token и вложение сохранены |
|
||||
| Остановка MinIO | API явно сообщает временную недоступность, затем восстанавливается | Upload вернул 503; после возврата MinIO API стал healthy | Потери volumes не было |
|
||||
| Restart API + PostgreSQL + Redis + MinIO + Nginx | Сохранить историю, вложение и существующую сессию | Persistence test PASS без повторного входа | Volumes сохранены |
|
||||
| Сверка старой upload reservation | Не менять свежие записи; сначала показать план, затем исправить устаревшую | Dry-run нашёл одну искусственно состаренную запись; apply пометил её `failed`; финальный dry-run ничего не менял | Удаление выполняется только для объектов/записей старше grace period |
|
||||
|
||||
RAM измерялась `docker stats` примерно раз в секунду, всего три sample за
|
||||
короткий запуск. Короткий пик мог быть пропущен; изменение RAM — измерение
|
||||
агрегата двух API-контейнеров, а не строгая верхняя граница. Это проверка
|
||||
ограниченного потокового профиля, не benchmark throughput и не track уровня 5.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- MinIO, PostgreSQL, Redis, Nginx и Docker host одноузловые. Сценарий
|
||||
сохраняет данные при отказе API и рестарте с volumes, но не при потере host
|
||||
или отдельного state volume.
|
||||
- PostgreSQL и S3 не имеют общей транзакции; компенсация и reconciler уменьшают
|
||||
последствия частичного сбоя, но не обеспечивают атомарность между системами.
|
||||
- Проверка чистого скачивания MinIO image из внешнего registry не выполнялась:
|
||||
этот digest был доступен локально в среде разработки.
|
||||
- Данные smoke — синтетические и намеренно остаются в локальных volumes до
|
||||
явного сброса.
|
||||
|
||||
@@ -0,0 +1,759 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unicodedata
|
||||
import uuid
|
||||
import warnings
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from auth import (
|
||||
create_session,
|
||||
hash_password,
|
||||
now_iso,
|
||||
parse_basic_auth,
|
||||
parse_bearer_token,
|
||||
validate_password,
|
||||
verify_password,
|
||||
)
|
||||
from db import ApiError, Store
|
||||
from s3store import ObjectStore, S3Unavailable
|
||||
from sessions import SessionStore
|
||||
|
||||
MAX_BODY_BYTES = 1024 * 1024
|
||||
MAX_UPLOAD_BYTES = min(10 * 1024 * 1024, int(os.environ.get('MAX_UPLOAD_BYTES', 10 * 1024 * 1024)))
|
||||
MAX_IMAGE_PIXELS = min(40_000_000, int(os.environ.get('MAX_IMAGE_PIXELS', 40_000_000)))
|
||||
MAX_CONCURRENT_UPLOADS = int(os.environ.get('MAX_CONCURRENT_UPLOADS', '16'))
|
||||
if (MAX_UPLOAD_BYTES < 1 or MAX_IMAGE_PIXELS < 1
|
||||
or not 1 <= MAX_CONCURRENT_UPLOADS <= 64):
|
||||
raise RuntimeError('Upload limits must be positive and concurrent uploads cannot exceed 64')
|
||||
UPLOAD_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS)
|
||||
USERNAME_PATTERN = re.compile(r'^[a-z0-9_]{3,32}$')
|
||||
CURSOR_TIMESTAMP_PATTERN = re.compile(
|
||||
r'^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z$'
|
||||
)
|
||||
IDEMPOTENCY_KEY_PATTERN = re.compile(r'^[A-Za-z0-9._:-]{1,128}$')
|
||||
LOG = logging.getLogger('messenger')
|
||||
OPENAPI_PATH = Path(__file__).resolve().parent / 'contracts' / 'openapi.json'
|
||||
SWAGGER_HTML = b'''<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>MTUSI Messenger API</title>
|
||||
<link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist@5.17.14/swagger-ui.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="swagger-ui"></div>
|
||||
<script src="https://unpkg.com/swagger-ui-dist@5.17.14/swagger-ui-bundle.js"></script>
|
||||
<script>
|
||||
window.onload = () => SwaggerUIBundle({
|
||||
url: "/openapi.json",
|
||||
dom_id: "#swagger-ui",
|
||||
deepLinking: true,
|
||||
presets: [SwaggerUIBundle.presets.apis],
|
||||
layout: "BaseLayout"
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
|
||||
|
||||
def cursor_encode(resource, item):
|
||||
value = json.dumps(
|
||||
{'resource': resource, 'created_at': item['created_at'], 'id': item['id']},
|
||||
separators=(',', ':'),
|
||||
).encode('utf-8')
|
||||
return base64.urlsafe_b64encode(value).rstrip(b'=').decode('ascii')
|
||||
|
||||
|
||||
def cursor_decode(cursor, resource):
|
||||
try:
|
||||
padding = '=' * (-len(cursor) % 4)
|
||||
decoded = base64.b64decode(cursor + padding, altchars=b'-_', validate=True)
|
||||
value = json.loads(decoded.decode('utf-8'))
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor') from None
|
||||
if (not isinstance(value, dict) or set(value) != {'resource', 'created_at', 'id'}
|
||||
or value.get('resource') != resource
|
||||
or not isinstance(value.get('created_at'), str)
|
||||
or not CURSOR_TIMESTAMP_PATTERN.fullmatch(value['created_at'])
|
||||
or not isinstance(value.get('id'), str)):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor')
|
||||
try:
|
||||
created_at = datetime.fromisoformat(value['created_at'][:-1] + '+00:00')
|
||||
item_id = str(uuid.UUID(value['id']))
|
||||
except (ValueError, AttributeError):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor') from None
|
||||
if item_id != value['id']:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor')
|
||||
return created_at, item_id
|
||||
|
||||
|
||||
def required_text(value, name, maximum):
|
||||
if not isinstance(value, str) or not value.strip() or len(value) > maximum:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid ' + name)
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid ' + name) from None
|
||||
return value
|
||||
|
||||
|
||||
def parse_uuid(value):
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid UUID')
|
||||
try:
|
||||
parsed = uuid.UUID(value)
|
||||
except (ValueError, AttributeError):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid UUID') from None
|
||||
if str(parsed) != value.lower():
|
||||
raise ApiError(400, 'invalid_request', 'Invalid UUID')
|
||||
return str(parsed)
|
||||
|
||||
|
||||
def attachment_filename(query):
|
||||
try:
|
||||
values = parse_qs(query, keep_blank_values=True, strict_parsing=True)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid attachment query') from None
|
||||
if set(values) != {'filename'} or len(values['filename']) != 1:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid attachment query')
|
||||
filename = values['filename'][0]
|
||||
if (not filename or len(filename) > 255 or '/' in filename or '\\' in filename
|
||||
or any(unicodedata.category(char) in {'Cc', 'Cf'} for char in filename)):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid filename')
|
||||
try:
|
||||
filename.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid filename') from None
|
||||
return filename
|
||||
|
||||
|
||||
class MessengerHandler(BaseHTTPRequestHandler):
|
||||
server_version = 'MTUSI-Messenger-Lab3'
|
||||
protocol_version = 'HTTP/1.1'
|
||||
|
||||
def log_message(self, _format, *args):
|
||||
LOG.info('request method=%s status=%s request_id=%s', self.command, self._status, self._request_id)
|
||||
|
||||
def _send_bytes(self, status, request_id, body, content_type, extra_headers=None):
|
||||
self._status = status
|
||||
self.send_response(status)
|
||||
self.send_header('X-Request-Id', request_id)
|
||||
self.send_header('Content-Length', str(len(body)))
|
||||
if status != 204:
|
||||
self.send_header('Content-Type', content_type)
|
||||
for name, value in (extra_headers or {}).items():
|
||||
self.send_header(name, value)
|
||||
self.send_header('X-Instance-Id', self.server.instance_id)
|
||||
self.end_headers()
|
||||
self._headers_sent = True
|
||||
if body:
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_stream(self, request_id, stream, length, content_type):
|
||||
self._status = 200
|
||||
self.send_response(200)
|
||||
self.send_header('X-Request-Id', request_id)
|
||||
self.send_header('Content-Length', str(length))
|
||||
self.send_header('Content-Type', content_type)
|
||||
self.send_header('X-Instance-Id', self.server.instance_id)
|
||||
self.end_headers()
|
||||
self._headers_sent = True
|
||||
remaining = length
|
||||
while remaining:
|
||||
chunk = stream.read(min(64 * 1024, remaining))
|
||||
if not chunk:
|
||||
raise OSError('S3 object ended before the stored content length')
|
||||
self.wfile.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
|
||||
def _send(self, status, request_id, payload=None, extra_headers=None):
|
||||
body = b'' if status == 204 else json.dumps(
|
||||
payload, ensure_ascii=False, separators=(',', ':')
|
||||
).encode('utf-8')
|
||||
self._send_bytes(
|
||||
status, request_id, body, 'application/json; charset=utf-8', extra_headers
|
||||
)
|
||||
|
||||
def _error(self, request_id, error, extra_headers=None):
|
||||
self._send(error.status, request_id, {
|
||||
'error': {
|
||||
'code': error.code,
|
||||
'message': error.message,
|
||||
'request_id': request_id,
|
||||
},
|
||||
}, extra_headers=extra_headers)
|
||||
|
||||
def _read_json(self):
|
||||
if self.headers.get_content_type() != 'application/json':
|
||||
raise ApiError(415, 'unsupported_media_type', 'Expected application/json')
|
||||
raw_length = self.headers.get('Content-Length')
|
||||
try:
|
||||
length = int(raw_length)
|
||||
except (TypeError, ValueError):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid Content-Length') from None
|
||||
if length < 0:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid Content-Length')
|
||||
if length > MAX_BODY_BYTES:
|
||||
raise ApiError(413, 'payload_too_large', 'Request body is too large')
|
||||
try:
|
||||
value = json.loads(self.rfile.read(length).decode('utf-8'))
|
||||
except (ValueError, UnicodeError):
|
||||
raise ApiError(400, 'invalid_request', 'Malformed JSON') from None
|
||||
if not isinstance(value, dict):
|
||||
raise ApiError(400, 'invalid_request', 'Expected a JSON object')
|
||||
return value
|
||||
|
||||
def _read_attachment(self):
|
||||
content_type = self.headers.get_content_type()
|
||||
if content_type not in ('image/png', 'image/jpeg'):
|
||||
self.close_connection = True
|
||||
raise ApiError(415, 'unsupported_media_type', 'Expected PNG or JPEG')
|
||||
try:
|
||||
length = int(self.headers.get('Content-Length'))
|
||||
except (TypeError, ValueError):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid Content-Length') from None
|
||||
if length < 0:
|
||||
self.close_connection = True
|
||||
raise ApiError(400, 'invalid_request', 'Invalid Content-Length')
|
||||
if length > MAX_UPLOAD_BYTES:
|
||||
self.close_connection = True
|
||||
raise ApiError(413, 'payload_too_large', 'Image exceeds the upload limit')
|
||||
if length == 0:
|
||||
raise ApiError(415, 'unsupported_media_type', 'Image is empty or invalid')
|
||||
|
||||
temp_file = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b')
|
||||
digest = hashlib.sha256()
|
||||
remaining = length
|
||||
try:
|
||||
while remaining:
|
||||
chunk = self.rfile.read(min(64 * 1024, remaining))
|
||||
if not chunk:
|
||||
raise ApiError(400, 'invalid_request', 'Incomplete image upload')
|
||||
temp_file.write(chunk)
|
||||
digest.update(chunk)
|
||||
remaining -= len(chunk)
|
||||
temp_file.seek(0)
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', Image.DecompressionBombWarning)
|
||||
image = Image.open(temp_file)
|
||||
detected_type = {
|
||||
'PNG': 'image/png',
|
||||
'JPEG': 'image/jpeg',
|
||||
}.get(image.format)
|
||||
width, height = image.size
|
||||
if width * height > MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, 'payload_too_large', 'Image dimensions exceed the pixel limit')
|
||||
if detected_type != content_type:
|
||||
raise ApiError(415, 'unsupported_media_type', 'Image content does not match Content-Type')
|
||||
image.verify()
|
||||
except Image.DecompressionBombError:
|
||||
raise ApiError(413, 'payload_too_large', 'Image dimensions exceed the pixel limit') from None
|
||||
except ApiError:
|
||||
raise
|
||||
except (UnidentifiedImageError, OSError, SyntaxError, ValueError):
|
||||
raise ApiError(415, 'unsupported_media_type', 'Image is invalid or unsupported') from None
|
||||
temp_file.seek(0)
|
||||
return temp_file, {
|
||||
'content_type': detected_type,
|
||||
'size_bytes': length,
|
||||
'sha256': digest.hexdigest(),
|
||||
'width': width,
|
||||
'height': height,
|
||||
}
|
||||
except Exception:
|
||||
temp_file.close()
|
||||
raise
|
||||
|
||||
def _dependencies_ok(self):
|
||||
try:
|
||||
self.server.store.ping()
|
||||
self.server.sessions.ping()
|
||||
self.server.objects.ping()
|
||||
return True
|
||||
except Exception:
|
||||
LOG.exception('dependency check failed')
|
||||
return False
|
||||
|
||||
def _unauthorized_bearer(self, request_id):
|
||||
self._error(
|
||||
request_id,
|
||||
ApiError(401, 'unauthorized', 'Authentication required'),
|
||||
extra_headers={'WWW-Authenticate': 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
def _unauthorized_basic(self, request_id):
|
||||
self._error(
|
||||
request_id,
|
||||
ApiError(401, 'unauthorized', 'Authentication required'),
|
||||
extra_headers={'WWW-Authenticate': 'Basic realm="messenger", charset="UTF-8"'},
|
||||
)
|
||||
|
||||
def _authenticated_user(self, request_id):
|
||||
auth = self.headers.get('Authorization', '')
|
||||
if auth.startswith('Basic '):
|
||||
self._unauthorized_bearer(request_id)
|
||||
return None
|
||||
token = parse_bearer_token(auth)
|
||||
if not token:
|
||||
self._unauthorized_bearer(request_id)
|
||||
return None
|
||||
try:
|
||||
user_id = self.server.sessions.get_user_id(token)
|
||||
except Exception:
|
||||
LOG.exception('session lookup failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return None
|
||||
if not user_id:
|
||||
self._unauthorized_bearer(request_id)
|
||||
return None
|
||||
try:
|
||||
user = self.server.store.get_user(user_id)
|
||||
except Exception:
|
||||
LOG.exception('user lookup failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return None
|
||||
if user is None:
|
||||
self._unauthorized_bearer(request_id)
|
||||
return None
|
||||
user['_token'] = token
|
||||
return user
|
||||
|
||||
@staticmethod
|
||||
def _query(query):
|
||||
try:
|
||||
values = parse_qs(query, keep_blank_values=True, strict_parsing=True)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid query parameters') from None
|
||||
if set(values) - {'limit', 'cursor'} or any(len(items) != 1 for items in values.values()):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid query parameters')
|
||||
try:
|
||||
limit = int(values.get('limit', ['50'])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid limit') from None
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid limit')
|
||||
cursor = values.get('cursor', [''])[0]
|
||||
if ('cursor' in values and not cursor) or len(cursor) > 2048:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor')
|
||||
return limit, cursor
|
||||
|
||||
def _create_session(self, request_id):
|
||||
credentials = parse_basic_auth(self.headers.get('Authorization'))
|
||||
if not credentials:
|
||||
self._unauthorized_basic(request_id)
|
||||
return
|
||||
username, password = credentials
|
||||
if not isinstance(username, str) or not USERNAME_PATTERN.fullmatch(username):
|
||||
self._unauthorized_basic(request_id)
|
||||
return
|
||||
if not validate_password(password):
|
||||
self._unauthorized_basic(request_id)
|
||||
return
|
||||
try:
|
||||
row = self.server.store.get_user_with_password(username)
|
||||
except Exception:
|
||||
LOG.exception('login lookup failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return
|
||||
if row is None or not verify_password(row['password_hash'], password):
|
||||
self._unauthorized_basic(request_id)
|
||||
return
|
||||
user = self.server.store.get_user(str(row['id']))
|
||||
try:
|
||||
token, expires_at = create_session(self.server.sessions, user['id'])
|
||||
except Exception:
|
||||
LOG.exception('session create failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return
|
||||
self._send(201, request_id, {
|
||||
'token': token,
|
||||
'token_type': 'Bearer',
|
||||
'expires_at': expires_at,
|
||||
'user': user,
|
||||
})
|
||||
|
||||
def _logout(self, request_id, user):
|
||||
try:
|
||||
self.server.sessions.revoke(user['_token'])
|
||||
except Exception:
|
||||
LOG.exception('session revoke failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return
|
||||
self._send_bytes(204, request_id, b'', 'application/json; charset=utf-8')
|
||||
|
||||
def _compensate_upload(self, attachment_id, object_key, remove_object):
|
||||
object_removed = True
|
||||
if remove_object:
|
||||
try:
|
||||
self.server.objects.delete_object(object_key)
|
||||
except S3Unavailable:
|
||||
LOG.error('upload compensation could not delete object id=%s', attachment_id)
|
||||
object_removed = False
|
||||
if not object_removed:
|
||||
return
|
||||
try:
|
||||
self.server.store.mark_attachment_failed(
|
||||
attachment_id, 'upload_failed', now_iso()
|
||||
)
|
||||
except ApiError as error:
|
||||
LOG.error(
|
||||
'upload compensation could not update metadata id=%s code=%s',
|
||||
attachment_id,
|
||||
error.code,
|
||||
)
|
||||
|
||||
def _upload_attachment(self, request_id, user, chat_id, query):
|
||||
filename = attachment_filename(query)
|
||||
self.server.store.get_chat(user['id'], chat_id)
|
||||
if not UPLOAD_SLOTS.acquire(timeout=5):
|
||||
raise ApiError(503, 'unavailable', 'Upload capacity is temporarily unavailable')
|
||||
temp_file = None
|
||||
try:
|
||||
temp_file, image = self._read_attachment()
|
||||
attachment_id = str(uuid.uuid4())
|
||||
object_key = 'objects/' + uuid.uuid4().hex
|
||||
created_at = now_iso()
|
||||
self.server.store.create_attachment_upload(
|
||||
attachment_id,
|
||||
chat_id,
|
||||
user['id'],
|
||||
object_key,
|
||||
filename,
|
||||
image['content_type'],
|
||||
image['size_bytes'],
|
||||
image['sha256'],
|
||||
image['width'],
|
||||
image['height'],
|
||||
created_at,
|
||||
)
|
||||
try:
|
||||
self.server.objects.put_file(
|
||||
object_key, temp_file, image['size_bytes'], image['content_type']
|
||||
)
|
||||
except S3Unavailable:
|
||||
self._compensate_upload(attachment_id, object_key, True)
|
||||
raise ApiError(503, 'unavailable', 'Attachment storage is unavailable') from None
|
||||
try:
|
||||
attachment = self.server.store.mark_attachment_ready(
|
||||
attachment_id, now_iso()
|
||||
)
|
||||
except ApiError:
|
||||
self._compensate_upload(attachment_id, object_key, True)
|
||||
raise
|
||||
self._send(201, request_id, attachment)
|
||||
finally:
|
||||
if temp_file is not None:
|
||||
temp_file.close()
|
||||
UPLOAD_SLOTS.release()
|
||||
|
||||
def _download_attachment(self, request_id, user, attachment_id, query):
|
||||
try:
|
||||
values = parse_qs(query, keep_blank_values=True, strict_parsing=True)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid attachment query') from None
|
||||
if set(values) - {'variant'} or any(len(items) != 1 for items in values.values()):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid attachment query')
|
||||
if values.get('variant', ['original'])[0] != 'original':
|
||||
raise ApiError(404, 'not_found', 'Attachment variant not found')
|
||||
attachment = self.server.store.get_attachment_for_download(
|
||||
user['id'], attachment_id
|
||||
)
|
||||
if attachment['status'] != 'ready':
|
||||
raise ApiError(409, 'conflict', 'Attachment is not ready')
|
||||
result = self.server.objects.get_object(attachment['object_key'])
|
||||
if result is None:
|
||||
self.server.store.mark_reconciled_missing(attachment_id, now_iso())
|
||||
raise ApiError(503, 'unavailable', 'Attachment content is unavailable')
|
||||
stream = result['Body']
|
||||
try:
|
||||
if result.get('ContentLength') != attachment['size_bytes']:
|
||||
self.server.store.mark_reconciled_missing(attachment_id, now_iso())
|
||||
raise ApiError(503, 'unavailable', 'Attachment content is inconsistent')
|
||||
self._send_stream(
|
||||
request_id,
|
||||
stream,
|
||||
attachment['size_bytes'],
|
||||
attachment['content_type'],
|
||||
)
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
def _register(self, request_id, body):
|
||||
if set(body) != {'username', 'display_name', 'password'}:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid user fields')
|
||||
username = body['username']
|
||||
if not isinstance(username, str) or not USERNAME_PATTERN.fullmatch(username):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid username')
|
||||
display_name = required_text(body['display_name'], 'display_name', 100)
|
||||
password = body['password']
|
||||
if not validate_password(password):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid password')
|
||||
created_at = now_iso()
|
||||
password_hash = hash_password(password)
|
||||
user = self.server.store.register_user(username, display_name, password_hash, created_at)
|
||||
self._send(201, request_id, user)
|
||||
|
||||
def _dispatch(self):
|
||||
request_id = str(uuid.uuid4())
|
||||
self._request_id = request_id
|
||||
self._status = 500
|
||||
self._headers_sent = False
|
||||
try:
|
||||
if self.server.stopping:
|
||||
raise ApiError(503, 'unavailable', 'Server is shutting down')
|
||||
route = urlsplit(self.path)
|
||||
path = route.path
|
||||
if self.command == 'GET' and path in ('/docs', '/docs/'):
|
||||
self._send_bytes(200, request_id, SWAGGER_HTML, 'text/html; charset=utf-8')
|
||||
return
|
||||
if self.command == 'GET' and path == '/openapi.json':
|
||||
self._send_bytes(
|
||||
200, request_id, self.server.openapi_json, 'application/json; charset=utf-8'
|
||||
)
|
||||
return
|
||||
if path == '/health/live' and self.command == 'GET':
|
||||
self._send(200, request_id, {'status': 'ok'})
|
||||
return
|
||||
if path == '/health/ready' and self.command == 'GET':
|
||||
if self._dependencies_ok():
|
||||
self._send(200, request_id, {'status': 'ok'})
|
||||
else:
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return
|
||||
if path == '/api/v1/users' and self.command == 'POST':
|
||||
self._register(request_id, self._read_json())
|
||||
return
|
||||
if path == '/api/v1/auth/sessions' and self.command == 'POST':
|
||||
self._create_session(request_id)
|
||||
return
|
||||
if path.startswith('/api/v1/'):
|
||||
user = self._authenticated_user(request_id)
|
||||
if user is None:
|
||||
return
|
||||
user_id = user['id']
|
||||
if path == '/api/v1/auth/sessions/current' and self.command == 'DELETE':
|
||||
self._logout(request_id, user)
|
||||
return
|
||||
if path == '/api/v1/users/me' and self.command == 'GET':
|
||||
self._send(200, request_id, {
|
||||
k: user[k] for k in ('id', 'username', 'display_name', 'created_at')
|
||||
})
|
||||
return
|
||||
if path == '/api/v1/chats' and self.command == 'POST':
|
||||
body = self._read_json()
|
||||
if set(body) != {'title', 'member_ids'}:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid chat fields')
|
||||
title = required_text(body['title'], 'title', 100)
|
||||
member_ids = body['member_ids']
|
||||
if (not isinstance(member_ids, list) or not member_ids or len(member_ids) > 100
|
||||
or any(not isinstance(value, str) for value in member_ids)):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid chat members')
|
||||
normalized = [parse_uuid(value) for value in member_ids]
|
||||
if len(set(normalized)) != len(normalized) or user_id not in normalized:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid chat members')
|
||||
chat = self.server.store.create_chat(
|
||||
user_id, title, normalized, now_iso()
|
||||
)
|
||||
self._send(201, request_id, chat)
|
||||
return
|
||||
if path == '/api/v1/chats' and self.command == 'GET':
|
||||
limit, cursor = self._query(route.query)
|
||||
key = cursor_decode(cursor, 'chats') if cursor else None
|
||||
items, next_key = self.server.store.list_chats(user_id, limit, key)
|
||||
next_cursor = cursor_encode('chats', items[-1]) if next_key else None
|
||||
self._send(200, request_id, {'items': items, 'next_cursor': next_cursor})
|
||||
return
|
||||
match = re.fullmatch(r'/api/v1/chats/([^/]+)', path)
|
||||
if match and self.command == 'GET':
|
||||
chat_id = parse_uuid(match.group(1))
|
||||
self._send(200, request_id, self.server.store.get_chat(user_id, chat_id))
|
||||
return
|
||||
match = re.fullmatch(r'/api/v1/chats/([^/]+)/messages', path)
|
||||
if match:
|
||||
chat_id = parse_uuid(match.group(1))
|
||||
resource = 'messages:' + chat_id
|
||||
if self.command == 'POST':
|
||||
body = self._read_json()
|
||||
if set(body) - {'text', 'attachment_ids'} or 'text' not in body:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid message fields')
|
||||
idempotency_key = self.headers.get('Idempotency-Key', '')
|
||||
if not IDEMPOTENCY_KEY_PATTERN.fullmatch(idempotency_key):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid Idempotency-Key')
|
||||
text = body['text']
|
||||
attachment_ids = body.get('attachment_ids', [])
|
||||
if (not isinstance(text, str) or len(text) > 4000
|
||||
or not isinstance(attachment_ids, list)
|
||||
or len(attachment_ids) > 10
|
||||
or any(not isinstance(value, str) for value in attachment_ids)):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid message fields')
|
||||
normalized_attachments = [
|
||||
parse_uuid(value) for value in attachment_ids
|
||||
]
|
||||
if len(set(normalized_attachments)) != len(normalized_attachments):
|
||||
raise ApiError(400, 'invalid_request', 'Duplicate attachment IDs')
|
||||
if not text.strip() and not normalized_attachments:
|
||||
raise ApiError(400, 'invalid_request', 'Message text or attachment is required')
|
||||
try:
|
||||
text.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid message text') from None
|
||||
message = self.server.store.create_message_idempotent(
|
||||
user_id,
|
||||
chat_id,
|
||||
text,
|
||||
normalized_attachments,
|
||||
idempotency_key,
|
||||
now_iso(),
|
||||
)
|
||||
self._send(201, request_id, message)
|
||||
return
|
||||
if self.command == 'GET':
|
||||
limit, cursor = self._query(route.query)
|
||||
key = cursor_decode(cursor, resource) if cursor else None
|
||||
items, next_key = self.server.store.list_messages(user_id, chat_id, limit, key)
|
||||
next_cursor = cursor_encode(resource, items[-1]) if next_key else None
|
||||
self._send(200, request_id, {'items': items, 'next_cursor': next_cursor})
|
||||
return
|
||||
match = re.fullmatch(r'/api/v1/chats/([^/]+)/attachments', path)
|
||||
if match and self.command == 'POST':
|
||||
self._upload_attachment(
|
||||
request_id, user, parse_uuid(match.group(1)), route.query
|
||||
)
|
||||
return
|
||||
match = re.fullmatch(r'/api/v1/attachments/([^/]+)', path)
|
||||
if match and self.command == 'GET':
|
||||
self._send(
|
||||
200,
|
||||
request_id,
|
||||
self.server.store.get_attachment(
|
||||
user_id, parse_uuid(match.group(1))
|
||||
),
|
||||
)
|
||||
return
|
||||
match = re.fullmatch(
|
||||
r'/api/v1/attachments/([^/]+)/content', path
|
||||
)
|
||||
if match and self.command == 'GET':
|
||||
self._download_attachment(
|
||||
request_id, user, parse_uuid(match.group(1)), route.query
|
||||
)
|
||||
return
|
||||
raise ApiError(404, 'not_found', 'Resource not found')
|
||||
except ApiError as error:
|
||||
self._error(request_id, error)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
LOG.info('request disconnected request_id=%s', request_id)
|
||||
except OSError:
|
||||
if self._headers_sent:
|
||||
LOG.info('response stream disconnected request_id=%s', request_id)
|
||||
else:
|
||||
LOG.exception('request I/O failed request_id=%s', request_id)
|
||||
self._error(
|
||||
request_id,
|
||||
ApiError(503, 'unavailable', 'Service unavailable'),
|
||||
)
|
||||
except Exception:
|
||||
LOG.exception('request failed request_id=%s', request_id)
|
||||
self._error(request_id, ApiError(500, 'internal_error', 'Internal server error'))
|
||||
|
||||
def do_GET(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_POST(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_PUT(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_PATCH(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_DELETE(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_OPTIONS(self):
|
||||
self._dispatch()
|
||||
|
||||
|
||||
class MessengerServer(ThreadingHTTPServer):
|
||||
daemon_threads = False
|
||||
block_on_close = True
|
||||
request_queue_size = 128
|
||||
|
||||
def __init__(self, address, store, sessions, objects):
|
||||
super().__init__(address, MessengerHandler)
|
||||
self.store = store
|
||||
self.sessions = sessions
|
||||
self.objects = objects
|
||||
self.stopping = False
|
||||
self.instance_id = os.environ.get('INSTANCE_ID') or socket.gethostname()
|
||||
try:
|
||||
spec = json.loads(OPENAPI_PATH.read_text(encoding='utf-8'))
|
||||
spec['servers'] = [{'url': '/', 'description': 'Текущий адрес API'}]
|
||||
self.openapi_json = json.dumps(
|
||||
spec, ensure_ascii=False, separators=(',', ':')
|
||||
).encode('utf-8')
|
||||
except (OSError, ValueError) as error:
|
||||
self.server_close()
|
||||
raise RuntimeError('Unable to load the OpenAPI contract') from error
|
||||
|
||||
|
||||
def main():
|
||||
host = os.environ.get('APP_HOST', '0.0.0.0')
|
||||
try:
|
||||
port = int(os.environ.get('APP_PORT', '8080'))
|
||||
except ValueError:
|
||||
raise SystemExit('APP_PORT must be an integer')
|
||||
if not 1 <= port <= 65535:
|
||||
raise SystemExit('APP_PORT must be between 1 and 65535')
|
||||
if os.environ.get('SESSION_STORE', 'redis').lower() != 'redis':
|
||||
raise SystemExit('SESSION_STORE must be redis for this implementation')
|
||||
|
||||
logging.basicConfig(
|
||||
level=os.environ.get('LOG_LEVEL', 'INFO').upper(),
|
||||
format='%(asctime)s %(levelname)s %(name)s %(message)s',
|
||||
stream=sys.stdout,
|
||||
)
|
||||
store = Store.connect(run_migrations=False)
|
||||
sessions = SessionStore.from_env()
|
||||
objects = ObjectStore.from_env()
|
||||
server = MessengerServer((host, port), store, sessions, objects)
|
||||
|
||||
def stop(_signum, _frame):
|
||||
server.stopping = True
|
||||
threading.Thread(target=server.shutdown, name='http-shutdown', daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
LOG.info('server started host=%s port=%d', host, port)
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.2)
|
||||
finally:
|
||||
server.server_close()
|
||||
store.close()
|
||||
objects.client.close()
|
||||
LOG.info('server stopped')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,77 @@
|
||||
import base64
|
||||
import binascii
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
|
||||
from sessions import SessionStore
|
||||
|
||||
PASSWORD_MIN = 12
|
||||
PASSWORD_MAX = 128
|
||||
_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.now(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def validate_password(password):
|
||||
if not isinstance(password, str):
|
||||
return False
|
||||
length = len(password)
|
||||
return PASSWORD_MIN <= length <= PASSWORD_MAX
|
||||
|
||||
|
||||
def hash_password(password):
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash, password):
|
||||
try:
|
||||
return _hasher.verify(password_hash, password)
|
||||
except VerifyMismatchError:
|
||||
return False
|
||||
|
||||
|
||||
def parse_basic_auth(header_value):
|
||||
if not header_value or not header_value.startswith('Basic '):
|
||||
return None
|
||||
encoded = header_value[6:].strip()
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode('utf-8')
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
return None
|
||||
if ':' not in decoded:
|
||||
return None
|
||||
username, password = decoded.split(':', 1)
|
||||
if not username:
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer_token(header_value):
|
||||
if not header_value or not header_value.startswith('Bearer '):
|
||||
return None
|
||||
token = header_value[7:].strip()
|
||||
return token or None
|
||||
|
||||
|
||||
def session_ttl_seconds():
|
||||
try:
|
||||
ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600'))
|
||||
except ValueError:
|
||||
ttl = 3600
|
||||
return max(1, ttl)
|
||||
|
||||
|
||||
def create_session(store: SessionStore, user_id):
|
||||
token = secrets.token_urlsafe(32)
|
||||
ttl = session_ttl_seconds()
|
||||
expires_at = (
|
||||
datetime.now(timezone.utc) + timedelta(seconds=ttl)
|
||||
).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
store.create(token, user_id, ttl)
|
||||
return token, expires_at
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.8-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- backend
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U messenger -d messenger']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${POSTGRES_CPU_LIMIT:-1.0}'
|
||||
memory: '${POSTGRES_MEMORY_LIMIT:-1G}'
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: ['redis-server', '--appendonly', 'yes', '--appendfsync', 'everysec']
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
networks:
|
||||
- backend
|
||||
healthcheck:
|
||||
test: ['CMD', 'redis-cli', 'ping']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${REDIS_CPU_LIMIT:-0.5}'
|
||||
memory: '${REDIS_MEMORY_LIMIT:-256M}'
|
||||
|
||||
minio:
|
||||
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||
entrypoint: ['/bin/sh', '/infra/minio-entrypoint.sh']
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
- ./infra/minio-entrypoint.sh:/infra/minio-entrypoint.sh:ro
|
||||
networks:
|
||||
- backend
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'curl -fsS http://127.0.0.1:9000/minio/health/ready >/dev/null']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${MINIO_CPU_LIMIT:-1.0}'
|
||||
memory: '${MINIO_MEMORY_LIMIT:-1G}'
|
||||
|
||||
minio-bootstrap:
|
||||
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||
entrypoint: ['/bin/sh', '/infra/minio-bootstrap.sh']
|
||||
environment:
|
||||
S3_BUCKET: messenger-attachments
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
volumes:
|
||||
- ./infra/minio-bootstrap.sh:/infra/minio-bootstrap.sh:ro
|
||||
- ./infra/s3-policy.json:/infra/s3-policy.json:ro
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
restart: 'no'
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${BOOTSTRAP_CPU_LIMIT:-0.25}'
|
||||
memory: '${BOOTSTRAP_MEMORY_LIMIT:-128M}'
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
command: ['python', 'scripts/migrate.py']
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: 'no'
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${MIGRATE_CPU_LIMIT:-0.5}'
|
||||
memory: '${MIGRATE_MEMORY_LIMIT:-256M}'
|
||||
|
||||
api:
|
||||
build: .
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
SESSION_STORE: redis
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-16}
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
minio-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
'CMD',
|
||||
'python',
|
||||
'-c',
|
||||
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
|
||||
]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
stop_grace_period: 15s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${API_CPU_LIMIT:-1.0}'
|
||||
memory: '${API_MEMORY_LIMIT:-512M}'
|
||||
|
||||
nginx:
|
||||
image: nginx:1.27.5-alpine
|
||||
ports:
|
||||
- '${APP_PORT:-8080}:8080'
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
networks:
|
||||
- frontend
|
||||
- backend
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${NGINX_CPU_LIMIT:-0.5}'
|
||||
memory: '${NGINX_MEMORY_LIMIT:-256M}'
|
||||
|
||||
networks:
|
||||
frontend:
|
||||
backend:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
minio_root_user:
|
||||
file: ./secrets/minio_root_user
|
||||
minio_root_password:
|
||||
file: ./secrets/minio_root_password
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
@@ -0,0 +1,532 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg import errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parent / 'migrations'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
def read_database_url():
|
||||
path = os.environ.get('DATABASE_URL_FILE')
|
||||
if path and os.path.isfile(path):
|
||||
return Path(path).read_text(encoding='utf-8').strip()
|
||||
url = os.environ.get('DATABASE_URL', '').strip()
|
||||
if not url:
|
||||
raise RuntimeError('DATABASE_URL or DATABASE_URL_FILE is required')
|
||||
return url
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, pool: ConnectionPool):
|
||||
self.pool = pool
|
||||
|
||||
@classmethod
|
||||
def connect(cls, run_migrations=True):
|
||||
pool = ConnectionPool(
|
||||
conninfo=read_database_url(),
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
timeout=5.0,
|
||||
kwargs={
|
||||
'row_factory': dict_row,
|
||||
'options': '-c statement_timeout=5000 -c lock_timeout=2000',
|
||||
},
|
||||
open=True,
|
||||
)
|
||||
store = cls(pool)
|
||||
if run_migrations:
|
||||
store.run_migrations()
|
||||
return store
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
|
||||
def ping(self):
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute('SELECT 1')
|
||||
|
||||
def run_migrations(self):
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
'CREATE TABLE IF NOT EXISTS schema_migrations '
|
||||
'(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())'
|
||||
)
|
||||
for path in sorted(MIGRATIONS_DIR.glob('*.sql')):
|
||||
version = path.name
|
||||
row = conn.execute(
|
||||
'SELECT 1 FROM schema_migrations WHERE version = %s',
|
||||
(version,),
|
||||
).fetchone()
|
||||
if row:
|
||||
continue
|
||||
sql = path.read_text(encoding='utf-8')
|
||||
with conn.transaction():
|
||||
conn.execute(sql)
|
||||
conn.execute(
|
||||
'INSERT INTO schema_migrations (version) VALUES (%s)',
|
||||
(version,),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _user_row(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'username': row['username'],
|
||||
'display_name': row['display_name'],
|
||||
'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
|
||||
}
|
||||
|
||||
def register_user(self, username, display_name, password_hash, created_at):
|
||||
user_id = str(uuid.uuid4())
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
'INSERT INTO users (id, username, display_name, password_hash, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(user_id, username, display_name, password_hash, created_at),
|
||||
)
|
||||
except errors.UniqueViolation:
|
||||
raise ApiError(409, 'conflict', 'Username already exists') from None
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
return {
|
||||
'id': user_id,
|
||||
'username': username,
|
||||
'display_name': display_name,
|
||||
'created_at': created_at,
|
||||
}
|
||||
|
||||
def get_user(self, user_id):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT id, username, display_name, created_at FROM users WHERE id = %s',
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return self._user_row(row)
|
||||
|
||||
def get_user_with_password(self, username):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s',
|
||||
(username,),
|
||||
).fetchone()
|
||||
return row
|
||||
|
||||
def create_chat(self, user_id, title, member_ids, created_at):
|
||||
chat_id = str(uuid.uuid4())
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
'SELECT id FROM users WHERE id = ANY(%s)',
|
||||
(member_ids,),
|
||||
).fetchall()
|
||||
if len(rows) != len(set(member_ids)):
|
||||
raise ApiError(400, 'invalid_request', 'Unknown chat member')
|
||||
conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(chat_id, title, created_at),
|
||||
)
|
||||
for index, member_id in enumerate(member_ids):
|
||||
conn.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)',
|
||||
(chat_id, member_id, index),
|
||||
)
|
||||
except ApiError:
|
||||
raise
|
||||
except errors.Error:
|
||||
raise ApiError(500, 'internal_error', 'Internal server error') from None
|
||||
return {
|
||||
'id': chat_id,
|
||||
'title': title,
|
||||
'member_ids': list(member_ids),
|
||||
'created_at': created_at,
|
||||
}
|
||||
|
||||
def _is_member(self, conn, chat_id, user_id):
|
||||
row = conn.execute(
|
||||
'SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s',
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def get_chat(self, user_id, chat_id):
|
||||
with self.pool.connection() as conn:
|
||||
if not self._is_member(conn, chat_id, user_id):
|
||||
raise ApiError(404, 'not_found', 'Chat not found')
|
||||
chat = conn.execute(
|
||||
'SELECT id, title, created_at FROM chats WHERE id = %s',
|
||||
(chat_id,),
|
||||
).fetchone()
|
||||
members = conn.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position',
|
||||
(chat_id,),
|
||||
).fetchall()
|
||||
return {
|
||||
'id': str(chat['id']),
|
||||
'title': chat['title'],
|
||||
'member_ids': [str(row['user_id']) for row in members],
|
||||
'created_at': chat['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
|
||||
}
|
||||
|
||||
def list_chats(self, user_id, limit, cursor_key):
|
||||
query = (
|
||||
'SELECT c.id, c.title, c.created_at '
|
||||
'FROM chats c '
|
||||
'JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = %s '
|
||||
)
|
||||
params = [user_id]
|
||||
if cursor_key:
|
||||
query += 'WHERE (c.created_at, c.id) > (%s, %s) '
|
||||
params.extend(cursor_key)
|
||||
query += 'ORDER BY c.created_at ASC, c.id ASC LIMIT %s'
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(query, params).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
items = []
|
||||
for row in page:
|
||||
chat_id = str(row['id'])
|
||||
items.append(self.get_chat(user_id, chat_id))
|
||||
next_cursor = None
|
||||
if has_more:
|
||||
last = page[-1]
|
||||
next_cursor = (last['created_at'], str(last['id']))
|
||||
return items, next_cursor
|
||||
|
||||
def create_message(self, user_id, chat_id, text, created_at):
|
||||
message_id = str(uuid.uuid4())
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
if not self._is_member(conn, chat_id, user_id):
|
||||
raise ApiError(404, 'not_found', 'Chat not found')
|
||||
conn.execute(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(message_id, chat_id, user_id, text, created_at),
|
||||
)
|
||||
return {
|
||||
'id': message_id,
|
||||
'chat_id': chat_id,
|
||||
'sender_id': user_id,
|
||||
'text': text,
|
||||
'created_at': created_at,
|
||||
'attachment_ids': [],
|
||||
}
|
||||
|
||||
def create_message_idempotent(
|
||||
self, user_id, chat_id, text, attachment_ids, idempotency_key, created_at
|
||||
):
|
||||
resource_path = '/api/v1/chats/' + chat_id + '/messages'
|
||||
request_hash = hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
'chat_id': chat_id,
|
||||
'text': text,
|
||||
'attachment_ids': attachment_ids,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(',', ':'),
|
||||
).encode('utf-8')
|
||||
).hexdigest()
|
||||
lock_key = '\x1f'.join(
|
||||
(user_id, 'POST', resource_path, idempotency_key)
|
||||
)
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
if not self._is_member(conn, chat_id, user_id):
|
||||
raise ApiError(404, 'not_found', 'Chat not found')
|
||||
conn.execute(
|
||||
'SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))',
|
||||
(lock_key,),
|
||||
)
|
||||
row = conn.execute(
|
||||
'SELECT request_hash, response FROM message_idempotency '
|
||||
'WHERE user_id = %s AND method = %s AND resource_path = %s '
|
||||
'AND idempotency_key = %s AND expires_at > NOW() FOR UPDATE',
|
||||
(user_id, 'POST', resource_path, idempotency_key),
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
if row['request_hash'] != request_hash:
|
||||
raise ApiError(409, 'conflict', 'Idempotency-Key was used with a different request')
|
||||
return row['response']
|
||||
|
||||
conn.execute(
|
||||
'DELETE FROM message_idempotency WHERE user_id = %s '
|
||||
'AND method = %s AND resource_path = %s AND idempotency_key = %s',
|
||||
(user_id, 'POST', resource_path, idempotency_key),
|
||||
)
|
||||
conn.execute(
|
||||
'DELETE FROM message_idempotency WHERE user_id = %s AND expires_at <= NOW()',
|
||||
(user_id,),
|
||||
)
|
||||
if attachment_ids:
|
||||
attachments = conn.execute(
|
||||
'SELECT id, status FROM attachments '
|
||||
'WHERE chat_id = %s AND id = ANY(%s) FOR SHARE',
|
||||
(chat_id, attachment_ids),
|
||||
).fetchall()
|
||||
if len(attachments) != len(attachment_ids):
|
||||
raise ApiError(404, 'not_found', 'Attachment not found')
|
||||
if any(row['status'] != 'ready' for row in attachments):
|
||||
raise ApiError(409, 'conflict', 'Attachment is not ready')
|
||||
conn.execute(
|
||||
'INSERT INTO message_idempotency '
|
||||
'(user_id, method, resource_path, idempotency_key, request_hash, expires_at) '
|
||||
"VALUES (%s, %s, %s, %s, %s, NOW() + INTERVAL '24 hours')",
|
||||
(user_id, 'POST', resource_path, idempotency_key, request_hash),
|
||||
)
|
||||
message_id = str(uuid.uuid4())
|
||||
conn.execute(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(message_id, chat_id, user_id, text, created_at),
|
||||
)
|
||||
for position, attachment_id in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
'INSERT INTO message_attachments '
|
||||
'(message_id, attachment_id, position) VALUES (%s, %s, %s)',
|
||||
(message_id, attachment_id, position),
|
||||
)
|
||||
message = {
|
||||
'id': message_id,
|
||||
'chat_id': chat_id,
|
||||
'sender_id': user_id,
|
||||
'text': text,
|
||||
'created_at': created_at,
|
||||
'attachment_ids': list(attachment_ids),
|
||||
}
|
||||
conn.execute(
|
||||
'UPDATE message_idempotency SET response = %s '
|
||||
'WHERE user_id = %s AND method = %s AND resource_path = %s '
|
||||
'AND idempotency_key = %s',
|
||||
(Jsonb(message), user_id, 'POST', resource_path, idempotency_key),
|
||||
)
|
||||
return message
|
||||
except ApiError:
|
||||
raise
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
|
||||
def list_messages(self, user_id, chat_id, limit, cursor_key):
|
||||
with self.pool.connection() as conn:
|
||||
if not self._is_member(conn, chat_id, user_id):
|
||||
raise ApiError(404, 'not_found', 'Chat not found')
|
||||
query = (
|
||||
'SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, '
|
||||
'COALESCE(array_agg(ma.attachment_id ORDER BY ma.position) '
|
||||
'FILTER (WHERE ma.attachment_id IS NOT NULL), ARRAY[]::UUID[]) AS attachment_ids '
|
||||
'FROM messages m LEFT JOIN message_attachments ma ON ma.message_id = m.id '
|
||||
'WHERE m.chat_id = %s '
|
||||
)
|
||||
params = [chat_id]
|
||||
if cursor_key:
|
||||
query += 'AND (m.created_at, m.id) > (%s, %s) '
|
||||
params.extend(cursor_key)
|
||||
query += (
|
||||
'GROUP BY m.id '
|
||||
'ORDER BY m.created_at ASC, m.id ASC LIMIT %s'
|
||||
)
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(query, params).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
items = [
|
||||
{
|
||||
'id': str(row['id']),
|
||||
'chat_id': str(row['chat_id']),
|
||||
'sender_id': str(row['sender_id']),
|
||||
'text': row['text'],
|
||||
'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
|
||||
'attachment_ids': [str(value) for value in row['attachment_ids']],
|
||||
}
|
||||
for row in page
|
||||
]
|
||||
next_cursor = None
|
||||
if has_more:
|
||||
last = page[-1]
|
||||
next_cursor = (last['created_at'], str(last['id']))
|
||||
return items, next_cursor
|
||||
|
||||
@staticmethod
|
||||
def _attachment_row(row, include_key=False):
|
||||
value = {
|
||||
'id': str(row['id']),
|
||||
'chat_id': str(row['chat_id']),
|
||||
'owner_id': str(row['owner_id']),
|
||||
'filename': row['filename'],
|
||||
'content_type': row['content_type'],
|
||||
'size_bytes': row['size_bytes'],
|
||||
'sha256': row['sha256'].strip(),
|
||||
'status': row['status'],
|
||||
'variants': row['variants'],
|
||||
'error_code': row['error_code'],
|
||||
'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
|
||||
'updated_at': row['updated_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
|
||||
}
|
||||
if include_key:
|
||||
value['object_key'] = row['object_key']
|
||||
value['width'] = row['width']
|
||||
value['height'] = row['height']
|
||||
return value
|
||||
|
||||
def create_attachment_upload(
|
||||
self, attachment_id, chat_id, owner_id, object_key, filename,
|
||||
content_type, size_bytes, sha256, width, height, created_at
|
||||
):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
if not self._is_member(conn, chat_id, owner_id):
|
||||
raise ApiError(404, 'not_found', 'Chat not found')
|
||||
conn.execute(
|
||||
'INSERT INTO attachments '
|
||||
'(id, chat_id, owner_id, object_key, filename, content_type, '
|
||||
'size_bytes, sha256, width, height, status, variants, error_code, '
|
||||
'created_at, updated_at) '
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, "
|
||||
"'processing', '[]'::jsonb, NULL, %s, %s)",
|
||||
(
|
||||
attachment_id, chat_id, owner_id, object_key, filename,
|
||||
content_type, size_bytes, sha256, width, height,
|
||||
created_at, created_at,
|
||||
),
|
||||
)
|
||||
except ApiError:
|
||||
raise
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
|
||||
def mark_attachment_ready(self, attachment_id, updated_at):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
'UPDATE attachments SET status = %s, error_code = NULL, updated_at = %s '
|
||||
"WHERE id = %s AND status = 'processing' RETURNING *",
|
||||
('ready', updated_at, attachment_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(503, 'unavailable', 'Attachment state could not be saved')
|
||||
return self._attachment_row(row)
|
||||
except ApiError:
|
||||
raise
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
|
||||
def mark_attachment_failed(self, attachment_id, error_code, updated_at):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
'UPDATE attachments SET status = %s, error_code = %s, updated_at = %s '
|
||||
"WHERE id = %s AND status = 'processing'",
|
||||
('failed', error_code, updated_at, attachment_id),
|
||||
)
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
|
||||
def get_attachment(self, user_id, attachment_id):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT a.* FROM attachments a '
|
||||
'JOIN chat_members cm ON cm.chat_id = a.chat_id '
|
||||
'WHERE a.id = %s AND cm.user_id = %s',
|
||||
(attachment_id, user_id),
|
||||
).fetchone()
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
if row is None:
|
||||
raise ApiError(404, 'not_found', 'Attachment not found')
|
||||
return self._attachment_row(row)
|
||||
|
||||
def get_attachment_for_download(self, user_id, attachment_id):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT a.* FROM attachments a '
|
||||
'JOIN chat_members cm ON cm.chat_id = a.chat_id '
|
||||
'WHERE a.id = %s AND cm.user_id = %s',
|
||||
(attachment_id, user_id),
|
||||
).fetchone()
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
if row is None:
|
||||
raise ApiError(404, 'not_found', 'Attachment not found')
|
||||
return self._attachment_row(row, include_key=True)
|
||||
|
||||
def list_stale_processing_attachments(self, cutoff):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM attachments WHERE status = 'processing' AND created_at < %s "
|
||||
'ORDER BY created_at, id',
|
||||
(cutoff,),
|
||||
).fetchall()
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
return [self._attachment_row(row, include_key=True) for row in rows]
|
||||
|
||||
def list_ready_attachments(self):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM attachments WHERE status = 'ready' ORDER BY id"
|
||||
).fetchall()
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
return [self._attachment_row(row, include_key=True) for row in rows]
|
||||
|
||||
def list_attachments_for_reconciliation(self):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
'SELECT * FROM attachments ORDER BY id'
|
||||
).fetchall()
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
return [self._attachment_row(row, include_key=True) for row in rows]
|
||||
|
||||
def get_attachment_by_object_key(self, object_key):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT * FROM attachments WHERE object_key = %s',
|
||||
(object_key,),
|
||||
).fetchone()
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
if row is None:
|
||||
return None
|
||||
return self._attachment_row(row, include_key=True)
|
||||
|
||||
def mark_reconciled_missing(self, attachment_id, updated_at):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
'UPDATE attachments SET status = %s, error_code = %s, updated_at = %s '
|
||||
"WHERE id = %s AND status = 'ready'",
|
||||
('failed', 'object_missing', updated_at, attachment_id),
|
||||
)
|
||||
except (errors.Error, PoolTimeout):
|
||||
raise ApiError(503, 'unavailable', 'Service unavailable') from None
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"concurrent_uploads": 10,
|
||||
"image_size_bytes": 2562448,
|
||||
"image_sha256": "b421436ca79ae3d6aa00a330f13e092c90cfb471fc944a8a5389445844952933",
|
||||
"elapsed_seconds": 2.315,
|
||||
"api_memory_baseline_bytes": 128198901,
|
||||
"api_memory_peak_sampled_bytes": 141882817,
|
||||
"api_memory_sampled_delta_bytes": 13683916,
|
||||
"memory_sample_interval_seconds": 1,
|
||||
"memory_sample_count": 3,
|
||||
"notes": "Docker stats streams aggregate API-container memory at about one-second intervals; short peaks may be missed."
|
||||
}
|
||||
+47
-4
@@ -1,7 +1,50 @@
|
||||
# Инфраструктура студента
|
||||
# Compose + Nginx + MinIO
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
The runnable topology is in the root `compose.yaml`. The example Compose file
|
||||
and `contracts/infrastructure.json` are teaching worksheets, not deployments.
|
||||
The supported local entry point is Nginx; API, PostgreSQL, Redis and MinIO ports
|
||||
are not published to the host.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
| Role | Service / image | Persistence and readiness |
|
||||
| --- | --- | --- |
|
||||
| API | `api`, built from `Dockerfile` / Python 3.12.11 | Two stateless replicas; `/health/ready`; no host port |
|
||||
| Load balancer | `nginx:1.27.5-alpine` | Host port `${APP_PORT:-8080}`; Docker DNS refresh 5 s; passive fail exclusion |
|
||||
| Database | `postgres:16.8-alpine` | `postgres_data` named volume; `pg_isready` |
|
||||
| Sessions | `redis:7.4.2-alpine` | `redis_data` named volume; AOF `everysec`; Redis `PING` |
|
||||
| Object storage | MinIO `RELEASE.2025-09-07T16-13-09Z`, pinned by image digest in Compose | `minio_data` named volume; MinIO readiness endpoint; S3 port 9000 and console 9001 are private |
|
||||
| Migration job | Same API image | Runs ordered SQL migrations once before API replicas |
|
||||
| Bucket bootstrap | Same pinned MinIO image, using its `mc` client | Idempotently creates the private bucket, versioned policy, and restricted API user; replaces the prior policy without deleting volumes |
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
PostgreSQL and S3 hold different parts of an attachment: PostgreSQL is the source
|
||||
of truth for metadata and access control; the private S3 bucket stores original
|
||||
bytes under a random server-generated key. The bucket name is fixed as
|
||||
`messenger-attachments`. No endpoint returns object keys or S3 credentials.
|
||||
|
||||
The MinIO root credentials are mounted only into MinIO and its one-shot bootstrap
|
||||
job. The API receives a separate access key restricted to `GetBucketLocation`,
|
||||
`ListBucket` with the `objects/` prefix, and Get/Put/Delete on `objects/*`.
|
||||
The bootstrap grants no anonymous policy. Verify
|
||||
that from inside the Compose network with
|
||||
`py -3 -X utf8 student-tests/check_private_bucket.py`; it expects an unsigned
|
||||
bucket-list request to receive HTTP 403 and confirms MinIO has no host port
|
||||
binding. The administrative console is not exposed.
|
||||
|
||||
Create local ignored secrets using `scripts/init-local-secrets.ps1`; never copy
|
||||
secrets from another lab. Do not rotate credentials while retaining the
|
||||
corresponding database or MinIO volumes. `scripts/restart.bat` / `.sh` restarts
|
||||
the API, database, Redis, MinIO and Nginx without deleting volumes.
|
||||
`scripts/stop-one.bat` / `.sh` stops only the API container whose hostname equals
|
||||
`TARGET_INSTANCE_ID` and leaves it stopped for the failover check.
|
||||
|
||||
Attachment writes reserve PostgreSQL metadata in `processing` state before
|
||||
writing S3. On success, the row becomes `ready` and the API returns 201. Errors
|
||||
attempt to delete the object and mark the reservation `failed`; if compensation
|
||||
cannot reach either store, the processing row remains visible to reconciliation.
|
||||
Run `scripts/reconcile_attachments.py` in dry-run mode first. It only acts on
|
||||
records/objects older than a 15-minute minimum grace period, marks stale uploads
|
||||
failed, deletes old unreferenced objects, and reports missing objects for ready
|
||||
metadata. `--apply` is required for changes.
|
||||
|
||||
MinIO, PostgreSQL, Redis, Nginx, and the Docker host are single-instance
|
||||
dependencies. This setup survives an API-replica failure and restarts with
|
||||
volumes, not loss of the Docker host or an individual state-store volume.
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
root_user=$(cat /run/secrets/minio_root_user)
|
||||
root_password=$(cat /run/secrets/minio_root_password)
|
||||
service_user=$(cat /run/secrets/s3_access_key_id)
|
||||
service_password=$(cat /run/secrets/s3_secret_access_key)
|
||||
|
||||
until mc alias set local http://minio:9000 "$root_user" "$root_password" >/dev/null 2>&1 \
|
||||
&& mc ready local >/dev/null 2>&1; do
|
||||
sleep 2
|
||||
done
|
||||
|
||||
bucket=${S3_BUCKET:-messenger-attachments}
|
||||
if [ "$bucket" != 'messenger-attachments' ]; then
|
||||
printf '%s\n' 'S3_BUCKET must be messenger-attachments' >&2
|
||||
exit 2
|
||||
fi
|
||||
policy=messenger-attachments-v2
|
||||
legacy_policy=messenger-attachments
|
||||
mc mb --ignore-existing "local/$bucket"
|
||||
if ! mc admin policy info local "$policy" >/dev/null 2>&1; then
|
||||
mc admin policy create local "$policy" /infra/s3-policy.json
|
||||
fi
|
||||
if ! mc admin user info local "$service_user" >/dev/null 2>&1; then
|
||||
mc admin user add local "$service_user" "$service_password" >/dev/null
|
||||
fi
|
||||
mc admin policy attach local "$policy" --user "$service_user" >/dev/null
|
||||
if mc admin policy info local "$legacy_policy" >/dev/null 2>&1; then
|
||||
mc admin policy detach local "$legacy_policy" --user "$service_user" >/dev/null
|
||||
mc admin policy remove local "$legacy_policy" >/dev/null
|
||||
fi
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
|
||||
exec minio server /data --address :9000 --console-address :9001
|
||||
@@ -0,0 +1,26 @@
|
||||
resolver 127.0.0.11 valid=5s ipv6=off;
|
||||
|
||||
upstream messenger_api {
|
||||
zone messenger_api 64k;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
server_name _;
|
||||
client_max_body_size 11m;
|
||||
client_body_timeout 20s;
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger_api;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_request_buffering off;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"],
|
||||
"Condition": {
|
||||
"StringLike": {
|
||||
"s3:prefix": ["objects/", "objects/*"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/objects/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS users_username_key ON users (username);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
position INT NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS chat_members_user_id_idx ON chat_members (user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS messages_chat_history_idx ON messages (chat_id, created_at ASC, id ASC);
|
||||
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE IF NOT EXISTS message_idempotency (
|
||||
user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
method TEXT NOT NULL CHECK (method = 'POST'),
|
||||
resource_path TEXT NOT NULL,
|
||||
idempotency_key VARCHAR(128) NOT NULL,
|
||||
request_hash CHAR(64) NOT NULL,
|
||||
response JSONB,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
PRIMARY KEY (user_id, method, resource_path, idempotency_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS message_idempotency_expiry_idx
|
||||
ON message_idempotency (user_id, expires_at);
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
width INT NOT NULL CHECK (width > 0),
|
||||
height INT NOT NULL CHECK (height > 0),
|
||||
status TEXT NOT NULL CHECK (status IN ('processing', 'ready', 'failed')),
|
||||
variants JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attachments_chat_created_idx
|
||||
ON attachments (chat_id, created_at, id);
|
||||
CREATE INDEX IF NOT EXISTS attachments_processing_created_idx
|
||||
ON attachments (created_at)
|
||||
WHERE status = 'processing';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT,
|
||||
position INT NOT NULL CHECK (position >= 0),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx
|
||||
ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,5 @@
|
||||
argon2-cffi==23.1.0
|
||||
psycopg[binary,pool]==3.2.9
|
||||
redis==5.2.1
|
||||
boto3==1.37.38
|
||||
Pillow==11.1.0
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
|
||||
class S3Unavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _read_secret(file_variable, value_variable):
|
||||
path = os.environ.get(file_variable)
|
||||
if path:
|
||||
return Path(path).read_text(encoding='utf-8').strip()
|
||||
value = os.environ.get(value_variable, '').strip()
|
||||
if not value:
|
||||
raise RuntimeError(value_variable + ' or ' + file_variable + ' is required')
|
||||
return value
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, client, bucket):
|
||||
self.client = client
|
||||
self.bucket = bucket
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
client = boto3.client(
|
||||
's3',
|
||||
endpoint_url=os.environ.get('S3_ENDPOINT', 'http://minio:9000'),
|
||||
region_name=os.environ.get('S3_REGION', 'us-east-1'),
|
||||
aws_access_key_id=_read_secret('S3_ACCESS_KEY_ID_FILE', 'S3_ACCESS_KEY_ID'),
|
||||
aws_secret_access_key=_read_secret(
|
||||
'S3_SECRET_ACCESS_KEY_FILE', 'S3_SECRET_ACCESS_KEY'
|
||||
),
|
||||
config=Config(
|
||||
signature_version='s3v4',
|
||||
s3={'addressing_style': 'path'},
|
||||
connect_timeout=2,
|
||||
read_timeout=10,
|
||||
max_pool_connections=32,
|
||||
retries={'max_attempts': 2, 'mode': 'standard'},
|
||||
),
|
||||
)
|
||||
return cls(client, os.environ.get('S3_BUCKET', 'messenger-attachments'))
|
||||
|
||||
def ping(self):
|
||||
try:
|
||||
self.client.get_bucket_location(Bucket=self.bucket)
|
||||
except (BotoCoreError, ClientError) as error:
|
||||
raise S3Unavailable('S3 bucket is unavailable') from error
|
||||
|
||||
def put_file(self, key, fileobj, size, content_type):
|
||||
fileobj.seek(0)
|
||||
try:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=fileobj,
|
||||
ContentLength=size,
|
||||
ContentType=content_type,
|
||||
)
|
||||
except (BotoCoreError, ClientError) as error:
|
||||
raise S3Unavailable('S3 upload failed') from error
|
||||
|
||||
def get_object(self, key):
|
||||
try:
|
||||
return self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as error:
|
||||
if error.response.get('Error', {}).get('Code') in {
|
||||
'404', 'NoSuchKey', 'NotFound',
|
||||
}:
|
||||
return None
|
||||
raise S3Unavailable('S3 download failed') from error
|
||||
except BotoCoreError as error:
|
||||
raise S3Unavailable('S3 download failed') from error
|
||||
|
||||
def delete_object(self, key):
|
||||
try:
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
except (BotoCoreError, ClientError) as error:
|
||||
raise S3Unavailable('S3 delete failed') from error
|
||||
|
||||
def object_exists(self, key):
|
||||
try:
|
||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except ClientError as error:
|
||||
if error.response.get('Error', {}).get('Code') in {
|
||||
'404', 'NoSuchKey', 'NotFound',
|
||||
}:
|
||||
return False
|
||||
raise S3Unavailable('S3 object check failed') from error
|
||||
except BotoCoreError as error:
|
||||
raise S3Unavailable('S3 object check failed') from error
|
||||
|
||||
def list_objects(self):
|
||||
try:
|
||||
paginator = self.client.get_paginator('list_objects_v2')
|
||||
for page in paginator.paginate(Bucket=self.bucket, Prefix='objects/'):
|
||||
for item in page.get('Contents', []):
|
||||
yield item
|
||||
except (BotoCoreError, ClientError) as error:
|
||||
raise S3Unavailable('S3 listing failed') from error
|
||||
@@ -40,7 +40,12 @@ def main():
|
||||
assert 'X-Instance-Id' in response['headers']
|
||||
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
|
||||
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
|
||||
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
|
||||
python_checks = (
|
||||
list((ROOT/'tests').glob('*.py'))
|
||||
+ list((ROOT/'scripts').glob('*.py'))
|
||||
+ list((ROOT/'student-tests').glob('*.py'))
|
||||
)
|
||||
for file in python_checks:
|
||||
ast.parse(file.read_text(),filename=str(file))
|
||||
for file in ROOT.rglob('*.json'):
|
||||
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
$secrets = Join-Path $root 'secrets'
|
||||
$names = @(
|
||||
'postgres_password',
|
||||
'database_url',
|
||||
'minio_root_user',
|
||||
'minio_root_password',
|
||||
's3_access_key_id',
|
||||
's3_secret_access_key'
|
||||
)
|
||||
|
||||
if (-not (Test-Path (Join-Path $root '.env'))) {
|
||||
Copy-Item (Join-Path $root '.env.example') (Join-Path $root '.env')
|
||||
}
|
||||
|
||||
$existing = @($names | Where-Object { Test-Path (Join-Path $secrets $_) })
|
||||
if ($existing.Count -gt 0) {
|
||||
if ($existing.Count -eq $names.Count) {
|
||||
Write-Output 'Local secrets already exist; leaving them unchanged.'
|
||||
exit 0
|
||||
}
|
||||
throw 'The local secret set is incomplete. Restore it as a set; do not rotate credentials against existing volumes.'
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force $secrets | Out-Null
|
||||
function New-HexSecret([int]$byteCount) {
|
||||
$bytes = New-Object byte[] $byteCount
|
||||
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
try {
|
||||
$rng.GetBytes($bytes)
|
||||
}
|
||||
finally {
|
||||
$rng.Dispose()
|
||||
}
|
||||
$builder = New-Object System.Text.StringBuilder
|
||||
foreach ($byte in $bytes) {
|
||||
[void]$builder.Append($byte.ToString('x2'))
|
||||
}
|
||||
return $builder.ToString()
|
||||
}
|
||||
|
||||
$databasePassword = New-HexSecret 32
|
||||
$values = @{
|
||||
'postgres_password' = $databasePassword
|
||||
'database_url' = "postgresql://messenger:$databasePassword@postgres:5432/messenger"
|
||||
'minio_root_user' = 'root-' + (New-HexSecret 6)
|
||||
'minio_root_password' = New-HexSecret 32
|
||||
's3_access_key_id' = New-HexSecret 10
|
||||
's3_secret_access_key' = New-HexSecret 32
|
||||
}
|
||||
foreach ($name in $names) {
|
||||
$path = Join-Path $secrets $name
|
||||
[System.IO.File]::WriteAllText(
|
||||
$path,
|
||||
$values[$name],
|
||||
[System.Text.Encoding]::ASCII
|
||||
)
|
||||
}
|
||||
Write-Output 'Created local Compose secrets. Their values were not printed.'
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply database migrations once before API replicas start."""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from db import Store
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
||||
store = Store.connect()
|
||||
try:
|
||||
logging.info('database migrations are current')
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dry-run first; remove only old, unreferenced objects or stale upload attempts."""
|
||||
import argparse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from db import ApiError, Store
|
||||
from s3store import ObjectStore, S3Unavailable
|
||||
|
||||
|
||||
def parse_timestamp(value):
|
||||
return datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--grace-seconds', type=int, default=900)
|
||||
parser.add_argument('--apply', action='store_true')
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 900:
|
||||
parser.error('--grace-seconds must be at least 900')
|
||||
|
||||
store = Store.connect(run_migrations=False)
|
||||
objects = ObjectStore.from_env()
|
||||
try:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds)
|
||||
records = store.list_attachments_for_reconciliation()
|
||||
by_key = {record['object_key']: record for record in records}
|
||||
summary = {
|
||||
'mode': 'apply' if args.apply else 'dry-run',
|
||||
'grace_seconds': args.grace_seconds,
|
||||
'stale_uploads': 0,
|
||||
'old_failed_objects': 0,
|
||||
'orphan_objects': 0,
|
||||
'missing_ready_objects': 0,
|
||||
'actions_applied': 0,
|
||||
}
|
||||
|
||||
for record in records:
|
||||
key = record['object_key']
|
||||
if (record['status'] == 'processing'
|
||||
and parse_timestamp(record['created_at']) < cutoff):
|
||||
summary['stale_uploads'] += 1
|
||||
exists = objects.object_exists(key)
|
||||
if args.apply:
|
||||
if exists:
|
||||
objects.delete_object(key)
|
||||
store.mark_attachment_failed(
|
||||
record['id'], 'upload_interrupted', datetime.now(timezone.utc)
|
||||
)
|
||||
summary['actions_applied'] += 1
|
||||
elif (record['status'] == 'failed'
|
||||
and parse_timestamp(record['updated_at']) < cutoff):
|
||||
if objects.object_exists(key):
|
||||
summary['old_failed_objects'] += 1
|
||||
if args.apply:
|
||||
objects.delete_object(key)
|
||||
summary['actions_applied'] += 1
|
||||
elif record['status'] == 'ready' and not objects.object_exists(key):
|
||||
summary['missing_ready_objects'] += 1
|
||||
if args.apply:
|
||||
store.mark_reconciled_missing(
|
||||
record['id'], datetime.now(timezone.utc)
|
||||
)
|
||||
summary['actions_applied'] += 1
|
||||
|
||||
for item in objects.list_objects():
|
||||
key = item['Key']
|
||||
if key in by_key or item['LastModified'] >= cutoff:
|
||||
continue
|
||||
summary['orphan_objects'] += 1
|
||||
if args.apply:
|
||||
objects.delete_object(key)
|
||||
summary['actions_applied'] += 1
|
||||
|
||||
print(json.dumps(summary, sort_keys=True))
|
||||
finally:
|
||||
store.close()
|
||||
objects.client.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (ApiError, S3Unavailable) as error:
|
||||
print('Reconciliation failed: ' + error.message if isinstance(error, ApiError) else
|
||||
'Reconciliation failed: S3 is unavailable', file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
@@ -0,0 +1,5 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart api postgres redis minio nginx
|
||||
exit /b %errorlevel%
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart api postgres redis minio nginx
|
||||
@@ -0,0 +1,21 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo TARGET_INSTANCE_ID is required 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
set TARGET_STOPPED=
|
||||
for /f "delims=" %%i in ('docker compose ps -q api') do if not defined TARGET_STOPPED call :stop_if_target %%i
|
||||
if defined TARGET_STOPPED exit /b 0
|
||||
echo No running API replica matches TARGET_INSTANCE_ID 1>&2
|
||||
exit /b 1
|
||||
|
||||
:stop_if_target
|
||||
for /f "delims=" %%h in ('docker inspect --format "{{.Config.Hostname}}" %1') do if "%%h"=="%TARGET_INSTANCE_ID%" (
|
||||
docker stop --time 2 %1 >nul
|
||||
if errorlevel 1 exit /b 1
|
||||
set TARGET_STOPPED=1
|
||||
exit /b 0
|
||||
)
|
||||
exit /b 1
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
for container_id in $(docker compose ps -q api); do
|
||||
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --time 2 "$container_id" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
printf '%s\n' 'No running API replica matches TARGET_INSTANCE_ID' >&2
|
||||
exit 1
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
import os
|
||||
|
||||
import redis
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, url, connect_timeout=2.0, socket_timeout=2.0):
|
||||
self._url = url
|
||||
self._client = redis.from_url(
|
||||
url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=connect_timeout,
|
||||
socket_timeout=socket_timeout,
|
||||
health_check_interval=30,
|
||||
)
|
||||
self._prefix = 'sess:'
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
url = _read_config('CACHE_URL_FILE', 'CACHE_URL')
|
||||
if not url:
|
||||
raise RuntimeError('CACHE_URL or CACHE_URL_FILE is required when SESSION_STORE=redis')
|
||||
return cls(url)
|
||||
|
||||
def ping(self):
|
||||
return self._client.ping()
|
||||
|
||||
def create(self, token, user_id, ttl_seconds):
|
||||
self._client.setex(self._prefix + token, ttl_seconds, user_id)
|
||||
|
||||
def get_user_id(self, token):
|
||||
return self._client.get(self._prefix + token)
|
||||
|
||||
def revoke(self, token):
|
||||
return self._client.delete(self._prefix + token) > 0
|
||||
|
||||
|
||||
def _read_config(file_env, value_env):
|
||||
path = os.environ.get(file_env)
|
||||
if path and os.path.isfile(path):
|
||||
with open(path, encoding='utf-8') as handle:
|
||||
return handle.read().strip()
|
||||
return os.environ.get(value_env, '').strip()
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check anonymous access to the private bucket from inside the Compose network."""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Failure, run
|
||||
|
||||
|
||||
def main():
|
||||
code = (
|
||||
"import os, urllib.error, urllib.request; "
|
||||
"bucket=os.environ['S3_BUCKET']; "
|
||||
"url='http://minio:9000/'+bucket+'?list-type=2&prefix=objects%2F'; "
|
||||
"req=urllib.request.Request(url); "
|
||||
"\ntry:\n urllib.request.urlopen(req, timeout=3)\n"
|
||||
"except urllib.error.HTTPError as error:\n"
|
||||
" raise SystemExit(0 if error.code == 403 else 1)\n"
|
||||
"raise SystemExit(1)"
|
||||
)
|
||||
result = subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode:
|
||||
raise Failure('anonymous MinIO bucket request did not return HTTP 403')
|
||||
container_id = subprocess.run(
|
||||
['docker', 'compose', 'ps', '-q', 'minio'],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
if not container_id:
|
||||
raise Failure('MinIO container is not running')
|
||||
bindings = subprocess.run(
|
||||
['docker', 'inspect', '--format', '{{json .HostConfig.PortBindings}}', container_id],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
if any(value for value in json.loads(bindings).values()):
|
||||
raise Failure('MinIO S3 endpoint must not be published on the host')
|
||||
print('PASS: anonymous bucket access is denied and the S3 port is not published.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Student negative and concurrency checks for the lab 4 attachment API."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import http.client
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
PARALLEL_UPLOADS = 10
|
||||
ANSI_ESCAPE = re.compile(r'\x1b\[[0-?]*[ -/]*[@-~]')
|
||||
|
||||
|
||||
def png_chunk(kind, data):
|
||||
return (
|
||||
struct.pack('>I', len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
|
||||
)
|
||||
|
||||
|
||||
def random_png(width=1600, height=1600):
|
||||
seed = 0x4C414234
|
||||
pixels = bytearray(width * height)
|
||||
for index in range(len(pixels)):
|
||||
seed = (1664525 * seed + 1013904223) & 0xffffffff
|
||||
pixels[index] = seed >> 24
|
||||
rows = b''.join(
|
||||
b'\x00' + pixels[row * width:(row + 1) * width]
|
||||
for row in range(height)
|
||||
)
|
||||
header = struct.pack('>IIBBBBB', width, height, 8, 0, 0, 0, 0)
|
||||
return (
|
||||
b'\x89PNG\r\n\x1a\n'
|
||||
+ png_chunk(b'IHDR', header)
|
||||
+ png_chunk(b'IDAT', zlib.compress(rows))
|
||||
+ png_chunk(b'IEND', b'')
|
||||
)
|
||||
|
||||
|
||||
def memory_bytes(value):
|
||||
match = re.match(r'\s*([\d.]+)\s*(B|KiB|MiB|GiB)', value)
|
||||
check(match is not None, 'could not parse docker stats memory value')
|
||||
factors = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3}
|
||||
return int(float(match.group(1)) * factors[match.group(2)])
|
||||
|
||||
|
||||
def api_memory(container_ids):
|
||||
output = subprocess.run(
|
||||
['docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *container_ids],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
lines = [line.strip() for line in output.splitlines() if line.strip()]
|
||||
check(len(lines) == len(container_ids), 'docker stats did not report every API replica')
|
||||
return sum(memory_bytes(line.split('/', 1)[0]) for line in lines)
|
||||
|
||||
|
||||
def make_upload(client, actor, chat_id, filename, content, content_type='image/png'):
|
||||
return client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + chat_id + '/attachments?filename=' + quote(filename, safe=''),
|
||||
201,
|
||||
actor=actor,
|
||||
raw=content,
|
||||
headers={'Content-Type': content_type},
|
||||
schema='Attachment',
|
||||
)['json']
|
||||
|
||||
|
||||
def slow_upload(base_url, actor, chat_id, index, content):
|
||||
endpoint = urlsplit(base_url or 'http://localhost:8080')
|
||||
check(endpoint.scheme == 'http', 'memory test currently requires the local HTTP endpoint')
|
||||
connection = http.client.HTTPConnection(
|
||||
endpoint.hostname, endpoint.port or 80, timeout=30
|
||||
)
|
||||
path = (
|
||||
'/api/v1/chats/' + chat_id + '/attachments?filename='
|
||||
+ quote('parallel-' + str(index) + '.png', safe='')
|
||||
)
|
||||
try:
|
||||
connection.putrequest('POST', path)
|
||||
connection.putheader('Authorization', 'Bearer ' + actor['token'])
|
||||
connection.putheader('Accept', 'application/json')
|
||||
connection.putheader('Content-Type', 'image/png')
|
||||
connection.putheader('Content-Length', str(len(content)))
|
||||
connection.putheader('Connection', 'close')
|
||||
connection.endheaders()
|
||||
chunk_size = 64 * 1024
|
||||
for offset in range(0, len(content), chunk_size):
|
||||
connection.send(content[offset:offset + chunk_size])
|
||||
time.sleep(0.05)
|
||||
response = connection.getresponse()
|
||||
payload = json.loads(response.read())
|
||||
check(response.status == 201, 'parallel upload did not return 201')
|
||||
check(bool(response.getheader('X-Request-Id')), 'parallel upload has no request ID')
|
||||
check(bool(response.getheader('X-Instance-Id')), 'parallel upload has no instance ID')
|
||||
from client import shape
|
||||
shape(payload, 'Attachment')
|
||||
return payload
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Attachment negative tests and 10-way upload memory measurement')
|
||||
client = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
fixture = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
|
||||
attachment = make_upload(client, alice, chat['id'], 'sample.png', fixture)
|
||||
|
||||
large = b'x' * (MAX_UPLOAD_BYTES + 1)
|
||||
client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + chat['id'] + '/attachments?filename=large.png',
|
||||
413,
|
||||
actor=alice,
|
||||
raw=large,
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
pixel_header = (
|
||||
b'\x89PNG\r\n\x1a\n'
|
||||
+ png_chunk(
|
||||
b'IHDR',
|
||||
struct.pack('>IIBBBBB', 10000, 5000, 8, 0, 0, 0, 0),
|
||||
)
|
||||
+ png_chunk(b'IDAT', zlib.compress(b''))
|
||||
+ png_chunk(b'IEND', b'')
|
||||
)
|
||||
client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + chat['id'] + '/attachments?filename=too-many-pixels.png',
|
||||
413,
|
||||
actor=alice,
|
||||
raw=pixel_header,
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + chat['id'] + '/attachments?filename=broken.png',
|
||||
415,
|
||||
actor=alice,
|
||||
raw=b'not a PNG image',
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + chat['id'] + '/attachments?filename=spoofed.jpg',
|
||||
415,
|
||||
actor=alice,
|
||||
raw=fixture,
|
||||
headers={'Content-Type': 'image/jpeg'},
|
||||
)
|
||||
client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + chat['id'] + '/attachments?filename=..%2Fprivate.png',
|
||||
400,
|
||||
actor=alice,
|
||||
raw=fixture,
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
|
||||
for path in (
|
||||
'/api/v1/attachments/' + attachment['id'],
|
||||
'/api/v1/attachments/' + attachment['id'] + '/content',
|
||||
):
|
||||
client.call('GET', path, 404, actor=eve)
|
||||
|
||||
other_chat = client.call(
|
||||
'POST',
|
||||
'/api/v1/chats',
|
||||
201,
|
||||
actor=alice,
|
||||
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]},
|
||||
schema='Chat',
|
||||
)['json']
|
||||
client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + other_chat['id'] + '/messages',
|
||||
404,
|
||||
actor=alice,
|
||||
body={'text': 'Чужое вложение', 'attachment_ids': [attachment['id']]},
|
||||
headers={'Idempotency-Key': 'cross-chat-attachment-check'},
|
||||
)
|
||||
|
||||
content = random_png()
|
||||
check(0 < len(content) <= MAX_UPLOAD_BYTES, 'generated stress image exceeds upload limit')
|
||||
container_ids = subprocess.run(
|
||||
['docker', 'compose', 'ps', '-q', 'api'],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.split()
|
||||
check(len(container_ids) >= 2, 'memory test requires at least two API containers')
|
||||
baseline = api_memory(container_ids)
|
||||
samples = []
|
||||
latest = {}
|
||||
stats_process = subprocess.Popen(
|
||||
[
|
||||
'docker', 'stats', '--format', '{{.Name}}|{{.MemUsage}}',
|
||||
*container_ids,
|
||||
],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
|
||||
def monitor():
|
||||
for line in stats_process.stdout:
|
||||
line = ANSI_ESCAPE.sub('', line)
|
||||
parts = line.strip().split('|', 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
try:
|
||||
latest[parts[0]] = memory_bytes(parts[1].split('/', 1)[0])
|
||||
except AssertionError:
|
||||
continue
|
||||
if len(latest) >= len(container_ids):
|
||||
samples.append(sum(latest.values()))
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor, daemon=True)
|
||||
monitor_thread.start()
|
||||
barrier = threading.Barrier(PARALLEL_UPLOADS)
|
||||
|
||||
def worker(index):
|
||||
barrier.wait()
|
||||
return slow_upload(
|
||||
args.base_url,
|
||||
alice,
|
||||
chat['id'],
|
||||
index,
|
||||
content,
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=PARALLEL_UPLOADS) as pool:
|
||||
results = list(pool.map(worker, range(PARALLEL_UPLOADS)))
|
||||
finally:
|
||||
stats_process.terminate()
|
||||
stats_process.wait(timeout=5)
|
||||
monitor_thread.join(timeout=5)
|
||||
elapsed = time.monotonic() - started
|
||||
check(len(results) == PARALLEL_UPLOADS, 'not all simultaneous uploads completed')
|
||||
check(samples, 'Docker stats did not produce any in-flight memory samples')
|
||||
check(all(item['status'] == 'ready' for item in results), 'an upload was not ready')
|
||||
check(len({item['id'] for item in results}) == PARALLEL_UPLOADS, 'upload IDs were reused')
|
||||
peak = max(samples + [baseline])
|
||||
report = {
|
||||
'concurrent_uploads': PARALLEL_UPLOADS,
|
||||
'image_size_bytes': len(content),
|
||||
'image_sha256': hashlib.sha256(content).hexdigest(),
|
||||
'elapsed_seconds': round(elapsed, 3),
|
||||
'api_memory_baseline_bytes': baseline,
|
||||
'api_memory_peak_sampled_bytes': peak,
|
||||
'api_memory_sampled_delta_bytes': peak - baseline,
|
||||
'memory_sample_interval_seconds': 1,
|
||||
'memory_sample_count': len(samples),
|
||||
'notes': 'Docker stats streams aggregate API-container memory at about one-second intervals; short peaks may be missed.',
|
||||
}
|
||||
output = ROOT / 'evidence' / 'attachment-concurrency.json'
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
||||
print(json.dumps(report, indent=2))
|
||||
print('PASS: attachment limits, image validation, access controls and 10 concurrent uploads.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicitly stop MinIO, verify uploads fail as unavailable, and restore it."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(
|
||||
['docker', 'compose', *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('MinIO outage behavior; stops and starts the lab MinIO service')
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
sample = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
|
||||
compose('stop', 'minio')
|
||||
try:
|
||||
time.sleep(2)
|
||||
response = client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + chat['id'] + '/attachments?filename=outage.png',
|
||||
503,
|
||||
actor=alice,
|
||||
raw=sample,
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
check(response['json']['error']['code'] == 'unavailable', 'S3 outage was not reported as unavailable')
|
||||
finally:
|
||||
compose('start', 'minio')
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
break
|
||||
except AssertionError:
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError('MinIO/API did not recover within 90 seconds')
|
||||
print('PASS: MinIO outage returned 503 and the service recovered without removing volumes.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -55,3 +55,30 @@ make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.
|
||||
## Что ещё остаётся доказать
|
||||
|
||||
Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production.
|
||||
|
||||
## Дополнительные проверки решения на MinIO
|
||||
|
||||
После запуска Compose можно выполнить собственные проверки уровня 4:
|
||||
|
||||
```powershell
|
||||
py -3 -X utf8 student-tests/test_attachments.py --base-url http://localhost:8080
|
||||
py -3 -X utf8 student-tests/test_s3_outage.py --base-url http://localhost:8080
|
||||
py -3 -X utf8 student-tests/check_private_bucket.py
|
||||
```
|
||||
|
||||
`test_attachments.py` проверяет лимит bytes, повреждённое изображение,
|
||||
несовпадающий MIME, опасное имя, чужой attachment ID, межчатовую привязку и
|
||||
10 параллельных загрузок. Он сохраняет sampled Docker memory в
|
||||
`evidence/attachment-concurrency.json`; сэмплирование раз в секунду может
|
||||
пропустить краткий пик. `test_s3_outage.py` явно останавливает и затем запускает
|
||||
MinIO, не удаляя volume. Запускайте его только на собственном учебном стенде.
|
||||
|
||||
Проверка согласованности по умолчанию безопасна и ничего не меняет:
|
||||
|
||||
```powershell
|
||||
docker compose exec -T api python scripts/reconcile_attachments.py --grace-seconds 900
|
||||
docker compose exec -T api python scripts/reconcile_attachments.py --grace-seconds 900 --apply
|
||||
```
|
||||
|
||||
`--apply` выполняйте только после просмотра dry-run сводки. Ни один шаг не
|
||||
удаляет volumes; строки с метаданными не удаляются, а помечаются `failed`.
|
||||
|
||||
Reference in New Issue
Block a user