This commit is contained in:
2026-11-05 18:08:31 +03:00
parent ec540a1a72
commit d6e976e4c0
33 changed files with 2789 additions and 52 deletions
+15 -13
View File
@@ -1,20 +1,22 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_CONCURRENT_UPLOADS=16
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
API_CPU_LIMIT=1.0
API_MEMORY_LIMIT=512M
POSTGRES_CPU_LIMIT=1.0
POSTGRES_MEMORY_LIMIT=1G
MINIO_CPU_LIMIT=1.0
MINIO_MEMORY_LIMIT=1G
REDIS_CPU_LIMIT=0.5
REDIS_MEMORY_LIMIT=256M
BOOTSTRAP_CPU_LIMIT=0.25
BOOTSTRAP_MEMORY_LIMIT=128M
MIGRATE_CPU_LIMIT=0.5
MIGRATE_MEMORY_LIMIT=256M
NGINX_CPU_LIMIT=0.5
NGINX_MEMORY_LIMIT=256M
+14 -14
View File
@@ -1,18 +1,18 @@
## Лабораторная 4
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Журавлев Алексей, БПИ2301
Целевая оценка (3/4/5): 4
Трек 5, если заявлен: не заявлен
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено: MinIO-вложения (upload/download), метаданные PostgreSQL, приватный bucket, две API-реплики, проверки и reconciliation. Перенесена основа lab3: API, авторизация, PostgreSQL/Redis, миграции и Nginx.
Команды запуска из чистого clone: `.\scripts\init-local-secrets.ps1`; `docker compose --env-file .env up -d --build` (подробности в REPORT.md).
Результаты проверок и ссылка на REPORT.md: smoke, проверки вложений/приватности, S3 outage, persistence и failover прошли; [REPORT.md](../REPORT.md).
Ограничения / известные ошибки: одноузловые MinIO/PostgreSQL/Redis; память измерена с периодом около 1 с, возможны непромеренные краткие пики.
Источники и использование ИИ: материалы и контракты lab3/lab4; использована помощь ИИ при разработке и проверке.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; результаты приведены в REPORT.md.
- [x] Сценарии отказов/безопасности уровня 4 воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+27
View File
@@ -0,0 +1,27 @@
FROM python:3.12.11-slim-bookworm
ARG BUILD_REVISION=local
LABEL org.opencontainers.image.revision=$BUILD_REVISION
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app.py auth.py db.py sessions.py s3store.py ./
COPY migrations ./migrations
COPY scripts/migrate.py scripts/reconcile_attachments.py ./scripts/
COPY contracts/openapi.json ./contracts/openapi.json
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser \
&& chown -R 10001:10001 /app
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" || exit 1
STOPSIGNAL SIGTERM
CMD ["python", "app.py"]
+13 -3
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover test-attachments test-s3-outage test-private-bucket reconcile-attachments
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -22,9 +22,19 @@ load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachments:
$(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-s3-outage:
$(PYTHON) student-tests/test_s3_outage.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-private-bucket:
$(PYTHON) student-tests/check_private_bucket.py
reconcile-attachments:
docker compose exec -T api python scripts/reconcile_attachments.py $(RECONCILE_ARGS)
+110 -17
View File
@@ -1,37 +1,130 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Журавлев Алексей, БПИ2301.
- Целевая оценка: 4; выбран MinIO, S3-совместимый сервер.
- Использованные источники и помощь: README/контракты лабораторных 3 и 4; собственная реализация; помощь ИИ при проверке.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Проверено на Windows 11, Docker Engine 29.8.2, Docker Compose v5.5.1,
Python 3.14.6 на хосте. Контейнер API собирается на Python 3.12.11.
Образы Postgres 16.8, Redis 7.4.2, Nginx 1.27.5 и MinIO
`RELEASE.2025-09-07T16-13-09Z` закреплены версиями; MinIO также закреплён
digest в `compose.yaml`.
PowerShell из корня проекта:
```powershell
.\scripts\init-local-secrets.ps1
# Если порт 8080 уже занят, изменить APP_PORT в локальном .env на 8081.
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
docker compose --env-file .env ps
py -3 -m venv .venv
.\.venv\Scripts\python.exe -m pip install -r requirements-dev.txt
make check PYTHON='py -3 -X utf8'
make validate PYTHON='.\.venv\Scripts\python.exe -X utf8'
make test PYTHON='py -3 -X utf8' BASE_URL=http://localhost:8081
make test-attachments PYTHON='py -3 -X utf8' BASE_URL=http://localhost:8081
make test-private-bucket PYTHON='py -3 -X utf8'
make test-s3-outage PYTHON='py -3 -X utf8' BASE_URL=http://localhost:8081
```
Скрипт создаёт локальные `.env` и `secrets/`; значения не выводятся и
исключены из Git. В этой среде порт `8080` занят стендом lab3, поэтому
используется `8081`. При отсутствии конфликта можно оставить порт `8080` и
передать соответствующий `BASE_URL`.
Для restart/failover checks используйте `make test-persistence` с
`ACTION_SCRIPT=scripts/restart.bat` и `make test-failover` с
`ACTION_SCRIPT=scripts/stop-one.bat`. Failover hook намеренно оставляет
реплику остановленной; после проверки восстановите две реплики:
`docker compose --env-file .env up -d --scale api=2`.
Остановка без удаления учебных данных: `docker compose --env-file .env down`.
Команда `docker compose --env-file .env down -v` намеренно удаляет volumes
PostgreSQL, Redis и MinIO и нужна только при осознанном сбросе тестового стенда.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
Клиент -> Nginx -> API x2 -> PostgreSQL (сессии вложений/сообщений)
-> Redis (сессии входа)
-> приватный MinIO bucket (байты вложений)
```
Из lab3 перенесены базовые API, авторизация, PostgreSQL/Redis, миграции,
идемпотентность сообщений, Dockerfile и Nginx. Метаданные вложений и связь
с сообщениями добавлены миграцией `003`; исходные байты хранятся только в
MinIO под случайным server-generated key. PostgreSQL определяет статус,
владельца и доступ; скачивание проходит через авторизованный API.
Загрузка ограничена 10 MiB и 40 млн пикселей. PNG/JPEG проверяются по
содержимому, MIME и имени файла. Тело читается чанками во временный
`SpooledTemporaryFile` с порогом 1 MiB; параллелизм настраивается
`MAX_CONCURRENT_UPLOADS` (по умолчанию 16, максимум 64). Есть таймауты S3.
Запись сначала резервируется в PostgreSQL как `processing`, затем пишется
объект, после чего metadata становится `ready`. Ошибки компенсируются
удалением объекта и переводом записи в `failed`.
У MinIO приватный bucket `messenger-attachments`, отдельный API user и
версионированная policy: `Get/Put/DeleteObject` только для `objects/*`,
`ListBucket` только с префиксом `objects/`, а `GetBucketLocation` разрешён
отдельно для readiness. Bootstrap повторно запускается без удаления volumes
и заменяет предыдущую policy. API, PostgreSQL, Redis и MinIO не имеют
опубликованных host ports; доступен только Nginx. MinIO console не опубликована.
Для старых `processing`/неиспользуемых объектов есть reconciler с dry-run
по умолчанию и обязательным grace period не менее 900 секунд:
```powershell
docker compose --env-file .env exec -T api python scripts/reconcile_attachments.py
docker compose --env-file .env exec -T api python scripts/reconcile_attachments.py --apply
```
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Контракт и OpenAPI | `make check`, `make validate` | Оба PASS; offline structure и OpenAPI 3.1/JSON Schema валидны | stdout проверок |
| Публичный smoke lab4 | `make test ... BASE_URL=http://localhost:8081` | PASS: публичный сквозной сценарий, включая вложение и контроль доступа | stdout `tests/smoke.py` |
| Вложения и ошибки | `make test-attachments ...` | PASS: >10 MiB и >40 MP отклонены; damaged image/MIME spoof отклонены; опасное имя и чужие чаты/ID не дают 500 | stdout `student-tests/test_attachments.py` |
| Загрузка и скачивание | Smoke + attachment tests | Метаданные `ready`; скачанный оригинал совпадает побайтно; доступ проверяется членством в чате | публичный smoke |
| Отказ одной API-реплики | `make test-failover ... ACTION_SCRIPT=scripts/stop-one.bat` | Наблюдались 2 реплики; поочерёдные тестовые запуски останавливали `api-1` и `api-2`; подтверждённые записи и вложение читались после отказа | stdout `tests/resilience.py`; Compose status |
| Restart всего стенда | `make test-persistence ... ACTION_SCRIPT=scripts/restart.bat` | PASS: данные, оригинал вложения и существующая сессия доступны после restart API, PostgreSQL, Redis, MinIO и Nginx без удаления volumes | stdout `tests/resilience.py`; `scripts/restart.bat` |
| Отказ MinIO | `make test-s3-outage ...` | Upload при отказе вернул 503; сервис восстановился с сохранёнными volumes | stdout `student-tests/test_s3_outage.py` |
| Приватность и права bucket | `make test-private-bucket` | PASS: анонимный list получает 403; порт S3 не опубликован | stdout `student-tests/check_private_bucket.py` |
| Bootstrap и ограниченная policy | Два повторных bootstrap; проверка `mc admin policy info` | PASS: policy создана/повторно применена, старая policy удалена; bucket и volumes сохранены | Compose bootstrap logs и policy в MinIO |
| Reconciliation | Dry-run, затем `--apply` на искусственно состаренной записи | Dry-run обнаружил одну stale `processing` запись; apply пометил её `failed`. Финальный dry-run: 0 действий, 0 missing/orphan | stdout `reconcile_attachments.py` |
| Потоковая обработка и память | 10 параллельных медленных загрузок PNG 2,562,448 байт | PASS за 2.315 с; суммарная sampled RAM API baseline 128,198,901 B, peak 141,882,817 B, delta 13,683,916 B | [attachment-concurrency.json](./evidence/attachment-concurrency.json) |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Failover сценарии на двух репликах восстановили успешные записи примерно за
0.68–0.87 с; переходные неопределённые POST повторялись с тем же
idempotency key. После тестов обе реплики восстановлены. Выбран только уровень
4; исследовательский трек уровня 5 не заявляется.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| Воздействие | Ожидание | Наблюдение | Восстановление / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| 10 одновременных медленных PNG upload | Ограниченный буфер RAM, успех всех валидных файлов | 10/10; файл 2.56 MB; sampled peak delta около 13.7 MB для двух API-контейнеров | 2.315 с; ошибок/потерь нет |
| Остановка API-реплики | LB продолжает обслуживать запросы другой репликой; данные не дублируются | В двух запусках проверены `api-1` и `api-2`; финальные повторы дали 102/102 и 120/120 успешных записей, время до первого успеха 0.804/0.788 с | Данные, существующий token и вложение сохранены |
| Остановка MinIO | API явно сообщает временную недоступность, затем восстанавливается | Upload вернул 503; после возврата MinIO API стал healthy | Потери volumes не было |
| Restart API + PostgreSQL + Redis + MinIO + Nginx | Сохранить историю, вложение и существующую сессию | Persistence test PASS без повторного входа | Volumes сохранены |
| Сверка старой upload reservation | Не менять свежие записи; сначала показать план, затем исправить устаревшую | Dry-run нашёл одну искусственно состаренную запись; apply пометил её `failed`; финальный dry-run ничего не менял | Удаление выполняется только для объектов/записей старше grace period |
RAM измерялась `docker stats` примерно раз в секунду, всего три sample за
короткий запуск. Короткий пик мог быть пропущен; изменение RAM — измерение
агрегата двух API-контейнеров, а не строгая верхняя граница. Это проверка
ограниченного потокового профиля, не benchmark throughput и не track уровня 5.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- MinIO, PostgreSQL, Redis, Nginx и Docker host одноузловые. Сценарий
сохраняет данные при отказе API и рестарте с volumes, но не при потере host
или отдельного state volume.
- PostgreSQL и S3 не имеют общей транзакции; компенсация и reconciler уменьшают
последствия частичного сбоя, но не обеспечивают атомарность между системами.
- Проверка чистого скачивания MinIO image из внешнего registry не выполнялась:
этот digest был доступен локально в среде разработки.
- Данные smoke — синтетические и намеренно остаются в локальных volumes до
явного сброса.
+759
View File
@@ -0,0 +1,759 @@
#!/usr/bin/env python3
import base64
import binascii
import hashlib
import json
import logging
import os
import re
import signal
import socket
import sys
import tempfile
import threading
import unicodedata
import uuid
import warnings
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from PIL import Image, UnidentifiedImageError
from auth import (
create_session,
hash_password,
now_iso,
parse_basic_auth,
parse_bearer_token,
validate_password,
verify_password,
)
from db import ApiError, Store
from s3store import ObjectStore, S3Unavailable
from sessions import SessionStore
MAX_BODY_BYTES = 1024 * 1024
MAX_UPLOAD_BYTES = min(10 * 1024 * 1024, int(os.environ.get('MAX_UPLOAD_BYTES', 10 * 1024 * 1024)))
MAX_IMAGE_PIXELS = min(40_000_000, int(os.environ.get('MAX_IMAGE_PIXELS', 40_000_000)))
MAX_CONCURRENT_UPLOADS = int(os.environ.get('MAX_CONCURRENT_UPLOADS', '16'))
if (MAX_UPLOAD_BYTES < 1 or MAX_IMAGE_PIXELS < 1
or not 1 <= MAX_CONCURRENT_UPLOADS <= 64):
raise RuntimeError('Upload limits must be positive and concurrent uploads cannot exceed 64')
UPLOAD_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS)
USERNAME_PATTERN = re.compile(r'^[a-z0-9_]{3,32}$')
CURSOR_TIMESTAMP_PATTERN = re.compile(
r'^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z$'
)
IDEMPOTENCY_KEY_PATTERN = re.compile(r'^[A-Za-z0-9._:-]{1,128}$')
LOG = logging.getLogger('messenger')
OPENAPI_PATH = Path(__file__).resolve().parent / 'contracts' / 'openapi.json'
SWAGGER_HTML = b'''<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>MTUSI Messenger API</title>
<link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist@5.17.14/swagger-ui.css">
</head>
<body>
<div id="swagger-ui"></div>
<script src="https://unpkg.com/swagger-ui-dist@5.17.14/swagger-ui-bundle.js"></script>
<script>
window.onload = () => SwaggerUIBundle({
url: "/openapi.json",
dom_id: "#swagger-ui",
deepLinking: true,
presets: [SwaggerUIBundle.presets.apis],
layout: "BaseLayout"
});
</script>
</body>
</html>
'''
def cursor_encode(resource, item):
value = json.dumps(
{'resource': resource, 'created_at': item['created_at'], 'id': item['id']},
separators=(',', ':'),
).encode('utf-8')
return base64.urlsafe_b64encode(value).rstrip(b'=').decode('ascii')
def cursor_decode(cursor, resource):
try:
padding = '=' * (-len(cursor) % 4)
decoded = base64.b64decode(cursor + padding, altchars=b'-_', validate=True)
value = json.loads(decoded.decode('utf-8'))
except (ValueError, UnicodeError, binascii.Error):
raise ApiError(400, 'invalid_request', 'Invalid cursor') from None
if (not isinstance(value, dict) or set(value) != {'resource', 'created_at', 'id'}
or value.get('resource') != resource
or not isinstance(value.get('created_at'), str)
or not CURSOR_TIMESTAMP_PATTERN.fullmatch(value['created_at'])
or not isinstance(value.get('id'), str)):
raise ApiError(400, 'invalid_request', 'Invalid cursor')
try:
created_at = datetime.fromisoformat(value['created_at'][:-1] + '+00:00')
item_id = str(uuid.UUID(value['id']))
except (ValueError, AttributeError):
raise ApiError(400, 'invalid_request', 'Invalid cursor') from None
if item_id != value['id']:
raise ApiError(400, 'invalid_request', 'Invalid cursor')
return created_at, item_id
def required_text(value, name, maximum):
if not isinstance(value, str) or not value.strip() or len(value) > maximum:
raise ApiError(400, 'invalid_request', 'Invalid ' + name)
try:
value.encode('utf-8')
except UnicodeEncodeError:
raise ApiError(400, 'invalid_request', 'Invalid ' + name) from None
return value
def parse_uuid(value):
if not isinstance(value, str):
raise ApiError(400, 'invalid_request', 'Invalid UUID')
try:
parsed = uuid.UUID(value)
except (ValueError, AttributeError):
raise ApiError(400, 'invalid_request', 'Invalid UUID') from None
if str(parsed) != value.lower():
raise ApiError(400, 'invalid_request', 'Invalid UUID')
return str(parsed)
def attachment_filename(query):
try:
values = parse_qs(query, keep_blank_values=True, strict_parsing=True)
except ValueError:
raise ApiError(400, 'invalid_request', 'Invalid attachment query') from None
if set(values) != {'filename'} or len(values['filename']) != 1:
raise ApiError(400, 'invalid_request', 'Invalid attachment query')
filename = values['filename'][0]
if (not filename or len(filename) > 255 or '/' in filename or '\\' in filename
or any(unicodedata.category(char) in {'Cc', 'Cf'} for char in filename)):
raise ApiError(400, 'invalid_request', 'Invalid filename')
try:
filename.encode('utf-8')
except UnicodeEncodeError:
raise ApiError(400, 'invalid_request', 'Invalid filename') from None
return filename
class MessengerHandler(BaseHTTPRequestHandler):
server_version = 'MTUSI-Messenger-Lab3'
protocol_version = 'HTTP/1.1'
def log_message(self, _format, *args):
LOG.info('request method=%s status=%s request_id=%s', self.command, self._status, self._request_id)
def _send_bytes(self, status, request_id, body, content_type, extra_headers=None):
self._status = status
self.send_response(status)
self.send_header('X-Request-Id', request_id)
self.send_header('Content-Length', str(len(body)))
if status != 204:
self.send_header('Content-Type', content_type)
for name, value in (extra_headers or {}).items():
self.send_header(name, value)
self.send_header('X-Instance-Id', self.server.instance_id)
self.end_headers()
self._headers_sent = True
if body:
self.wfile.write(body)
def _send_stream(self, request_id, stream, length, content_type):
self._status = 200
self.send_response(200)
self.send_header('X-Request-Id', request_id)
self.send_header('Content-Length', str(length))
self.send_header('Content-Type', content_type)
self.send_header('X-Instance-Id', self.server.instance_id)
self.end_headers()
self._headers_sent = True
remaining = length
while remaining:
chunk = stream.read(min(64 * 1024, remaining))
if not chunk:
raise OSError('S3 object ended before the stored content length')
self.wfile.write(chunk)
remaining -= len(chunk)
def _send(self, status, request_id, payload=None, extra_headers=None):
body = b'' if status == 204 else json.dumps(
payload, ensure_ascii=False, separators=(',', ':')
).encode('utf-8')
self._send_bytes(
status, request_id, body, 'application/json; charset=utf-8', extra_headers
)
def _error(self, request_id, error, extra_headers=None):
self._send(error.status, request_id, {
'error': {
'code': error.code,
'message': error.message,
'request_id': request_id,
},
}, extra_headers=extra_headers)
def _read_json(self):
if self.headers.get_content_type() != 'application/json':
raise ApiError(415, 'unsupported_media_type', 'Expected application/json')
raw_length = self.headers.get('Content-Length')
try:
length = int(raw_length)
except (TypeError, ValueError):
raise ApiError(400, 'invalid_request', 'Invalid Content-Length') from None
if length < 0:
raise ApiError(400, 'invalid_request', 'Invalid Content-Length')
if length > MAX_BODY_BYTES:
raise ApiError(413, 'payload_too_large', 'Request body is too large')
try:
value = json.loads(self.rfile.read(length).decode('utf-8'))
except (ValueError, UnicodeError):
raise ApiError(400, 'invalid_request', 'Malformed JSON') from None
if not isinstance(value, dict):
raise ApiError(400, 'invalid_request', 'Expected a JSON object')
return value
def _read_attachment(self):
content_type = self.headers.get_content_type()
if content_type not in ('image/png', 'image/jpeg'):
self.close_connection = True
raise ApiError(415, 'unsupported_media_type', 'Expected PNG or JPEG')
try:
length = int(self.headers.get('Content-Length'))
except (TypeError, ValueError):
raise ApiError(400, 'invalid_request', 'Invalid Content-Length') from None
if length < 0:
self.close_connection = True
raise ApiError(400, 'invalid_request', 'Invalid Content-Length')
if length > MAX_UPLOAD_BYTES:
self.close_connection = True
raise ApiError(413, 'payload_too_large', 'Image exceeds the upload limit')
if length == 0:
raise ApiError(415, 'unsupported_media_type', 'Image is empty or invalid')
temp_file = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b')
digest = hashlib.sha256()
remaining = length
try:
while remaining:
chunk = self.rfile.read(min(64 * 1024, remaining))
if not chunk:
raise ApiError(400, 'invalid_request', 'Incomplete image upload')
temp_file.write(chunk)
digest.update(chunk)
remaining -= len(chunk)
temp_file.seek(0)
try:
with warnings.catch_warnings():
warnings.simplefilter('ignore', Image.DecompressionBombWarning)
image = Image.open(temp_file)
detected_type = {
'PNG': 'image/png',
'JPEG': 'image/jpeg',
}.get(image.format)
width, height = image.size
if width * height > MAX_IMAGE_PIXELS:
raise ApiError(413, 'payload_too_large', 'Image dimensions exceed the pixel limit')
if detected_type != content_type:
raise ApiError(415, 'unsupported_media_type', 'Image content does not match Content-Type')
image.verify()
except Image.DecompressionBombError:
raise ApiError(413, 'payload_too_large', 'Image dimensions exceed the pixel limit') from None
except ApiError:
raise
except (UnidentifiedImageError, OSError, SyntaxError, ValueError):
raise ApiError(415, 'unsupported_media_type', 'Image is invalid or unsupported') from None
temp_file.seek(0)
return temp_file, {
'content_type': detected_type,
'size_bytes': length,
'sha256': digest.hexdigest(),
'width': width,
'height': height,
}
except Exception:
temp_file.close()
raise
def _dependencies_ok(self):
try:
self.server.store.ping()
self.server.sessions.ping()
self.server.objects.ping()
return True
except Exception:
LOG.exception('dependency check failed')
return False
def _unauthorized_bearer(self, request_id):
self._error(
request_id,
ApiError(401, 'unauthorized', 'Authentication required'),
extra_headers={'WWW-Authenticate': 'Bearer realm="messenger"'},
)
def _unauthorized_basic(self, request_id):
self._error(
request_id,
ApiError(401, 'unauthorized', 'Authentication required'),
extra_headers={'WWW-Authenticate': 'Basic realm="messenger", charset="UTF-8"'},
)
def _authenticated_user(self, request_id):
auth = self.headers.get('Authorization', '')
if auth.startswith('Basic '):
self._unauthorized_bearer(request_id)
return None
token = parse_bearer_token(auth)
if not token:
self._unauthorized_bearer(request_id)
return None
try:
user_id = self.server.sessions.get_user_id(token)
except Exception:
LOG.exception('session lookup failed')
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
return None
if not user_id:
self._unauthorized_bearer(request_id)
return None
try:
user = self.server.store.get_user(user_id)
except Exception:
LOG.exception('user lookup failed')
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
return None
if user is None:
self._unauthorized_bearer(request_id)
return None
user['_token'] = token
return user
@staticmethod
def _query(query):
try:
values = parse_qs(query, keep_blank_values=True, strict_parsing=True)
except ValueError:
raise ApiError(400, 'invalid_request', 'Invalid query parameters') from None
if set(values) - {'limit', 'cursor'} or any(len(items) != 1 for items in values.values()):
raise ApiError(400, 'invalid_request', 'Invalid query parameters')
try:
limit = int(values.get('limit', ['50'])[0])
except ValueError:
raise ApiError(400, 'invalid_request', 'Invalid limit') from None
if not 1 <= limit <= 100:
raise ApiError(400, 'invalid_request', 'Invalid limit')
cursor = values.get('cursor', [''])[0]
if ('cursor' in values and not cursor) or len(cursor) > 2048:
raise ApiError(400, 'invalid_request', 'Invalid cursor')
return limit, cursor
def _create_session(self, request_id):
credentials = parse_basic_auth(self.headers.get('Authorization'))
if not credentials:
self._unauthorized_basic(request_id)
return
username, password = credentials
if not isinstance(username, str) or not USERNAME_PATTERN.fullmatch(username):
self._unauthorized_basic(request_id)
return
if not validate_password(password):
self._unauthorized_basic(request_id)
return
try:
row = self.server.store.get_user_with_password(username)
except Exception:
LOG.exception('login lookup failed')
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
return
if row is None or not verify_password(row['password_hash'], password):
self._unauthorized_basic(request_id)
return
user = self.server.store.get_user(str(row['id']))
try:
token, expires_at = create_session(self.server.sessions, user['id'])
except Exception:
LOG.exception('session create failed')
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
return
self._send(201, request_id, {
'token': token,
'token_type': 'Bearer',
'expires_at': expires_at,
'user': user,
})
def _logout(self, request_id, user):
try:
self.server.sessions.revoke(user['_token'])
except Exception:
LOG.exception('session revoke failed')
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
return
self._send_bytes(204, request_id, b'', 'application/json; charset=utf-8')
def _compensate_upload(self, attachment_id, object_key, remove_object):
object_removed = True
if remove_object:
try:
self.server.objects.delete_object(object_key)
except S3Unavailable:
LOG.error('upload compensation could not delete object id=%s', attachment_id)
object_removed = False
if not object_removed:
return
try:
self.server.store.mark_attachment_failed(
attachment_id, 'upload_failed', now_iso()
)
except ApiError as error:
LOG.error(
'upload compensation could not update metadata id=%s code=%s',
attachment_id,
error.code,
)
def _upload_attachment(self, request_id, user, chat_id, query):
filename = attachment_filename(query)
self.server.store.get_chat(user['id'], chat_id)
if not UPLOAD_SLOTS.acquire(timeout=5):
raise ApiError(503, 'unavailable', 'Upload capacity is temporarily unavailable')
temp_file = None
try:
temp_file, image = self._read_attachment()
attachment_id = str(uuid.uuid4())
object_key = 'objects/' + uuid.uuid4().hex
created_at = now_iso()
self.server.store.create_attachment_upload(
attachment_id,
chat_id,
user['id'],
object_key,
filename,
image['content_type'],
image['size_bytes'],
image['sha256'],
image['width'],
image['height'],
created_at,
)
try:
self.server.objects.put_file(
object_key, temp_file, image['size_bytes'], image['content_type']
)
except S3Unavailable:
self._compensate_upload(attachment_id, object_key, True)
raise ApiError(503, 'unavailable', 'Attachment storage is unavailable') from None
try:
attachment = self.server.store.mark_attachment_ready(
attachment_id, now_iso()
)
except ApiError:
self._compensate_upload(attachment_id, object_key, True)
raise
self._send(201, request_id, attachment)
finally:
if temp_file is not None:
temp_file.close()
UPLOAD_SLOTS.release()
def _download_attachment(self, request_id, user, attachment_id, query):
try:
values = parse_qs(query, keep_blank_values=True, strict_parsing=True)
except ValueError:
raise ApiError(400, 'invalid_request', 'Invalid attachment query') from None
if set(values) - {'variant'} or any(len(items) != 1 for items in values.values()):
raise ApiError(400, 'invalid_request', 'Invalid attachment query')
if values.get('variant', ['original'])[0] != 'original':
raise ApiError(404, 'not_found', 'Attachment variant not found')
attachment = self.server.store.get_attachment_for_download(
user['id'], attachment_id
)
if attachment['status'] != 'ready':
raise ApiError(409, 'conflict', 'Attachment is not ready')
result = self.server.objects.get_object(attachment['object_key'])
if result is None:
self.server.store.mark_reconciled_missing(attachment_id, now_iso())
raise ApiError(503, 'unavailable', 'Attachment content is unavailable')
stream = result['Body']
try:
if result.get('ContentLength') != attachment['size_bytes']:
self.server.store.mark_reconciled_missing(attachment_id, now_iso())
raise ApiError(503, 'unavailable', 'Attachment content is inconsistent')
self._send_stream(
request_id,
stream,
attachment['size_bytes'],
attachment['content_type'],
)
finally:
stream.close()
def _register(self, request_id, body):
if set(body) != {'username', 'display_name', 'password'}:
raise ApiError(400, 'invalid_request', 'Invalid user fields')
username = body['username']
if not isinstance(username, str) or not USERNAME_PATTERN.fullmatch(username):
raise ApiError(400, 'invalid_request', 'Invalid username')
display_name = required_text(body['display_name'], 'display_name', 100)
password = body['password']
if not validate_password(password):
raise ApiError(400, 'invalid_request', 'Invalid password')
created_at = now_iso()
password_hash = hash_password(password)
user = self.server.store.register_user(username, display_name, password_hash, created_at)
self._send(201, request_id, user)
def _dispatch(self):
request_id = str(uuid.uuid4())
self._request_id = request_id
self._status = 500
self._headers_sent = False
try:
if self.server.stopping:
raise ApiError(503, 'unavailable', 'Server is shutting down')
route = urlsplit(self.path)
path = route.path
if self.command == 'GET' and path in ('/docs', '/docs/'):
self._send_bytes(200, request_id, SWAGGER_HTML, 'text/html; charset=utf-8')
return
if self.command == 'GET' and path == '/openapi.json':
self._send_bytes(
200, request_id, self.server.openapi_json, 'application/json; charset=utf-8'
)
return
if path == '/health/live' and self.command == 'GET':
self._send(200, request_id, {'status': 'ok'})
return
if path == '/health/ready' and self.command == 'GET':
if self._dependencies_ok():
self._send(200, request_id, {'status': 'ok'})
else:
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
return
if path == '/api/v1/users' and self.command == 'POST':
self._register(request_id, self._read_json())
return
if path == '/api/v1/auth/sessions' and self.command == 'POST':
self._create_session(request_id)
return
if path.startswith('/api/v1/'):
user = self._authenticated_user(request_id)
if user is None:
return
user_id = user['id']
if path == '/api/v1/auth/sessions/current' and self.command == 'DELETE':
self._logout(request_id, user)
return
if path == '/api/v1/users/me' and self.command == 'GET':
self._send(200, request_id, {
k: user[k] for k in ('id', 'username', 'display_name', 'created_at')
})
return
if path == '/api/v1/chats' and self.command == 'POST':
body = self._read_json()
if set(body) != {'title', 'member_ids'}:
raise ApiError(400, 'invalid_request', 'Invalid chat fields')
title = required_text(body['title'], 'title', 100)
member_ids = body['member_ids']
if (not isinstance(member_ids, list) or not member_ids or len(member_ids) > 100
or any(not isinstance(value, str) for value in member_ids)):
raise ApiError(400, 'invalid_request', 'Invalid chat members')
normalized = [parse_uuid(value) for value in member_ids]
if len(set(normalized)) != len(normalized) or user_id not in normalized:
raise ApiError(400, 'invalid_request', 'Invalid chat members')
chat = self.server.store.create_chat(
user_id, title, normalized, now_iso()
)
self._send(201, request_id, chat)
return
if path == '/api/v1/chats' and self.command == 'GET':
limit, cursor = self._query(route.query)
key = cursor_decode(cursor, 'chats') if cursor else None
items, next_key = self.server.store.list_chats(user_id, limit, key)
next_cursor = cursor_encode('chats', items[-1]) if next_key else None
self._send(200, request_id, {'items': items, 'next_cursor': next_cursor})
return
match = re.fullmatch(r'/api/v1/chats/([^/]+)', path)
if match and self.command == 'GET':
chat_id = parse_uuid(match.group(1))
self._send(200, request_id, self.server.store.get_chat(user_id, chat_id))
return
match = re.fullmatch(r'/api/v1/chats/([^/]+)/messages', path)
if match:
chat_id = parse_uuid(match.group(1))
resource = 'messages:' + chat_id
if self.command == 'POST':
body = self._read_json()
if set(body) - {'text', 'attachment_ids'} or 'text' not in body:
raise ApiError(400, 'invalid_request', 'Invalid message fields')
idempotency_key = self.headers.get('Idempotency-Key', '')
if not IDEMPOTENCY_KEY_PATTERN.fullmatch(idempotency_key):
raise ApiError(400, 'invalid_request', 'Invalid Idempotency-Key')
text = body['text']
attachment_ids = body.get('attachment_ids', [])
if (not isinstance(text, str) or len(text) > 4000
or not isinstance(attachment_ids, list)
or len(attachment_ids) > 10
or any(not isinstance(value, str) for value in attachment_ids)):
raise ApiError(400, 'invalid_request', 'Invalid message fields')
normalized_attachments = [
parse_uuid(value) for value in attachment_ids
]
if len(set(normalized_attachments)) != len(normalized_attachments):
raise ApiError(400, 'invalid_request', 'Duplicate attachment IDs')
if not text.strip() and not normalized_attachments:
raise ApiError(400, 'invalid_request', 'Message text or attachment is required')
try:
text.encode('utf-8')
except UnicodeEncodeError:
raise ApiError(400, 'invalid_request', 'Invalid message text') from None
message = self.server.store.create_message_idempotent(
user_id,
chat_id,
text,
normalized_attachments,
idempotency_key,
now_iso(),
)
self._send(201, request_id, message)
return
if self.command == 'GET':
limit, cursor = self._query(route.query)
key = cursor_decode(cursor, resource) if cursor else None
items, next_key = self.server.store.list_messages(user_id, chat_id, limit, key)
next_cursor = cursor_encode(resource, items[-1]) if next_key else None
self._send(200, request_id, {'items': items, 'next_cursor': next_cursor})
return
match = re.fullmatch(r'/api/v1/chats/([^/]+)/attachments', path)
if match and self.command == 'POST':
self._upload_attachment(
request_id, user, parse_uuid(match.group(1)), route.query
)
return
match = re.fullmatch(r'/api/v1/attachments/([^/]+)', path)
if match and self.command == 'GET':
self._send(
200,
request_id,
self.server.store.get_attachment(
user_id, parse_uuid(match.group(1))
),
)
return
match = re.fullmatch(
r'/api/v1/attachments/([^/]+)/content', path
)
if match and self.command == 'GET':
self._download_attachment(
request_id, user, parse_uuid(match.group(1)), route.query
)
return
raise ApiError(404, 'not_found', 'Resource not found')
except ApiError as error:
self._error(request_id, error)
except (BrokenPipeError, ConnectionResetError):
LOG.info('request disconnected request_id=%s', request_id)
except OSError:
if self._headers_sent:
LOG.info('response stream disconnected request_id=%s', request_id)
else:
LOG.exception('request I/O failed request_id=%s', request_id)
self._error(
request_id,
ApiError(503, 'unavailable', 'Service unavailable'),
)
except Exception:
LOG.exception('request failed request_id=%s', request_id)
self._error(request_id, ApiError(500, 'internal_error', 'Internal server error'))
def do_GET(self):
self._dispatch()
def do_POST(self):
self._dispatch()
def do_PUT(self):
self._dispatch()
def do_PATCH(self):
self._dispatch()
def do_DELETE(self):
self._dispatch()
def do_OPTIONS(self):
self._dispatch()
class MessengerServer(ThreadingHTTPServer):
daemon_threads = False
block_on_close = True
request_queue_size = 128
def __init__(self, address, store, sessions, objects):
super().__init__(address, MessengerHandler)
self.store = store
self.sessions = sessions
self.objects = objects
self.stopping = False
self.instance_id = os.environ.get('INSTANCE_ID') or socket.gethostname()
try:
spec = json.loads(OPENAPI_PATH.read_text(encoding='utf-8'))
spec['servers'] = [{'url': '/', 'description': 'Текущий адрес API'}]
self.openapi_json = json.dumps(
spec, ensure_ascii=False, separators=(',', ':')
).encode('utf-8')
except (OSError, ValueError) as error:
self.server_close()
raise RuntimeError('Unable to load the OpenAPI contract') from error
def main():
host = os.environ.get('APP_HOST', '0.0.0.0')
try:
port = int(os.environ.get('APP_PORT', '8080'))
except ValueError:
raise SystemExit('APP_PORT must be an integer')
if not 1 <= port <= 65535:
raise SystemExit('APP_PORT must be between 1 and 65535')
if os.environ.get('SESSION_STORE', 'redis').lower() != 'redis':
raise SystemExit('SESSION_STORE must be redis for this implementation')
logging.basicConfig(
level=os.environ.get('LOG_LEVEL', 'INFO').upper(),
format='%(asctime)s %(levelname)s %(name)s %(message)s',
stream=sys.stdout,
)
store = Store.connect(run_migrations=False)
sessions = SessionStore.from_env()
objects = ObjectStore.from_env()
server = MessengerServer((host, port), store, sessions, objects)
def stop(_signum, _frame):
server.stopping = True
threading.Thread(target=server.shutdown, name='http-shutdown', daemon=True).start()
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
LOG.info('server started host=%s port=%d', host, port)
try:
server.serve_forever(poll_interval=0.2)
finally:
server.server_close()
store.close()
objects.client.close()
LOG.info('server stopped')
if __name__ == '__main__':
main()
+77
View File
@@ -0,0 +1,77 @@
import base64
import binascii
import os
import secrets
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError
from sessions import SessionStore
PASSWORD_MIN = 12
PASSWORD_MAX = 128
_hasher = PasswordHasher()
def now_iso():
return datetime.now(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def validate_password(password):
if not isinstance(password, str):
return False
length = len(password)
return PASSWORD_MIN <= length <= PASSWORD_MAX
def hash_password(password):
return _hasher.hash(password)
def verify_password(password_hash, password):
try:
return _hasher.verify(password_hash, password)
except VerifyMismatchError:
return False
def parse_basic_auth(header_value):
if not header_value or not header_value.startswith('Basic '):
return None
encoded = header_value[6:].strip()
try:
decoded = base64.b64decode(encoded, validate=True).decode('utf-8')
except (ValueError, UnicodeError, binascii.Error):
return None
if ':' not in decoded:
return None
username, password = decoded.split(':', 1)
if not username:
return None
return username, password
def parse_bearer_token(header_value):
if not header_value or not header_value.startswith('Bearer '):
return None
token = header_value[7:].strip()
return token or None
def session_ttl_seconds():
try:
ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600'))
except ValueError:
ttl = 3600
return max(1, ttl)
def create_session(store: SessionStore, user_id):
token = secrets.token_urlsafe(32)
ttl = session_ttl_seconds()
expires_at = (
datetime.now(timezone.utc) + timedelta(seconds=ttl)
).isoformat(timespec='microseconds').replace('+00:00', 'Z')
store.create(token, user_id, ttl)
return token, expires_at
+211
View File
@@ -0,0 +1,211 @@
services:
postgres:
image: postgres:16.8-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
volumes:
- postgres_data:/var/lib/postgresql/data
networks:
- backend
healthcheck:
test: ['CMD-SHELL', 'pg_isready -U messenger -d messenger']
interval: 5s
timeout: 3s
retries: 12
start_period: 5s
restart: unless-stopped
deploy:
resources:
limits:
cpus: '${POSTGRES_CPU_LIMIT:-1.0}'
memory: '${POSTGRES_MEMORY_LIMIT:-1G}'
redis:
image: redis:7.4.2-alpine
command: ['redis-server', '--appendonly', 'yes', '--appendfsync', 'everysec']
volumes:
- redis_data:/data
networks:
- backend
healthcheck:
test: ['CMD', 'redis-cli', 'ping']
interval: 5s
timeout: 3s
retries: 12
restart: unless-stopped
deploy:
resources:
limits:
cpus: '${REDIS_CPU_LIMIT:-0.5}'
memory: '${REDIS_MEMORY_LIMIT:-256M}'
minio:
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
entrypoint: ['/bin/sh', '/infra/minio-entrypoint.sh']
secrets:
- minio_root_user
- minio_root_password
volumes:
- minio_data:/data
- ./infra/minio-entrypoint.sh:/infra/minio-entrypoint.sh:ro
networks:
- backend
healthcheck:
test: ['CMD-SHELL', 'curl -fsS http://127.0.0.1:9000/minio/health/ready >/dev/null']
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
restart: unless-stopped
deploy:
resources:
limits:
cpus: '${MINIO_CPU_LIMIT:-1.0}'
memory: '${MINIO_MEMORY_LIMIT:-1G}'
minio-bootstrap:
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
entrypoint: ['/bin/sh', '/infra/minio-bootstrap.sh']
environment:
S3_BUCKET: messenger-attachments
secrets:
- minio_root_user
- minio_root_password
- s3_access_key_id
- s3_secret_access_key
volumes:
- ./infra/minio-bootstrap.sh:/infra/minio-bootstrap.sh:ro
- ./infra/s3-policy.json:/infra/s3-policy.json:ro
networks:
- backend
depends_on:
minio:
condition: service_healthy
restart: 'no'
deploy:
resources:
limits:
cpus: '${BOOTSTRAP_CPU_LIMIT:-0.25}'
memory: '${BOOTSTRAP_MEMORY_LIMIT:-128M}'
migrate:
build: .
command: ['python', 'scripts/migrate.py']
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
networks:
- backend
depends_on:
postgres:
condition: service_healthy
restart: 'no'
deploy:
resources:
limits:
cpus: '${MIGRATE_CPU_LIMIT:-0.5}'
memory: '${MIGRATE_MEMORY_LIMIT:-256M}'
api:
build: .
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
SESSION_STORE: redis
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL: redis://redis:6379/0
S3_ENDPOINT: http://minio:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-16}
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
networks:
- backend
depends_on:
migrate:
condition: service_completed_successfully
minio-bootstrap:
condition: service_completed_successfully
redis:
condition: service_healthy
healthcheck:
test:
[
'CMD',
'python',
'-c',
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
]
interval: 3s
timeout: 3s
retries: 3
start_period: 10s
stop_grace_period: 15s
restart: unless-stopped
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: '${API_CPU_LIMIT:-1.0}'
memory: '${API_MEMORY_LIMIT:-512M}'
nginx:
image: nginx:1.27.5-alpine
ports:
- '${APP_PORT:-8080}:8080'
volumes:
- ./infra/nginx.conf:/etc/nginx/conf.d/default.conf:ro
networks:
- frontend
- backend
depends_on:
api:
condition: service_healthy
healthcheck:
test: ['CMD-SHELL', 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
interval: 5s
timeout: 3s
retries: 5
restart: unless-stopped
deploy:
resources:
limits:
cpus: '${NGINX_CPU_LIMIT:-0.5}'
memory: '${NGINX_MEMORY_LIMIT:-256M}'
networks:
frontend:
backend:
internal: true
volumes:
postgres_data:
redis_data:
minio_data:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
minio_root_user:
file: ./secrets/minio_root_user
minio_root_password:
file: ./secrets/minio_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
+532
View File
@@ -0,0 +1,532 @@
import hashlib
import json
import os
import uuid
from pathlib import Path
from psycopg import errors
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from psycopg.types.json import Jsonb
MIGRATIONS_DIR = Path(__file__).resolve().parent / 'migrations'
class ApiError(Exception):
def __init__(self, status, code, message):
super().__init__(message)
self.status = status
self.code = code
self.message = message
def read_database_url():
path = os.environ.get('DATABASE_URL_FILE')
if path and os.path.isfile(path):
return Path(path).read_text(encoding='utf-8').strip()
url = os.environ.get('DATABASE_URL', '').strip()
if not url:
raise RuntimeError('DATABASE_URL or DATABASE_URL_FILE is required')
return url
class Store:
def __init__(self, pool: ConnectionPool):
self.pool = pool
@classmethod
def connect(cls, run_migrations=True):
pool = ConnectionPool(
conninfo=read_database_url(),
min_size=1,
max_size=10,
timeout=5.0,
kwargs={
'row_factory': dict_row,
'options': '-c statement_timeout=5000 -c lock_timeout=2000',
},
open=True,
)
store = cls(pool)
if run_migrations:
store.run_migrations()
return store
def close(self):
self.pool.close()
def ping(self):
with self.pool.connection() as conn:
conn.execute('SELECT 1')
def run_migrations(self):
with self.pool.connection() as conn:
conn.execute(
'CREATE TABLE IF NOT EXISTS schema_migrations '
'(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())'
)
for path in sorted(MIGRATIONS_DIR.glob('*.sql')):
version = path.name
row = conn.execute(
'SELECT 1 FROM schema_migrations WHERE version = %s',
(version,),
).fetchone()
if row:
continue
sql = path.read_text(encoding='utf-8')
with conn.transaction():
conn.execute(sql)
conn.execute(
'INSERT INTO schema_migrations (version) VALUES (%s)',
(version,),
)
@staticmethod
def _user_row(row):
return {
'id': str(row['id']),
'username': row['username'],
'display_name': row['display_name'],
'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
}
def register_user(self, username, display_name, password_hash, created_at):
user_id = str(uuid.uuid4())
try:
with self.pool.connection() as conn:
conn.execute(
'INSERT INTO users (id, username, display_name, password_hash, created_at) '
'VALUES (%s, %s, %s, %s, %s)',
(user_id, username, display_name, password_hash, created_at),
)
except errors.UniqueViolation:
raise ApiError(409, 'conflict', 'Username already exists') from None
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
return {
'id': user_id,
'username': username,
'display_name': display_name,
'created_at': created_at,
}
def get_user(self, user_id):
with self.pool.connection() as conn:
row = conn.execute(
'SELECT id, username, display_name, created_at FROM users WHERE id = %s',
(user_id,),
).fetchone()
if row is None:
return None
return self._user_row(row)
def get_user_with_password(self, username):
with self.pool.connection() as conn:
row = conn.execute(
'SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s',
(username,),
).fetchone()
return row
def create_chat(self, user_id, title, member_ids, created_at):
chat_id = str(uuid.uuid4())
try:
with self.pool.connection() as conn:
with conn.transaction():
rows = conn.execute(
'SELECT id FROM users WHERE id = ANY(%s)',
(member_ids,),
).fetchall()
if len(rows) != len(set(member_ids)):
raise ApiError(400, 'invalid_request', 'Unknown chat member')
conn.execute(
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
(chat_id, title, created_at),
)
for index, member_id in enumerate(member_ids):
conn.execute(
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)',
(chat_id, member_id, index),
)
except ApiError:
raise
except errors.Error:
raise ApiError(500, 'internal_error', 'Internal server error') from None
return {
'id': chat_id,
'title': title,
'member_ids': list(member_ids),
'created_at': created_at,
}
def _is_member(self, conn, chat_id, user_id):
row = conn.execute(
'SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s',
(chat_id, user_id),
).fetchone()
return row is not None
def get_chat(self, user_id, chat_id):
with self.pool.connection() as conn:
if not self._is_member(conn, chat_id, user_id):
raise ApiError(404, 'not_found', 'Chat not found')
chat = conn.execute(
'SELECT id, title, created_at FROM chats WHERE id = %s',
(chat_id,),
).fetchone()
members = conn.execute(
'SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position',
(chat_id,),
).fetchall()
return {
'id': str(chat['id']),
'title': chat['title'],
'member_ids': [str(row['user_id']) for row in members],
'created_at': chat['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
}
def list_chats(self, user_id, limit, cursor_key):
query = (
'SELECT c.id, c.title, c.created_at '
'FROM chats c '
'JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = %s '
)
params = [user_id]
if cursor_key:
query += 'WHERE (c.created_at, c.id) > (%s, %s) '
params.extend(cursor_key)
query += 'ORDER BY c.created_at ASC, c.id ASC LIMIT %s'
params.append(limit + 1)
with self.pool.connection() as conn:
rows = conn.execute(query, params).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
items = []
for row in page:
chat_id = str(row['id'])
items.append(self.get_chat(user_id, chat_id))
next_cursor = None
if has_more:
last = page[-1]
next_cursor = (last['created_at'], str(last['id']))
return items, next_cursor
def create_message(self, user_id, chat_id, text, created_at):
message_id = str(uuid.uuid4())
with self.pool.connection() as conn:
with conn.transaction():
if not self._is_member(conn, chat_id, user_id):
raise ApiError(404, 'not_found', 'Chat not found')
conn.execute(
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
'VALUES (%s, %s, %s, %s, %s)',
(message_id, chat_id, user_id, text, created_at),
)
return {
'id': message_id,
'chat_id': chat_id,
'sender_id': user_id,
'text': text,
'created_at': created_at,
'attachment_ids': [],
}
def create_message_idempotent(
self, user_id, chat_id, text, attachment_ids, idempotency_key, created_at
):
resource_path = '/api/v1/chats/' + chat_id + '/messages'
request_hash = hashlib.sha256(
json.dumps(
{
'chat_id': chat_id,
'text': text,
'attachment_ids': attachment_ids,
},
ensure_ascii=False,
sort_keys=True,
separators=(',', ':'),
).encode('utf-8')
).hexdigest()
lock_key = '\x1f'.join(
(user_id, 'POST', resource_path, idempotency_key)
)
try:
with self.pool.connection() as conn:
with conn.transaction():
if not self._is_member(conn, chat_id, user_id):
raise ApiError(404, 'not_found', 'Chat not found')
conn.execute(
'SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))',
(lock_key,),
)
row = conn.execute(
'SELECT request_hash, response FROM message_idempotency '
'WHERE user_id = %s AND method = %s AND resource_path = %s '
'AND idempotency_key = %s AND expires_at > NOW() FOR UPDATE',
(user_id, 'POST', resource_path, idempotency_key),
).fetchone()
if row is not None:
if row['request_hash'] != request_hash:
raise ApiError(409, 'conflict', 'Idempotency-Key was used with a different request')
return row['response']
conn.execute(
'DELETE FROM message_idempotency WHERE user_id = %s '
'AND method = %s AND resource_path = %s AND idempotency_key = %s',
(user_id, 'POST', resource_path, idempotency_key),
)
conn.execute(
'DELETE FROM message_idempotency WHERE user_id = %s AND expires_at <= NOW()',
(user_id,),
)
if attachment_ids:
attachments = conn.execute(
'SELECT id, status FROM attachments '
'WHERE chat_id = %s AND id = ANY(%s) FOR SHARE',
(chat_id, attachment_ids),
).fetchall()
if len(attachments) != len(attachment_ids):
raise ApiError(404, 'not_found', 'Attachment not found')
if any(row['status'] != 'ready' for row in attachments):
raise ApiError(409, 'conflict', 'Attachment is not ready')
conn.execute(
'INSERT INTO message_idempotency '
'(user_id, method, resource_path, idempotency_key, request_hash, expires_at) '
"VALUES (%s, %s, %s, %s, %s, NOW() + INTERVAL '24 hours')",
(user_id, 'POST', resource_path, idempotency_key, request_hash),
)
message_id = str(uuid.uuid4())
conn.execute(
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
'VALUES (%s, %s, %s, %s, %s)',
(message_id, chat_id, user_id, text, created_at),
)
for position, attachment_id in enumerate(attachment_ids):
conn.execute(
'INSERT INTO message_attachments '
'(message_id, attachment_id, position) VALUES (%s, %s, %s)',
(message_id, attachment_id, position),
)
message = {
'id': message_id,
'chat_id': chat_id,
'sender_id': user_id,
'text': text,
'created_at': created_at,
'attachment_ids': list(attachment_ids),
}
conn.execute(
'UPDATE message_idempotency SET response = %s '
'WHERE user_id = %s AND method = %s AND resource_path = %s '
'AND idempotency_key = %s',
(Jsonb(message), user_id, 'POST', resource_path, idempotency_key),
)
return message
except ApiError:
raise
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
def list_messages(self, user_id, chat_id, limit, cursor_key):
with self.pool.connection() as conn:
if not self._is_member(conn, chat_id, user_id):
raise ApiError(404, 'not_found', 'Chat not found')
query = (
'SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, '
'COALESCE(array_agg(ma.attachment_id ORDER BY ma.position) '
'FILTER (WHERE ma.attachment_id IS NOT NULL), ARRAY[]::UUID[]) AS attachment_ids '
'FROM messages m LEFT JOIN message_attachments ma ON ma.message_id = m.id '
'WHERE m.chat_id = %s '
)
params = [chat_id]
if cursor_key:
query += 'AND (m.created_at, m.id) > (%s, %s) '
params.extend(cursor_key)
query += (
'GROUP BY m.id '
'ORDER BY m.created_at ASC, m.id ASC LIMIT %s'
)
params.append(limit + 1)
with self.pool.connection() as conn:
rows = conn.execute(query, params).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
items = [
{
'id': str(row['id']),
'chat_id': str(row['chat_id']),
'sender_id': str(row['sender_id']),
'text': row['text'],
'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
'attachment_ids': [str(value) for value in row['attachment_ids']],
}
for row in page
]
next_cursor = None
if has_more:
last = page[-1]
next_cursor = (last['created_at'], str(last['id']))
return items, next_cursor
@staticmethod
def _attachment_row(row, include_key=False):
value = {
'id': str(row['id']),
'chat_id': str(row['chat_id']),
'owner_id': str(row['owner_id']),
'filename': row['filename'],
'content_type': row['content_type'],
'size_bytes': row['size_bytes'],
'sha256': row['sha256'].strip(),
'status': row['status'],
'variants': row['variants'],
'error_code': row['error_code'],
'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
'updated_at': row['updated_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
}
if include_key:
value['object_key'] = row['object_key']
value['width'] = row['width']
value['height'] = row['height']
return value
def create_attachment_upload(
self, attachment_id, chat_id, owner_id, object_key, filename,
content_type, size_bytes, sha256, width, height, created_at
):
try:
with self.pool.connection() as conn:
with conn.transaction():
if not self._is_member(conn, chat_id, owner_id):
raise ApiError(404, 'not_found', 'Chat not found')
conn.execute(
'INSERT INTO attachments '
'(id, chat_id, owner_id, object_key, filename, content_type, '
'size_bytes, sha256, width, height, status, variants, error_code, '
'created_at, updated_at) '
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, "
"'processing', '[]'::jsonb, NULL, %s, %s)",
(
attachment_id, chat_id, owner_id, object_key, filename,
content_type, size_bytes, sha256, width, height,
created_at, created_at,
),
)
except ApiError:
raise
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
def mark_attachment_ready(self, attachment_id, updated_at):
try:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
'UPDATE attachments SET status = %s, error_code = NULL, updated_at = %s '
"WHERE id = %s AND status = 'processing' RETURNING *",
('ready', updated_at, attachment_id),
).fetchone()
if row is None:
raise ApiError(503, 'unavailable', 'Attachment state could not be saved')
return self._attachment_row(row)
except ApiError:
raise
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
def mark_attachment_failed(self, attachment_id, error_code, updated_at):
try:
with self.pool.connection() as conn:
conn.execute(
'UPDATE attachments SET status = %s, error_code = %s, updated_at = %s '
"WHERE id = %s AND status = 'processing'",
('failed', error_code, updated_at, attachment_id),
)
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
def get_attachment(self, user_id, attachment_id):
try:
with self.pool.connection() as conn:
row = conn.execute(
'SELECT a.* FROM attachments a '
'JOIN chat_members cm ON cm.chat_id = a.chat_id '
'WHERE a.id = %s AND cm.user_id = %s',
(attachment_id, user_id),
).fetchone()
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
if row is None:
raise ApiError(404, 'not_found', 'Attachment not found')
return self._attachment_row(row)
def get_attachment_for_download(self, user_id, attachment_id):
try:
with self.pool.connection() as conn:
row = conn.execute(
'SELECT a.* FROM attachments a '
'JOIN chat_members cm ON cm.chat_id = a.chat_id '
'WHERE a.id = %s AND cm.user_id = %s',
(attachment_id, user_id),
).fetchone()
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
if row is None:
raise ApiError(404, 'not_found', 'Attachment not found')
return self._attachment_row(row, include_key=True)
def list_stale_processing_attachments(self, cutoff):
try:
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT * FROM attachments WHERE status = 'processing' AND created_at < %s "
'ORDER BY created_at, id',
(cutoff,),
).fetchall()
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
return [self._attachment_row(row, include_key=True) for row in rows]
def list_ready_attachments(self):
try:
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT * FROM attachments WHERE status = 'ready' ORDER BY id"
).fetchall()
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
return [self._attachment_row(row, include_key=True) for row in rows]
def list_attachments_for_reconciliation(self):
try:
with self.pool.connection() as conn:
rows = conn.execute(
'SELECT * FROM attachments ORDER BY id'
).fetchall()
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
return [self._attachment_row(row, include_key=True) for row in rows]
def get_attachment_by_object_key(self, object_key):
try:
with self.pool.connection() as conn:
row = conn.execute(
'SELECT * FROM attachments WHERE object_key = %s',
(object_key,),
).fetchone()
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
if row is None:
return None
return self._attachment_row(row, include_key=True)
def mark_reconciled_missing(self, attachment_id, updated_at):
try:
with self.pool.connection() as conn:
conn.execute(
'UPDATE attachments SET status = %s, error_code = %s, updated_at = %s '
"WHERE id = %s AND status = 'ready'",
('failed', 'object_missing', updated_at, attachment_id),
)
except (errors.Error, PoolTimeout):
raise ApiError(503, 'unavailable', 'Service unavailable') from None
+12
View File
@@ -0,0 +1,12 @@
{
"concurrent_uploads": 10,
"image_size_bytes": 2562448,
"image_sha256": "b421436ca79ae3d6aa00a330f13e092c90cfb471fc944a8a5389445844952933",
"elapsed_seconds": 2.315,
"api_memory_baseline_bytes": 128198901,
"api_memory_peak_sampled_bytes": 141882817,
"api_memory_sampled_delta_bytes": 13683916,
"memory_sample_interval_seconds": 1,
"memory_sample_count": 3,
"notes": "Docker stats streams aggregate API-container memory at about one-second intervals; short peaks may be missed."
}
+47 -4
View File
@@ -1,7 +1,50 @@
# Инфраструктура студента
# Compose + Nginx + MinIO
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
The runnable topology is in the root `compose.yaml`. The example Compose file
and `contracts/infrastructure.json` are teaching worksheets, not deployments.
The supported local entry point is Nginx; API, PostgreSQL, Redis and MinIO ports
are not published to the host.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
| Role | Service / image | Persistence and readiness |
| --- | --- | --- |
| API | `api`, built from `Dockerfile` / Python 3.12.11 | Two stateless replicas; `/health/ready`; no host port |
| Load balancer | `nginx:1.27.5-alpine` | Host port `${APP_PORT:-8080}`; Docker DNS refresh 5 s; passive fail exclusion |
| Database | `postgres:16.8-alpine` | `postgres_data` named volume; `pg_isready` |
| Sessions | `redis:7.4.2-alpine` | `redis_data` named volume; AOF `everysec`; Redis `PING` |
| Object storage | MinIO `RELEASE.2025-09-07T16-13-09Z`, pinned by image digest in Compose | `minio_data` named volume; MinIO readiness endpoint; S3 port 9000 and console 9001 are private |
| Migration job | Same API image | Runs ordered SQL migrations once before API replicas |
| Bucket bootstrap | Same pinned MinIO image, using its `mc` client | Idempotently creates the private bucket, versioned policy, and restricted API user; replaces the prior policy without deleting volumes |
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
PostgreSQL and S3 hold different parts of an attachment: PostgreSQL is the source
of truth for metadata and access control; the private S3 bucket stores original
bytes under a random server-generated key. The bucket name is fixed as
`messenger-attachments`. No endpoint returns object keys or S3 credentials.
The MinIO root credentials are mounted only into MinIO and its one-shot bootstrap
job. The API receives a separate access key restricted to `GetBucketLocation`,
`ListBucket` with the `objects/` prefix, and Get/Put/Delete on `objects/*`.
The bootstrap grants no anonymous policy. Verify
that from inside the Compose network with
`py -3 -X utf8 student-tests/check_private_bucket.py`; it expects an unsigned
bucket-list request to receive HTTP 403 and confirms MinIO has no host port
binding. The administrative console is not exposed.
Create local ignored secrets using `scripts/init-local-secrets.ps1`; never copy
secrets from another lab. Do not rotate credentials while retaining the
corresponding database or MinIO volumes. `scripts/restart.bat` / `.sh` restarts
the API, database, Redis, MinIO and Nginx without deleting volumes.
`scripts/stop-one.bat` / `.sh` stops only the API container whose hostname equals
`TARGET_INSTANCE_ID` and leaves it stopped for the failover check.
Attachment writes reserve PostgreSQL metadata in `processing` state before
writing S3. On success, the row becomes `ready` and the API returns 201. Errors
attempt to delete the object and mark the reservation `failed`; if compensation
cannot reach either store, the processing row remains visible to reconciliation.
Run `scripts/reconcile_attachments.py` in dry-run mode first. It only acts on
records/objects older than a 15-minute minimum grace period, marks stale uploads
failed, deletes old unreferenced objects, and reports missing objects for ready
metadata. `--apply` is required for changes.
MinIO, PostgreSQL, Redis, Nginx, and the Docker host are single-instance
dependencies. This setup survives an API-replica failure and restarts with
volumes, not loss of the Docker host or an individual state-store volume.
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
set -eu
root_user=$(cat /run/secrets/minio_root_user)
root_password=$(cat /run/secrets/minio_root_password)
service_user=$(cat /run/secrets/s3_access_key_id)
service_password=$(cat /run/secrets/s3_secret_access_key)
until mc alias set local http://minio:9000 "$root_user" "$root_password" >/dev/null 2>&1 \
&& mc ready local >/dev/null 2>&1; do
sleep 2
done
bucket=${S3_BUCKET:-messenger-attachments}
if [ "$bucket" != 'messenger-attachments' ]; then
printf '%s\n' 'S3_BUCKET must be messenger-attachments' >&2
exit 2
fi
policy=messenger-attachments-v2
legacy_policy=messenger-attachments
mc mb --ignore-existing "local/$bucket"
if ! mc admin policy info local "$policy" >/dev/null 2>&1; then
mc admin policy create local "$policy" /infra/s3-policy.json
fi
if ! mc admin user info local "$service_user" >/dev/null 2>&1; then
mc admin user add local "$service_user" "$service_password" >/dev/null
fi
mc admin policy attach local "$policy" --user "$service_user" >/dev/null
if mc admin policy info local "$legacy_policy" >/dev/null 2>&1; then
mc admin policy detach local "$legacy_policy" --user "$service_user" >/dev/null
mc admin policy remove local "$legacy_policy" >/dev/null
fi
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
exec minio server /data --address :9000 --console-address :9001
+26
View File
@@ -0,0 +1,26 @@
resolver 127.0.0.11 valid=5s ipv6=off;
upstream messenger_api {
zone messenger_api 64k;
server api:8080 resolve max_fails=1 fail_timeout=2s;
}
server {
listen 8080;
server_name _;
client_max_body_size 11m;
client_body_timeout 20s;
location / {
proxy_pass http://messenger_api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_request_buffering off;
proxy_connect_timeout 2s;
proxy_send_timeout 20s;
proxy_read_timeout 20s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
}
}
+25
View File
@@ -0,0 +1,25 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
},
{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger-attachments"],
"Condition": {
"StringLike": {
"s3:prefix": ["objects/", "objects/*"]
}
}
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/objects/*"]
}
]
}
+34
View File
@@ -0,0 +1,34 @@
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username TEXT NOT NULL,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS users_username_key ON users (username);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
position INT NOT NULL,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX IF NOT EXISTS chat_members_user_id_idx ON chat_members (user_id);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS messages_chat_history_idx ON messages (chat_id, created_at ASC, id ASC);
+14
View File
@@ -0,0 +1,14 @@
CREATE TABLE IF NOT EXISTS message_idempotency (
user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
method TEXT NOT NULL CHECK (method = 'POST'),
resource_path TEXT NOT NULL,
idempotency_key VARCHAR(128) NOT NULL,
request_hash CHAR(64) NOT NULL,
response JSONB,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL,
PRIMARY KEY (user_id, method, resource_path, idempotency_key)
);
CREATE INDEX IF NOT EXISTS message_idempotency_expiry_idx
ON message_idempotency (user_id, expires_at);
+34
View File
@@ -0,0 +1,34 @@
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
owner_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
object_key TEXT NOT NULL UNIQUE,
filename TEXT NOT NULL,
content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 CHAR(64) NOT NULL,
width INT NOT NULL CHECK (width > 0),
height INT NOT NULL CHECK (height > 0),
status TEXT NOT NULL CHECK (status IN ('processing', 'ready', 'failed')),
variants JSONB NOT NULL DEFAULT '[]'::jsonb,
error_code TEXT,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS attachments_chat_created_idx
ON attachments (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS attachments_processing_created_idx
ON attachments (created_at)
WHERE status = 'processing';
CREATE TABLE IF NOT EXISTS message_attachments (
message_id UUID NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT,
position INT NOT NULL CHECK (position >= 0),
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id, position)
);
CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx
ON message_attachments (attachment_id);
+5
View File
@@ -0,0 +1,5 @@
argon2-cffi==23.1.0
psycopg[binary,pool]==3.2.9
redis==5.2.1
boto3==1.37.38
Pillow==11.1.0
+106
View File
@@ -0,0 +1,106 @@
import os
from pathlib import Path
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
class S3Unavailable(RuntimeError):
pass
def _read_secret(file_variable, value_variable):
path = os.environ.get(file_variable)
if path:
return Path(path).read_text(encoding='utf-8').strip()
value = os.environ.get(value_variable, '').strip()
if not value:
raise RuntimeError(value_variable + ' or ' + file_variable + ' is required')
return value
class ObjectStore:
def __init__(self, client, bucket):
self.client = client
self.bucket = bucket
@classmethod
def from_env(cls):
client = boto3.client(
's3',
endpoint_url=os.environ.get('S3_ENDPOINT', 'http://minio:9000'),
region_name=os.environ.get('S3_REGION', 'us-east-1'),
aws_access_key_id=_read_secret('S3_ACCESS_KEY_ID_FILE', 'S3_ACCESS_KEY_ID'),
aws_secret_access_key=_read_secret(
'S3_SECRET_ACCESS_KEY_FILE', 'S3_SECRET_ACCESS_KEY'
),
config=Config(
signature_version='s3v4',
s3={'addressing_style': 'path'},
connect_timeout=2,
read_timeout=10,
max_pool_connections=32,
retries={'max_attempts': 2, 'mode': 'standard'},
),
)
return cls(client, os.environ.get('S3_BUCKET', 'messenger-attachments'))
def ping(self):
try:
self.client.get_bucket_location(Bucket=self.bucket)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable('S3 bucket is unavailable') from error
def put_file(self, key, fileobj, size, content_type):
fileobj.seek(0)
try:
self.client.put_object(
Bucket=self.bucket,
Key=key,
Body=fileobj,
ContentLength=size,
ContentType=content_type,
)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable('S3 upload failed') from error
def get_object(self, key):
try:
return self.client.get_object(Bucket=self.bucket, Key=key)
except ClientError as error:
if error.response.get('Error', {}).get('Code') in {
'404', 'NoSuchKey', 'NotFound',
}:
return None
raise S3Unavailable('S3 download failed') from error
except BotoCoreError as error:
raise S3Unavailable('S3 download failed') from error
def delete_object(self, key):
try:
self.client.delete_object(Bucket=self.bucket, Key=key)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable('S3 delete failed') from error
def object_exists(self, key):
try:
self.client.head_object(Bucket=self.bucket, Key=key)
return True
except ClientError as error:
if error.response.get('Error', {}).get('Code') in {
'404', 'NoSuchKey', 'NotFound',
}:
return False
raise S3Unavailable('S3 object check failed') from error
except BotoCoreError as error:
raise S3Unavailable('S3 object check failed') from error
def list_objects(self):
try:
paginator = self.client.get_paginator('list_objects_v2')
for page in paginator.paginate(Bucket=self.bucket, Prefix='objects/'):
for item in page.get('Contents', []):
yield item
except (BotoCoreError, ClientError) as error:
raise S3Unavailable('S3 listing failed') from error
+6 -1
View File
@@ -40,7 +40,12 @@ def main():
assert 'X-Instance-Id' in response['headers']
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
python_checks = (
list((ROOT/'tests').glob('*.py'))
+ list((ROOT/'scripts').glob('*.py'))
+ list((ROOT/'student-tests').glob('*.py'))
)
for file in python_checks:
ast.parse(file.read_text(),filename=str(file))
for file in ROOT.rglob('*.json'):
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
+60
View File
@@ -0,0 +1,60 @@
$ErrorActionPreference = 'Stop'
$root = Split-Path -Parent $PSScriptRoot
$secrets = Join-Path $root 'secrets'
$names = @(
'postgres_password',
'database_url',
'minio_root_user',
'minio_root_password',
's3_access_key_id',
's3_secret_access_key'
)
if (-not (Test-Path (Join-Path $root '.env'))) {
Copy-Item (Join-Path $root '.env.example') (Join-Path $root '.env')
}
$existing = @($names | Where-Object { Test-Path (Join-Path $secrets $_) })
if ($existing.Count -gt 0) {
if ($existing.Count -eq $names.Count) {
Write-Output 'Local secrets already exist; leaving them unchanged.'
exit 0
}
throw 'The local secret set is incomplete. Restore it as a set; do not rotate credentials against existing volumes.'
}
New-Item -ItemType Directory -Force $secrets | Out-Null
function New-HexSecret([int]$byteCount) {
$bytes = New-Object byte[] $byteCount
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
try {
$rng.GetBytes($bytes)
}
finally {
$rng.Dispose()
}
$builder = New-Object System.Text.StringBuilder
foreach ($byte in $bytes) {
[void]$builder.Append($byte.ToString('x2'))
}
return $builder.ToString()
}
$databasePassword = New-HexSecret 32
$values = @{
'postgres_password' = $databasePassword
'database_url' = "postgresql://messenger:$databasePassword@postgres:5432/messenger"
'minio_root_user' = 'root-' + (New-HexSecret 6)
'minio_root_password' = New-HexSecret 32
's3_access_key_id' = New-HexSecret 10
's3_secret_access_key' = New-HexSecret 32
}
foreach ($name in $names) {
$path = Join-Path $secrets $name
[System.IO.File]::WriteAllText(
$path,
$values[$name],
[System.Text.Encoding]::ASCII
)
}
Write-Output 'Created local Compose secrets. Their values were not printed.'
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env python3
"""Apply database migrations once before API replicas start."""
import logging
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from db import Store
def main():
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
store = Store.connect()
try:
logging.info('database migrations are current')
finally:
store.close()
if __name__ == '__main__':
main()
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
"""Dry-run first; remove only old, unreferenced objects or stale upload attempts."""
import argparse
from datetime import datetime, timedelta, timezone
import json
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from db import ApiError, Store
from s3store import ObjectStore, S3Unavailable
def parse_timestamp(value):
return datetime.fromisoformat(value.replace('Z', '+00:00'))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--grace-seconds', type=int, default=900)
parser.add_argument('--apply', action='store_true')
args = parser.parse_args()
if args.grace_seconds < 900:
parser.error('--grace-seconds must be at least 900')
store = Store.connect(run_migrations=False)
objects = ObjectStore.from_env()
try:
cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds)
records = store.list_attachments_for_reconciliation()
by_key = {record['object_key']: record for record in records}
summary = {
'mode': 'apply' if args.apply else 'dry-run',
'grace_seconds': args.grace_seconds,
'stale_uploads': 0,
'old_failed_objects': 0,
'orphan_objects': 0,
'missing_ready_objects': 0,
'actions_applied': 0,
}
for record in records:
key = record['object_key']
if (record['status'] == 'processing'
and parse_timestamp(record['created_at']) < cutoff):
summary['stale_uploads'] += 1
exists = objects.object_exists(key)
if args.apply:
if exists:
objects.delete_object(key)
store.mark_attachment_failed(
record['id'], 'upload_interrupted', datetime.now(timezone.utc)
)
summary['actions_applied'] += 1
elif (record['status'] == 'failed'
and parse_timestamp(record['updated_at']) < cutoff):
if objects.object_exists(key):
summary['old_failed_objects'] += 1
if args.apply:
objects.delete_object(key)
summary['actions_applied'] += 1
elif record['status'] == 'ready' and not objects.object_exists(key):
summary['missing_ready_objects'] += 1
if args.apply:
store.mark_reconciled_missing(
record['id'], datetime.now(timezone.utc)
)
summary['actions_applied'] += 1
for item in objects.list_objects():
key = item['Key']
if key in by_key or item['LastModified'] >= cutoff:
continue
summary['orphan_objects'] += 1
if args.apply:
objects.delete_object(key)
summary['actions_applied'] += 1
print(json.dumps(summary, sort_keys=True))
finally:
store.close()
objects.client.close()
if __name__ == '__main__':
try:
main()
except (ApiError, S3Unavailable) as error:
print('Reconciliation failed: ' + error.message if isinstance(error, ApiError) else
'Reconciliation failed: S3 is unavailable', file=sys.stderr)
raise SystemExit(1) from None
+5
View File
@@ -0,0 +1,5 @@
@echo off
setlocal
cd /d "%~dp0.."
docker compose restart api postgres redis minio nginx
exit /b %errorlevel%
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose restart api postgres redis minio nginx
+21
View File
@@ -0,0 +1,21 @@
@echo off
setlocal
cd /d "%~dp0.."
if "%TARGET_INSTANCE_ID%"=="" (
echo TARGET_INSTANCE_ID is required 1>&2
exit /b 2
)
set TARGET_STOPPED=
for /f "delims=" %%i in ('docker compose ps -q api') do if not defined TARGET_STOPPED call :stop_if_target %%i
if defined TARGET_STOPPED exit /b 0
echo No running API replica matches TARGET_INSTANCE_ID 1>&2
exit /b 1
:stop_if_target
for /f "delims=" %%h in ('docker inspect --format "{{.Config.Hostname}}" %1') do if "%%h"=="%TARGET_INSTANCE_ID%" (
docker stop --time 2 %1 >nul
if errorlevel 1 exit /b 1
set TARGET_STOPPED=1
exit /b 0
)
exit /b 1
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
cd "$(dirname "$0")/.."
for container_id in $(docker compose ps -q api); do
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 2 "$container_id" >/dev/null
exit 0
fi
done
printf '%s\n' 'No running API replica matches TARGET_INSTANCE_ID' >&2
exit 1
+43
View File
@@ -0,0 +1,43 @@
import os
import redis
class SessionStore:
def __init__(self, url, connect_timeout=2.0, socket_timeout=2.0):
self._url = url
self._client = redis.from_url(
url,
decode_responses=True,
socket_connect_timeout=connect_timeout,
socket_timeout=socket_timeout,
health_check_interval=30,
)
self._prefix = 'sess:'
@classmethod
def from_env(cls):
url = _read_config('CACHE_URL_FILE', 'CACHE_URL')
if not url:
raise RuntimeError('CACHE_URL or CACHE_URL_FILE is required when SESSION_STORE=redis')
return cls(url)
def ping(self):
return self._client.ping()
def create(self, token, user_id, ttl_seconds):
self._client.setex(self._prefix + token, ttl_seconds, user_id)
def get_user_id(self, token):
return self._client.get(self._prefix + token)
def revoke(self, token):
return self._client.delete(self._prefix + token) > 0
def _read_config(file_env, value_env):
path = os.environ.get(file_env)
if path and os.path.isfile(path):
with open(path, encoding='utf-8') as handle:
return handle.read().strip()
return os.environ.get(value_env, '').strip()
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Check anonymous access to the private bucket from inside the Compose network."""
from pathlib import Path
import json
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Failure, run
def main():
code = (
"import os, urllib.error, urllib.request; "
"bucket=os.environ['S3_BUCKET']; "
"url='http://minio:9000/'+bucket+'?list-type=2&prefix=objects%2F'; "
"req=urllib.request.Request(url); "
"\ntry:\n urllib.request.urlopen(req, timeout=3)\n"
"except urllib.error.HTTPError as error:\n"
" raise SystemExit(0 if error.code == 403 else 1)\n"
"raise SystemExit(1)"
)
result = subprocess.run(
['docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code],
cwd=ROOT,
capture_output=True,
text=True,
)
if result.returncode:
raise Failure('anonymous MinIO bucket request did not return HTTP 403')
container_id = subprocess.run(
['docker', 'compose', 'ps', '-q', 'minio'],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
).stdout.strip()
if not container_id:
raise Failure('MinIO container is not running')
bindings = subprocess.run(
['docker', 'inspect', '--format', '{{json .HostConfig.PortBindings}}', container_id],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
).stdout.strip()
if any(value for value in json.loads(bindings).values()):
raise Failure('MinIO S3 endpoint must not be published on the host')
print('PASS: anonymous bucket access is denied and the S3 port is not published.')
if __name__ == '__main__':
run(main)
+287
View File
@@ -0,0 +1,287 @@
#!/usr/bin/env python3
"""Student negative and concurrency checks for the lab 4 attachment API."""
from concurrent.futures import ThreadPoolExecutor
import http.client
import hashlib
import json
from pathlib import Path
import re
import struct
import subprocess
import sys
import threading
import time
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
from urllib.parse import quote, urlsplit
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
PARALLEL_UPLOADS = 10
ANSI_ESCAPE = re.compile(r'\x1b\[[0-?]*[ -/]*[@-~]')
def png_chunk(kind, data):
return (
struct.pack('>I', len(data))
+ kind
+ data
+ struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
)
def random_png(width=1600, height=1600):
seed = 0x4C414234
pixels = bytearray(width * height)
for index in range(len(pixels)):
seed = (1664525 * seed + 1013904223) & 0xffffffff
pixels[index] = seed >> 24
rows = b''.join(
b'\x00' + pixels[row * width:(row + 1) * width]
for row in range(height)
)
header = struct.pack('>IIBBBBB', width, height, 8, 0, 0, 0, 0)
return (
b'\x89PNG\r\n\x1a\n'
+ png_chunk(b'IHDR', header)
+ png_chunk(b'IDAT', zlib.compress(rows))
+ png_chunk(b'IEND', b'')
)
def memory_bytes(value):
match = re.match(r'\s*([\d.]+)\s*(B|KiB|MiB|GiB)', value)
check(match is not None, 'could not parse docker stats memory value')
factors = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3}
return int(float(match.group(1)) * factors[match.group(2)])
def api_memory(container_ids):
output = subprocess.run(
['docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *container_ids],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
).stdout
lines = [line.strip() for line in output.splitlines() if line.strip()]
check(len(lines) == len(container_ids), 'docker stats did not report every API replica')
return sum(memory_bytes(line.split('/', 1)[0]) for line in lines)
def make_upload(client, actor, chat_id, filename, content, content_type='image/png'):
return client.call(
'POST',
'/api/v1/chats/' + chat_id + '/attachments?filename=' + quote(filename, safe=''),
201,
actor=actor,
raw=content,
headers={'Content-Type': content_type},
schema='Attachment',
)['json']
def slow_upload(base_url, actor, chat_id, index, content):
endpoint = urlsplit(base_url or 'http://localhost:8080')
check(endpoint.scheme == 'http', 'memory test currently requires the local HTTP endpoint')
connection = http.client.HTTPConnection(
endpoint.hostname, endpoint.port or 80, timeout=30
)
path = (
'/api/v1/chats/' + chat_id + '/attachments?filename='
+ quote('parallel-' + str(index) + '.png', safe='')
)
try:
connection.putrequest('POST', path)
connection.putheader('Authorization', 'Bearer ' + actor['token'])
connection.putheader('Accept', 'application/json')
connection.putheader('Content-Type', 'image/png')
connection.putheader('Content-Length', str(len(content)))
connection.putheader('Connection', 'close')
connection.endheaders()
chunk_size = 64 * 1024
for offset in range(0, len(content), chunk_size):
connection.send(content[offset:offset + chunk_size])
time.sleep(0.05)
response = connection.getresponse()
payload = json.loads(response.read())
check(response.status == 201, 'parallel upload did not return 201')
check(bool(response.getheader('X-Request-Id')), 'parallel upload has no request ID')
check(bool(response.getheader('X-Instance-Id')), 'parallel upload has no instance ID')
from client import shape
shape(payload, 'Attachment')
return payload
finally:
connection.close()
def main():
args = arguments('Attachment negative tests and 10-way upload memory measurement')
client = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, eve, chat = client.bootstrap()
fixture = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
attachment = make_upload(client, alice, chat['id'], 'sample.png', fixture)
large = b'x' * (MAX_UPLOAD_BYTES + 1)
client.call(
'POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=large.png',
413,
actor=alice,
raw=large,
headers={'Content-Type': 'image/png'},
)
pixel_header = (
b'\x89PNG\r\n\x1a\n'
+ png_chunk(
b'IHDR',
struct.pack('>IIBBBBB', 10000, 5000, 8, 0, 0, 0, 0),
)
+ png_chunk(b'IDAT', zlib.compress(b''))
+ png_chunk(b'IEND', b'')
)
client.call(
'POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=too-many-pixels.png',
413,
actor=alice,
raw=pixel_header,
headers={'Content-Type': 'image/png'},
)
client.call(
'POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=broken.png',
415,
actor=alice,
raw=b'not a PNG image',
headers={'Content-Type': 'image/png'},
)
client.call(
'POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=spoofed.jpg',
415,
actor=alice,
raw=fixture,
headers={'Content-Type': 'image/jpeg'},
)
client.call(
'POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=..%2Fprivate.png',
400,
actor=alice,
raw=fixture,
headers={'Content-Type': 'image/png'},
)
for path in (
'/api/v1/attachments/' + attachment['id'],
'/api/v1/attachments/' + attachment['id'] + '/content',
):
client.call('GET', path, 404, actor=eve)
other_chat = client.call(
'POST',
'/api/v1/chats',
201,
actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]},
schema='Chat',
)['json']
client.call(
'POST',
'/api/v1/chats/' + other_chat['id'] + '/messages',
404,
actor=alice,
body={'text': 'Чужое вложение', 'attachment_ids': [attachment['id']]},
headers={'Idempotency-Key': 'cross-chat-attachment-check'},
)
content = random_png()
check(0 < len(content) <= MAX_UPLOAD_BYTES, 'generated stress image exceeds upload limit')
container_ids = subprocess.run(
['docker', 'compose', 'ps', '-q', 'api'],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
).stdout.split()
check(len(container_ids) >= 2, 'memory test requires at least two API containers')
baseline = api_memory(container_ids)
samples = []
latest = {}
stats_process = subprocess.Popen(
[
'docker', 'stats', '--format', '{{.Name}}|{{.MemUsage}}',
*container_ids,
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
bufsize=1,
)
def monitor():
for line in stats_process.stdout:
line = ANSI_ESCAPE.sub('', line)
parts = line.strip().split('|', 1)
if len(parts) != 2:
continue
try:
latest[parts[0]] = memory_bytes(parts[1].split('/', 1)[0])
except AssertionError:
continue
if len(latest) >= len(container_ids):
samples.append(sum(latest.values()))
monitor_thread = threading.Thread(target=monitor, daemon=True)
monitor_thread.start()
barrier = threading.Barrier(PARALLEL_UPLOADS)
def worker(index):
barrier.wait()
return slow_upload(
args.base_url,
alice,
chat['id'],
index,
content,
)
started = time.monotonic()
try:
with ThreadPoolExecutor(max_workers=PARALLEL_UPLOADS) as pool:
results = list(pool.map(worker, range(PARALLEL_UPLOADS)))
finally:
stats_process.terminate()
stats_process.wait(timeout=5)
monitor_thread.join(timeout=5)
elapsed = time.monotonic() - started
check(len(results) == PARALLEL_UPLOADS, 'not all simultaneous uploads completed')
check(samples, 'Docker stats did not produce any in-flight memory samples')
check(all(item['status'] == 'ready' for item in results), 'an upload was not ready')
check(len({item['id'] for item in results}) == PARALLEL_UPLOADS, 'upload IDs were reused')
peak = max(samples + [baseline])
report = {
'concurrent_uploads': PARALLEL_UPLOADS,
'image_size_bytes': len(content),
'image_sha256': hashlib.sha256(content).hexdigest(),
'elapsed_seconds': round(elapsed, 3),
'api_memory_baseline_bytes': baseline,
'api_memory_peak_sampled_bytes': peak,
'api_memory_sampled_delta_bytes': peak - baseline,
'memory_sample_interval_seconds': 1,
'memory_sample_count': len(samples),
'notes': 'Docker stats streams aggregate API-container memory at about one-second intervals; short peaks may be missed.',
}
output = ROOT / 'evidence' / 'attachment-concurrency.json'
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps(report, indent=2))
print('PASS: attachment limits, image validation, access controls and 10 concurrent uploads.')
if __name__ == '__main__':
run(main)
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python3
"""Explicitly stop MinIO, verify uploads fail as unavailable, and restore it."""
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
def compose(*args):
return subprocess.run(
['docker', 'compose', *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def main():
args = arguments('MinIO outage behavior; stops and starts the lab MinIO service')
client = Client(args.base_url, args.ca_file)
alice, _bob, _eve, chat = client.bootstrap()
sample = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
compose('stop', 'minio')
try:
time.sleep(2)
response = client.call(
'POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=outage.png',
503,
actor=alice,
raw=sample,
headers={'Content-Type': 'image/png'},
)
check(response['json']['error']['code'] == 'unavailable', 'S3 outage was not reported as unavailable')
finally:
compose('start', 'minio')
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
client.call('GET', '/health/ready', schema='Health')
break
except AssertionError:
time.sleep(1)
else:
raise RuntimeError('MinIO/API did not recover within 90 seconds')
print('PASS: MinIO outage returned 503 and the service recovered without removing volumes.')
if __name__ == '__main__':
run(main)
+27
View File
@@ -55,3 +55,30 @@ make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.
## Что ещё остаётся доказать
Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production.
## Дополнительные проверки решения на MinIO
После запуска Compose можно выполнить собственные проверки уровня 4:
```powershell
py -3 -X utf8 student-tests/test_attachments.py --base-url http://localhost:8080
py -3 -X utf8 student-tests/test_s3_outage.py --base-url http://localhost:8080
py -3 -X utf8 student-tests/check_private_bucket.py
```
`test_attachments.py` проверяет лимит bytes, повреждённое изображение,
несовпадающий MIME, опасное имя, чужой attachment ID, межчатовую привязку и
10 параллельных загрузок. Он сохраняет sampled Docker memory в
`evidence/attachment-concurrency.json`; сэмплирование раз в секунду может
пропустить краткий пик. `test_s3_outage.py` явно останавливает и затем запускает
MinIO, не удаляя volume. Запускайте его только на собственном учебном стенде.
Проверка согласованности по умолчанию безопасна и ничего не меняет:
```powershell
docker compose exec -T api python scripts/reconcile_attachments.py --grace-seconds 900
docker compose exec -T api python scripts/reconcile_attachments.py --grace-seconds 900 --apply
```
`--apply` выполняйте только после просмотра dry-run сводки. Ни один шаг не
удаляет volumes; строки с метаданными не удаляются, а помечаются `failed`.