72 changed files with 4413 additions and 23 deletions
+5
View File
@@ -23,3 +23,8 @@ WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
PUBLIC_PORT=18082
# Maximum consecutive processing attempts before the job moves to failed.
JOB_MAX_ATTEMPTS=5
+59
View File
@@ -0,0 +1,59 @@
# syntax=docker/dockerfile:1
#
# MTUSI Messenger, lab5 — PostgreSQL + Redis/Valkey HTTP API.
# Base image is pinned by tag *and* digest (multi-arch manifest list), so a rebuild
# on another machine with the same platform yields the same layers.
#
# Runtime dependencies: psycopg2-binary for PostgreSQL, redis for the session cache.
# Development-only tooling lives in requirements-dev.txt and never enters this image.
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONPATH=/app/src
# Pinned runtime dependencies (versions fixed for reproducible builds).
RUN pip install --no-cache-dir \
psycopg2-binary==2.9.11 \
redis==7.0.1 \
bcrypt==4.0.1 \
boto3==1.42.97 \
Pillow==12.3.0 \
pika==1.3.2
# Unprivileged runtime account; the image's default root user is never used at runtime.
RUN groupadd --gid 10001 app \
&& useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin app
ARG APP_REVISION=local
ENV APP_REVISION=$APP_REVISION
WORKDIR /app
COPY --chown=10001:10001 src/ /app/src/
COPY --chown=10001:10001 migrations/ /app/migrations/
USER 10001:10001
# Configuration comes from the environment; secrets are read from files at runtime.
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
LOG_LEVEL=info \
SHUTDOWN_GRACE_SECONDS=8 \
SESSION_TTL_SECONDS=3600 \
SESSION_STORE=redis \
DB_POOL_MIN=2 \
DB_POOL_MAX=10 \
DB_CONNECT_TIMEOUT=5 \
DB_STATEMENT_TIMEOUT=10
EXPOSE 8080
# Python (not curl) keeps the image slim; /health/ready is the synchronous-path probe.
HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"]
STOPSIGNAL SIGTERM
# The HTTP process is PID 1 and handles SIGTERM itself (graceful drain, exit code 0).
CMD ["python", "-m", "messenger"]
+2 -1
View File
@@ -1,5 +1,6 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
PUBLIC_PORT ?= $(or $(shell sed -n 's/^PUBLIC_PORT=//p' .env 2>/dev/null),18082)
BASE_URL ?= http://localhost:$(PUBLIC_PORT)
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
+102 -22
View File
@@ -1,37 +1,117 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор — Любченко Никита, БПИ2303.
- Целевая оценка — 4.
- Использованы Python 3.12, Pillow, boto3, pika, PostgreSQL, Redis, Nginx, MinIO и RabbitMQ. ИИ использован при разработке и проверке реализации.
- Результаты прогонов и их ограничения приведены ниже. Ссылку на PR добавляет автор.
## Запуск из чистого clone
## Запуск из чистого клона
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Engine с Compose, Make и Python 3.9+ для публичных тестов. Первая сборка обращается к реестрам образов, GitHub, PyPI и прокси Go для зависимостей MinIO.
## Архитектура и решения
```sh
cp -n .env.example .env
python3 scripts/bootstrap.py
docker compose config --quiet
docker compose up -d --build --wait --wait-timeout 120
make check
make test
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Стенд открывается на http://localhost:18082. Порт задаёт PUBLIC_PORT. Секреты создаёт bootstrap. Команда `docker compose down` сохраняет тома. `docker compose down -v` удаляет учебные данные PostgreSQL, Redis, MinIO и RabbitMQ.
## Архитектура и согласованность
Две API-реплики за Nginx используют PostgreSQL для метаданных и MinIO для исходников. Redis сохраняет роль кэша сессий. До ответа 202 оригинал сохранён в staging, а метаданные, задача и outbox зафиксированы в одной транзакции PostgreSQL. Отдельный издатель использует устойчивую очередь RabbitMQ и publisher confirms. При повторной публикации сохраняются ID задачи и номер попытки. Повтор завершённой работы не меняет метаданные.
Worker забирает задачи с prefetch 1. После перехода в processing он получает оригинал по ключу из БД и создаёт три варианта. Файлы каждой попытки имеют отдельный префикс. Только текущая попытка с действующим lease фиксирует ready и варианты одной транзакцией. Consumer ACK следует после устойчивого решения по задаче. До ready скачивание возвращает 409. Участник чата может отправить сообщение с queued-вложением.
Worker ограничен 1 CPU и 768 MiB, таймер обработки составляет 20 с. Lease действует 60 с. Временные ошибки и истечение аренды обрабатываются через ограниченный backoff с jitter. Некорректный файл сразу переводит задачу в failed. Операторский replay сохраняет аудит. Cleaner удаляет staging после ready, а для failed сохраняет его на 24 часа. Старые результаты отдельных попыток удаляются после периода ожидания.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Критерий | Результат | Артефакт |
| --- | --- | --- |
| Структура и OpenAPI | PASS | `evidence/check.txt`, `evidence/validate.txt` |
| Перенос lab4 | Текстовый и файловый сценарий, общая сессия, приватный MinIO | `evidence/lab4-baseline.txt` |
| Устойчивый ответ 202 и привязка до готовности | 202 queued, 409 при скачивании, одна транзакция задачи и outbox | `evidence/queued-upload.txt` |
| Форматы, размеры и побайтный оригинал | Thumbnail 256×160, crop 128×128, WebP 320×200, повтор не меняет результат. Задача передана обработчику напрямую | `evidence/worker-variants.txt` |
| Retry, failed, replay, очистка | Проверены на PostgreSQL и MinIO. Истечение lease смоделировано после записи файлов | `evidence/job-lifecycle.txt` |
| Откат при отказе publish | Запись осталась неопубликованной на реальном PostgreSQL после имитации NACK | `evidence/outbox-nack.txt` |
| Публичный smoke через RabbitMQ | PASS | `evidence/worker-smoke.txt` |
| Разрыв между COMMIT и publish | Задача, принятая при остановленном издателе, опубликована после его запуска | `evidence/outbox-live.txt` |
| Остановка worker и рестарт API/RabbitMQ | Та же задача дошла до ready, исходник совпал побайтно | `evidence/broker-resilience.txt` |
| Остановка worker после записи в S3 до COMMIT/ACK | После истечения lease восстановление заняло 60,90 с; опубликованы три варианта одного результата | `evidence/broker-resilience.txt` |
| Публичный persistence и failover | PASS, данные и действующая сессия сохранены | `evidence/persistence.txt`, `evidence/failover.txt` |
| Приватность бакета | Чтение с учётной записью API прошло, анонимный GET получил 403 | `evidence/s3-private-final.txt` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Эксперименты и отказы
## Сценарии защиты
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
```sh
docker compose stop worker
sh student-tests/outbox.sh
python3 student-tests/broker_resilience.py --base-url http://localhost:18082
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 scripts/manage.py restore
```
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Сценарий outbox останавливает издателя и подтверждает публикацию после его запуска. Его нужно запускать с остановленным worker. Сценарий broker_resilience останавливает worker, проверяет переживание рестарта API и RabbitMQ, затем убивает отдельный тестовый worker после записи в S3 и до COMMIT/ACK. Оба сценария рассчитаны на учебный стенд и создают синтетические данные. Проверка отдельной попытки использует fixture PNG 320×200.
## Ограничения и следующие шаги
Прогон выполнен в Docker Engine 29.5.3, Compose 5.1.4, aarch64. Повторные проверки с актуальным кодом сохраняются в файлах `*-final.txt`. После повторной загрузки образа RabbitMQ 4.1.4 запущен. Публичный smoke и подтверждённая публикация прошли на реальном брокере.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
## История реализации
Записи ниже отражают состояние проверок на соответствующем этапе. Итоговые результаты приведены в таблице выше.
## Этап 1. Перенос lab4
Проект `mtusi-lab5` слушает `http://localhost:18082`. PostgreSQL, Redis и MinIO получили отдельные тома. Локальные секреты сгенерированы для lab5.
```sh
cp -n .env.example .env
python3 scripts/bootstrap.py
docker compose config --quiet
docker compose up -d --build
make check
```
Для проверки переноса использовались существующие образы lab4 с локальными тегами lab5. Публичный smoke lab4 прошёл на новом стенде. Отдельно проверены сессия на обеих репликах и отсутствие анонимного доступа к объекту в MinIO. Результаты сохранены в `evidence/lab4-baseline.txt`. На этом этапе upload ещё отвечает 201 по контракту lab4. Smoke lab5 рассчитан на переход к 202 и обработку в worker.
## Этап 2. Устойчивый приём вложения
Миграция `0004_attachment_jobs.sql` добавляет состояния обработки и таблицы задач, вариантов и outbox. После успешного PUT под `attachments/staging/` API в одной транзакции сохраняет задачу, событие outbox и состояние `queued`. До готовности вложение можно прикрепить к сообщению, а скачивание возвращает 409.
Проверка на Compose с PostgreSQL и MinIO: `python3 student-tests/queued_upload.py --base-url http://localhost:18082`. SQL дополнительно подтвердил одну задачу и одну неопубликованную запись outbox. Результаты в `evidence/queued-upload.txt`. Из-за TLS-таймаута реестра для прогона поверх локального API-образа lab4 скопированы актуальные исходники и миграции lab5.
## Этап 3. RabbitMQ и публикация outbox
В Compose добавлены RabbitMQ 4.1.4 с постоянным томом и отдельный outbox-publisher. Издатель берёт готовую строку outbox через FOR UPDATE SKIP LOCKED, публикует постоянное сообщение в durable-очередь и только после publisher confirm записывает published_at. При сбое между confirm и коммитом возможна повторная доставка с тем же job_id.
После доступной загрузки образа запустить `docker compose build api-a`, затем `docker compose up -d --no-build` и `sh student-tests/outbox.sh`. Сценарий останавливает издателя, создаёт вложение, проверяет неопубликованную запись и ждёт подтверждения после запуска издателя. Worker появится на следующем этапе.
Конфигурация Compose и make check прошли. На настоящем PostgreSQL проверен откат после имитации broker NACK, см. evidence/outbox-nack.txt. Прогон с реальным RabbitMQ пока открыт: загрузка образа не удалась из-за ошибок доступа к реестрам.
## Этап 4. Worker и производные изображения
Worker обрабатывает по одной доставке (prefetch 1) с лимитом контейнера 768 MiB и таймером обработки 20 с. Он скачивает исходник из staging с проверкой размера и SHA-256, повторно валидирует изображение и сохраняет побайтный оригинал. Производные файлы записываются в ключи, зависящие от ID вложения и версии обработки. JPEG использует качество 85, WebP — 82. EXIF-ориентация применяется перед преобразованием, метаданные исходника не копируются, прозрачность сводится на белый фон.
На этом этапе блокировка строки PostgreSQL защищает обработку от дублирования. Статус ready и три строки вариантов фиксируются одной транзакцией после записи всех файлов. ACK после COMMIT. Отдельно наблюдаемое processing, lease, управляемые повторы и очистка staging относятся к этапу 5.
`python3 student-tests/worker_variants.py --base-url http://localhost:18082` проверяет обработчик на реальных PostgreSQL и MinIO через прямую передачу job, затем сверяет скачанные байты и метаданные через Nginx. Результат в `evidence/worker-variants.txt`.
## Этап 5. Lease, retry и очистка
Worker транзакционно захватывает queued-задачу на 60 секунд и переводит её в processing. Одна доставка обрабатывается за раз. Таймер worker — 20 секунд. Публикация результата проверяет номер попытки и действительность lease. Результаты попыток имеют разные S3-префиксы, поэтому запоздавшая попытка не перезаписывает актуальный вариант.
Временная ошибка и истёкший lease планируют новую попытку через outbox с задержкой 2^n секунд (исходная задержка ограничена 60 с) и jitter ±25%. По умолчанию JOB_MAX_ATTEMPTS=5. Некорректный оригинал завершает задачу с invalid_image. После исчерпания попыток возвращается retry_exhausted. Реестр failed хранится в PostgreSQL. Replay увеличивает поколение попытки, сбрасывает счётчик retry и пишет аудит. Replay доступен для failed до удаления staging.
```sh
docker compose exec -T api-a python -m messenger.failed_jobs list
docker compose exec -T api-a python -m messenger.failed_jobs replay JOB_UUID --operator local
```
Cleaner удаляет staging после ready или спустя 24 часа после failed. Файлы старых и окончательно неуспешных попыток удаляются после 15-минутного периода ожидания. Операции очистки можно повторять после сбоя.
Проверка `python3 student-tests/job_lifecycle.py --base-url http://localhost:18082` выполнена с реальными PostgreSQL и MinIO. Истечение lease моделировалось обновлением дедлайна после записи файлов. Результаты в `evidence/job-lifecycle.txt`. Брокерный сценарий физической остановки worker до ACK остаётся для итогового прогона.
+267
View File
@@ -0,0 +1,267 @@
name: mtusi-lab5
x-api: &api
build:
context: .
args:
APP_REVISION: ${API_IMAGE_TAG:-local}
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
environment: &api-env
APP_HOST: 0.0.0.0
APP_PORT: '8080'
APP_ENV: lab5
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: redis
SHUTDOWN_GRACE_SECONDS: '8'
DB_POOL_MIN: '2'
DB_POOL_MAX: '16'
DB_CONNECT_TIMEOUT: '1'
DB_STATEMENT_TIMEOUT: '4'
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
networks: [backend]
cpus: 0.50
mem_limit: 256m
stop_grace_period: 9s
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1)"]
interval: 2s
timeout: 2s
retries: 2
start_period: 3s
# Deliberately no auto-restart: failover hook must leave its target stopped.
restart: 'no'
services:
rabbitmq:
image: rabbitmq:4.1.4-alpine
hostname: rabbitmq
entrypoint: [/bin/sh, -ec]
command:
- |
export RABBITMQ_DEFAULT_USER=lab5
RABBITMQ_DEFAULT_PASS=$$(cat /run/secrets/rabbitmq_password)
export RABBITMQ_DEFAULT_PASS
exec docker-entrypoint.sh rabbitmq-server
secrets: [rabbitmq_password]
volumes: [rabbitmq_data:/var/lib/rabbitmq]
networks: [backend]
healthcheck:
test: [CMD, rabbitmq-diagnostics, -q, check_port_connectivity]
interval: 5s
timeout: 5s
retries: 12
outbox-publisher:
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
command: [python, -m, messenger.publish_jobs]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
RABBITMQ_PASSWORD_FILE: /run/secrets/rabbitmq_password
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
DB_POOL_MIN: '1'
DB_POOL_MAX: '2'
secrets: [database_url, rabbitmq_password]
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
networks: [backend]
restart: unless-stopped
healthcheck:
disable: true
s3:
image: mtusi-lab5-s3:2025-09-07
build: &s3-build
context: .
dockerfile: infra/Dockerfile.s3
entrypoint: [/bin/sh, -ec]
command:
- |
MINIO_ROOT_USER=$$(cat /run/secrets/s3_root_user)
MINIO_ROOT_PASSWORD=$$(cat /run/secrets/s3_root_password)
export MINIO_ROOT_USER MINIO_ROOT_PASSWORD
exec minio server /data
environment:
MINIO_BROWSER: "off"
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
networks: [backend]
s3-bootstrap:
image: mtusi-lab5-s3:2025-09-07
build: *s3-build
entrypoint: [/bin/sh, /bootstrap.sh]
command: []
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
volumes: [./infra/s3-bootstrap.sh:/bootstrap.sh:ro]
depends_on: [s3]
networks: [backend]
postgres:
image: postgres:15-alpine@sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [pg_data:/var/lib/postgresql/data]
networks: [backend]
cpus: 1.0
mem_limit: 512m
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 20
redis:
image: redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f
command: [redis-server, --appendonly, 'yes', --maxmemory, 96mb, --maxmemory-policy, allkeys-lru]
volumes: [redis_data:/data]
networks: [backend]
cpus: 0.25
mem_limit: 128m
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 1s
retries: 10
migrate:
build:
context: .
args:
APP_REVISION: ${API_IMAGE_TAG:-local}
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
command: [python, -m, messenger.migrate]
environment: *api-env
secrets: [database_url, cache_url, cursor_secret]
depends_on:
postgres:
condition: service_healthy
networks: [backend]
cpus: 0.50
mem_limit: 256m
restart: 'no'
healthcheck:
disable: true
attachment-cleaner:
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
build:
context: .
args:
APP_REVISION: ${API_IMAGE_TAG:-local}
command: [python, -m, messenger.clean_attachments]
environment: *api-env
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
networks: [backend]
cpus: 0.25
mem_limit: 128m
restart: unless-stopped
healthcheck:
disable: true
worker:
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
command: [python, -m, messenger.worker]
environment:
<<: *api-env
RABBITMQ_PASSWORD_FILE: /run/secrets/rabbitmq_password
DB_POOL_MIN: '1'
DB_POOL_MAX: '2'
secrets: [database_url, s3_access_key_id, s3_secret_access_key, rabbitmq_password]
depends_on:
outbox-publisher:
condition: service_started
s3-bootstrap:
condition: service_completed_successfully
networks: [backend]
cpus: 1.0
mem_limit: 768m
restart: unless-stopped
healthcheck:
disable: true
api-a:
<<: *api
api-b:
<<: *api
nginx:
image: nginx:1.28.2-alpine@sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e
entrypoint: [/bin/sh, -ec]
command:
- 'cp /etc/nginx/lab/upstreams.conf /etc/nginx/upstreams.conf; /bin/sh /etc/nginx/lab/healthwatch.sh & exec nginx -g "daemon off;"'
volumes:
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra:/etc/nginx/lab:ro
ports: ['127.0.0.1:${PUBLIC_PORT:-18082}:8080']
networks: [frontend, backend]
depends_on:
api-a:
condition: service_healthy
api-b:
condition: service_healthy
cpus: 0.50
mem_limit: 128m
stop_grace_period: 10s
healthcheck:
test: [CMD-SHELL, 'kill -0 $$(cat /tmp/healthwatch.pid) && wget -T 1 -q -O /dev/null http://127.0.0.1:8080/health/ready']
interval: 2s
timeout: 2s
retries: 3
networks:
frontend:
backend:
internal: true
volumes:
pg_data:
redis_data:
s3_data:
rabbitmq_data:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
cursor_secret:
file: ./secrets/cursor_secret
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
rabbitmq_password:
file: ./secrets/rabbitmq_password
+2
View File
@@ -3,3 +3,5 @@
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
Файлы `*-final.txt` содержат повторный прогон на итоговом коде. `queued-upload-final.txt` проверяет HTTP-приём, `worker-variants-final.txt` — результаты обработки с прямой передачей задания, `job-lifecycle-final.txt` — lease, retry, replay и очистку на PostgreSQL и MinIO. Прямой сценарий не измеряет доставку RabbitMQ.
+2
View File
@@ -0,0 +1,2 @@
name durable messages_ready messages_unacknowledged
attachment.process.v1 true 0 0
+69
View File
@@ -0,0 +1,69 @@
Container mtusi-lab5-worker-1 Stopping
Container mtusi-lab5-worker-1 Stopped
Container mtusi-lab5-api-a-1 Restarting
Container mtusi-lab5-api-b-1 Restarting
Container mtusi-lab5-rabbitmq-1 Restarting
Container mtusi-lab5-api-a-1 Started
Container mtusi-lab5-api-b-1 Started
Container mtusi-lab5-rabbitmq-1 Started
Container mtusi-lab5-api-b-1 Running
Container mtusi-lab5-api-a-1 Running
Container mtusi-lab5-api-a-1 Waiting
Container mtusi-lab5-api-b-1 Waiting
Container mtusi-lab5-api-a-1 Healthy
Container mtusi-lab5-api-b-1 Healthy
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-s3-bootstrap-1 Starting
Container mtusi-lab5-s3-bootstrap-1 Started
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-rabbitmq-1 Waiting
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-rabbitmq-1 Healthy
Container mtusi-lab5-s3-bootstrap-1 Waiting
Container mtusi-lab5-s3-bootstrap-1 Exited
Container mtusi-lab5-worker-1 Starting
Container mtusi-lab5-worker-1 Started
PASS: worker stopped; after API and RabbitMQ restart the same attachment became ready
Container mtusi-lab5-worker-1 Stopping
Container mtusi-lab5-worker-1 Stopped
Container lab5-fault-e0d0159d35 Creating
Container lab5-fault-e0d0159d35 Created
558b991a978ef6f8660d068af2a737cea3e0fe7366185b65069dad119981ec7c
INFO:pika.adapters.utils.connection_workflow:Pika version 1.3.2 connecting to ('172.23.0.9', 5672)
INFO:pika.adapters.utils.io_services_utils:Socket connected: <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.23.0.11', 41090), raddr=('172.23.0.9', 5672)>
INFO:pika.adapters.utils.connection_workflow:Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
INFO:pika.adapters.utils.connection_workflow:AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
INFO:pika.adapters.utils.connection_workflow:AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
INFO:pika.adapters.blocking_connection:Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
INFO:pika.adapters.blocking_connection:Created channel=1
lab5-fault-e0d0159d35
Container mtusi-lab5-s3-bootstrap-1 Starting
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-s3-bootstrap-1 Started
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-rabbitmq-1 Waiting
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-rabbitmq-1 Healthy
Container mtusi-lab5-s3-bootstrap-1 Waiting
Container mtusi-lab5-s3-bootstrap-1 Exited
Container mtusi-lab5-worker-1 Starting
Container mtusi-lab5-worker-1 Started
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-s3-bootstrap-1 Starting
Container mtusi-lab5-s3-bootstrap-1 Started
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-rabbitmq-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-rabbitmq-1 Healthy
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-s3-bootstrap-1 Waiting
Container mtusi-lab5-s3-bootstrap-1 Exited
PASS: worker killed after S3 write before COMMIT/ACK; recovered in 60.90s
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18082"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.682}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+12
View File
@@ -0,0 +1,12 @@
api-a running healthy
api-b running healthy
attachment-cleaner running
migrate exited
nginx running healthy
outbox-publisher running
postgres running healthy
rabbitmq running healthy
redis running healthy
s3 running
s3-bootstrap exited
worker running
+2
View File
@@ -0,0 +1,2 @@
PASS: expired claim fenced, retry exhausted, audited replay, stale objects and staging cleaned
PASS: ready original survives cleanup and replay requests
+2
View File
@@ -0,0 +1,2 @@
PASS: expired claim fenced, retry exhausted, audited replay, stale objects and staging cleaned
PASS: ready original survives cleanup and replay requests
+6
View File
@@ -0,0 +1,6 @@
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
Container mtusi-lab5-s3-1 Running
Container mtusi-lab5-s3-bootstrap-run-63e5e364daea Creating
Container mtusi-lab5-s3-bootstrap-run-63e5e364daea Created
PASS: authenticated round trip; anonymous GET returned 403
PASS: shared session on two lab5 API replicas: 74e294abbbf5 d55768f2410a
+16
View File
@@ -0,0 +1,16 @@
Container mtusi-lab5-outbox-publisher-1 Stopping
Container mtusi-lab5-outbox-publisher-1 Stopped
PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked
Attachment ID: 168907b8-2152-42a4-ad05-d9e40aa211d8
PASS: upload committed while publisher was stopped
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-rabbitmq-1 Waiting
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-rabbitmq-1 Healthy
Container mtusi-lab5-outbox-publisher-1 Starting
Container mtusi-lab5-outbox-publisher-1 Started
PASS: same attachment job confirmed after publisher startup
+2
View File
@@ -0,0 +1,2 @@
PASS: PostgreSQL outbox remains unpublished after simulated broker NACK
The test used the real lab5 PostgreSQL and publisher code with an injected NACK in place of the broker channel.
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18082"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked
Attachment ID: df03f2ef-0193-4b3d-8591-ef0f432a948e
+3
View File
@@ -0,0 +1,3 @@
PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked
Attachment ID: 3f72ec80-cf76-4e14-a373-3946735437d7
PASS: exactly one queued attachment, job and unpublished outbox row
+2
View File
@@ -0,0 +1,2 @@
ba452b9cf1c8e766d58801c2767f5bd719a36b93b39db8c58875cf09ac4b996b
3d7a006ce123e652c1754efc4d7fcd5dbac77605d47a313ed6c5501fb3880d8c
+4
View File
@@ -0,0 +1,4 @@
Container mtusi-lab5-s3-1 Running
Container mtusi-lab5-s3-bootstrap-run-6191998ad078 Creating
Container mtusi-lab5-s3-bootstrap-run-6191998ad078 Created
PASS: authenticated round trip; anonymous GET returned 403
+4
View File
@@ -0,0 +1,4 @@
/Users/atmose/Documents/Codex/2026-11-06/new-chat/work/lab3-venv/bin/python scripts/validate_openapi.py
/Users/atmose/Documents/Codex/2026-11-06/new-chat/work/lab3-venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
warnings.warn(
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:18082"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+2
View File
@@ -0,0 +1,2 @@
PASS: original unchanged, thumbnail 256x160, crop 128x128, WebP 320x200
PASS: processing the same committed job twice keeps one set of variants
+3
View File
@@ -0,0 +1,3 @@
PASS: original unchanged, thumbnail 256x160, crop 128x128, WebP 320x200
PASS: processing the same committed job twice keeps one set of variants
PASS: EXIF rotation applied, EXIF stripped, transparency on white, no upscaling
+13
View File
@@ -0,0 +1,13 @@
# syntax=docker/dockerfile:1
FROM public.ecr.aws/docker/library/golang:1.24.2-alpine@sha256:7772cb5322baa875edd74705556d08f0eeca7b9c4b5367754ce3f2f00041ccee AS build
ENV CGO_ENABLED=0 GOPROXY="https://proxy.golang.org|https://goproxy.io|direct"
WORKDIR /src/minio
ADD https://codeload.github.com/minio/minio/tar.gz/refs/tags/RELEASE.2025-09-07T16-13-09Z /tmp/minio.tar.gz
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/minio.tar.gz --strip-components=1 && go build -o /out/minio .
WORKDIR /src/mc
ADD https://codeload.github.com/minio/mc/tar.gz/refs/tags/RELEASE.2025-08-13T08-35-41Z /tmp/mc.tar.gz
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/mc.tar.gz --strip-components=1 && go build -o /out/mc .
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
COPY --from=build /out/minio /out/mc /usr/local/bin/
ENTRYPOINT ["minio"]
+53
View File
@@ -0,0 +1,53 @@
#!/bin/sh
# Active readiness bridge for open-source Nginx. No Docker socket or credentials.
set -eu
probe() {
if wget -T 1 -q -O /dev/null "http://$1:8080/health/ready" 2>/dev/null; then
echo up > "/tmp/$1.health"
else
echo down > "/tmp/$1.health"
fi
}
control() { cat "/tmp/control-$1" 2>/dev/null || echo serve:boot; }
echo $$ > /tmp/healthwatch.pid
while :; do
sleep 1
probe api-a &
first=$!
probe api-b &
second=$!
wait "$first"
wait "$second"
# Read desired state AFTER probes. A generation is acknowledged only after
# the matching upstream configuration has been applied, never from a stale probe.
a_control=$(control api-a)
b_control=$(control api-b)
a_state=$(cat /tmp/api-a.health)
b_state=$(cat /tmp/api-b.health)
case "$a_control" in drain:*) a_state=down;; esac
case "$b_control" in drain:*) b_state=down;; esac
: > /tmp/upstreams.next
for service in api-a api-b; do
if [ "$service" = api-a ]; then state=$a_state; else state=$b_state; fi
suffix=''
[ "$state" = up ] || suffix=' down'
printf 'server %s:8080 resolve max_fails=1 fail_timeout=2s%s;\n' "$service" "$suffix" >> /tmp/upstreams.next
done
if ! cmp -s /tmp/upstreams.next /etc/nginx/upstreams.conf; then
cp /etc/nginx/upstreams.conf /tmp/upstreams.previous
# Rename on the same filesystem: readers never see missing/partial config.
cp /tmp/upstreams.next /etc/nginx/upstreams.next
mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf
if nginx -t && nginx -s reload; then
echo 'readiness upstream set updated'
else
cp /tmp/upstreams.previous /etc/nginx/upstreams.next
mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf
exit 1
fi
fi
printf '%s %s\n' "$a_control" "$a_state" > /tmp/applied-api-a.next
mv -f /tmp/applied-api-a.next /tmp/applied-api-a
printf '%s %s\n' "$b_control" "$b_state" > /tmp/applied-api-b.next
mv -f /tmp/applied-api-b.next /tmp/applied-api-b
done
+49
View File
@@ -0,0 +1,49 @@
worker_processes 1;
worker_shutdown_timeout 6s;
events { worker_connections 1024; }
http {
resolver 127.0.0.11 valid=1s ipv6=off;
resolver_timeout 1s;
log_format lab '$time_iso8601 $request_method $uri $status upstream=$upstream_addr '
'instance=$upstream_http_x_instance_id time=$request_time';
access_log /dev/stdout lab;
error_log /dev/stderr warn;
upstream api {
zone api 64k;
include /etc/nginx/upstreams.conf;
}
server {
listen 8080;
client_max_body_size 1m;
client_body_timeout 5s;
proxy_http_version 1.1;
proxy_set_header Connection close;
proxy_set_header Host $host;
proxy_set_header X-Request-Id $http_x_request_id;
proxy_connect_timeout 1s;
proxy_read_timeout 5s;
proxy_send_timeout 5s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 6s;
location ~ ^/api/v1/chats/[^/]+/attachments$ {
# Keep a proxy ceiling above the API limit so the API can return its JSON 413.
client_max_body_size 11m;
proxy_read_timeout 45s;
proxy_send_timeout 30s;
proxy_next_upstream off;
proxy_pass http://api;
}
location ~ ^/api/v1/attachments/[^/]+/content$ {
proxy_buffering off;
proxy_read_timeout 10s;
# A shared S3 outage returns 503 without marking the API itself down.
proxy_next_upstream error timeout http_502 http_504;
proxy_pass http://api;
}
location / {
proxy_pass http://api;
}
}
}
+73
View File
@@ -0,0 +1,73 @@
#!/bin/sh
set -eu
umask 077
bucket=${S3_BUCKET:-messenger-attachments}
case "$bucket" in
''|*[!a-z0-9.-]*) echo 'Invalid S3 bucket name' >&2; exit 1 ;;
esac
config=$(mktemp -d)
trap 'rm -rf "$config"' EXIT
export MC_CONFIG_DIR="$config"
root_user=$(cat /run/secrets/s3_root_user)
root_password=$(cat /run/secrets/s3_root_password)
api_user=$(cat /run/secrets/s3_access_key_id)
api_password=$(cat /run/secrets/s3_secret_access_key)
endpoint=${S3_ENDPOINT:-http://s3:9000}
if [ "${1:-}" = verify ]; then
export MC_HOST_api="http://$api_user:$api_password@${endpoint#http://}"
key="attachments/probe-${config##*/}"
object="api/$bucket/$key"
trap 'mc rm "$object" >/dev/null 2>&1 || true; rm -rf "$config"' EXIT
printf '%s' 'lab5-private-bucket-probe' | mc pipe "$object" >/dev/null
test "$(mc cat "$object")" = 'lab5-private-bucket-probe'
python3 - "$endpoint/$bucket/$key" <<'PYTEST'
import sys
from urllib.error import HTTPError
from urllib.request import urlopen
try:
with urlopen(sys.argv[1], timeout=5):
raise SystemExit("FAIL: anonymous GET succeeded")
except HTTPError as exc:
if exc.code != 403:
raise
print("PASS: authenticated round trip; anonymous GET returned 403")
PYTEST
mc rm "$object" >/dev/null
exit 0
fi
attempt=0
until mc alias set storage "$endpoint" "$root_user" "$root_password" >/dev/null 2>&1 &&
mc admin info storage >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 30 ]; then
echo 'S3 bootstrap could not connect or authenticate' >&2
exit 1
fi
sleep 1
done
mc mb --ignore-existing "storage/$bucket"
mc anonymous set none "storage/$bucket"
cat > "$config/api-policy.json" <<POLICY
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::$bucket/attachments/*"]
}, {
"Effect": "Allow",
"Action": ["s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::$bucket"]
}]
}
POLICY
mc admin policy create storage lab5-attachments "$config/api-policy.json"
mc admin user add storage "$api_user" "$api_password" >/dev/null
mc admin policy attach storage lab5-attachments --user "$api_user"
echo 'PASS: private S3 bucket and scoped API policy configured'
+2
View File
@@ -0,0 +1,2 @@
server api-a:8080 resolve max_fails=1 fail_timeout=2s;
server api-b:8080 resolve max_fails=1 fail_timeout=2s;
+51
View File
@@ -0,0 +1,51 @@
-- Migration 0001: initial schema for lab2.
-- Idempotent: safe to re-run on an existing database.
CREATE TABLE IF NOT EXISTS users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE,
password_hash text NOT NULL,
display_name text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
-- Keyset pagination index: history by (chat_id, created_at, id) avoids
-- full table scan on every page read.
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
ON messages (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_messages_chat_id
ON messages (chat_id);
CREATE TABLE IF NOT EXISTS sessions (
token_hash text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_sessions_user_id
ON sessions (user_id);
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
ON sessions (expires_at);
+14
View File
@@ -0,0 +1,14 @@
-- Additive migration: compatible with the lab2 schema and existing messages.
CREATE TABLE IF NOT EXISTS message_idempotency (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method text NOT NULL,
path text NOT NULL,
key text NOT NULL,
request_hash text NOT NULL,
response_status integer NOT NULL,
response_body jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, method, path, key)
);
-- Results are retained indefinitely in this lab (therefore at least 24 hours).
-- There is intentionally no independent cache expiry that could allow duplicates.
+37
View File
@@ -0,0 +1,37 @@
-- Additive migration. Existing text messages have no attachment links.
ALTER TABLE messages
ADD CONSTRAINT messages_id_chat_id_key UNIQUE (id, chat_id);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
object_key text NOT NULL UNIQUE CHECK (object_key LIKE 'attachments/%'),
-- pending is internal. The upload API exposes ready after committing metadata.
status text NOT NULL CHECK (status IN ('pending', 'ready', 'failed')),
error_code text,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (id, chat_id),
CHECK ((status = 'failed') = (error_code IS NOT NULL))
);
CREATE INDEX attachments_pending_updated_idx
ON attachments (updated_at, id) WHERE status = 'pending';
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
attachment_id uuid NOT NULL,
chat_id uuid NOT NULL,
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, position),
UNIQUE (message_id, attachment_id),
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE,
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
+33
View File
@@ -0,0 +1,33 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments ADD CHECK (status IN ('pending','queued','processing','ready','failed'));
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
attempt integer NOT NULL DEFAULT 1 CHECK (attempt > 0),
created_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (attachment_id, pipeline_version)
);
CREATE TABLE attachment_outbox (
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE,
attempt integer NOT NULL CHECK (attempt > 0),
available_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
PRIMARY KEY (job_id, attempt)
);
CREATE INDEX attachment_outbox_due_idx ON attachment_outbox (available_at) WHERE published_at IS NULL;
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
variant text NOT NULL CHECK (variant IN ('thumbnail','crop','optimized')),
object_key text NOT NULL UNIQUE,
content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/webp')),
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
width integer NOT NULL CHECK (width > 0),
height integer NOT NULL CHECK (height > 0),
PRIMARY KEY (attachment_id, pipeline_version, variant)
);
+12
View File
@@ -0,0 +1,12 @@
ALTER TABLE attachment_jobs ADD COLUMN lease_until timestamptz;
ALTER TABLE attachment_jobs ADD COLUMN attempts_used integer NOT NULL DEFAULT 0;
ALTER TABLE attachment_jobs ADD COLUMN staging_cleaned_at timestamptz;
ALTER TABLE attachment_outbox ADD COLUMN objects_cleaned_at timestamptz;
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE lease_until IS NOT NULL;
CREATE TABLE attachment_replays (
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE,
attempt integer NOT NULL,
operator text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (job_id, attempt)
);
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
"""Generate local credentials once, never print them or overwrite existing files."""
from pathlib import Path
import os
import secrets
root = Path(__file__).resolve().parents[1]
folder = root / 'secrets'
folder.mkdir(mode=0o700, exist_ok=True)
password_file = folder / 'postgres_password'
def save(name, value):
path = folder / name
if path.exists():
return
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, 'w') as handle:
handle.write(value + '\n')
save('postgres_password', secrets.token_hex(24))
password = password_file.read_text().strip()
save('database_url', 'postgresql://messenger:%s@postgres:5432/messenger' % password)
save('cache_url', 'redis://redis:6379/0')
save('cursor_secret', secrets.token_hex(32))
save('s3_root_user', 'lab5-admin-' + secrets.token_hex(8))
save('s3_root_password', secrets.token_hex(32))
save('s3_access_key_id', 'lab5-api-' + secrets.token_hex(8))
save('s3_secret_access_key', secrets.token_hex(24))
save('rabbitmq_password', secrets.token_hex(24))
print('PASS: local secret files ready (existing values preserved)')
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env python3
"""Explicit, bounded operator actions for this lab's Compose project only."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
SERVICES = ('api-a', 'api-b')
def command(args, capture=False):
result = subprocess.run(args, cwd=ROOT, check=True, text=True,
stdout=subprocess.PIPE if capture else None)
return result.stdout.strip() if capture else None
def compose(*args, capture=False):
return command(['docker', 'compose', '--env-file', '.env', *args], capture)
def inspect(container):
return json.loads(command(['docker', 'inspect', container], True))[0]
def containers(service):
return compose('ps', '-a', '-q', service, capture=True).splitlines()
def start_services(*services):
for service in services:
for container in containers(service):
command(['docker', 'start', container])
def wait_healthy(service, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
ids = containers(service)
if ids and all(inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' for c in ids):
return
time.sleep(.5)
raise RuntimeError('%s did not become healthy' % service)
def target_service(target):
if not target:
raise RuntimeError('TARGET_INSTANCE_ID is required')
for service in SERVICES:
for container in containers(service):
data = inspect(container)
if data['Config']['Hostname'] == target:
return service, container
raise RuntimeError('Unknown API instance alias')
def set_backend(service, enabled):
# Fixed allowlist; shell receives no untrusted instance or credential input.
if service not in SERVICES:
raise ValueError('Unknown service')
token = ('serve:' if enabled else 'drain:') + uuid.uuid4().hex
path = '/tmp/control-' + service
compose('exec', '-T', 'nginx', 'sh', '-ec',
"printf '%s\\n' '" + token + "' > " + path + '.next; mv -f ' + path + '.next ' + path)
deadline = time.monotonic() + 10
expected = token + (' up' if enabled else ' down')
while time.monotonic() < deadline:
ack = compose('exec', '-T', 'nginx', 'sh', '-c',
'cat /tmp/applied-' + service + ' 2>/dev/null || true', capture=True)
if ack == expected:
return
time.sleep(.2)
raise RuntimeError('Readiness controller did not acknowledge backend state for ' + service)
def stop_one():
service, container = target_service(os.environ.get('TARGET_INSTANCE_ID'))
running = [c for name in SERVICES for c in containers(name)
if inspect(c)['State'].get('Health', {}).get('Status') == 'healthy'
and inspect(c)['State']['Running']]
if len(running) < 2:
raise RuntimeError('Restore the other replica before this experiment')
print('Stopping API replica:', service, container[:12], flush=True)
command(['docker', 'stop', '--time', '8', container])
if inspect(container)['State']['Running']:
raise RuntimeError('Target is still running')
def restart():
# Never start the one-shot migration service as part of persistence testing.
compose('restart', 'postgres', 'redis', 's3', 'rabbitmq', *SERVICES, 'nginx', 'outbox-publisher', 'worker')
for service in (*SERVICES, 'nginx'):
wait_healthy(service)
def rolling_update():
# Run additive migrations once before replacing either API version.
compose('run', '--rm', '--no-deps', 'migrate')
for service in SERVICES:
other = SERVICES[1] if service == SERVICES[0] else SERVICES[0]
wait_healthy(other)
set_backend(other, True)
before = containers(service)
set_backend(service, False)
# The old nginx worker can retain the old upstream table until it exits.
# worker_shutdown_timeout=6s bounds that drain; no new work then reaches it.
time.sleep(6.5)
compose('up', '-d', '--no-deps', '--force-recreate', '--no-build', service)
wait_healthy(service)
set_backend(service, True)
time.sleep(2) # DNS valid=1s + readiness before moving to the next replica.
after = containers(service)
if before == after:
raise RuntimeError('Replica was not replaced')
print(json.dumps({'service': service, 'before': before, 'after': after}), flush=True)
def restore():
start_services(*SERVICES)
for service in SERVICES:
wait_healthy(service)
for service in SERVICES:
set_backend(service, True)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('stop-one', 'restart', 'restore', 'rolling-update', 'status'))
args = parser.parse_args()
if args.action == 'stop-one':
stop_one()
elif args.action == 'restart':
restart()
elif args.action == 'rolling-update':
rolling_update()
elif args.action == 'restore':
restore()
else:
for service in ('postgres', 'redis', *SERVICES, 'nginx'):
for container in containers(service):
data = inspect(container)
print(json.dumps({'service': service, 'id': data['Id'][:12],
'instance': data['Config']['Hostname'],
'running': data['State']['Running'],
'started_at': data['State']['StartedAt'],
'health': data['State'].get('Health', {}).get('Status'),
'image_id': data['Image'],
'cpu_limit': data['HostConfig']['NanoCpus'] / 1e9,
'memory_limit_bytes': data['HostConfig']['Memory'],
'published_ports': data['HostConfig'].get('PortBindings')}))
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
exec python3 scripts/manage.py restart
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
exec python3 scripts/manage.py rolling-update
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
exec python3 scripts/manage.py stop-one
+3
View File
@@ -0,0 +1,3 @@
"""MTUSI Messenger lab1: in-memory HTTP API (standard library only)."""
__version__ = '1.0.0'
+7
View File
@@ -0,0 +1,7 @@
"""Entry point: python -m messenger."""
import sys
from .server import main
if __name__ == '__main__':
sys.exit(main())
+299
View File
@@ -0,0 +1,299 @@
"""Domain logic and routing for the lab2 API: PostgreSQL state, Bearer sessions."""
from __future__ import annotations
import base64
import datetime as dt
import json
import re
import uuid
from . import cursor as cursor_codec
from . import security
from . import validation
from .attachments import AttachmentStorage, filename_from, upload_file
from .db import Database
from .errors import ApiError, Conflict, InvalidRequest, NotFound, Unauthorized, Unavailable, UnsupportedMediaType
from .store import Store, utc_now, rfc3339
SCOPE_CHATS = 'chats'
SCOPE_MESSAGES = 'chat/{chat_id}/messages'
class Request:
"""One parsed HTTP request; the API layer never touches sockets."""
__slots__ = ('method', 'path', 'query', 'headers', 'body', 'request_id', 'client')
def __init__(self, method, path, query, headers, body, request_id, client):
self.method = method
self.path = path
self.query = query
self.headers = headers
self.body = body
self.request_id = request_id
self.client = client
class Api:
UPLOAD_PATH = re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/attachments$')
ROUTES = (
('POST', UPLOAD_PATH, 'upload_attachment'),
('GET', re.compile(r'^/api/v1/attachments/(?P<id>[^/]+)$'), 'get_attachment'),
('GET', re.compile(r'^/api/v1/attachments/(?P<id>[^/]+)/content$'), 'download_attachment'),
('GET', re.compile(r'^/health/live$'), 'health_live'),
('GET', re.compile(r'^/health/ready$'), 'health_ready'),
('POST', re.compile(r'^/api/v1/users$'), 'create_user'),
('GET', re.compile(r'^/api/v1/users/me$'), 'current_user'),
('GET', re.compile(r'^/api/v1/chats$'), 'list_chats'),
('POST', re.compile(r'^/api/v1/chats$'), 'create_chat'),
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)$'), 'get_chat'),
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'list_messages'),
('POST', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'post_message'),
('POST', re.compile(r'^/api/v1/auth/sessions$'), 'create_session'),
('DELETE', re.compile(r'^/api/v1/auth/sessions/current$'), 'revoke_session'),
)
def __init__(self, config, store: Store, database: Database):
self.config = config
self.store = store
self.database = database
self.attachments = AttachmentStorage()
def dispatch(self, request):
"""Return (status, payload). Only ApiError and payloads leave this method."""
path_matched = False
for method, pattern, handler in self.ROUTES:
match = pattern.match(request.path)
if match:
path_matched = True
if method == request.method:
return getattr(self, handler)(request, match.groupdict())
raise NotFound('Маршрут не найден' if not path_matched else 'Метод не поддерживается')
# ------------------------------------------------------------- health
def health_live(self, request, path_params):
return 200, {'status': 'ok'}
def health_ready(self, request, path_params):
try:
self.database.ping()
except Exception:
raise Unavailable('База данных недоступна')
return 200, {'status': 'ok'}
# -------------------------------------------------------------- users
def create_user(self, request, path_params):
payload = self._json_object(request)
validation.require_fields(payload, ('username', 'display_name', 'password'),
('username', 'display_name', 'password'))
username = validation.require_username(payload)
display_name = validation.require_text_field(
payload, 'display_name', 1, self.config.max_display_name_code_points)
password = validation.require_text_field(
payload, 'password', 12, 128, strip_check=False)
password_hash = security.hash_password(password)
return 201, self.store.create_user(username, display_name, password_hash)
def current_user(self, request, path_params):
return 200, self._authenticate(request)
# -------------------------------------------------------------- chats
def create_chat(self, request, path_params):
actor = self._authenticate(request)
payload = self._json_object(request)
validation.require_fields(payload, ('title', 'member_ids'), ('title', 'member_ids'))
title = validation.require_text_field(payload, 'title', 1,
self.config.max_title_code_points)
members = validation.require_member_ids(payload, 100)
if actor['id'] not in members:
raise InvalidRequest('Создатель чата должен быть указан в member_ids')
if not self.store.users_exist(members):
raise InvalidRequest('member_ids содержит неизвестных пользователей')
return 201, self.store.create_chat(title, members)
def list_chats(self, request, path_params):
actor = self._authenticate(request)
limit, after_key = self._pagination(request, SCOPE_CHATS)
items, has_more = self.store.page_chats(actor['id'], after_key, limit)
for item in items:
item['member_ids'] = self._chat_member_ids(item['id'])
return 200, {'items': items, 'next_cursor': self._next_cursor(SCOPE_CHATS, items, has_more)}
def get_chat(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
chat['member_ids'] = self._chat_member_ids(chat['id'])
return 200, chat
# ----------------------------------------------------------- messages
def post_message(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
key = validation.require_idempotency_key(request.headers.get('Idempotency-Key'))
payload = self._json_object(request)
validation.require_fields(payload, ('text',), ('text', 'attachment_ids'))
attachment_ids = validation.require_attachment_ids(payload)
text = validation.require_text_field(payload, 'text', 0 if attachment_ids else 1,
self.config.max_text_code_points,
strip_check=not attachment_ids)
return 201, self.store.add_message(chat['id'], actor['id'], text, key, attachment_ids)
def list_messages(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
scope = SCOPE_MESSAGES.format(chat_id=chat['id'])
limit, after_key = self._pagination(request, scope)
items, has_more = self.store.page_messages(chat['id'], after_key, limit)
return 200, {'items': items, 'next_cursor': self._next_cursor(scope, items, has_more)}
def upload_attachment(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
filename = filename_from(request.query)
mime = request.headers.get('Content-Type', '').split(';', 1)[0].strip().lower()
with upload_file(request.body, mime) as (file, size, digest):
aid = str(uuid.uuid4())
key = 'attachments/staging/' + aid
self.store.reserve_attachment(aid, chat['id'], actor['id'], filename,
mime, size, digest, key)
row = self.store.queue_attachment(
aid, lambda object_key: self.attachments.put(object_key, file, size, mime))
return 202, self.store.public_attachment(row)
def _visible_attachment(self, request, path_params):
actor = self._authenticate(request)
aid = validation.require_uuid_string(path_params['id'], 'attachment_id')
row = self.store.visible_attachment(aid, actor['id'])
if row is None:
raise NotFound('Вложение не найдено')
return row
def get_attachment(self, request, path_params):
row = self._visible_attachment(request, path_params)
variants = self.store.attachment_variants(row['id']) if row['status'] == 'ready' else ()
return 200, self.store.public_attachment(row, variants)
def download_attachment(self, request, path_params):
row = self._visible_attachment(request, path_params)
if row['status'] != 'ready':
raise Conflict('Вложение не готово')
variant = request.query.get('variant', ['original'])
if variant != ['original']:
row = next((item for item in self.store.attachment_variants(row['id'])
if [item['name']] == variant), None)
if row is None:
raise NotFound('Вариант вложения не найден')
return 200, self.attachments.download(row)
# ----------------------------------------------------------- sessions
def create_session(self, request, path_params):
"""Login: only Authorization: Basic username:password, no JSON body."""
authorization = request.headers.get('Authorization') or ''
if not authorization.lower().startswith('basic '):
raise Unauthorized('Требуется авторизация Basic')
try:
raw = base64.b64decode(authorization[len('basic '):].strip()).decode('utf-8')
except Exception:
raise Unauthorized('Некорректные учётные данные') from None
if ':' not in raw:
raise Unauthorized('Некорректные учётные данные')
username, password = raw.split(':', 1)
row = self.store.user_by_username(username)
if row is None or not security.verify_password(password, row['password_hash']):
raise Unauthorized('Неверное имя пользователя или пароль', www_authenticate='Basic realm="messenger", charset="UTF-8"')
token = security.generate_token()
token_hash = security.hash_token(token)
expires_at = utc_now() + dt.timedelta(seconds=self.config.session_ttl_seconds)
self.store.create_session(token_hash, row['id'], expires_at)
user = self.store._public_user({
'id': row['id'],
'username': row['username'],
'display_name': row['display_name'],
'created_at': row['created_at'],
})
return 201, {
'token': token,
'token_type': 'Bearer',
'expires_at': rfc3339(expires_at),
'user': user,
}
def revoke_session(self, request, path_params):
actor = self._authenticate(request)
token = self._bearer_token(request)
self.store.delete_session(security.hash_token(token))
return 204, None
# ------------------------------------------------------------- helpers
def _authenticate(self, request):
"""Lab2 identity: only Authorization: Bearer <token> identifies the actor.
X-User-Id is ignored: it does not change the session owner and alone
grants no access. Basic on business endpoints is 401.
"""
token = self._bearer_token(request)
if token is None:
raise Unauthorized('Требуется идентификация пользователя',
www_authenticate='Bearer realm="messenger"')
user_id = self.store.get_session(security.hash_token(token))
if user_id is None:
raise Unauthorized('Сессия недействительна или истекла',
www_authenticate='Bearer realm="messenger"')
user = self.store.user(user_id)
if user is None:
raise Unauthorized('Пользователь не найден',
www_authenticate='Bearer realm="messenger"')
return user
def _bearer_token(self, request):
authorization = request.headers.get('Authorization') or ''
if not authorization.lower().startswith('bearer '):
return None
return authorization[len('bearer '):].strip()
def _chat_member_ids(self, chat_id):
with self.database.connection() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
return [str(row['user_id']) for row in cur.fetchall()]
def _member_chat(self, actor, raw_chat_id):
"""Foreign and missing chats are indistinguishable: both are 404."""
chat_id = validation.require_uuid_string(raw_chat_id, 'chat_id')
chat = self.store.chat(chat_id)
if chat is None or not self.store.is_member(chat, actor['id']):
raise NotFound('Чат не найден')
return chat
def _json_object(self, request):
content_type = request.headers.get('Content-Type') or ''
if content_type.split(';')[0].strip().lower() != 'application/json':
raise UnsupportedMediaType('Ожидается Content-Type: application/json')
if not request.body:
raise InvalidRequest('Требуется JSON-тело запроса')
try:
payload = json.loads(request.body.decode('utf-8'))
except (UnicodeError, ValueError):
raise InvalidRequest('Тело запроса не является корректным JSON') from None
return validation.require_object(payload)
def _pagination(self, request, scope):
raw_limit = (request.query.get('limit') or [None])[0]
limit = validation.parse_limit(raw_limit, self.config.default_page_limit,
self.config.max_page_limit)
raw_cursor = validation.parse_cursor((request.query.get('cursor') or [None])[0])
if raw_cursor is None:
return limit, None
try:
stamp, identifier = cursor_codec.decode(raw_cursor, scope, self.config.cursor_secret)
except cursor_codec.InvalidCursor:
raise InvalidRequest('cursor некорректен или относится к другому ресурсу') from None
return limit, (stamp, identifier)
def _next_cursor(self, scope, items, has_more):
if not has_more or not items:
return None
last = items[-1]
return cursor_codec.encode(scope, last['created_at'], last['id'],
self.config.cursor_secret)
+142
View File
@@ -0,0 +1,142 @@
"""Bounded image validation and private S3 storage."""
from contextlib import contextmanager
import hashlib
import os
import struct
import tempfile
import threading
import unicodedata
from functools import cached_property
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image
from .errors import InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
MAX_IMAGE_PIXELS = 40_000_000
CHUNK_SIZE = 64 * 1024
FORMATS = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
# Pillow can allocate a full decoded image. Allow only one decode per API process.
_decode_slot = threading.Lock()
_upload_slots = threading.BoundedSemaphore(10)
def filename_from(query):
names = query.get('filename', [])
if len(names) != 1:
raise InvalidRequest('Укажите одно имя файла')
name = names[0]
if (not 1 <= len(name) <= 255 or not name.strip()
or any(c in '/\\' or unicodedata.category(c) in {'Cc', 'Cs'} for c in name)):
raise InvalidRequest('Некорректное имя файла')
return name
def validate_image(file, declared_type):
try:
with _decode_slot:
file.seek(0)
with Image.open(file, formats=list(FORMATS)) as image:
if image.width * image.height > MAX_IMAGE_PIXELS:
raise PayloadTooLarge('Изображение превышает 40 млн пикселей')
if FORMATS[image.format] != declared_type:
raise UnsupportedMediaType('Тип изображения не совпадает с Content-Type')
image.verify()
file.seek(0)
with Image.open(file, formats=list(FORMATS)) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise PayloadTooLarge('Изображение превышает 40 млн пикселей') from None
except (OSError, SyntaxError, ValueError, OverflowError, EOFError, struct.error):
raise UnsupportedMediaType('Повреждённое или неподдерживаемое изображение') from None
@contextmanager
def upload_file(read_chunks, content_type):
if content_type not in FORMATS.values():
raise UnsupportedMediaType('Ожидается PNG или JPEG')
if not _upload_slots.acquire(blocking=False):
raise Unavailable('Слишком много одновременных загрузок')
try:
with tempfile.TemporaryFile() as file:
digest = hashlib.sha256()
size = 0
for chunk in read_chunks():
size += len(chunk)
if size > MAX_UPLOAD_BYTES:
raise PayloadTooLarge()
digest.update(chunk)
file.write(chunk)
if size == 0:
raise UnsupportedMediaType('Пустое изображение')
validate_image(file, content_type)
file.seek(0)
yield file, size, digest.hexdigest()
finally:
_upload_slots.release()
class BinaryResponse:
def __init__(self, stream, content_type, size):
self.stream = stream
self.content_type = content_type
self.size = size
def chunks(self):
try:
yield from self.stream.iter_chunks(CHUNK_SIZE)
except (BotoCoreError, OSError) as exc:
raise OSError('S3 response interrupted') from exc
def close(self):
self.stream.close()
class AttachmentStorage:
@cached_property
def client(self):
def secret(name):
with open(os.environ[name + '_FILE'], encoding='utf-8') as file:
return file.read().strip()
return boto3.session.Session().client(
's3', endpoint_url=os.environ['S3_ENDPOINT'],
region_name=os.environ.get('S3_REGION', 'us-east-1'),
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1},
request_checksum_calculation="when_required",
s3={"addressing_style": "path"}))
@property
def bucket(self):
return os.environ['S3_BUCKET']
def put(self, key, file, size, content_type):
try:
self.client.put_object(Bucket=self.bucket, Key=key, Body=file,
ContentLength=size, ContentType=content_type)
except (BotoCoreError, ClientError, OSError, KeyError):
raise Unavailable('Хранилище вложений недоступно') from None
def delete(self, key):
try:
self.client.delete_object(Bucket=self.bucket, Key=key)
except (BotoCoreError, ClientError, OSError, KeyError):
raise Unavailable('Хранилище вложений недоступно') from None
def download(self, row):
try:
obj = self.client.get_object(Bucket=self.bucket, Key=row['object_key'])
except ClientError as exc:
if exc.response.get('Error', {}).get('Code') in {'NoSuchKey', '404'}:
raise NotFound('Оригинал вложения не найден') from None
raise Unavailable('Хранилище вложений недоступно') from None
except (BotoCoreError, OSError, KeyError):
raise Unavailable('Хранилище вложений недоступно') from None
if obj['ContentLength'] != row['size_bytes']:
obj['Body'].close()
raise Unavailable('Размер оригинала не совпадает с метаданными')
return BinaryResponse(obj['Body'], row['content_type'], row['size_bytes'])
+95
View File
@@ -0,0 +1,95 @@
"""Redis session cache with TTL and graceful degradation.
PostgreSQL is the source of truth for sessions. Redis is a look-aside cache:
- write: session is persisted in PostgreSQL first, then copied to Redis with TTL
- read: Redis first (fast); on miss, read PostgreSQL and backfill Redis
- when Redis is unavailable: fall back to PostgreSQL only, never treat an
anonymous request as trusted
"""
from __future__ import annotations
import logging
import redis
from .config import Config
logger = logging.getLogger('messenger.cache')
class SessionCache:
"""Look-aside cache for session tokens in Redis/Valkey."""
def __init__(self, config: Config):
self._config = config
self._client = None
self._available = False
def open(self):
if not self._config.cache_url:
self._available = False
return
try:
self._client = redis.Redis.from_url(
self._config.cache_url,
socket_connect_timeout=2,
socket_timeout=2,
socket_keepalive=True,
health_check_interval=30,
decode_responses=True,
)
self._client.ping()
self._available = True
logger.info('cache_connected', extra={'event': 'cache_connected'})
except Exception as exc: # noqa: BLE001 - Redis is optional, degrade safely
self._available = False
self._client = None
logger.warning('cache_unavailable', extra={
'event': 'cache_unavailable', 'detail': 'falling back to PostgreSQL'})
def close(self):
if self._client is not None:
try:
self._client.close()
except Exception:
pass
self._client = None
self._available = False
@property
def available(self):
return self._available and self._client is not None
def get(self, token_hash):
"""Return cached user_id or None (miss or unavailable)."""
if not self.available:
return None
try:
value = self._client.get('session:%s' % token_hash)
return value
except Exception as exc: # noqa: BLE001
logger.warning('cache_read_failed', extra={
'event': 'cache_read_failed', 'detail': 'falling back to PostgreSQL'})
self._available = False
return None
def set(self, token_hash, user_id, ttl_seconds):
"""Cache a session with TTL. Failure is non-fatal."""
if not self.available:
return
try:
self._client.setex('session:%s' % token_hash, ttl_seconds, user_id)
except Exception as exc: # noqa: BLE001
logger.warning('cache_write_failed', extra={
'event': 'cache_write_failed', 'detail': 'continuing without cache'})
self._available = False
def delete(self, token_hash):
"""Remove a session from the cache (logout). Failure is non-fatal."""
if not self.available:
return
try:
self._client.delete('session:%s' % token_hash)
except Exception as exc: # noqa: BLE001
logger.warning('cache_delete_failed', extra={
'event': 'cache_delete_failed'})
+58
View File
@@ -0,0 +1,58 @@
"""Remove expired upload reservations and their S3 objects."""
import argparse
import logging
import time
from .attachments import AttachmentStorage
from .config import Config
from .db import Database
from .errors import Unavailable
from .store import Store
from .job_lifecycle import clean_staging, clean_abandoned_outputs
GRACE_SECONDS = 15 * 60
INTERVAL_SECONDS = 60
BATCH_SIZE = 25
def clean_batch(store, storage):
count = 0
while count < BATCH_SIZE and store.clean_pending_attachment(GRACE_SECONDS, storage.delete):
count += 1
return count
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--once', action='store_true', help='Process one bounded batch and exit')
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
config = Config.from_env()
database = Database(config)
store = Store(config, database, None)
storage = AttachmentStorage()
try:
database.open()
while True:
try:
deleted = clean_batch(store, storage)
for cleanup in (clean_staging, clean_abandoned_outputs):
for _ in range(BATCH_SIZE):
if not cleanup(database, storage.delete):
break
deleted += 1
if deleted or args.once:
logging.info('Expired attachment reservations removed: %s', deleted)
except Unavailable:
logging.warning('Attachment cleanup deferred because a dependency is unavailable')
if args.once:
return 1
if args.once:
return 0
time.sleep(INTERVAL_SECONDS)
finally:
database.close()
if __name__ == '__main__':
raise SystemExit(main())
+149
View File
@@ -0,0 +1,149 @@
"""Runtime configuration. Every value is read from the environment."""
from __future__ import annotations
import os
import secrets
# Contract limits: they cannot be raised by configuration.
CONTRACT_MAX_TEXT_CODE_POINTS = 4000
CONTRACT_MAX_PAGE_LIMIT = 100
CONTRACT_SESSION_TTL_MIN = 1
CONTRACT_SESSION_TTL_MAX = 86400
class ConfigError(ValueError):
"""Raised when an environment value cannot be used."""
def _str(environ, name, default):
value = environ.get(name)
if value is None or value == '':
return default
return value
def _int(environ, name, default, minimum, maximum):
raw = environ.get(name)
if raw is None or raw == '':
return default
try:
value = int(raw)
except ValueError:
raise ConfigError('%s must be an integer, got %r' % (name, raw)) from None
if not minimum <= value <= maximum:
raise ConfigError('%s must be in %d..%d, got %d' % (name, minimum, maximum, value))
return value
def _log_level(environ, name, default):
value = _str(environ, name, default).upper()
if value not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
raise ConfigError('%s must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL' % name)
return value
def _secret_from_file(environ, name, default=None):
"""Читает секрет из файла, путь к которому задаёт переменная окружения.
Секреты не лежат в переменных окружения напрямую: контейнер монтирует
файл (например, Docker secret), а приложение читает его содержимое.
Если файл не задан или не существует, используется значение переменной
окружения без суффикса _FILE (удобно для локального запуска без секретов).
"""
path_name = name + '_FILE'
path = environ.get(path_name)
if path:
try:
with open(path, 'r', encoding='utf-8') as handle:
return handle.read().strip()
except OSError:
pass
return environ.get(name) or default
class Config:
"""Effective configuration of one process instance."""
def __init__(self, host, port, env, log_level, cursor_secret, shutdown_grace_seconds,
max_text_code_points, max_display_name_code_points, max_title_code_points,
default_page_limit, max_page_limit, max_body_bytes,
database_url, cache_url, session_ttl_seconds, session_store,
pool_min, pool_max, connect_timeout, statement_timeout):
self.host = host
self.port = port
self.env = env
self.log_level = log_level
self.cursor_secret = cursor_secret
self.shutdown_grace_seconds = shutdown_grace_seconds
self.max_text_code_points = max_text_code_points
self.max_display_name_code_points = max_display_name_code_points
self.max_title_code_points = max_title_code_points
self.default_page_limit = default_page_limit
self.max_page_limit = max_page_limit
self.max_body_bytes = max_body_bytes
self.database_url = database_url
self.cache_url = cache_url
self.session_ttl_seconds = session_ttl_seconds
self.session_store = session_store
self.pool_min = pool_min
self.pool_max = pool_max
self.connect_timeout = connect_timeout
self.statement_timeout = statement_timeout
@classmethod
def from_env(cls, environ=None):
environ = os.environ if environ is None else environ
secret = _secret_from_file(environ, 'CURSOR_SECRET', '')
if not secret:
secret = secrets.token_hex(32)
session_store = _str(environ, 'SESSION_STORE', 'postgres')
if session_store not in ('postgres', 'redis'):
raise ConfigError('SESSION_STORE должен быть postgres или redis, got %r' % session_store)
return cls(
host=_str(environ, 'APP_HOST', '0.0.0.0'),
port=_int(environ, 'APP_PORT', 8080, 1, 65535),
env=_str(environ, 'APP_ENV', 'lab'),
log_level=_log_level(environ, 'LOG_LEVEL', 'INFO'),
cursor_secret=secret.encode('utf-8'),
shutdown_grace_seconds=_int(environ, 'SHUTDOWN_GRACE_SECONDS', 8, 1, 10),
max_text_code_points=_int(environ, 'MAX_TEXT_CODE_POINTS', CONTRACT_MAX_TEXT_CODE_POINTS,
1, CONTRACT_MAX_TEXT_CODE_POINTS),
max_display_name_code_points=_int(environ, 'MAX_DISPLAY_NAME_CODE_POINTS', 100, 1, 100),
max_title_code_points=_int(environ, 'MAX_TITLE_CODE_POINTS', 100, 1, 100),
default_page_limit=_int(environ, 'DEFAULT_PAGE_LIMIT', 50, 1, CONTRACT_MAX_PAGE_LIMIT),
max_page_limit=_int(environ, 'MAX_PAGE_LIMIT', CONTRACT_MAX_PAGE_LIMIT, 1,
CONTRACT_MAX_PAGE_LIMIT),
max_body_bytes=_int(environ, 'MAX_BODY_BYTES', 1048576, 1024, 16 * 1048576),
database_url=_secret_from_file(environ, 'DATABASE_URL', None),
cache_url=_secret_from_file(environ, 'CACHE_URL', None),
session_ttl_seconds=_int(environ, 'SESSION_TTL_SECONDS', 3600,
CONTRACT_SESSION_TTL_MIN, CONTRACT_SESSION_TTL_MAX),
session_store=session_store,
pool_min=_int(environ, 'DB_POOL_MIN', 2, 1, 50),
pool_max=_int(environ, 'DB_POOL_MAX', 10, 1, 100),
connect_timeout=_int(environ, 'DB_CONNECT_TIMEOUT', 5, 1, 60),
statement_timeout=_int(environ, 'DB_STATEMENT_TIMEOUT', 10, 1, 300),
)
def as_log_fields(self):
"""Configuration summary safe for logs (no secrets)."""
return {
'app_env': self.env,
'host': self.host,
'port': self.port,
'log_level': self.log_level,
'shutdown_grace_seconds': self.shutdown_grace_seconds,
'default_page_limit': self.default_page_limit,
'max_page_limit': self.max_page_limit,
'max_text_code_points': self.max_text_code_points,
'max_body_bytes': self.max_body_bytes,
'cursor_secret_configured': bool(os.environ.get('CURSOR_SECRET')),
'database_url_configured': bool(self.database_url),
'cache_url_configured': bool(self.cache_url),
'session_ttl_seconds': self.session_ttl_seconds,
'session_store': self.session_store,
'db_pool_min': self.pool_min,
'db_pool_max': self.pool_max,
'db_connect_timeout': self.connect_timeout,
'db_statement_timeout': self.statement_timeout,
}
+66
View File
@@ -0,0 +1,66 @@
"""Opaque keyset cursors: HMAC-protected (created_at, id) plus a resource scope."""
from __future__ import annotations
import base64
import binascii
import datetime as dt
import hashlib
import hmac
import json
import uuid
VERSION = 1
MAC_BYTES = 16
class InvalidCursor(ValueError):
"""Cursor is malformed, forged, expired or belongs to another resource."""
def _key(secret):
return hashlib.sha256(b'mtusi-messenger-cursor-v1' + secret).digest()
def encode(scope, created_at, object_id, secret):
"""Serialize a keyset position bound to one resource scope."""
payload = json.dumps(
{'v': VERSION, 'scope': scope, 'created_at': created_at, 'id': str(object_id)},
separators=(',', ':'), sort_keys=True).encode('utf-8')
mac = hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]
return base64.urlsafe_b64encode(payload + mac).decode('ascii').rstrip('=')
def decode(token, scope, secret):
"""Return (created_at datetime, uuid) or raise InvalidCursor."""
if not isinstance(token, str) or not token:
raise InvalidCursor('empty cursor')
padding = '=' * (-len(token) % 4)
try:
raw = base64.urlsafe_b64decode(token + padding)
except (binascii.Error, ValueError):
raise InvalidCursor('cursor is not base64url') from None
if len(raw) <= MAC_BYTES:
raise InvalidCursor('cursor is too short')
payload, mac = raw[:-MAC_BYTES], raw[-MAC_BYTES:]
if not hmac.compare_digest(mac, hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]):
raise InvalidCursor('cursor signature mismatch')
try:
data = json.loads(payload.decode('utf-8'))
except (ValueError, UnicodeError):
raise InvalidCursor('cursor payload is not JSON') from None
if not isinstance(data, dict) or data.get('v') != VERSION:
raise InvalidCursor('unsupported cursor version')
if data.get('scope') != scope:
raise InvalidCursor('cursor belongs to another resource')
created_at = data.get('created_at')
object_id = data.get('id')
if not isinstance(created_at, str) or not isinstance(object_id, str):
raise InvalidCursor('cursor payload types')
try:
stamp = dt.datetime.fromisoformat(created_at.replace('Z', '+00:00'))
identifier = uuid.UUID(object_id)
except ValueError:
raise InvalidCursor('cursor payload values') from None
if stamp.tzinfo is None:
raise InvalidCursor('cursor timestamp has no timezone')
return stamp.astimezone(dt.timezone.utc), identifier
+97
View File
@@ -0,0 +1,97 @@
"""PostgreSQL connection pool and schema migrations."""
from __future__ import annotations
import os
from contextlib import contextmanager
from pathlib import Path
from psycopg2 import pool, OperationalError, InterfaceError
from psycopg2.extras import RealDictCursor
from .config import Config
from .errors import Unavailable
_ROOT = Path(__file__).resolve().parents[2]
_MIGRATIONS_DIR = _ROOT / 'migrations'
class Database:
"""Thread-safe connection pool with prepared statement timeouts.
All queries are parameterised (%s). Connections are returned to the pool
after every operation; the pool itself is closed on shutdown.
"""
def __init__(self, config: Config):
self._config = config
self._pool = None
def open(self):
if self._pool is not None:
return
try:
self._pool = pool.ThreadedConnectionPool(
minconn=self._config.pool_min,
maxconn=self._config.pool_max,
dsn=self._config.database_url,
connect_timeout=self._config.connect_timeout,
cursor_factory=RealDictCursor,
)
except Exception as exc: # psycopg2.Error or configuration error
raise Unavailable('Не удалось подключиться к PostgreSQL: %s' % exc) from None
def close(self):
if self._pool is not None:
self._pool.closeall()
self._pool = None
@contextmanager
def connection(self):
"""Yield a connection; rollback on exception, return to pool on exit."""
if self._pool is None:
raise Unavailable('База данных не инициализирована')
try:
conn = self._pool.getconn()
except (pool.PoolError, OperationalError, InterfaceError):
raise Unavailable('Пул соединений PostgreSQL недоступен') from None
try:
with conn.cursor() as cur:
cur.execute('SET statement_timeout = %s',
(self._config.statement_timeout * 1000,))
yield conn
except Exception as exc:
try:
conn.rollback()
except Exception:
pass
if isinstance(exc, (OperationalError, InterfaceError)):
raise Unavailable('PostgreSQL недоступен') from None
raise
finally:
self._pool.putconn(conn, close=bool(conn.closed))
@contextmanager
def cursor(self):
"""Yield a cursor inside a connection; commit on success, rollback on failure."""
with self.connection() as conn:
with conn.cursor() as cur:
yield cur
conn.commit()
def ping(self):
with self.cursor() as cur:
cur.execute('SELECT 1')
cur.fetchone()
def run_migrations(self):
"""Apply versioned migrations once, under a transaction advisory lock."""
with self.cursor() as cur:
cur.execute('SELECT pg_advisory_xact_lock(3072026)')
cur.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
'(name text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())')
for path in sorted(_MIGRATIONS_DIR.glob('*.sql')):
cur.execute('SELECT 1 FROM schema_migrations WHERE name = %s', (path.name,))
if cur.fetchone() is not None:
continue
cur.execute(path.read_text(encoding='utf-8'))
cur.execute('INSERT INTO schema_migrations(name) VALUES (%s)', (path.name,))
+73
View File
@@ -0,0 +1,73 @@
"""API error taxonomy; every message is safe to return to a client."""
class ApiError(Exception):
"""Base class for errors that map onto the public error payload."""
status = 500
code = 'internal_error'
default_message = 'Внутренняя ошибка сервера'
def __init__(self, message=None):
self.message = message or self.default_message
super().__init__(self.message)
class InvalidRequest(ApiError):
status = 400
code = 'invalid_request'
default_message = 'Некорректный запрос'
class Unauthorized(ApiError):
status = 401
code = 'unauthorized'
default_message = 'Требуется идентификация пользователя'
www_authenticate = None
def __init__(self, message=None, www_authenticate=None):
self.message = message or self.default_message
self.www_authenticate = www_authenticate
super().__init__(self.message)
class NotFound(ApiError):
status = 404
code = 'not_found'
default_message = 'Объект не найден'
class Conflict(ApiError):
status = 409
code = 'conflict'
default_message = 'Конфликт состояния'
class PayloadTooLarge(ApiError):
status = 413
code = 'payload_too_large'
default_message = 'Тело запроса слишком велико'
class UnsupportedMediaType(ApiError):
status = 415
code = 'unsupported_media_type'
default_message = 'Поддерживается только application/json'
class RateLimited(ApiError):
status = 429
code = 'rate_limited'
default_message = 'Слишком много запросов'
class Unavailable(ApiError):
status = 503
code = 'unavailable'
default_message = 'Сервис временно недоступен'
class InternalError(ApiError):
status = 500
code = 'internal_error'
default_message = 'Внутренняя ошибка сервера'
+39
View File
@@ -0,0 +1,39 @@
"""List failed jobs and explicitly replay a failed job with retained staging."""
import argparse
import json
from uuid import UUID
from .config import Config
from .db import Database
from .errors import ApiError
from .job_lifecycle import replay
def main():
parser = argparse.ArgumentParser(description=__doc__)
actions = parser.add_subparsers(dest='action', required=True)
actions.add_parser('list')
command = actions.add_parser('replay')
command.add_argument('job_id', type=UUID)
command.add_argument('--operator', required=True)
args = parser.parse_args()
database = Database(Config.from_env())
database.open()
try:
if args.action == 'replay':
print(json.dumps({'job_id': str(args.job_id), 'attempt': replay(database, str(args.job_id), args.operator)}))
else:
with database.cursor() as cur:
cur.execute("SELECT j.id AS job_id,a.id AS attachment_id,a.error_code,j.attempt, "
"j.staging_cleaned_at IS NULL AS staging_retained "
"FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE a.status='failed' ORDER BY a.updated_at DESC LIMIT 100")
print(json.dumps(cur.fetchall(), default=str))
except ApiError as exc:
parser.exit(1, str(exc) + '\n')
finally:
database.close()
if __name__ == '__main__':
main()
+135
View File
@@ -0,0 +1,135 @@
"""Transactional claims, bounded retries and explicit failed-job recovery."""
import os
import random
from .errors import Conflict, NotFound
LEASE_SECONDS = 60
STAGING_RETENTION_SECONDS = 86400
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
if not 1 <= MAX_ATTEMPTS <= 20:
raise ValueError('JOB_MAX_ATTEMPTS must be between 1 and 20')
def _locked_job(cur, job_id):
cur.execute(
'SELECT j.*, a.status, a.updated_at, a.error_code, '
'j.lease_until > clock_timestamp() AS lease_valid FROM attachment_jobs j '
'JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s FOR UPDATE OF j,a',
(job_id,))
return cur.fetchone()
def claim(db, delivery):
with db.cursor() as cur:
row = _locked_job(cur, delivery['job_id'])
if (not row or row['status'] != 'queued' or row['attempt'] != delivery['attempt']
or str(row['attachment_id']) != delivery['attachment_id']
or row['pipeline_version'] != delivery['pipeline_version']):
return None
cur.execute('UPDATE attachment_jobs SET lease_until=clock_timestamp()+make_interval(secs => %s), '
'attempts_used=attempts_used+1 WHERE id=%s', (LEASE_SECONDS, row['id']))
cur.execute("UPDATE attachments SET status='processing', updated_at=now() "
"WHERE id=%s RETURNING *", (row['attachment_id'],))
return cur.fetchone()
def _requeue(cur, row, delay, reset=False):
attempt = row['attempt'] + 1
cur.execute('UPDATE attachment_jobs SET attempt=%s, lease_until=NULL, attempts_used=%s WHERE id=%s',
(attempt, 0 if reset else row['attempts_used'], row['id']))
cur.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s",
(row['attachment_id'],))
cur.execute('INSERT INTO attachment_outbox (job_id,attempt,available_at) '
'VALUES (%s,%s,now()+make_interval(secs => %s))', (row['id'], attempt, delay))
return attempt
def _fail_or_retry(cur, row, error_code):
if error_code is None and row['attempts_used'] < MAX_ATTEMPTS:
delay = min(60, 2 ** row['attempts_used']) * random.uniform(.75, 1.25)
_requeue(cur, row, delay)
return
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s",
(error_code or 'retry_exhausted', row['attachment_id']))
def complete(db, delivery, result=None, error_code=None):
with db.cursor() as cur:
row = _locked_job(cur, delivery['job_id'])
if (not row or row['status'] != 'processing'
or row['attempt'] != delivery['attempt'] or not row['lease_valid']):
return False
if result is None:
_fail_or_retry(cur, row, error_code)
return True
key, variants = result
cur.executemany(
'INSERT INTO attachment_variants (attachment_id,pipeline_version,variant,object_key,'
'content_type,size_bytes,sha256,width,height) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)',
[(row['attachment_id'], row['pipeline_version'], *item) for item in variants])
cur.execute("UPDATE attachments SET object_key=%s,status='ready',updated_at=now() WHERE id=%s",
(key, row['attachment_id']))
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
return True
def recover_expired(db):
with db.cursor() as cur:
cur.execute(
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE a.status='processing' AND j.lease_until <= now() "
"ORDER BY j.lease_until LIMIT 10 FOR UPDATE OF j,a SKIP LOCKED")
for row in cur.fetchall():
_fail_or_retry(cur, row, None)
def replay(db, job_id, operator):
with db.cursor() as cur:
row = _locked_job(cur, job_id)
if row is None:
raise NotFound('Задача не найдена')
if row['status'] != 'failed' or row['staging_cleaned_at'] is not None:
raise Conflict('Повтор доступен только для failed с сохранённым исходником')
attempt = _requeue(cur, row, 0, reset=True)
cur.execute('INSERT INTO attachment_replays (job_id,attempt,operator) VALUES (%s,%s,%s)',
(job_id, attempt, operator))
return attempt
def clean_staging(db, remove):
with db.cursor() as cur:
cur.execute(
"SELECT j.id,j.attachment_id FROM attachment_jobs j "
"JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.staging_cleaned_at IS NULL AND (a.status='ready' OR "
"(a.status='failed' AND a.updated_at < now()-make_interval(secs => %s))) "
"LIMIT 1 FOR UPDATE OF j,a SKIP LOCKED", (STAGING_RETENTION_SECONDS,))
row = cur.fetchone()
if not row:
return False
remove('attachments/staging/' + str(row['attachment_id']))
cur.execute('UPDATE attachment_jobs SET staging_cleaned_at=now() WHERE id=%s', (row['id'],))
return True
def clean_abandoned_outputs(db, remove):
with db.cursor() as cur:
cur.execute(
"SELECT j.id,j.attachment_id,j.pipeline_version,o.attempt "
"FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
"JOIN attachments a ON a.id=j.attachment_id "
"WHERE o.objects_cleaned_at IS NULL AND o.available_at < now()-interval '15 minutes' "
"AND (o.attempt < j.attempt OR (a.status='failed' AND "
"a.updated_at < now()-interval '15 minutes')) "
"ORDER BY o.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED")
row = cur.fetchone()
if not row:
return False
prefix = 'attachments/final/{attachment_id}/v{pipeline_version}/a{attempt}/'.format(**row)
for name in ('original', 'thumbnail.jpeg', 'crop.jpeg', 'optimized.webp'):
remove(prefix + name)
cur.execute('UPDATE attachment_outbox SET objects_cleaned_at=now() WHERE job_id=%s AND attempt=%s',
(row['id'], row['attempt']))
return True
+45
View File
@@ -0,0 +1,45 @@
"""Structured stdout logging without payload contents."""
from __future__ import annotations
import json
import logging
import sys
import time
_FIELDS = ('event', 'request_id', 'method', 'route', 'status', 'duration_ms', 'actor_id',
'error_code', 'client', 'outcome', 'count', 'detail', 'signal', 'version')
class JsonFormatter(logging.Formatter):
"""One JSON object per line; message text of API payloads is never logged."""
def format(self, record):
payload = {
'ts': time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(record.created)) +
'.%03dZ' % int(record.msecs),
'level': record.levelname.lower(),
'logger': record.name,
'message': record.getMessage(),
}
for field in _FIELDS:
value = getattr(record, field, None)
if value is not None:
payload[field] = value
if record.exc_info and record.levelno >= logging.ERROR:
payload['exception'] = record.exc_info[0].__name__
return json.dumps(payload, ensure_ascii=False, sort_keys=True)
def configure(level_name):
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger()
for existing in list(root.handlers):
root.removeHandler(existing)
root.addHandler(handler)
root.setLevel(getattr(logging, level_name))
return logging.getLogger('messenger')
def access_logger():
return logging.getLogger('messenger.access')
+17
View File
@@ -0,0 +1,17 @@
"""One-shot schema migration entry point; API processes never migrate."""
from .config import Config
from .db import Database
def main():
database = Database(Config.from_env())
try:
database.open()
database.run_migrations()
print('PASS: migrations applied')
finally:
database.close()
if __name__ == '__main__':
main()
+88
View File
@@ -0,0 +1,88 @@
"""Publish committed attachment jobs using RabbitMQ publisher confirms."""
import json
import logging
import os
from pathlib import Path
import time
import pika
from .config import Config
from .db import Database
from .errors import Unavailable
from .store import rfc3339
from .job_lifecycle import recover_expired
QUEUE = 'attachment.process.v1'
log = logging.getLogger(__name__)
def connect():
password = Path(os.environ['RABBITMQ_PASSWORD_FILE']).read_text().strip()
return pika.BlockingConnection(pika.ConnectionParameters(
host='rabbitmq', credentials=pika.PlainCredentials('lab5', password),
heartbeat=30, socket_timeout=3, stack_timeout=5, blocked_connection_timeout=5))
def publish_one(database, channel):
with database.cursor() as cur:
cur.execute(
'SELECT j.id, j.attachment_id, j.pipeline_version, j.created_at, o.attempt '
'FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id '
'WHERE o.published_at IS NULL AND o.available_at <= now() '
'ORDER BY o.available_at, o.job_id LIMIT 1 FOR UPDATE OF o SKIP LOCKED')
row = cur.fetchone()
if not row:
return False
job = {
'schema_version': 1, 'job_id': str(row['id']), 'type': QUEUE,
'attachment_id': str(row['attachment_id']),
'pipeline_version': row['pipeline_version'], 'attempt': row['attempt'],
'created_at': rfc3339(row['created_at']),
}
channel.basic_publish(
exchange='', routing_key=QUEUE, body=json.dumps(job).encode(),
properties=pika.BasicProperties(content_type='application/json',
delivery_mode=2, message_id=job['job_id']),
mandatory=True)
cur.execute('UPDATE attachment_outbox SET published_at=now() WHERE job_id=%s AND attempt=%s',
(row['id'], row['attempt']))
return True
def main():
logging.basicConfig(level=logging.INFO)
database = Database(Config.from_env())
database.open()
try:
while True:
connection = None
try:
connection = connect()
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.confirm_delivery()
next_recovery = 0
while True:
if time.monotonic() >= next_recovery:
recover_expired(database)
next_recovery = time.monotonic() + 1
if not publish_one(database, channel):
connection.sleep(.25)
else:
connection.process_data_events()
except (pika.exceptions.AMQPError, OSError, Unavailable) as exc:
log.warning('Publisher will retry: %s', type(exc).__name__)
time.sleep(2)
finally:
if connection and connection.is_open:
try:
connection.close()
except pika.exceptions.AMQPError:
pass
finally:
database.close()
if __name__ == '__main__':
main()
+99
View File
@@ -0,0 +1,99 @@
"""Password hashing and session tokens.
Passwords are hashed with a library password KDF: scrypt from the standard
library when available, otherwise bcrypt with a SHA-256 pre-hash so that
passwords longer than bcrypt's 72-byte limit are still supported.
Session tokens are opaque random values (>= 256 bits of entropy); only their
SHA-256 is stored in the database.
"""
from __future__ import annotations
import base64
import hashlib
import hmac
import secrets
import bcrypt
try: # scrypt is part of hashlib on OpenSSL-enabled builds
hashlib.scrypt(b'x', salt=b's', n=16, r=8, p=1, dklen=16)
_HAS_SCRYPT = True
except Exception:
_HAS_SCRYPT = False
if not _HAS_SCRYPT:
import bcrypt # type: ignore
SCRYPT_N = 16384
SCRYPT_R = 8
SCRYPT_P = 1
SCRYPT_DKLEN = 32
SALT_BYTES = 16
def hash_password(password: str) -> str:
"""Hash a UTF-8 password with a library password KDF.
Returns a single storable string:
scrypt:$n:$r:$p:$salt_b64:$hash_b64
or
bcrypt:$salt_b64:$hash_b64
"""
raw = password.encode('utf-8')
if _HAS_SCRYPT:
salt = secrets.token_bytes(SALT_BYTES)
dk = hashlib.scrypt(
raw,
salt=salt,
n=SCRYPT_N,
r=SCRYPT_R,
p=SCRYPT_P,
dklen=SCRYPT_DKLEN,
)
return 'scrypt:%d:%d:%d:%s:%s' % (
SCRYPT_N, SCRYPT_R, SCRYPT_P,
base64.b64encode(salt).decode('ascii'),
base64.b64encode(dk).decode('ascii'),
)
# bcrypt limits input to 72 bytes; pre-hash with SHA-256 so long passwords
# (up to 128 code points per contract) are fully covered.
digest = hashlib.sha256(raw).digest()
hashed = bcrypt.hashpw(digest, bcrypt.gensalt(rounds=12))
return 'bcrypt:%s' % hashed.decode('ascii')
def verify_password(password: str, password_hash: str) -> bool:
"""Verify a password against a KDF hash string. Constant-time compare."""
try:
raw = password.encode('utf-8')
if password_hash.startswith('scrypt:'):
parts = password_hash.split(':')
if len(parts) != 6 or parts[0] != 'scrypt':
return False
_, n, r, p, salt_b64, hash_b64 = parts
salt = base64.b64decode(salt_b64)
expected = base64.b64decode(hash_b64)
dk = hashlib.scrypt(
raw,
salt=salt,
n=int(n), r=int(r), p=int(p),
dklen=len(expected),
)
return hmac.compare_digest(dk, expected)
if password_hash.startswith('bcrypt:'):
stored = password_hash[len('bcrypt:'):].encode('ascii')
digest = hashlib.sha256(raw).digest()
return bcrypt.checkpw(digest, stored)
return False
except Exception:
return False
def generate_token() -> str:
"""Opaque random bearer token, at least 256 bits of entropy."""
return secrets.token_urlsafe(32)
def hash_token(token: str) -> str:
"""SHA-256 of a bearer token, stored in the database instead of the token itself."""
return hashlib.sha256(token.encode('utf-8')).hexdigest()
+305
View File
@@ -0,0 +1,305 @@
"""HTTP/1.1 front end: request ids, structured logs, graceful SIGTERM shutdown."""
from __future__ import annotations
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
import platform
import signal
import sys
import threading
import time
import uuid
from urllib.parse import parse_qs, urlsplit
from . import __version__, logging_utils
from .api import Api, Request
from .attachments import BinaryResponse, CHUNK_SIZE, MAX_UPLOAD_BYTES
from .cache import SessionCache
from .config import Config, ConfigError
from .db import Database
from .errors import ApiError, InvalidRequest, PayloadTooLarge, Unavailable
from .store import Store
MAX_REQUEST_ID_LENGTH = 128
MAX_LOGGED_LINE = 200
def sanitize_request_id(raw):
"""Echo a safe client id, otherwise generate one. Never trust raw input."""
if raw and len(raw) <= MAX_REQUEST_ID_LENGTH and all(0x21 <= ord(ch) <= 0x7E for ch in raw):
return raw
return uuid.uuid4().hex
def route_template(path):
for _method, pattern, _handler in Api.ROUTES:
if pattern.match(path):
return pattern.pattern.replace('[^/]+', '{chat_id}').strip('^$')
return 'unmatched'
def encode_payload(payload):
return json.dumps(payload, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
class ApiServer(ThreadingHTTPServer):
"""Threaded server that tracks in-flight requests and stops accepting on demand."""
daemon_threads = True
allow_reuse_address = True
request_queue_size = 128
def __init__(self, address, api, config, logger, database):
self.api = api
self.config = config
self.logger = logger
self.database = database
self.instance_id = os.environ.get("HOSTNAME") or uuid.uuid4().hex
self.shutting_down = False
self._active = 0
self._idle = threading.Condition()
super().__init__(address, ApiHandler)
def enter(self):
with self._idle:
self._active += 1
def leave(self):
with self._idle:
self._active -= 1
if self._active == 0:
self._idle.notify_all()
def wait_idle(self, timeout):
deadline = time.monotonic() + timeout
with self._idle:
while self._active > 0:
remaining = deadline - time.monotonic()
if remaining <= 0:
return False
self._idle.wait(remaining)
return True
class ApiHandler(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
server_version = 'mtusi-messenger/' + __version__
sys_version = ''
timeout = 30
# --------------------------------------------------------- entry points
def do_GET(self):
self._handle()
do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_GET
def log_message(self, fmt, *args):
"""Base-class chatter stays debug-only; access logs are structured."""
self.server.logger.debug('http_note', extra={
'event': 'http_note', 'detail': (fmt % args)[:MAX_LOGGED_LINE]})
# ------------------------------------------------------------- pipeline
def _handle(self):
started = time.monotonic()
server = self.server
request_id = sanitize_request_id(self.headers.get('X-Request-Id'))
head_only = self.command == 'HEAD'
path = urlsplit(self.path).path
status, error_code = 500, None
self._last_exc = None
server.enter()
try:
if server.shutting_down:
raise Unavailable('Сервер завершает работу')
if self.command == 'POST' and Api.UPLOAD_PATH.fullmatch(path):
# Rejected uploads may leave unread bytes in the connection.
self.close_connection = True
body = self._upload_chunks
else:
body = self._read_body()
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
request = Request('GET' if head_only else self.command, path, query, self.headers,
body, request_id, self.client_address[0])
status, payload = server.api.dispatch(request)
encoded = payload if isinstance(payload, BinaryResponse) else encode_payload(payload)
except ApiError as exc:
self._last_exc = exc
status, error_code = exc.status, exc.code
encoded = self._error_payload(exc, request_id)
except Exception: # last resort: log privately, answer generically
server.logger.exception('unhandled_error', extra={
'event': 'unhandled_error', 'request_id': request_id,
'method': self.command, 'route': route_template(path)})
status, error_code = 500, 'internal_error'
encoded = self._error_payload(ApiError(), request_id)
finally:
server.logger.info('http_request', extra={
'event': 'http_request', 'request_id': request_id, 'method': self.command,
'route': route_template(path), 'status': status,
'duration_ms': round(1000 * (time.monotonic() - started), 3),
'error_code': error_code, 'client': self.client_address[0]})
try:
self._send(status, encoded, request_id, head_only)
finally:
if isinstance(encoded, BinaryResponse):
encoded.close()
server.leave()
def _read_body(self):
if self.headers.get('Transfer-Encoding'):
self.close_connection = True
raise InvalidRequest('Transfer-Encoding не поддерживается')
raw_length = self.headers.get('Content-Length')
if raw_length is None:
return b''
try:
length = int(raw_length.strip())
except ValueError:
raise InvalidRequest('Некорректный Content-Length') from None
if length < 0:
raise InvalidRequest('Некорректный Content-Length')
if length > self.server.config.max_body_bytes:
self.close_connection = True
raise PayloadTooLarge('Тело запроса больше %d байт' % self.server.config.max_body_bytes)
if length == 0:
return b''
return self.rfile.read(length)
def _upload_chunks(self):
if self.headers.get('Transfer-Encoding'):
raise InvalidRequest('Требуется Content-Length')
lengths = self.headers.get_all('Content-Length') or []
try:
length = int(lengths[0]) if len(lengths) == 1 else -1
except ValueError:
length = -1
if length < 0:
raise InvalidRequest('Некорректный Content-Length')
if length > MAX_UPLOAD_BYTES:
raise PayloadTooLarge()
deadline = time.monotonic() + 30
while length:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise InvalidRequest('Истекло время загрузки')
self.connection.settimeout(min(5, remaining))
try:
chunk = self.rfile.read1(min(CHUNK_SIZE, length))
except OSError:
raise InvalidRequest('Загрузка прервана') from None
finally:
self.connection.settimeout(self.timeout)
if not chunk:
raise InvalidRequest('Загрузка прервана')
length -= len(chunk)
yield chunk
@staticmethod
def _error_payload(exc, request_id):
return encode_payload({'error': {
'code': exc.code, 'message': exc.message, 'request_id': request_id}})
def _send(self, status, encoded, request_id, head_only):
try:
self.send_response(status)
binary = isinstance(encoded, BinaryResponse)
self.send_header('Content-Type', encoded.content_type if binary else 'application/json; charset=utf-8')
self.send_header('X-Request-Id', request_id)
self.send_header('X-Instance-Id', self.server.instance_id)
self.send_header('Cache-Control', 'no-store')
if status == 204:
self.send_header('Content-Length', '0')
if status == 503:
self.send_header('Retry-After', '1')
self.end_headers()
return
self.send_header('Content-Length', str(encoded.size if binary else len(encoded)))
if status == 503:
self.send_header('Retry-After', '1')
if status == 401 and self._last_exc is not None:
www_auth = getattr(self._last_exc, 'www_authenticate', None)
if www_auth:
self.send_header('WWW-Authenticate', www_auth)
if self.close_connection:
self.send_header('Connection', 'close')
self.end_headers()
if not head_only and encoded:
if binary:
for chunk in encoded.chunks():
self.wfile.write(chunk)
else:
self.wfile.write(encoded)
except OSError:
self.close_connection = True
def build_server(config, logger, api, database):
"""Create the listening server; raises OSError when the port is taken."""
return ApiServer((config.host, config.port), api, config, logger, database)
def run(config, logger, database, cache):
"""Serve until SIGTERM/SIGINT, then drain in-flight work within the grace period."""
api = Api(config, Store(config, database, cache), database)
try:
server = build_server(config, logger, api, database)
except OSError as exc:
logger.error('listen_failed', extra={'event': 'listen_failed', 'detail': str(exc)})
return 1
stop = threading.Event()
def handle_signal(signum, _frame):
if stop.is_set():
return
stop.set()
server.shutting_down = True
logger.warning('shutdown_signal', extra={
'event': 'shutdown_signal', 'signal': signum,
'detail': 'stop accepting new requests'})
threading.Thread(target=server.shutdown, name='shutdown', daemon=True).start()
for name in ('SIGTERM', 'SIGINT'):
if hasattr(signal, name):
signal.signal(getattr(signal, name), handle_signal)
logger.info('server_started', extra=dict(
{'event': 'server_started', 'version': __version__,
'detail': 'pid %d, python %s' % (os.getpid(), platform.python_version())},
**config.as_log_fields()))
started = time.monotonic()
try:
server.serve_forever(poll_interval=0.2)
finally:
server.server_close()
drained = server.wait_idle(config.shutdown_grace_seconds)
database.close()
logger.info('server_stopped', extra={
'event': 'server_stopped',
'outcome': 'graceful' if drained else 'grace_period_exceeded',
'duration_ms': round(1000 * (time.monotonic() - started), 3)})
return 0
def main(argv=None):
try:
config = Config.from_env()
except ConfigError as exc:
print('configuration error: %s' % exc, file=sys.stderr)
return 2
logger = logging_utils.configure(config.log_level)
database = Database(config)
cache = SessionCache(config)
try:
database.open()
cache.open()
except Exception as exc: # noqa: BLE001 - startup must fail loud and clear
logger.error('database_start_failed', extra={
'event': 'database_start_failed', 'detail': str(exc)})
database.close()
cache.close()
return 1
try:
return run(config, logger, database, cache)
finally:
cache.close()
+390
View File
@@ -0,0 +1,390 @@
"""PostgreSQL-backed state store. All writes go through parameterised queries
inside explicit transactions; reads use keyset pagination over indexed columns."""
from __future__ import annotations
import datetime as dt
import hashlib
import json
from psycopg2.extras import Json
from psycopg2.errors import UniqueViolation, ForeignKeyViolation
import uuid
from . import cursor as cursor_codec
from . import security
from .cache import SessionCache
from .config import Config
from .db import Database
from .errors import Conflict, InvalidRequest, NotFound
def utc_now():
return dt.datetime.now(dt.timezone.utc)
def rfc3339(stamp):
"""RFC 3339 UTC with fixed microsecond precision and the Z suffix."""
return stamp.astimezone(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
class Store:
"""PostgreSQL state: users, chats, membership, messages, sessions."""
def __init__(self, config: Config, database: Database, cache: SessionCache):
self._config = config
self._db = database
self._cache = cache
# ---------------------------------------------------------------- reads
def stats(self):
with self._db.cursor() as cur:
cur.execute('SELECT (SELECT count(*) FROM users) AS users,'
' (SELECT count(*) FROM chats) AS chats,'
' (SELECT count(*) FROM messages) AS messages')
row = cur.fetchone()
return {
'users': row['users'],
'chats': row['chats'],
'messages': row['messages'],
}
def user(self, user_id):
with self._db.cursor() as cur:
cur.execute(
'SELECT id, username, display_name, created_at FROM users WHERE id = %s',
(str(user_id),))
row = cur.fetchone()
return self._public_user(row) if row else None
def user_by_username(self, username):
"""Return user row including password_hash, or None."""
with self._db.cursor() as cur:
cur.execute(
'SELECT id, username, password_hash, display_name, created_at FROM users '
'WHERE username = %s', (username,))
return cur.fetchone()
def users_exist(self, identifiers):
if not identifiers:
return True
with self._db.cursor() as cur:
cur.execute(
'SELECT count(*) AS c FROM users WHERE id = ANY(%s::uuid[])',
([str(i) for i in identifiers],))
row = cur.fetchone()
return row['c'] == len(identifiers)
# --------------------------------------------------------------- writes
def create_user(self, username, display_name, password_hash):
with self._db.cursor() as cur:
try:
cur.execute(
'INSERT INTO users (id, username, password_hash, display_name, created_at) '
'VALUES (%s, %s, %s, %s, %s) '
'RETURNING id, username, display_name, created_at',
(str(uuid.uuid4()), username, password_hash, display_name, utc_now()))
except UniqueViolation:
# Unique violation on username -> conflict, rolled back by cursor context.
raise Conflict('username уже занят') from None
row = cur.fetchone()
return self._public_user(row)
def create_chat(self, title, member_ids):
"""Atomically create a chat and its membership. Rollback on any failure."""
identifier = str(uuid.uuid4())
stamp = utc_now()
with self._db.cursor() as cur:
try:
cur.execute(
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
(identifier, title, stamp))
for member_id in member_ids:
cur.execute(
'INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)',
(identifier, str(member_id)))
except ForeignKeyViolation:
raise InvalidRequest('member_ids содержит неизвестных пользователей') from None
return {
'id': identifier,
'title': title,
'member_ids': list(member_ids),
'created_at': rfc3339(stamp),
}
def reserve_attachment(self, aid, chat_id, owner_id, filename, mime, size, digest, key):
with self._db.cursor() as cur:
cur.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, "
"size_bytes, sha256, object_key, status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')",
(aid, chat_id, owner_id, filename, mime, size, digest, key))
def queue_attachment(self, aid, put_object):
"""Keep a row lock from before PUT until staging metadata, job and outbox commit."""
with self._db.cursor() as cur:
cur.execute("SELECT object_key FROM attachments WHERE id=%s "
"AND status='pending' FOR UPDATE", (aid,))
pending = cur.fetchone()
if pending is None:
raise Conflict('Загрузка вложения не может быть завершена')
put_object(pending['object_key'])
job_id = str(uuid.uuid4())
cur.execute('INSERT INTO attachment_jobs (id, attachment_id, pipeline_version) '
'VALUES (%s,%s,1)', (job_id, aid))
cur.execute('INSERT INTO attachment_outbox (job_id, attempt) VALUES (%s,1)', (job_id,))
cur.execute("UPDATE attachments SET status='queued', updated_at=now() "
"WHERE id=%s RETURNING *", (aid,))
return cur.fetchone()
def clean_pending_attachment(self, grace_seconds, delete_object):
"""Remove one expired reservation and its object, or leave it for retry."""
with self._db.cursor() as cur:
cur.execute(
"SELECT id, object_key FROM attachments WHERE status='pending' "
"AND updated_at < now() - make_interval(secs => %s) "
"ORDER BY updated_at, id LIMIT 1 FOR UPDATE SKIP LOCKED",
(grace_seconds,))
row = cur.fetchone()
if row is None:
return False
delete_object(row['object_key'])
cur.execute('DELETE FROM attachments WHERE id=%s', (row['id'],))
return True
def visible_attachment(self, aid, user_id):
with self._db.cursor() as cur:
cur.execute(
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s AND a.status<>'pending'", (aid, user_id))
return cur.fetchone()
def attachment_variants(self, aid):
with self._db.cursor() as cur:
cur.execute(
'SELECT DISTINCT ON (variant) variant AS name, object_key, content_type, '
'size_bytes, sha256, width, height FROM attachment_variants '
'WHERE attachment_id=%s ORDER BY variant, pipeline_version DESC', (aid,))
return cur.fetchall()
@staticmethod
def public_attachment(row, variants=()):
return {
'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']),
'filename': row['filename'], 'content_type': row['content_type'],
'size_bytes': row['size_bytes'], 'sha256': row['sha256'], 'status': row['status'],
'error_code': row['error_code'],
'variants': [{k: v for k, v in item.items() if k != 'object_key'} for item in variants],
'created_at': rfc3339(row['created_at']), 'updated_at': rfc3339(row['updated_at']),
}
def add_message(self, chat_id, sender_id, text, key, attachment_ids=()):
"""One transaction owns both the message and its replayable result.
The advisory lock serializes only the same user/method/path/key across
processes. Hash collisions serialize extra requests, never alias keys:
the complete key is still checked against the primary key in the table.
A crash before COMMIT rolls back the message, links and replay record. A lost reply after COMMIT
is recovered by replay. Transient failures are never recorded.
"""
path = '/api/v1/chats/%s/messages' % chat_id
body = {'text': text}
# Keep the lab3 digest for text-only requests, including explicit [].
if attachment_ids:
body['attachment_ids'] = list(attachment_ids)
body_hash = hashlib.sha256(json.dumps(body, ensure_ascii=False,
sort_keys=True).encode('utf-8')).hexdigest()
scope = json.dumps([str(sender_id), 'POST', path, key], separators=(',', ':'))
lock_id = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], 'big', signed=True)
identifier = str(uuid.uuid4())
stamp = utc_now()
with self._db.cursor() as cur:
cur.execute('SELECT pg_advisory_xact_lock(%s)', (lock_id,))
cur.execute('SELECT request_hash, response_body FROM message_idempotency '
'WHERE user_id=%s AND method=%s AND path=%s AND key=%s',
(str(sender_id), 'POST', path, key))
replay = cur.fetchone()
if replay is not None:
if replay['request_hash'] != body_hash:
raise Conflict('Idempotency-Key уже использован с другим телом запроса')
return {'attachment_ids': [], **replay['response_body']}
if attachment_ids:
cur.execute(
'SELECT id, status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[]) FOR SHARE',
(str(chat_id), list(attachment_ids)))
attachments = cur.fetchall()
if len(attachments) != len(attachment_ids):
raise NotFound('Вложение не найдено')
if any(row['status'] not in ('ready', 'queued', 'processing') for row in attachments):
raise Conflict('Вложение не готово')
cur.execute(
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
'VALUES (%s, %s, %s, %s, %s) '
'RETURNING id, chat_id, sender_id, text, created_at',
(identifier, str(chat_id), str(sender_id), text, stamp))
result = self._public_message(cur.fetchone())
if attachment_ids:
cur.executemany(
'INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) '
'VALUES (%s,%s,%s,%s)',
[(identifier, aid, str(chat_id), pos) for pos, aid in enumerate(attachment_ids)])
result['attachment_ids'] = list(attachment_ids)
cur.execute('INSERT INTO message_idempotency '
'(user_id, method, path, key, request_hash, response_status, response_body) '
'VALUES (%s,%s,%s,%s,%s,201,%s)',
(str(sender_id), 'POST', path, key, body_hash, Json(result)))
return result
# --------------------------------------------------------------- pages
def page_messages(self, chat_id, after_key, limit):
"""Return (public messages, has_more) for a keyset window."""
with self._db.cursor() as cur:
if after_key is None:
cur.execute(
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
'WHERE chat_id = %s '
'ORDER BY created_at, id '
'LIMIT %s', (str(chat_id), limit + 1))
else:
stamp, identifier = after_key
cur.execute(
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
'WHERE chat_id = %s '
'AND (created_at, id) > (%s, %s) '
'ORDER BY created_at, id '
'LIMIT %s',
(str(chat_id), stamp, str(identifier), limit + 1))
rows = cur.fetchall()
items = [self._public_message(row) for row in rows[:limit]]
if items:
by_id = {item['id']: item for item in items}
cur.execute(
'SELECT message_id, attachment_id FROM message_attachments '
'WHERE message_id=ANY(%s::uuid[]) ORDER BY message_id, position',
(list(by_id),))
for row in cur.fetchall():
by_id[str(row['message_id'])]['attachment_ids'].append(str(row['attachment_id']))
has_more = len(rows) > limit
return items, has_more
def page_chats(self, user_id, after_key, limit):
"""Return (chats visible to the user, has_more) ordered by (created_at, id)."""
with self._db.cursor() as cur:
if after_key is None:
cur.execute(
'SELECT c.id, c.title, c.created_at FROM chats c '
'JOIN chat_members cm ON cm.chat_id = c.id '
'WHERE cm.user_id = %s '
'ORDER BY c.created_at, c.id '
'LIMIT %s', (str(user_id), limit + 1))
else:
stamp, identifier = after_key
cur.execute(
'SELECT c.id, c.title, c.created_at FROM chats c '
'JOIN chat_members cm ON cm.chat_id = c.id '
'WHERE cm.user_id = %s '
'AND (c.created_at, c.id) > (%s, %s) '
'ORDER BY c.created_at, c.id '
'LIMIT %s',
(str(user_id), stamp, str(identifier), limit + 1))
rows = cur.fetchall()
has_more = len(rows) > limit
items = [self._public_chat(dict(row)) for row in rows[:limit]]
return items, has_more
# ----------------------------------------------------------- projection
def chat(self, chat_id):
with self._db.cursor() as cur:
cur.execute(
'SELECT id, title, created_at FROM chats WHERE id = %s', (str(chat_id),))
row = cur.fetchone()
if row is None:
return None
chat = self._public_chat(dict(row))
cur.execute(
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
chat['member_ids'] = [str(r['user_id']) for r in cur.fetchall()]
return chat
@staticmethod
def is_member(chat, user_id):
return str(user_id) in chat['member_ids']
@staticmethod
def _public_user(row):
return {
'id': str(row['id']),
'username': row['username'],
'display_name': row['display_name'],
'created_at': rfc3339(row['created_at']),
}
@staticmethod
def _public_chat(row):
return {
'id': str(row['id']),
'title': row['title'],
'member_ids': [], # filled by API when needed
'created_at': rfc3339(row['created_at']),
}
@staticmethod
def _public_message(row):
return {
'id': str(row['id']),
'chat_id': str(row['chat_id']),
'sender_id': str(row['sender_id']),
'text': row['text'],
'attachment_ids': [],
'created_at': rfc3339(row['created_at']),
}
# --------------------------------------------------------------- sessions
def create_session(self, token_hash, user_id, expires_at):
with self._db.cursor() as cur:
cur.execute(
'INSERT INTO sessions (token_hash, user_id, expires_at, created_at) '
'VALUES (%s, %s, %s, %s)',
(token_hash, str(user_id), expires_at, utc_now()))
ttl = max(1, int((expires_at - utc_now()).total_seconds()))
self._cache.set(token_hash, str(user_id), ttl)
def get_session(self, token_hash):
"""Return user_id for a valid non-expired session, or None.
The cache is consulted first, but every cache hit is re-validated
against PostgreSQL so that a revoked or expired session is never
trusted from a stale Redis key. On miss, read PostgreSQL and
backfill the cache. Expired sessions found during lookup are deleted.
"""
cached = self._cache.get(token_hash)
if cached is not None:
with self._db.cursor() as cur:
cur.execute(
'SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > %s',
(token_hash, utc_now()))
if cur.fetchone() is None:
self._cache.delete(token_hash)
return None
return cached
with self._db.cursor() as cur:
cur.execute(
'SELECT user_id, expires_at FROM sessions WHERE token_hash = %s',
(token_hash,))
row = cur.fetchone()
if row is None:
return None
if row['expires_at'] <= utc_now():
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
return None
user_id = str(row['user_id'])
ttl = max(1, int((row['expires_at'] - utc_now()).total_seconds()))
self._cache.set(token_hash, user_id, ttl)
return user_id
def delete_session(self, token_hash):
self._cache.delete(token_hash)
with self._db.cursor() as cur:
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
def cleanup_expired_sessions(self):
with self._db.cursor() as cur:
cur.execute('DELETE FROM sessions WHERE expires_at <= %s', (utc_now(),))
+142
View File
@@ -0,0 +1,142 @@
"""Strict request payload validation; any deviation is 400 invalid_request."""
from __future__ import annotations
import re
import uuid
from .errors import InvalidRequest
USERNAME_PATTERN = re.compile(r'[a-z0-9_]{3,32}')
DIGITS_PATTERN = re.compile(r'[0-9]+')
def safe_label(value):
"""Render untrusted text for an error message without breaking its encoding.
Field names come straight from the client JSON, so they may contain the same
lone surrogates that the payload itself is rejected for.
"""
text = value if isinstance(value, str) else repr(value)
return text.encode('utf-8', 'replace').decode('utf-8')
def require_encodable(value, name):
"""Reject text that cannot survive a UTF-8 round trip.
`json.loads` happily produces lone surrogates from `\ud800` escapes. Such a
string cannot be UTF-8 encoded, so it must be rejected before it is stored;
otherwise the write is committed and the response encoding fails afterwards.
"""
try:
value.encode('utf-8')
except UnicodeEncodeError:
raise InvalidRequest(
'Поле %s содержит недопустимые символы Unicode' % name) from None
return value
def require_object(payload):
if not isinstance(payload, dict):
raise InvalidRequest('Тело запроса должно быть JSON-объектом')
return payload
def require_fields(payload, required, allowed):
"""Reject missing required fields and any unknown field."""
missing = [name for name in required if name not in payload]
if missing:
raise InvalidRequest('Отсутствуют обязательные поля: ' + ', '.join(sorted(missing)))
extra = [name for name in payload if name not in allowed]
if extra:
raise InvalidRequest('Неизвестные поля: ' + ', '.join(sorted(safe_label(n) for n in extra)))
return payload
def require_text_field(payload, name, minimum, maximum, strip_check=True):
value = payload.get(name)
if not isinstance(value, str):
raise InvalidRequest('Поле %s должно быть строкой' % name)
require_encodable(value, name)
if strip_check and not value.strip():
raise InvalidRequest('Поле %s не может быть пустым или состоять из пробелов' % name)
if len(value) < minimum:
raise InvalidRequest('Поле %s короче %d символов' % (name, minimum))
if len(value) > maximum:
raise InvalidRequest('Поле %s длиннее %d символов' % (name, maximum))
return value
def require_username(payload):
username = payload.get('username')
# fullmatch, not match: a trailing "$" would also accept a final newline.
if not isinstance(username, str) or USERNAME_PATTERN.fullmatch(username) is None:
raise InvalidRequest('username должен соответствовать ^[a-z0-9_]{3,32}$')
return require_encodable(username, 'username')
def require_uuid_string(value, name):
if not isinstance(value, str):
raise InvalidRequest('Поле %s должно быть строкой UUID' % name)
try:
return str(uuid.UUID(value))
except ValueError:
raise InvalidRequest('Поле %s не является корректным UUID' % name) from None
def require_attachment_ids(payload):
value = payload.get('attachment_ids', [])
if not isinstance(value, list) or len(value) > 10:
raise InvalidRequest('attachment_ids должен содержать от 0 до 10 UUID')
identifiers = [require_uuid_string(item, 'attachment_ids') for item in value]
if len(set(identifiers)) != len(identifiers):
raise InvalidRequest('attachment_ids содержит повторяющиеся UUID')
return identifiers
def require_member_ids(payload, maximum, minimum=1):
value = payload.get('member_ids')
if not isinstance(value, list):
raise InvalidRequest('member_ids должен быть массивом UUID')
if not minimum <= len(value) <= maximum:
raise InvalidRequest('member_ids должен содержать от %d до %d участников' % (minimum, maximum))
identifiers = [require_uuid_string(item, 'member_ids') for item in value]
if len(set(identifiers)) != len(identifiers):
raise InvalidRequest('member_ids содержит повторяющихся участников')
return identifiers
def parse_limit(raw, default, maximum):
"""Parse a limit query value strictly: absent means default, anything else must be a plain integer.
An empty value is a client error rather than an absent parameter, and the
digit run is length-bounded before int() because Python refuses to convert
strings longer than its integer-string limit, which would surface as a 500.
"""
if raw is None:
return default
if not DIGITS_PATTERN.fullmatch(raw):
raise InvalidRequest('limit должен быть целым числом от 1 до %d' % maximum)
# int() refuses absurdly long digit strings (ValueError -> 500). Comparing the
# significant digits against the maximum first rejects them without converting.
significant = raw.lstrip('0') or '0'
if len(significant) > len(str(maximum)):
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
value = int(significant)
if not 1 <= value <= maximum:
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
return value
def parse_cursor(raw):
if raw is None:
return None
if not isinstance(raw, str) or not raw:
raise InvalidRequest('cursor должен быть непустой строкой')
return raw
def require_idempotency_key(value):
"""Only the contract's bounded ASCII token is accepted."""
if not isinstance(value, str) or re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', value) is None:
raise InvalidRequest('Требуется корректный Idempotency-Key (1–128 ASCII символов)')
return value
+148
View File
@@ -0,0 +1,148 @@
"""Process attachment jobs and acknowledge only committed results."""
import hashlib
from io import BytesIO
import json
import logging
import signal
import tempfile
from uuid import UUID
from PIL import Image, ImageOps
import pika
from .attachments import AttachmentStorage, MAX_UPLOAD_BYTES, validate_image
from .config import Config
from .db import Database
from .errors import NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
from .job_lifecycle import claim, complete
from .publish_jobs import QUEUE, connect
log = logging.getLogger(__name__)
def fitted(image, bound):
width, height = image.size
longest = max(width, height)
if longest <= bound:
return image.copy()
return image.resize((max(1, width * bound // longest),
max(1, height * bound // longest)), Image.Resampling.LANCZOS)
def save_variants(source, storage, prefix):
with Image.open(source, formats=['PNG', 'JPEG']) as original:
with ImageOps.exif_transpose(original) as corrected:
with corrected.convert('RGBA') as rgba:
rgb = Image.new('RGB', rgba.size, 'white')
rgb.paste(rgba, mask=rgba.getchannel('A'))
with rgb:
specs = (
('thumbnail', 'jpeg', 85, lambda: fitted(rgb, 256)),
('crop', 'jpeg', 85, lambda: ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS)),
('optimized', 'webp', 82, lambda: fitted(rgb, 1280)),
)
results = []
for name, fmt, quality, render in specs:
with render() as image, BytesIO() as file:
image.save(file, format=fmt.upper(), quality=quality)
size = file.tell()
digest = hashlib.sha256(file.getbuffer()).hexdigest()
file.seek(0)
key = f'{prefix}/{name}.{fmt}'
mime = f'image/{fmt}'
storage.put(key, file, size, mime)
results.append((name, key, mime, size, digest, image.width, image.height))
return results
def materialize(attachment, storage, version, attempt):
prefix = f"attachments/final/{attachment['id']}/v{version}/a{attempt}"
key = prefix + '/original'
stream = storage.download(attachment)
try:
with tempfile.TemporaryFile() as source:
digest, size = hashlib.sha256(), 0
for chunk in stream.chunks():
size += len(chunk)
if size > MAX_UPLOAD_BYTES:
raise PayloadTooLarge()
digest.update(chunk)
source.write(chunk)
if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']:
raise UnsupportedMediaType('Контрольная сумма оригинала не совпадает')
validate_image(source, attachment['content_type'])
source.seek(0)
variants = save_variants(source, storage, prefix)
source.seek(0)
storage.put(key, source, size, attachment['content_type'])
finally:
stream.close()
return key, variants
def process(database, storage, job):
attachment = claim(database, job)
if attachment is None:
return
try:
result = materialize(attachment, storage, job['pipeline_version'], job['attempt'])
except (PayloadTooLarge, UnsupportedMediaType, NotFound):
complete(database, job, error_code='invalid_image')
except (Unavailable, OSError):
complete(database, job)
else:
complete(database, job, result=result)
def parse_job(body):
job = json.loads(body)
if not isinstance(job, dict) or job.get('type') != QUEUE or job.get('schema_version') != 1:
raise ValueError('Unknown job format')
for field in ('job_id', 'attachment_id'):
UUID(job[field])
for field in ('pipeline_version', 'attempt'):
if type(job[field]) is not int or job[field] < 1:
raise ValueError('Invalid job number')
return job
def main():
logging.basicConfig(level=logging.INFO)
database = Database(Config.from_env())
database.open()
storage = AttachmentStorage()
connection = None
def timeout(_signal, _frame):
raise TimeoutError('Attachment processing exceeded 20 seconds')
signal.signal(signal.SIGALRM, timeout)
def consume(channel, method, _properties, body):
try:
job = parse_job(body)
except (ValueError, KeyError, TypeError, AttributeError):
channel.basic_reject(method.delivery_tag, requeue=False)
return
signal.alarm(20)
try:
process(database, storage, job)
finally:
signal.alarm(0)
channel.basic_ack(method.delivery_tag)
try:
connection = connect()
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue=QUEUE, on_message_callback=consume)
channel.start_consuming()
finally:
if connection and connection.is_open:
connection.close()
database.close()
if __name__ == '__main__':
main()
+106
View File
@@ -0,0 +1,106 @@
"""Run cleanup scenarios against a disposable PostgreSQL database."""
import os
import sys
from pathlib import Path
from uuid import uuid4
import psycopg2
from psycopg2 import sql
from psycopg2.extensions import make_dsn
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
from messenger.config import Config
from messenger.db import Database
from messenger.errors import Unavailable
from messenger.store import Store
GRACE_SECONDS = 900
def unavailable(_key):
raise Unavailable('synthetic S3 failure')
def expect_unavailable(operation):
try:
operation()
except Unavailable:
return
raise AssertionError('Expected Unavailable')
def exercise(database):
store = Store(None, database, None)
user, chat = str(uuid4()), str(uuid4())
with database.cursor() as cur:
cur.execute("INSERT INTO users VALUES (%s,'cleanup','unused','Cleanup')", (user,))
cur.execute("INSERT INTO chats VALUES (%s,'Cleanup')", (chat,))
def reserve(expired=False):
aid = str(uuid4())
key = 'attachments/' + aid
store.reserve_attachment(aid, chat, user, 'x.png', 'image/png', 1, 'a' * 64, key)
if expired:
with database.cursor() as cur:
cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (aid,))
return aid, key
objects = set()
fresh, fresh_key = reserve()
objects.add(fresh_key)
ready, ready_key = reserve(expired=True)
store.finish_attachment(ready, objects.add)
with database.cursor() as cur:
cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (ready,))
active, active_key = reserve(expired=True)
def active_put(key):
objects.add(key)
assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
store.finish_attachment(active, active_put)
failed, failed_key = reserve(expired=True)
def put_then_fail(key):
objects.add(key)
# Inject a failure after the object exists and before the ready update.
raise Unavailable('synthetic failure after PUT')
expect_unavailable(lambda: store.finish_attachment(failed, put_then_fail))
assert failed_key in objects
expect_unavailable(lambda: store.clean_pending_attachment(GRACE_SECONDS, unavailable))
with database.cursor() as cur:
cur.execute('SELECT status FROM attachments WHERE id=%s', (failed,))
assert cur.fetchone()['status'] == 'pending'
assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
assert objects == {fresh_key, ready_key, active_key}
reserve(expired=True) # Reservation whose PUT never created an object.
assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
with database.cursor() as cur:
cur.execute('SELECT id::text AS id, status FROM attachments')
assert {row['id']: row['status'] for row in cur.fetchall()} == {
fresh: 'pending', ready: 'ready', active: 'ready',
}
print('PASS: PostgreSQL row lock protects active PUT; fresh and ready attachments preserved')
print('PASS: interrupted PUT and failure after PUT cleaned; S3 failure retries safely')
def main():
dsn = os.environ['LAB4_TEST_DATABASE_URL']
schema = 'cleanup_' + uuid4().hex
with psycopg2.connect(dsn) as conn, conn.cursor() as cur:
cur.execute(sql.SQL('CREATE SCHEMA {}').format(sql.Identifier(schema)))
config = Config.from_env({'DATABASE_URL': make_dsn(dsn, options='-csearch_path=' + schema)})
database = Database(config)
try:
database.open()
database.run_migrations()
exercise(database)
finally:
database.close()
with psycopg2.connect(dsn) as conn, conn.cursor() as cur:
cur.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(schema)))
if __name__ == '__main__':
main()
+68
View File
@@ -0,0 +1,68 @@
"""Check upload errors through Nginx and exercise an actual S3 outage."""
from io import BytesIO
from pathlib import Path
import subprocess
import sys
import time
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, arguments, check, run
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file, timeout=45)
alice, _, eve, chat = client.bootstrap()
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
path = f"/api/v1/chats/{chat['id']}/attachments"
good = path + '?filename=sample.png'
item, _ = client.attachment(alice, chat)
content = f"/api/v1/attachments/{item['id']}/content"
def upload(url, payload, expected, mime='image/png', actor=alice):
return client.call('POST', url, expected, actor=actor, raw=payload,
headers={'Content-Type': mime})
huge = BytesIO()
Image.new('1', (8001, 5000)).save(huge, 'PNG')
for payload, expected, mime in [
(b'x' * (10 * 1024 * 1024 + 1), 413, 'image/png'),
(huge.getvalue(), 413, 'image/png'),
(data[:30], 415, 'image/png'),
(data, 415, 'image/jpeg'),
]:
upload(good, payload, expected, mime)
upload(path + '?filename=a%2Fb.png', data, 400)
upload(good, data, 404, actor=eve)
client.call('GET', content, 404, actor=eve)
print('PASS: size, pixel count, corruption, MIME, filename and chat ACL over Nginx')
def s3_command(action):
subprocess.run(['docker', 'compose', action, 's3'], cwd=ROOT, check=True, timeout=30)
try:
s3_command('stop')
upload(good, data, 503)
client.call('GET', content, 503, actor=alice)
client.message(alice, chat, 'Текст при недоступном S3')
finally:
s3_command('start')
started = time.monotonic()
for _ in range(40):
try:
downloaded = client.call('GET', content, actor=alice)
break
except Failure:
time.sleep(.25)
else:
raise Failure('S3 download did not recover after restart')
check(downloaded['bytes'] == data, 'original changed after S3 restart')
print(f'PASS: S3 outage returned 503, text remained available, original recovered in {time.monotonic()-started:.2f}s')
if __name__ == '__main__':
run(main)
+54
View File
@@ -0,0 +1,54 @@
"""Stop the actual upload replica and check original bytes after failover and restart."""
from hashlib import sha256
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file, timeout=55)
alice, bob, _, chat = client.bootstrap()
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
original = (ROOT / 'tests/fixtures/sample.png').read_bytes()
reply = client.call('POST', path, 201, actor=alice, raw=original,
headers={'Content-Type': 'image/png'}, schema='Attachment')
aid = reply['json']['id']
upload_instance = reply['headers']['X-Instance-Id']
digest = sha256(original).hexdigest()
content = '/api/v1/attachments/' + aid + '/content'
text = client.message(alice, chat, '', attachments=[aid])['json']
def verify():
received = client.call('GET', content, actor=bob)
check(received['bytes'] == original and sha256(received['bytes']).hexdigest() == digest,
'original changed')
history = client.call('GET', f"/api/v1/chats/{chat['id']}/messages",
actor=bob, schema='MessagePage')['json']
check(text in history['items'], 'message attachment link lost')
return received['headers']['X-Instance-Id']
start = time.monotonic()
try:
subprocess.run(['scripts/stop-one.sh'], cwd=ROOT, check=True, timeout=30,
env={**os.environ, 'TARGET_INSTANCE_ID': upload_instance})
instance = verify()
check(instance != upload_instance, 'download was served by the stopped uploader')
print(f'PASS: {upload_instance} -> {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s')
finally:
subprocess.run([sys.executable, 'scripts/manage.py', 'restore'], cwd=ROOT, check=True, timeout=100)
start = time.monotonic()
subprocess.run(['scripts/restart.sh'], cwd=ROOT, check=True, timeout=100)
instance = verify()
print(f'PASS: after full restart via {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s')
if __name__ == '__main__':
run(main)
+141
View File
@@ -0,0 +1,141 @@
from datetime import datetime, timezone
from hashlib import sha256
from io import BytesIO
import logging
import os
import http.client
import socket
from pathlib import Path
import threading
from urllib.error import HTTPError
from urllib.request import Request, urlopen
from uuid import uuid4
import json
import sys
from botocore.response import StreamingBody
from PIL import Image
root = Path(__file__).resolve().parents[1]
os.environ['S3_BUCKET'] = 'test'
sys.path.insert(0, str(root / 'src'))
from messenger.api import Api
from messenger.config import Config
from messenger.security import hash_token
from messenger.server import build_server
from messenger.store import Store
chat, member, outsider = [str(uuid4()) for _ in range(3)]
class Metadata:
rows = {}
def get_session(self, token):
return {hash_token('member'): member, hash_token('outsider'): outsider}.get(token)
def user(self, uid):
return {'id': uid}
def chat(self, cid):
return {'id': cid} if cid == chat else None
def is_member(self, _chat, uid):
return uid == member
def reserve_attachment(self, aid, cid, uid, filename, mime, size, digest, key):
now = datetime.now(timezone.utc)
self.rows[aid] = dict(id=aid, chat_id=cid, owner_id=uid, filename=filename,
content_type=mime, size_bytes=size, sha256=digest,
object_key=key, status='pending', error_code=None,
created_at=now, updated_at=now)
def finish_attachment(self, aid, put_object):
put_object(self.rows[aid]['object_key'])
self.rows[aid]['status'] = 'ready'
return self.rows[aid]
def visible_attachment(self, aid, uid):
row = self.rows.get(aid)
if uid == member and row and row['status'] == 'ready':
return row
return None
public_attachment = staticmethod(Store.public_attachment)
class S3:
def __init__(self):
self.objects = {}
self.fail = False
def put_object(self, **kw):
if self.fail:
from botocore.exceptions import EndpointConnectionError
raise EndpointConnectionError(endpoint_url='http://s3.invalid')
data = kw['Body'].read()
assert len(data) == kw['ContentLength']
self.objects[kw['Key']] = data
def get_object(self, **kw):
data = self.objects[kw['Key']]
return {'ContentLength': len(data), 'Body': StreamingBody(BytesIO(data), len(data))}
config = Config.from_env({'APP_HOST':'127.0.0.1','APP_PORT':'8080'})
config.port = 0
api = Api(config, Metadata(), None)
s3 = S3()
api.attachments.client = s3
server = build_server(config, logging.getLogger('lab4-check'), api, None)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
base = 'http://127.0.0.1:' + str(server.server_address[1])
upload = f'/api/v1/chats/{chat}/attachments?filename=x.png'
def call(path, method='GET', data=None, mime='image/png', token='member'):
headers = {'Authorization': 'Bearer ' + token, 'Content-Type': mime}
try:
response = urlopen(Request(base + path, data=data, headers=headers, method=method), timeout=12)
except HTTPError as exc:
response = exc
with response:
return response.code, response.headers, response.read()
try:
for image_format, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]:
buf = BytesIO()
Image.new('RGB', (10, 10), 'green').save(buf, format=image_format)
data = buf.getvalue()
status, _, raw = call(upload, 'POST', data, mime)
assert status == 201, (status, raw)
item = json.loads(raw)
assert item['status'] == 'ready' and item['sha256'] == sha256(data).hexdigest()
assert item['size_bytes'] == len(data) and 'object_key' not in item
path = '/api/v1/attachments/' + item['id']
assert json.loads(call(path)[2]) == item
status, headers, actual = call(path + '/content')
assert (status, headers['Content-Type'], actual) == (200, mime, data)
assert call(path, token='outsider')[0] == 404
assert call(path + '/content', token='outsider')[0] == 404
assert call(path + '/content?variant=thumbnail')[0] == 404
png = (root / 'tests/fixtures/sample.png').read_bytes()
assert call(upload, 'POST', png)[0] == 201
assert call(upload, 'POST', png, 'image/jpeg')[0] == 415
assert call(upload, 'POST', png[:30])[0] == 415
assert call(upload.replace('x.png','a%2Fb.png'), 'POST', png)[0] == 400
before = len(api.store.rows)
for length, partial, expected in [(10 * 1024 * 1024 + 1, b'', 413), (64, b'abc', 400)]:
conn = http.client.HTTPConnection(*server.server_address, timeout=5)
try:
conn.request('POST', upload, body=partial, headers={
'Authorization': 'Bearer member', 'Content-Type': 'image/png',
'Content-Length': str(length)})
if partial:
conn.sock.shutdown(socket.SHUT_WR)
with conn.getresponse() as response:
assert response.status == expected, response.status
response.read()
finally:
conn.close()
assert len(api.store.rows) == before
buf = BytesIO()
Image.new('1', (8001, 5000)).save(buf, 'PNG')
assert call(upload, 'POST', buf.getvalue())[0] == 413
s3.fail = True
assert call(upload, 'POST', png)[0] == 503
assert list(api.store.rows.values())[-1]['status'] == 'pending'
print('PASS: local HTTP PNG/JPEG round trip, checksum, metadata, ACL and missing variant')
print('PASS: malformed image, mismatched MIME, unsafe filename, byte/pixel limits and S3 error mapping')
print('NOTE: metadata and S3 are test substitutes; real Compose integration not exercised')
finally:
server.shutdown()
server.server_close()
thread.join()
+98
View File
@@ -0,0 +1,98 @@
"""Exercise broker persistence and kill a worker after S3 writes, before SQL commit/ACK."""
from pathlib import Path
import subprocess
import sys
import time
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
def cmd(*args, capture=False):
return subprocess.run(args, cwd=ROOT, text=True, check=True, timeout=100,
stdout=subprocess.PIPE if capture else None).stdout
def compose(*args, capture=False):
return cmd('docker', 'compose', *args, capture=capture)
def wait_for(probe, timeout, label):
until = time.monotonic() + timeout
while time.monotonic() < until:
if probe():
return
time.sleep(.25)
raise RuntimeError(label)
def confirmed(aid):
count = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-At', '-c', f"SELECT count(*) FROM attachment_outbox o JOIN attachment_jobs j "
f"ON j.id=o.job_id WHERE j.attachment_id='{aid}' AND o.published_at IS NOT NULL",
capture=True)
return int(count.strip()) > 0
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
marker = 'AFTER_S3_BEFORE_COMMIT'
fault_worker = 'lab5-fault-' + uuid4().hex[:10]
compose('stop', 'worker')
try:
alice, bob, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
client.message(alice, chat, '', attachments=[attachment['id']])
path = '/api/v1/attachments/' + attachment['id']
client.call('GET', path + '/content', 409, actor=bob)
wait_for(lambda: confirmed(attachment['id']), 30, 'no publisher confirm')
compose('restart', 'rabbitmq', 'api-a', 'api-b')
compose('up', '-d', '--no-build', '--no-deps', '--wait', 'api-a', 'api-b')
queued = client.call('GET', path, actor=bob, schema='Attachment')['json']
check(queued['status'] == 'queued', 'job lost during restart')
compose('start', 'worker')
client.wait_ready(bob, attachment)
check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original lost')
print('PASS: worker stopped; after API and RabbitMQ restart the same attachment became ready', flush=True)
compose('stop', 'worker')
attachment, original = client.attachment(alice, chat)
path = '/api/v1/attachments/' + attachment['id']
code = """
import sys,time
from messenger import worker
commit = worker.complete
def pause(db, job, result=None, error_code=None):
if job['attachment_id'] == sys.argv[1] and result is not None:
print('AFTER_S3_BEFORE_COMMIT', flush=True)
time.sleep(120)
return commit(db, job, result, error_code)
worker.complete = pause
worker.main()
"""
compose('run', '-d', '--no-deps', '--name', fault_worker, 'worker',
'python', '-c', code, attachment['id'])
wait_for(lambda: marker in cmd('docker', 'logs', fault_worker, capture=True),
40, 'worker did not reach fault point')
cmd('docker', 'kill', fault_worker)
compose('start', 'worker')
start = time.monotonic()
def is_ready():
state = client.call('GET', path, actor=bob, schema='Attachment')['json']
check(state['status'] != 'failed', 'processing failed after crash')
return state if state['status'] == 'ready' else None
wait_for(is_ready, 90, 'expired processing lease did not recover')
ready = is_ready()
check(len(ready['variants']) == 3, 'expected exactly one set of variants')
check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original changed')
print(f'PASS: worker killed after S3 write before COMMIT/ACK; recovered in {time.monotonic()-start:.2f}s')
finally:
subprocess.run(['docker', 'rm', '-f', fault_worker], cwd=ROOT, capture_output=True, timeout=15)
compose('start', 'worker')
if __name__ == '__main__':
run(main)
+89
View File
@@ -0,0 +1,89 @@
"""Exercise job fencing, failed replay and cleanup on the stage-5 stand."""
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
code = """
import sys
from messenger.attachments import AttachmentStorage
from messenger.config import Config
from messenger.db import Database
from messenger.errors import Conflict
import messenger.job_lifecycle as lifecycle
from messenger.worker import materialize, process
db = Database(Config.from_env())
db.open()
s3 = AttachmentStorage()
aid = sys.argv[1]
def delivery():
with db.cursor() as cur:
cur.execute('SELECT id AS job_id,attachment_id,pipeline_version,attempt FROM attachment_jobs WHERE attachment_id=%s', (aid,))
return cur.fetchone()
try:
first = delivery()
row = lifecycle.claim(db, first)
assert row['status'] == 'processing' and lifecycle.claim(db, first) is None
stale_result = materialize(row, s3, first['pipeline_version'], first['attempt'])
with db.cursor() as cur:
cur.execute("UPDATE attachment_jobs SET lease_until=now()-interval '1 second' WHERE id=%s", (first['job_id'],))
lifecycle.recover_expired(db)
second = delivery()
assert second['attempt'] == first['attempt'] + 1
assert not lifecycle.complete(db, first, result=stale_result)
assert lifecycle.claim(db, second)
lifecycle.MAX_ATTEMPTS = 2
assert lifecycle.complete(db, second)
with db.cursor() as cur:
cur.execute('SELECT status,error_code FROM attachments WHERE id=%s', (aid,))
assert cur.fetchone() == {'status':'failed','error_code':'retry_exhausted'}
assert lifecycle.replay(db, first['job_id'], 'stage5-test') == 3
third = delivery()
process(db, s3, third)
process(db, s3, third)
try:
lifecycle.replay(db, first['job_id'], 'stage5-test')
except Conflict:
pass
else:
raise AssertionError('ready job was replayed')
with db.cursor() as cur:
cur.execute('SELECT count(*) AS n FROM attachment_variants WHERE attachment_id=%s', (aid,))
assert cur.fetchone()['n'] == 3
cur.execute('SELECT count(*) AS n FROM attachment_replays WHERE job_id=%s', (first['job_id'],))
assert cur.fetchone()['n'] == 1
cur.execute("UPDATE attachment_outbox SET available_at=now()-interval '16 minutes' WHERE job_id=%s AND attempt=1",
(first['job_id'],))
# Check that the cleaner can remove an obsolete attempt even after replay succeeded.
removed = []
while lifecycle.clean_abandoned_outputs(db, lambda key: (s3.delete(key), removed.append(key))):
pass
assert stale_result[0] in removed, removed
while lifecycle.clean_staging(db, s3.delete):
pass
with db.cursor() as cur:
cur.execute('SELECT staging_cleaned_at,lease_until FROM attachment_jobs WHERE id=%s', (first['job_id'],))
row = cur.fetchone()
assert row['staging_cleaned_at'] is not None and row['lease_until'] is None
print('PASS: expired claim fenced, retry exhausted, audited replay, stale objects and staging cleaned')
finally:
db.close()
"""
subprocess.run(['docker', 'compose', 'exec', '-T', 'api-a', 'python', '-c',
code, attachment['id']], cwd=ROOT, check=True, timeout=45)
path = f"/api/v1/attachments/{attachment['id']}/content"
check(client.call('GET', path, actor=bob)['bytes'] == original, 'cleanup damaged original')
print('PASS: ready original survives cleanup and replay requests')
if __name__ == '__main__':
run(main)
+53
View File
@@ -0,0 +1,53 @@
"""Exercise attachment links and message replay over the running lab4 API."""
import sys
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments('Message attachment links and replay')
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
first, _ = client.attachment(alice, chat)
second, _ = client.attachment(bob, chat)
ids = [second['id'], first['id']]
key = uuid.uuid4().hex
sent = client.message(bob, chat, '', key=key, attachments=ids)['json']
replay = client.message(bob, chat, '', key=key, attachments=ids)['json']
check(sent == replay and sent['attachment_ids'] == ids, 'ordered attachment replay mismatch')
path = f"/api/v1/chats/{chat['id']}/messages"
client.call('POST', path, 409, actor=bob, body={'text': '', 'attachment_ids': ids[::-1]},
headers={'Idempotency-Key': key})
for invalid in [[ids[0]] * 2, [str(uuid.uuid4()) for _ in range(11)], ['bad UUID'], None]:
client.call('POST', path, 400, actor=bob, body={'text': '', 'attachment_ids': invalid},
headers={'Idempotency-Key': uuid.uuid4().hex})
client.call('POST', path, 400, actor=bob, body={'text': ''},
headers={'Idempotency-Key': uuid.uuid4().hex})
client.call('POST', path, 404, actor=bob,
body={'text': '', 'attachment_ids': [str(uuid.uuid4())]},
headers={'Idempotency-Key': uuid.uuid4().hex})
other = client.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id'], eve['id']]},
schema='Chat')['json']
foreign, _ = client.attachment(alice, other)
client.call('POST', path, 404, actor=alice,
body={'text': '', 'attachment_ids': [foreign['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex})
plain_key = uuid.uuid4().hex
plain = client.message(alice, chat, 'Обычный текст', key=plain_key)['json']
check(client.message(alice, chat, 'Обычный текст', key=plain_key, attachments=[])['json'] == plain,
'omitted and empty attachment_ids must replay identically')
page = client.call('GET', path, actor=alice, schema='MessagePage')['json']
check([item for item in page['items'] if item['id'] == sent['id']] == [sent],
'ordered attachment IDs missing or duplicated in history')
print('PASS: ordered attachments, replay, invalid input, cross-chat rejection and history')
if __name__ == '__main__':
run(main)
+27
View File
@@ -0,0 +1,27 @@
#!/bin/sh
# Run against a lab5 stage-3 stand without a worker.
set -eu
cd "$(dirname "$0")/.."
base_url=${BASE_URL:-http://localhost:18082}
sql() {
docker compose exec -T postgres psql -U messenger -d messenger -At -v ON_ERROR_STOP=1 -c "$1"
}
docker compose stop outbox-publisher
trap 'docker compose start outbox-publisher >/dev/null' EXIT
upload=$(PYTHONDONTWRITEBYTECODE=1 python3 student-tests/queued_upload.py --base-url "$base_url")
printf '%s\n' "$upload"
aid=$(printf '%s\n' "$upload" | awk '/Attachment ID:/ {print $3}')
query="SELECT count(*) FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='$aid' AND o.published_at IS"
test "$(sql "$query NULL")" = 1
echo 'PASS: upload committed while publisher was stopped'
docker compose start outbox-publisher
trap - EXIT
for i in $(seq 1 40); do
if test "$(sql "$query NOT NULL")" = 1; then
echo 'PASS: same attachment job confirmed after publisher startup'
exit 0
fi
sleep 1
done
echo 'FAIL: outbox row not confirmed' >&2
exit 1
+27
View File
@@ -0,0 +1,27 @@
"""Verify durable handoff before a publisher or worker is enabled."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
attachment, _ = client.attachment(alice, chat)
path = '/api/v1/attachments/' + attachment['id']
check(attachment['status'] == 'queued', 'expected queued with worker disabled')
metadata = client.call('GET', path, actor=bob, schema='Attachment')['json']
check(metadata == attachment, 'queued metadata differs')
client.call('GET', path, 404, actor=eve)
client.call('GET', path + '/content', 409, actor=bob)
sent = client.message(alice, chat, '', attachments=[attachment['id']])['json']
check(sent['attachment_ids'] == [attachment['id']], 'queued attachment not linked')
print('PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked')
print('Attachment ID:', attachment['id'])
if __name__ == '__main__':
run(main)
+72
View File
@@ -0,0 +1,72 @@
"""Run via docker compose exec -T api-a python - with this file on stdin."""
from hashlib import sha256
from io import BytesIO
from uuid import uuid4
from botocore.exceptions import ClientError
from PIL import Image
from messenger.attachments import AttachmentStorage
from messenger.clean_attachments import GRACE_SECONDS, clean_batch
from messenger.config import Config
from messenger.db import Database
from messenger.errors import Unavailable
from messenger.store import Store
def main():
config = Config.from_env()
db = Database(config)
db.open()
store = Store(config, db, None)
s3 = AttachmentStorage()
user, chat, aid = [str(uuid4()) for _ in range(3)]
key = 'attachments/' + aid
payload = BytesIO()
Image.new('RGB', (2, 2), 'green').save(payload, format='PNG')
data = payload.getvalue()
try:
with db.cursor() as cur:
cur.execute('INSERT INTO users (id,username,password_hash,display_name) VALUES (%s,%s,%s,%s)',
(user, 'gc_' + user[:8], 'unused', 'Cleanup check'))
cur.execute("INSERT INTO chats (id,title) VALUES (%s,'Cleanup check')", (chat,))
store.reserve_attachment(aid, chat, user, 'probe.png', 'image/png',
len(data), sha256(data).hexdigest(), key)
def put_then_fail(object_key):
s3.put(object_key, BytesIO(data), len(data), 'image/png')
raise Unavailable('synthetic failure after successful PUT')
try:
store.finish_attachment(aid, put_then_fail)
except Unavailable as exc:
assert str(exc) == 'synthetic failure after successful PUT'
else:
raise AssertionError('fault did not run')
obj = s3.client.get_object(Bucket=s3.bucket, Key=key)
with obj['Body'] as body:
assert body.read() == data
with db.cursor() as cur:
cur.execute('SELECT status FROM attachments WHERE id=%s', (aid,))
assert cur.fetchone()['status'] == 'pending'
cur.execute("UPDATE attachments SET updated_at=now()-make_interval(secs => %s) WHERE id=%s",
(GRACE_SECONDS + 1, aid))
clean_batch(store, s3)
with db.cursor() as cur:
cur.execute('SELECT 1 FROM attachments WHERE id=%s', (aid,))
assert cur.fetchone() is None
try:
obj = s3.client.get_object(Bucket=s3.bucket, Key=key)
except ClientError as exc:
assert exc.response['ResponseMetadata']['HTTPStatusCode'] == 404, exc
else:
obj['Body'].close()
raise AssertionError('orphan object still exists')
print('PASS: actual MinIO object existed after failed commit and was removed by cleanup')
finally:
db.close()
if __name__ == '__main__':
main()
+104
View File
@@ -0,0 +1,104 @@
"""Measure sampled cgroup memory during ten simultaneous HTTP uploads."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
from io import BytesIO
import json
import math
from pathlib import Path
import random
import subprocess
import sys
from threading import Barrier
import time
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, arguments, check, run
COUNT = 10
SAMPLER = """
import select, sys, time
from pathlib import Path
usage = Path('/sys/fs/cgroup/memory.current')
limit = Path('/sys/fs/cgroup/memory.max')
cpu = Path('/sys/fs/cgroup/cpu.stat')
print(limit.read_text().strip(), flush=True)
while True:
stats = dict(line.split() for line in cpu.read_text().splitlines())
print(usage.read_text().strip(), stats['usage_usec'], flush=True)
if select.select([sys.stdin], [], [], .05)[0]:
break
"""
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file, timeout=60)
alice, _, _, chat = client.bootstrap()
width = 1700
source = Image.frombytes('RGB', (width, width), random.Random(2026).randbytes(width * width * 3))
buffer = BytesIO()
source.save(buffer, format='PNG', compress_level=0)
payload = buffer.getvalue()
digest = hashlib.sha256(payload).hexdigest()
check(len(payload) <= 10 * 1024 * 1024, 'fixture exceeds upload size limit')
path = f"/api/v1/chats/{chat['id']}/attachments?filename=load.png"
barrier = Barrier(COUNT + 1)
samplers = {}
def upload():
local = Client(args.base_url, args.ca_file, timeout=60)
barrier.wait(timeout=30)
start = time.monotonic()
try:
reply = local.call('POST', path, 201, actor=alice, raw=payload,
headers={'Content-Type': 'image/png'}, schema='Attachment')
check(reply['json']['sha256'] == digest, 'upload hash mismatch')
return {'instance': reply['headers']['X-Instance-Id'], 'id': reply['json']['id'],
'seconds': time.monotonic() - start}
except Failure as exc:
return {'error': str(exc), 'seconds': time.monotonic() - start}
try:
for service in ('api-a', 'api-b'):
proc = subprocess.Popen(
['docker', 'compose', 'exec', '-T', service, 'python', '-u', '-c', SAMPLER],
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
samplers[service] = proc
with ThreadPoolExecutor(max_workers=COUNT) as pool:
futures = [pool.submit(upload) for _ in range(COUNT)]
limits = {service: int(proc.stdout.readline()) for service, proc in samplers.items()}
initial = {service: tuple(map(int, proc.stdout.readline().split()))
for service, proc in samplers.items()}
start = time.monotonic()
barrier.wait(timeout=30)
requests = [future.result() for future in futures]
elapsed = time.monotonic() - start
finally:
for proc in samplers.values():
if proc.poll() is None:
proc.stdin.close()
resources = {}
for service, proc in samplers.items():
samples = [initial[service]] + [tuple(map(int, line.split())) for line in proc.stdout]
check(proc.wait(timeout=10) == 0 and len(samples) > 1, 'memory sampler failed')
resources[service] = {
'baseline_bytes': samples[0][0], 'peak_sample_bytes': max(row[0] for row in samples),
'limit_bytes': limits[service], 'samples': len(samples),
'cpu_seconds': (samples[-1][1] - samples[0][1]) / 1e6,
}
successes = [row for row in requests if 'id' in row]
durations = sorted(row['seconds'] for row in successes)
percentiles = {f'p{p}': durations[math.ceil(p / 100 * len(durations)) - 1]
for p in (50, 95, 99)} if durations else {}
print(json.dumps({'concurrency': COUNT, 'payload_bytes': len(payload), 'sha256': digest,
'elapsed_seconds': elapsed, 'successful_rps': len(successes) / elapsed,
'latency_seconds': percentiles, 'requests': requests,
'memory_sample_interval_seconds': .05, 'resources': resources}, indent=2))
check(len({row['id'] for row in successes}) == COUNT, 'not all uploads succeeded uniquely')
if __name__ == '__main__':
run(main)
+62
View File
@@ -0,0 +1,62 @@
"""Verify worker effects through HTTP and MinIO with a direct job handoff."""
import hashlib
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
from image_probe import jpeg_size
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
code = """
from messenger.attachments import AttachmentStorage
from messenger.config import Config
from messenger.db import Database
from messenger.worker import process
import sys
db = Database(Config.from_env())
db.open()
try:
with db.cursor() as cur:
cur.execute('SELECT id AS job_id, attachment_id, pipeline_version, attempt '
'FROM attachment_jobs WHERE attachment_id=%s', (sys.argv[1],))
job = cur.fetchone()
storage = AttachmentStorage()
process(db, storage, job)
process(db, storage, job)
finally:
db.close()
"""
subprocess.run(['docker', 'compose', 'exec', '-T', 'api-a', 'python', '-c',
code, attachment['id']], cwd=ROOT, check=True, timeout=30)
path = '/api/v1/attachments/' + attachment['id']
ready = client.call('GET', path, actor=bob, schema='Attachment')['json']
check(ready['status'] == 'ready' and len(ready['variants']) == 3, 'expected three variants')
check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original changed')
expected = {'thumbnail': (256, 160, 'image/jpeg'), 'crop': (128, 128, 'image/jpeg'),
'optimized': (320, 200, 'image/webp')}
for item in ready['variants']:
width, height, mime = expected[item['name']]
reply = client.call('GET', path + '/content?variant=' + item['name'], actor=bob)
data = reply['bytes']
check((item['width'], item['height'], item['content_type']) == (width, height, mime),
'incorrect variant metadata')
check(reply['headers'].get_content_type() == mime and len(data) == item['size_bytes']
and hashlib.sha256(data).hexdigest() == item['sha256'], 'incorrect variant payload')
if mime == 'image/jpeg':
check(jpeg_size(data) == (width, height), 'invalid JPEG dimensions')
else:
check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'invalid WebP signature')
print('PASS: original unchanged, thumbnail 256x160, crop 128x128, WebP 320x200')
print('PASS: processing the same committed job twice keeps one set of variants')
if __name__ == '__main__':
run(main)