98 changed files with 5837 additions and 18 deletions
+23
View File
@@ -20,6 +20,29 @@ MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
WORKER_PREFETCH=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_LEASE_SECONDS=90
RETRY_BASE_SECONDS=2
RETRY_MAX_SECONDS=60
STAGING_RETENTION_SECONDS=86400
THUMBNAIL_JPEG_QUALITY=85
CROP_JPEG_QUALITY=85
OPTIMIZED_WEBP_QUALITY=82
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change-me-local
POSTGRES_DB=messenger
DATABASE_URL=postgresql+asyncpg://messenger:change-me-local@postgres:5432/messenger
REDIS_URL=redis://redis:6379/0
BROKER_USER=messenger
BROKER_PASSWORD=change-me-rabbitmq
BROKER_URL=amqp://messenger:change-me-rabbitmq@broker:5672/
MINIO_ROOT_USER=local-minio-admin
MINIO_ROOT_PASSWORD=change-me-minio-root
S3_ACCESS_KEY_ID=messenger-api
S3_SECRET_ACCESS_KEY=change-me-minio-api
LOG_LEVEL=info
+17
View File
@@ -0,0 +1,17 @@
FROM python:3.13.15-slim
WORKDIR /app
COPY requirements.txt .
RUN python -m pip install -r requirements.txt \
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app
COPY app ./app
COPY alembic ./alembic
COPY alembic.ini .
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=3).read()"]
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"]
+15
View File
@@ -0,0 +1,15 @@
FROM python:3.13.15-slim
WORKDIR /tests
COPY requirements.txt requirements-dev.txt ./
RUN python -m pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt \
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin tests
COPY lab.json .
COPY app ./app
COPY contracts ./contracts
COPY tests ./tests
RUN chown -R tests:tests /tests
USER 10001:10001
CMD ["python", "-m", "pytest", "-q"]
+56 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test test-attachments test-pipeline test-pipeline-smoke test-outbox test-publisher-confirm-crash test-worker-crash test-failed-replay test-worker-outage test-broker-persistence test-docker load test-persistence test-failover replay-failed cleanup-staging
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -18,6 +18,54 @@ validate:
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachments:
$(PYTHON) -m pytest -q tests/test_attachments.py
test-pipeline:
$(PYTHON) -m pytest -q tests/test_pipeline.py
test-pipeline-smoke:
$(PYTHON) tests/pipeline_smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-outbox:
$(PYTHON) tests/outbox_recovery.py --base-url "$(BASE_URL)" \
--stop-publisher-script scripts/stop-publisher.sh \
--start-publisher-script scripts/start-publisher.sh $(TLS_ARGS)
test-publisher-confirm-crash:
$(PYTHON) tests/publisher_confirm_crash.py --base-url "$(BASE_URL)" \
--stop-publisher-script scripts/stop-publisher.sh \
--crash-publisher-script scripts/crash-publisher-after-confirm.sh $(TLS_ARGS)
test-worker-crash:
$(PYTHON) tests/worker_crash.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--crash-worker-script scripts/crash-worker.sh --mode objects $(TLS_ARGS)
$(PYTHON) tests/worker_crash.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--crash-worker-script scripts/crash-worker.sh --mode commit $(TLS_ARGS)
test-failed-replay:
$(PYTHON) tests/failed_replay.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--crash-worker-script scripts/crash-worker.sh \
--replay-script scripts/replay-attachment.sh $(TLS_ARGS)
test-worker-outage:
$(PYTHON) tests/worker_outage.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--restart-api-script scripts/restart-api.sh \
--start-worker-script scripts/start-worker.sh $(TLS_ARGS)
test-broker-persistence:
$(PYTHON) tests/broker_persistence.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--restart-broker-script scripts/restart-broker.sh \
--start-worker-script scripts/start-worker.sh $(TLS_ARGS)
test-docker:
docker compose -f compose.tests.yaml run --rm tests
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
@@ -28,3 +76,10 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
replay-failed:
@test -n "$(JOB_ID)" || (echo "Set JOB_ID"; exit 2)
docker compose --env-file .env run --rm publisher python -m app.tools.replay_failed --job-id "$(JOB_ID)"
cleanup-staging:
docker compose --env-file .env run --rm worker python -m app.tools.cleanup_staging --dry-run
+115 -17
View File
@@ -1,37 +1,135 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Сергей Возжин, БПИ2301.
- Целевая оценка: 4.
- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/101
- Библиотеки: FastAPI, SQLAlchemy/asyncpg, aio-pika, Pillow, boto3, Redis.
При реализации использовалась помощь ИИ; решения проверены тестами из репозитория.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker с Compose v2, Make и Python 3.9+ для внешних smoke-скриптов.
Образы закреплены в Compose: Python 3.13.15, PostgreSQL 17.4, Redis 7.4.2,
RabbitMQ 4.1.4, HAProxy 3.2.26 и MinIO по digest.
```sh
cp .env.example .env
# Заменить все change-me-* локальными значениями; .env не коммитить.
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
docker compose --env-file .env ps
make test BASE_URL=http://localhost:8080
make test-pipeline-smoke BASE_URL=http://localhost:8080
```
Миграции выполняет отдельный `migrate` service до старта API/publisher/worker.
Обычная остановка сохраняет данные:
```sh
docker compose --env-file .env down
```
Намеренное удаление всех учебных данных выполняется только явной командой
`docker compose --env-file .env down -v`. Она необратимо удаляет volumes PostgreSQL,
Redis, RabbitMQ и MinIO.
Для воспроизводимых тестов используется изолированный стенд:
```sh
docker compose -f compose.tests.yaml up -d --build --wait
docker compose -f compose.tests.yaml run --rm tests
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Поток данных:
```text
HTTP API -> private S3 staging -> PostgreSQL attachment/job/outbox -> 202
|
v
outbox publisher -> RabbitMQ
|
v
worker pool
|
final S3 objects <- variants/original -> PostgreSQL ready
```
PostgreSQL — источник истины для состояния job, failed registry и решения о replay.
RabbitMQ обеспечивает durable очередь, publisher confirms, manual consumer ack и DLQ.
Сообщение содержит только versioned IDs и номера попытки, без байтов, credentials и
клиентского пути. Ответ 202 отправляется лишь после записи исходника в private staging
и одной транзакции `attachment + job + outbox`; broker может быть временно недоступен.
Идемпотентность основана на уникальной паре `(attachment_id, pipeline_version)` и
детерминированных S3 keys. Варианты upsert-ятся по
`(attachment_id, pipeline_version, name)`. `ready` проверяет полноту эффекта и не
исполняется повторно. Это at-least-once, а не exactly-once: delivery может повториться,
но логический результат остаётся один.
Worker создаёт неизменённый original, JPEG thumbnail в bounding box 256×256 без
увеличения, центрированный JPEG crop 128×128 и WebP optimized в bounding box
1280×1280 без увеличения. EXIF orientation применяется перед resize; alpha
компонуется на белый, входные metadata не копируются. Quality задаются через
`THUMBNAIL_JPEG_QUALITY=85`, `CROP_JPEG_QUALITY=85` и
`OPTIMIZED_WEBP_QUALITY=82`.
Временные ошибки повторяются с exponential backoff и jitter до `JOB_MAX_ATTEMPTS`.
Постоянные ошибки сразу переходят в `failed` с безопасным code и RabbitMQ DLQ.
Lease возвращает брошенный `processing` в очередь. Обработка изображения вынесена в
дочерний процесс с timeout и лимитом адресного пространства 512 MiB; дополнительно
ограничены upload bytes, decoded pixels, worker concurrency и prefetch.
Staging удаляется только после DB commit состояния `ready`. Отдельный cleanup удаляет
лишь terminal staging старше retention; `--dry-run` включён для операторской проверки.
Replay разрешён только для failed job с существующим staging, записывается в audit и
повторный replay ready job отклоняется.
Из lab4 сохранены auth/session, чаты/сообщения, идемпотентность HTTP, private S3 и
доступ к attachment только членам чата. Изменён только объявленный контракт upload
`201 -> 202` и асинхронные состояния. Старые lab4 benchmark/evidence и проверки
S3 outage/failover не копировались: они не доказывают новые критерии lab5; вместо них
добавлены pipeline/outbox/broker/worker/DLQ fault tests.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Полный журнал команд и кратких результатов находится в
[`evidence/lab5-grade4.md`](evidence/lab5-grade4.md).
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий | Команда/сценарий | Наблюдаемый результат |
| --- | --- | --- |
| Контракт и unit/negative | `make check`; Docker pytest | PASS; 19 passed |
| Публичный API | `python3 tests/smoke.py ...` | upload 202, polling ready, original/thumbnail и прежний API проходят |
| Grade-4 variants | `tests/pipeline_smoke.py`; `tests/test_pipeline.py` | original + thumbnail/crop/optimized, EXIF/alpha/no-upscale/checksum и большой IPC-result |
| COMMIT → publish | `tests/outbox_recovery.py` | после старта publisher прежняя job стала ready без нового upload |
| Confirm → outbox mark | `tests/publisher_confirm_crash.py` | повтор publish дал один логический результат |
| Worker outage | `tests/worker_outage.py` | stop → 202 → message → API restart → worker → ready |
| Broker persistence | `tests/broker_persistence.py` | durable delivery пережила restart broker с volume |
| Crash до ack | `tests/worker_crash.py`, оба режима | recovery после записи S3 и после DB commit; без дублей |
| failed/DLQ/replay | `tests/failed_replay.py`; `rabbitmqctl list_queues` | failed + safe code, DLQ=1, replay → ready, второй replay отклонён |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Целевая оценка 4, поэтому performance-исследование уровня 5 не заявляется. Числа RPS,
latency и RSS не приводятся: короткие функциональные fault tests для таких выводов
недостаточны.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| Воздействие | Ожидание | Наблюдение | Потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Publisher выключен после DB commit | pending outbox публикуется после восстановления | job дошла до ready без нового upload | 0 подтверждённых job |
| Worker выключен и API перезапущен | 202 не означает зависимость от процесса API | после запуска worker получен ready | 0 |
| Broker restart с volume | durable delivery остаётся | исходная job обработана | 0 |
| Worker crash после S3 effects | lease/retry повторяет идемпотентно | один original и три варианта | 0 |
| Worker crash после DB commit до ack | stale redelivery распознаётся | ready не откатился, дублей нет | 0 |
| Постоянная ошибка | failed registry + DLQ | safe `fault_injected`, DLQ message, guarded replay | 0; staging сохранён |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Это single-host Compose: отказ хоста, quorum broker/DB и multi-AZ не проверялись.
- RLIMIT_AS зависит от ОС; основной внешний предел остаётся Docker memory limit.
- Cleanup запускается оператором/планировщиком, отдельный scheduler service не добавлен.
- DLQ используется для наблюдаемости, а replay берёт решение из PostgreSQL; stale DLQ
сообщения намеренно не являются источником истины.
- Для оценки 5 нужны отдельный нагрузочный эксперимент или расширенная матрица reorder и
смены pipeline version; здесь такие выводы не заявляются.
+36
View File
@@ -0,0 +1,36 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = placeholder://configured-at-runtime
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
import asyncio
from logging.config import fileConfig
from alembic import context
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import async_engine_from_config
from app.config import load_settings
from app.db.postgres import Base
from app import models # noqa: F401 - imports ORM models into Base.metadata
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def _url() -> str:
return load_settings().database_url
def run_migrations_offline() -> None:
context.configure(url=_url(), target_metadata=target_metadata, literal_binds=True, dialect_opts={'paramstyle': 'named'})
with context.begin_transaction():
context.run_migrations()
def _run_migrations(connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def _run_migrations_async() -> None:
configuration = config.get_section(config.config_ini_section, {})
configuration['sqlalchemy.url'] = _url()
connectable = async_engine_from_config(configuration, prefix='sqlalchemy.', poolclass=pool.NullPool)
async with connectable.connect() as connection:
await connection.run_sync(_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(_run_migrations_async())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+22
View File
@@ -0,0 +1,22 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,64 @@
"""Initial PostgreSQL schema for lab 2.
Revision ID: 20261019_0001
Revises:
Create Date: 2026-10-19
"""
from alembic import op
import sqlalchemy as sa
revision = '20261019_0001'
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
'users',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('username', sa.String(length=32), nullable=False),
sa.Column('display_name', sa.String(length=100), nullable=False),
sa.Column('password_hash', sa.Text(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('username'),
)
op.create_table(
'chats',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('title', sa.String(length=100), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint('id'),
)
op.create_table(
'chat_members',
sa.Column('chat_id', sa.Uuid(), nullable=False),
sa.Column('user_id', sa.Uuid(), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('chat_id', 'user_id'),
)
op.create_index('chat_members_user_chat_idx', 'chat_members', ['user_id', 'chat_id'])
op.create_table(
'messages',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('chat_id', sa.Uuid(), nullable=False),
sa.Column('sender_id', sa.Uuid(), nullable=False),
sa.Column('text', sa.String(length=4000), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['sender_id'], ['users.id']),
sa.PrimaryKeyConstraint('id'),
)
op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id'])
def downgrade() -> None:
op.drop_index('messages_history_idx', table_name='messages')
op.drop_table('messages')
op.drop_index('chat_members_user_chat_idx', table_name='chat_members')
op.drop_table('chat_members')
op.drop_table('chats')
op.drop_table('users')
@@ -0,0 +1,35 @@
"""Add shared message idempotency for lab 3.
Revision ID: 20261023_0002
Revises: 20261019_0001
Create Date: 2026-10-23
"""
from alembic import op
import sqlalchemy as sa
revision = '20261023_0002'
down_revision = '20261019_0001'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
'messages',
sa.Column('idempotency_key', sa.String(length=128), nullable=True),
)
op.create_unique_constraint(
'messages_sender_chat_idempotency_key',
'messages',
['sender_id', 'chat_id', 'idempotency_key'],
)
def downgrade() -> None:
op.drop_constraint(
'messages_sender_chat_idempotency_key',
'messages',
type_='unique',
)
op.drop_column('messages', 'idempotency_key')
@@ -0,0 +1,69 @@
"""Add S3 attachment metadata and message links for lab 4.
Revision ID: 20261030_0003
Revises: 20261023_0002
Create Date: 2026-10-30
"""
from alembic import op
import sqlalchemy as sa
revision = '20261030_0003'
down_revision = '20261023_0002'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
'attachments',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('chat_id', sa.Uuid(), nullable=False),
sa.Column('owner_id', sa.Uuid(), nullable=False),
sa.Column('filename', sa.String(length=255), nullable=False),
sa.Column('object_key', sa.String(length=255), nullable=False),
sa.Column('content_type', sa.String(length=32), nullable=False),
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
sa.Column('sha256', sa.String(length=64), nullable=False),
sa.Column('width', sa.Integer(), nullable=False),
sa.Column('height', sa.Integer(), nullable=False),
sa.Column('status', sa.String(length=16), nullable=False),
sa.Column('error_code', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['owner_id'], ['users.id']),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('object_key', name='attachments_object_key_key'),
)
op.create_index(
'attachments_chat_created_idx',
'attachments',
['chat_id', 'created_at', 'id'],
)
op.create_table(
'message_attachments',
sa.Column('message_id', sa.Uuid(), nullable=False),
sa.Column('attachment_id', sa.Uuid(), nullable=False),
sa.Column('position', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('message_id', 'attachment_id'),
sa.UniqueConstraint(
'message_id',
'position',
name='message_attachments_message_position_key',
),
)
op.create_index(
'message_attachments_attachment_idx',
'message_attachments',
['attachment_id'],
)
def downgrade() -> None:
op.drop_index('message_attachments_attachment_idx', table_name='message_attachments')
op.drop_table('message_attachments')
op.drop_index('attachments_chat_created_idx', table_name='attachments')
op.drop_table('attachments')
@@ -0,0 +1,126 @@
"""Add asynchronous attachment jobs, variants and transactional outbox.
Revision ID: 20261116_0004
Revises: 20261030_0003
Create Date: 2026-11-16
"""
from alembic import op
import sqlalchemy as sa
revision = '20261116_0004'
down_revision = '20261030_0003'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.alter_column('attachments', 'object_key', new_column_name='original_key')
op.alter_column('attachments', 'original_key', nullable=True)
op.add_column('attachments', sa.Column('staging_key', sa.String(length=255), nullable=True))
op.add_column(
'attachments',
sa.Column('pipeline_version', sa.Integer(), nullable=False, server_default='1'),
)
op.create_unique_constraint('attachments_staging_key_key', 'attachments', ['staging_key'])
op.create_table(
'attachment_variants',
sa.Column('attachment_id', sa.Uuid(), nullable=False),
sa.Column('pipeline_version', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=16), nullable=False),
sa.Column('object_key', sa.String(length=255), nullable=False),
sa.Column('content_type', sa.String(length=32), nullable=False),
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
sa.Column('width', sa.Integer(), nullable=False),
sa.Column('height', sa.Integer(), nullable=False),
sa.Column('sha256', sa.String(length=64), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')", name='variant_name_check'),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('attachment_id', 'pipeline_version', 'name'),
sa.UniqueConstraint('object_key', name='attachment_variants_object_key_key'),
)
op.create_table(
'attachment_jobs',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('attachment_id', sa.Uuid(), nullable=False),
sa.Column('pipeline_version', sa.Integer(), nullable=False),
sa.Column('state', sa.String(length=16), nullable=False),
sa.Column('attempt', sa.Integer(), nullable=False),
sa.Column('max_attempts', sa.Integer(), nullable=False),
sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('lease_owner', sa.String(length=128), nullable=True),
sa.Column('lease_until', sa.DateTime(timezone=True), nullable=True),
sa.Column('last_error_code', sa.String(length=64), nullable=True),
sa.Column('replay_count', sa.Integer(), nullable=False, server_default='0'),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint(
"state IN ('queued', 'processing', 'ready', 'failed')",
name='attachment_job_state_check',
),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint(
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
),
)
op.create_index(
'attachment_jobs_lease_idx',
'attachment_jobs',
['state', 'lease_until'],
)
op.create_table(
'outbox_events',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('event_type', sa.String(length=64), nullable=False),
sa.Column('job_id', sa.Uuid(), nullable=False),
sa.Column('payload', sa.JSON(), nullable=False),
sa.Column('available_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('claim_owner', sa.String(length=128), nullable=True),
sa.Column('claim_until', sa.DateTime(timezone=True), nullable=True),
sa.Column('publish_attempts', sa.Integer(), nullable=False, server_default='0'),
sa.Column('last_error_code', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
)
op.create_index(
'outbox_events_pending_idx',
'outbox_events',
['published_at', 'available_at'],
)
op.create_table(
'job_events',
sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column('job_id', sa.Uuid(), nullable=False),
sa.Column('kind', sa.String(length=32), nullable=False),
sa.Column('attempt', sa.Integer(), nullable=False),
sa.Column('error_code', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
)
op.create_index('job_events_job_created_idx', 'job_events', ['job_id', 'created_at'])
def downgrade() -> None:
op.drop_index('job_events_job_created_idx', table_name='job_events')
op.drop_table('job_events')
op.drop_index('outbox_events_pending_idx', table_name='outbox_events')
op.drop_table('outbox_events')
op.drop_index('attachment_jobs_lease_idx', table_name='attachment_jobs')
op.drop_table('attachment_jobs')
op.drop_table('attachment_variants')
op.drop_constraint('attachments_staging_key_key', 'attachments', type_='unique')
op.drop_column('attachments', 'pipeline_version')
op.drop_column('attachments', 'staging_key')
op.alter_column('attachments', 'original_key', nullable=False)
op.alter_column('attachments', 'original_key', new_column_name='object_key')
+1
View File
@@ -0,0 +1 @@
"""Мессенджер на PostgreSQL и Redis, лабораторная 2."""
+1
View File
@@ -0,0 +1 @@
"""HTTP-граница: FastAPI, коды состояния, формат ошибок, курсоры."""
+28
View File
@@ -0,0 +1,28 @@
import base64
import json
from uuid import UUID
from ..domain.errors import Invalid
from ..domain.models import Keyset
from .timefmt import from_rfc3339, to_rfc3339
MAX_LENGTH = 512
def encode(scope: str, position: Keyset) -> str:
payload = {'s': scope, 't': to_rfc3339(position.created_at), 'i': str(position.id)}
raw = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8')
return base64.urlsafe_b64encode(raw).decode('ascii').rstrip('=')
def decode(scope: str, value: str) -> Keyset:
if not value or len(value) > MAX_LENGTH:
raise Invalid('cursor недействителен для этого запроса')
try:
raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4))
payload = json.loads(raw)
if not isinstance(payload, dict) or payload.get('s') != scope:
raise ValueError('scope mismatch')
return Keyset(created_at=from_rfc3339(payload['t']), id=UUID(payload['i']))
except Exception:
raise Invalid('cursor недействителен для этого запроса') from None
+78
View File
@@ -0,0 +1,78 @@
from __future__ import annotations
from dataclasses import dataclass
from fastapi import Depends, Header, HTTPException, Request, status
from ..db.redis import RedisSessionStore
from ..domain.errors import Unauthorized
from ..domain.models import User
from ..services.chats import ChatService
from ..services.attachments import AttachmentService
from ..services.messages import MessageService
from ..services.users import UserService
_BEARER_CHALLENGE = 'Bearer realm="messenger"'
@dataclass(frozen=True, slots=True)
class Services:
attachments: AttachmentService
users: UserService
chats: ChatService
messages: MessageService
sessions: RedisSessionStore
def services(request: Request) -> Services:
return request.app.state.services
def user_service(container: Services = Depends(services)) -> UserService:
return container.users
def chat_service(container: Services = Depends(services)) -> ChatService:
return container.chats
def message_service(container: Services = Depends(services)) -> MessageService:
return container.messages
def attachment_service(container: Services = Depends(services)) -> AttachmentService:
return container.attachments
def session_store(container: Services = Depends(services)) -> RedisSessionStore:
return container.sessions
def _bearer_error() -> HTTPException:
return HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='требуется действующая Bearer-сессия',
headers={'WWW-Authenticate': _BEARER_CHALLENGE},
)
async def bearer_token(authorization: str | None = Header(None)) -> str:
if not authorization:
raise _bearer_error()
scheme, separator, token = authorization.partition(' ')
if scheme.lower() != 'bearer' or not separator or not token or token.strip() != token:
raise _bearer_error()
return token
async def current_user(
token: str = Depends(bearer_token),
container: Services = Depends(services),
) -> User:
user_id = await container.sessions.get_user_id(token)
if user_id is None:
raise _bearer_error()
try:
return await container.users.identify(user_id)
except Unauthorized:
raise _bearer_error() from None
+107
View File
@@ -0,0 +1,107 @@
import logging
from fastapi import FastAPI
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from sqlalchemy.exc import SQLAlchemyError
from starlette.exceptions import HTTPException as StarletteHTTPException
from ..domain.errors import (
Conflict,
DomainError,
Invalid,
NotFound,
PayloadTooLarge,
Unauthorized,
Unavailable,
UnsupportedMediaType,
)
logger = logging.getLogger('app.error')
CODE_BY_STATUS = {
400: 'invalid_request',
401: 'unauthorized',
404: 'not_found',
409: 'conflict',
413: 'payload_too_large',
415: 'unsupported_media_type',
429: 'rate_limited',
500: 'internal_error',
503: 'unavailable',
}
STATUS_BY_DOMAIN = {
Invalid: 400,
Unauthorized: 401,
NotFound: 404,
Conflict: 409,
PayloadTooLarge: 413,
UnsupportedMediaType: 415,
Unavailable: 503,
}
def code_for(status: int) -> str:
if status in CODE_BY_STATUS:
return CODE_BY_STATUS[status]
return 'invalid_request' if status < 500 else 'internal_error'
def error_response(status: int, message: str, request_id: str, code: str = '') -> JSONResponse:
return JSONResponse(
status_code=status,
content={'error': {'code': code or code_for(status), 'message': message, 'request_id': request_id}},
)
def _domain_status(error: DomainError) -> int:
for kind, status in STATUS_BY_DOMAIN.items():
if isinstance(error, kind):
return status
return 500
def _describe(error: RequestValidationError) -> str:
"""Краткое указание на пол, без stack trace и внутреннего устройства"""
for item in error.errors():
if item.get('type') == 'json_invalid':
return 'тело запроса не является корректным JSON'
location = '.'.join(str(part) for part in item.get('loc', ()) if part != 'body')
return f'некорректное поле {location}' if location else 'некорректный запрос'
return 'некорректный запрос'
def request_id(request) -> str:
return getattr(request.state, 'request_id', 'unknown')
def install(app: FastAPI) -> None:
@app.exception_handler(DomainError)
async def _on_domain(request, error: DomainError) -> JSONResponse:
return error_response(_domain_status(error), error.message, request_id(request))
@app.exception_handler(RequestValidationError)
async def _on_validation(request, error: RequestValidationError) -> JSONResponse:
return error_response(400, _describe(error), request_id(request))
@app.exception_handler(SQLAlchemyError)
async def _on_database(request, error: SQLAlchemyError) -> JSONResponse:
logger.warning('database error request_id=%s type=%s', request_id(request), type(error).__name__)
return error_response(503, 'база данных недоступна', request_id(request))
@app.exception_handler(OSError)
async def _on_storage_socket(request, error: OSError) -> JSONResponse:
logger.warning('storage socket error request_id=%s type=%s', request_id(request), type(error).__name__)
return error_response(503, 'обязательное хранилище недоступно', request_id(request))
@app.exception_handler(StarletteHTTPException)
async def _on_http(request, error: StarletteHTTPException) -> JSONResponse:
message = error.detail if isinstance(error.detail, str) and error.detail else 'ошибка запроса'
response = error_response(error.status_code, message, request_id(request))
for name, value in (error.headers or {}).items():
response.headers[name] = value
return response
__all__ = ['code_for', 'error_response', 'install']
+43
View File
@@ -0,0 +1,43 @@
import logging
import os
import socket
from uuid import uuid4
from starlette.datastructures import MutableHeaders
from .errors import error_response
logger = logging.getLogger('app')
class RequestContext:
def __init__(self, app) -> None:
self.app = app
self.instance_id = os.getenv('INSTANCE_ID') or socket.gethostname()
async def __call__(self, scope, receive, send) -> None:
if scope['type'] != 'http':
await self.app(scope, receive, send)
return
request_id = uuid4().hex
scope.setdefault('state', {})['request_id'] = request_id
started = False
async def send_wrapper(message) -> None:
nonlocal started
if message['type'] == 'http.response.start':
started = True
headers = MutableHeaders(scope=message)
headers['X-Request-Id'] = request_id
headers['X-Instance-Id'] = self.instance_id
await send(message)
try:
await self.app(scope, receive, send_wrapper)
except Exception:
logger.exception(f'unhandled error request_id={request_id} path={scope.get("path")}')
if started:
raise
response = error_response(500, 'внутренняя ошибка', request_id)
await response(scope, receive, send_wrapper)
+3
View File
@@ -0,0 +1,3 @@
from . import auth, chats, health, messages, users
__all__ = ['auth', 'chats', 'health', 'messages', 'users']
+64
View File
@@ -0,0 +1,64 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, Header, Query, Request
from fastapi.responses import StreamingResponse
from ...domain.models import User
from ...services.attachments import AttachmentService
from ..deps import attachment_service, current_user
from ..schemas import AttachmentOut
router = APIRouter(tags=['attachments'])
@router.post(
'/api/v1/chats/{chat_id}/attachments',
status_code=202,
response_model=AttachmentOut,
)
async def upload(
chat_id: UUID,
request: Request,
filename: str = Query(..., min_length=1, max_length=255),
content_type: Annotated[str | None, Header(alias='Content-Type')] = None,
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> AttachmentOut:
attachment = await attachments.upload(
actor,
chat_id,
filename,
content_type or '',
request.stream(),
)
return AttachmentOut.from_domain(attachment)
@router.get('/api/v1/attachments/{attachment_id}', response_model=AttachmentOut)
async def metadata(
attachment_id: UUID,
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> AttachmentOut:
return AttachmentOut.from_domain(await attachments.get(actor, attachment_id))
@router.get('/api/v1/attachments/{attachment_id}/content')
async def content(
attachment_id: UUID,
variant: str = Query('original'),
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> StreamingResponse:
attachment, stream, content_type, size_bytes = await attachments.content(
actor, attachment_id, variant
)
return StreamingResponse(
stream.chunks(),
media_type=content_type,
headers={
'Content-Length': str(size_bytes),
'Content-Disposition': f'inline; filename="attachment-{attachment.id}"',
},
)
+71
View File
@@ -0,0 +1,71 @@
from __future__ import annotations
import base64
from binascii import Error as Base64Error
from fastapi import APIRouter, Depends, Header, HTTPException, status
from ...auth.tokens import create_token
from ...db.redis import RedisSessionStore
from ...domain.models import User
from ...services.users import UserService
from ..deps import bearer_token, current_user, session_store, user_service
from ..schemas import SessionOut, UserOut
from ..timefmt import to_rfc3339
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
_BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
def _unauthorized_basic() -> HTTPException:
return HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='неверные учётные данные',
headers={'WWW-Authenticate': _BASIC_CHALLENGE},
)
def _parse_basic(authorization: str | None) -> tuple[str, str]:
if not authorization:
raise _unauthorized_basic()
scheme, separator, encoded = authorization.partition(' ')
if scheme.lower() != 'basic' or not separator or not encoded:
raise _unauthorized_basic()
try:
decoded = base64.b64decode(encoded, validate=True).decode('utf-8')
except (Base64Error, UnicodeDecodeError):
raise _unauthorized_basic() from None
username, separator, password = decoded.partition(':')
if not separator or not username:
raise _unauthorized_basic()
return username, password
@router.post('', status_code=status.HTTP_201_CREATED, response_model=SessionOut)
async def create_session(
authorization: str | None = Header(None),
users: UserService = Depends(user_service),
sessions: RedisSessionStore = Depends(session_store),
) -> SessionOut:
username, password = _parse_basic(authorization)
user = await users.authenticate(username, password)
if user is None:
raise _unauthorized_basic()
token = create_token()
expires_at = await sessions.create(user.id, token)
return SessionOut(
token=token,
token_type='Bearer',
expires_at=to_rfc3339(expires_at),
user=UserOut.from_domain(user),
)
@router.delete('/current', status_code=status.HTTP_204_NO_CONTENT, response_model=None)
async def revoke_session(
token: str = Depends(bearer_token),
_: User = Depends(current_user),
sessions: RedisSessionStore = Depends(session_store),
) -> None:
await sessions.revoke(token)
+49
View File
@@ -0,0 +1,49 @@
from typing import Optional
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from ...domain.models import User
from ...services.chats import ChatService
from .. import cursors
from ..deps import chat_service, current_user
from ..schemas import (
LIMIT_DEFAULT,
LIMIT_MAX,
LIMIT_MIN,
ChatOut,
ChatPageOut,
CreateChat,
)
router = APIRouter(prefix='/api/v1/chats', tags=['chats'])
@router.post('', status_code=201, response_model=ChatOut)
async def create(
body: CreateChat,
actor: User = Depends(current_user),
chats: ChatService = Depends(chat_service),
) -> ChatOut:
return ChatOut.from_domain(await chats.create(actor, body.title, body.member_ids))
@router.get('', response_model=ChatPageOut)
async def page(
actor: User = Depends(current_user),
chats: ChatService = Depends(chat_service),
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
) -> ChatPageOut:
scope = f'chats:{actor.id}'
after = cursors.decode(scope, cursor) if cursor is not None else None
return ChatPageOut.from_domain(await chats.page(actor, after, limit), scope)
@router.get('/{chat_id}', response_model=ChatOut)
async def get(
chat_id: UUID,
actor: User = Depends(current_user),
chats: ChatService = Depends(chat_service),
) -> ChatOut:
return ChatOut.from_domain(await chats.get(actor, chat_id))
+29
View File
@@ -0,0 +1,29 @@
from fastapi import APIRouter, HTTPException, Request
from redis.exceptions import RedisError
from sqlalchemy import text
from sqlalchemy.exc import SQLAlchemyError
from ...domain.errors import Unavailable
from ..schemas import HealthOut
router = APIRouter(tags=['health'])
@router.get('/health/live', response_model=HealthOut)
async def live() -> HealthOut:
return HealthOut(status='ok')
@router.get('/health/ready', response_model=HealthOut)
async def ready(request: Request) -> HealthOut:
if not request.app.state.ready:
raise HTTPException(status_code=503, detail='приложение не готово принимать трафик')
try:
async with request.app.state.engine.connect() as connection:
await connection.execute(text('SELECT 1'))
await request.app.state.redis.ping()
await request.app.state.storage.ping()
except (SQLAlchemyError, RedisError, Unavailable, OSError):
raise HTTPException(status_code=503, detail='обязательное хранилище недоступно') from None
return HealthOut(status='ok')
+53
View File
@@ -0,0 +1,53 @@
from typing import Annotated, Optional
from uuid import UUID
from fastapi import APIRouter, Depends, Header, Query
from ...domain.models import User
from ...services.messages import MessageService
from .. import cursors
from ..deps import current_user, message_service
from ..schemas import (
LIMIT_DEFAULT,
LIMIT_MAX,
LIMIT_MIN,
CreateMessage,
MessageOut,
MessagePageOut,
)
router = APIRouter(prefix='/api/v1/chats/{chat_id}/messages', tags=['messages'])
@router.post('', status_code=201, response_model=MessageOut)
async def send(
chat_id: UUID,
body: CreateMessage,
idempotency_key: Annotated[
str,
Header(
alias='Idempotency-Key',
min_length=1,
max_length=128,
pattern=r'^[A-Za-z0-9._:-]+$',
),
],
actor: User = Depends(current_user),
messages: MessageService = Depends(message_service),
) -> MessageOut:
return MessageOut.from_domain(
await messages.send(actor, chat_id, body.text, body.attachment_ids, idempotency_key)
)
@router.get('', response_model=MessagePageOut)
async def page(
chat_id: UUID,
actor: User = Depends(current_user),
messages: MessageService = Depends(message_service),
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
) -> MessagePageOut:
scope = f'messages:{chat_id}'
after = cursors.decode(scope, cursor) if cursor is not None else None
return MessagePageOut.from_domain(await messages.history(actor, chat_id, after, limit), scope)
+18
View File
@@ -0,0 +1,18 @@
from fastapi import APIRouter, Depends
from ...domain.models import User
from ...services.users import UserService
from ..deps import current_user, user_service
from ..schemas import CreateUser, UserOut
router = APIRouter(prefix='/api/v1', tags=['users'])
@router.post('/users', status_code=201, response_model=UserOut)
async def register(body: CreateUser, users: UserService = Depends(user_service)) -> UserOut:
return UserOut.from_domain(await users.register(body.username, body.display_name, body.password))
@router.get('/users/me', response_model=UserOut)
async def me(actor: User = Depends(current_user)) -> UserOut:
return UserOut.from_domain(actor)
+209
View File
@@ -0,0 +1,209 @@
from typing import Optional
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from ..domain.models import Attachment, Chat, Message, Page, User
from . import cursors
from .timefmt import to_rfc3339
NAME_MAX = 100
TEXT_MAX = 4000
MEMBERS_MAX = 100
LIMIT_MIN = 1
LIMIT_MAX = 100
LIMIT_DEFAULT = 50
def not_blank(value: str) -> str:
if not value.strip():
raise ValueError('строка не может состоять только из пробельных символов')
return value
class Strict(BaseModel):
model_config = ConfigDict(extra='forbid')
class HealthOut(Strict):
status: str = 'ok'
class CreateUser(Strict):
username: str = Field(min_length=3, max_length=32, pattern=r'^[a-z0-9_]{3,32}$')
display_name: str = Field(min_length=1, max_length=NAME_MAX)
password: str = Field(min_length=12, max_length=128)
@field_validator('display_name')
@classmethod
def _display_name(cls, value: str) -> str:
return not_blank(value)
class UserOut(Strict):
id: UUID
username: str
display_name: str
created_at: str
@classmethod
def from_domain(cls, user: User) -> 'UserOut':
return cls(
id=user.id,
username=user.username,
display_name=user.display_name,
created_at=to_rfc3339(user.created_at),
)
class SessionOut(Strict):
token: str = Field(min_length=32)
token_type: str = 'Bearer'
expires_at: str
user: UserOut
class CreateChat(Strict):
title: str = Field(min_length=1, max_length=NAME_MAX)
member_ids: list[UUID] = Field(min_length=1, max_length=MEMBERS_MAX)
@field_validator('title')
@classmethod
def _title(cls, value: str) -> str:
return not_blank(value)
@field_validator('member_ids')
@classmethod
def _members_unique(cls, value: list[UUID]) -> list[UUID]:
if len(set(value)) != len(value):
raise ValueError('member_ids не должен содержать дубликаты')
return value
class ChatOut(Strict):
id: UUID
title: str
member_ids: list[UUID]
created_at: str
@classmethod
def from_domain(cls, chat: Chat) -> 'ChatOut':
return cls(
id=chat.id,
title=chat.title,
member_ids=list(chat.member_ids),
created_at=to_rfc3339(chat.created_at),
)
class CreateMessage(Strict):
text: str = Field(max_length=TEXT_MAX)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@field_validator('attachment_ids')
@classmethod
def _attachments_unique(cls, value: list[UUID]) -> list[UUID]:
if len(set(value)) != len(value):
raise ValueError('attachment_ids не должен содержать дубликаты')
return value
@model_validator(mode='after')
def _content_present(self) -> 'CreateMessage':
if not self.text.strip() and not self.attachment_ids:
raise ValueError('сообщение должно содержать текст или вложение')
return self
class MessageOut(Strict):
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: str
attachment_ids: list[UUID]
@classmethod
def from_domain(cls, message: Message) -> 'MessageOut':
return cls(
id=message.id,
chat_id=message.chat_id,
sender_id=message.sender_id,
text=message.text,
created_at=to_rfc3339(message.created_at),
attachment_ids=list(message.attachment_ids),
)
class VariantOut(Strict):
name: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
class AttachmentOut(Strict):
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: str
variants: list[VariantOut] = Field(default_factory=list)
error_code: str | None
created_at: str
updated_at: str
@classmethod
def from_domain(cls, attachment: Attachment) -> 'AttachmentOut':
return cls(
id=attachment.id,
chat_id=attachment.chat_id,
owner_id=attachment.owner_id,
filename=attachment.filename,
content_type=attachment.content_type,
size_bytes=attachment.size_bytes,
sha256=attachment.sha256,
status=attachment.status,
variants=[
VariantOut(
name=item.name,
content_type=item.content_type,
size_bytes=item.size_bytes,
width=item.width,
height=item.height,
sha256=item.sha256,
)
for item in attachment.variants
],
error_code=attachment.error_code,
created_at=to_rfc3339(attachment.created_at),
updated_at=to_rfc3339(attachment.updated_at),
)
class ChatPageOut(Strict):
items: list[ChatOut]
next_cursor: Optional[str]
@classmethod
def from_domain(cls, page: Page, scope: str) -> 'ChatPageOut':
return cls(
items=[ChatOut.from_domain(chat) for chat in page.items],
next_cursor=cursors.encode(scope, page.next) if page.next else None,
)
class MessagePageOut(Strict):
items: list[MessageOut]
next_cursor: Optional[str]
@classmethod
def from_domain(cls, page: Page, scope: str) -> 'MessagePageOut':
return cls(
items=[MessageOut.from_domain(message) for message in page.items],
next_cursor=cursors.encode(scope, page.next) if page.next else None,
)
+11
View File
@@ -0,0 +1,11 @@
from datetime import datetime, timezone
def to_rfc3339(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def from_rfc3339(value: str) -> datetime:
if not isinstance(value, str) or not value.endswith('Z'):
raise ValueError('timestamp must end with Z')
return datetime.fromisoformat(value[:-1]).replace(tzinfo=timezone.utc)
+4
View File
@@ -0,0 +1,4 @@
from .passwords import hash_password, password_matches
from .tokens import create_token
__all__ = ['create_token', 'hash_password', 'password_matches']
+15
View File
@@ -0,0 +1,15 @@
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError, VerifyMismatchError
_hasher = PasswordHasher()
def hash_password(password: str) -> str:
return _hasher.hash(password)
def password_matches(stored_hash: str, password: str) -> bool:
try:
return _hasher.verify(stored_hash, password)
except (VerifyMismatchError, VerificationError):
return False
+11
View File
@@ -0,0 +1,11 @@
import hashlib
import secrets
def create_token() -> str:
return secrets.token_urlsafe(32)
def session_key(token: str) -> str:
digest = hashlib.sha256(token.encode('utf-8')).hexdigest()
return f'session:{digest}'
+3
View File
@@ -0,0 +1,3 @@
from .settings import Settings, load_settings
__all__ = ['Settings', 'load_settings']
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
def _read_value(name: str, file_name: str) -> str:
value = os.getenv(name)
if value:
return value
path = os.getenv(file_name)
if path:
return Path(path).read_text(encoding='utf-8').strip()
raise RuntimeError(f'не задана настройка {name} или {file_name}')
@dataclass(frozen=True, slots=True)
class Settings:
database_url: str
redis_url: str
session_ttl_seconds: int
s3_endpoint: str
s3_region: str
s3_bucket: str
s3_access_key_id: str
s3_secret_access_key: str
max_upload_bytes: int
max_image_pixels: int
broker_url: str
worker_concurrency: int
worker_prefetch: int
job_max_attempts: int
job_timeout_seconds: int
job_lease_seconds: int
retry_base_seconds: int
retry_max_seconds: int
staging_retention_seconds: int
thumbnail_jpeg_quality: int
crop_jpeg_quality: int
optimized_webp_quality: int
def _positive_int(name: str, default: str) -> int:
try:
value = int(os.getenv(name, default))
except ValueError as exc:
raise RuntimeError(f'{name} должен быть целым числом') from exc
if value <= 0:
raise RuntimeError(f'{name} должен быть больше нуля')
return value
def _quality(name: str, default: str) -> int:
value = _positive_int(name, default)
if value > 100:
raise RuntimeError(f'{name} должен быть не больше 100')
return value
def load_settings() -> Settings:
return Settings(
database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'),
redis_url=_read_value('REDIS_URL', 'CACHE_URL_FILE'),
session_ttl_seconds=_positive_int('SESSION_TTL_SECONDS', '3600'),
s3_endpoint=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
s3_region=os.getenv('S3_REGION', 'us-east-1'),
s3_bucket=os.getenv('S3_BUCKET', 'messenger-attachments'),
s3_access_key_id=_read_value('S3_ACCESS_KEY_ID', 'S3_ACCESS_KEY_ID_FILE'),
s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'),
max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'),
max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'),
broker_url=_read_value('BROKER_URL', 'BROKER_URL_FILE'),
worker_concurrency=_positive_int('WORKER_CONCURRENCY', '2'),
worker_prefetch=_positive_int('WORKER_PREFETCH', '2'),
job_max_attempts=_positive_int('JOB_MAX_ATTEMPTS', '5'),
job_timeout_seconds=_positive_int('JOB_TIMEOUT_SECONDS', '60'),
job_lease_seconds=_positive_int('JOB_LEASE_SECONDS', '90'),
retry_base_seconds=_positive_int('RETRY_BASE_SECONDS', '2'),
retry_max_seconds=_positive_int('RETRY_MAX_SECONDS', '60'),
staging_retention_seconds=_positive_int('STAGING_RETENTION_SECONDS', '86400'),
thumbnail_jpeg_quality=_quality('THUMBNAIL_JPEG_QUALITY', '85'),
crop_jpeg_quality=_quality('CROP_JPEG_QUALITY', '85'),
optimized_webp_quality=_quality('OPTIMIZED_WEBP_QUALITY', '82'),
)
+4
View File
@@ -0,0 +1,4 @@
from .postgres import Base, create_engine, create_session_factory
from .redis import RedisSessionStore, create_redis
__all__ = ['Base', 'RedisSessionStore', 'create_engine', 'create_redis', 'create_session_factory']
+23
View File
@@ -0,0 +1,23 @@
from __future__ import annotations
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase
class Base(DeclarativeBase):
pass
def create_engine(url: str) -> AsyncEngine:
return create_async_engine(
url,
pool_size=5,
max_overflow=5,
pool_timeout=5,
pool_pre_ping=True,
connect_args={'server_settings': {'statement_timeout': '5000'}},
)
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
return async_sessionmaker(engine, expire_on_commit=False)
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from uuid import UUID
from redis.asyncio import Redis
from redis.exceptions import RedisError
from ..auth.tokens import session_key
from ..domain.errors import Unavailable
def create_redis(url: str) -> Redis:
return Redis.from_url(url, decode_responses=True, socket_connect_timeout=3, socket_timeout=3)
class RedisSessionStore:
def __init__(self, client: Redis, ttl_seconds: int) -> None:
self._client = client
self._ttl_seconds = ttl_seconds
async def create(self, user_id: UUID, token: str) -> datetime:
try:
await self._client.set(session_key(token), str(user_id), ex=self._ttl_seconds)
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
return datetime.now(timezone.utc) + timedelta(seconds=self._ttl_seconds)
async def get_user_id(self, token: str) -> UUID | None:
try:
value = await self._client.get(session_key(token))
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
if value is None:
return None
try:
return UUID(value)
except ValueError:
return None
async def revoke(self, token: str) -> None:
try:
await self._client.delete(session_key(token))
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
async def ping(self) -> None:
try:
await self._client.ping()
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
View File
+37
View File
@@ -0,0 +1,37 @@
from __future__ import annotations
class DomainError(Exception):
"""Сообщение попадает клиенту, поэтому не содержит внутренних деталей."""
def __init__(self, message: str) -> None:
super().__init__(message)
self.message = message
class Invalid(DomainError):
"""Запрос нарушает правила предметной области."""
class Unauthorized(DomainError):
"""Нет действительной идентификации."""
class NotFound(DomainError):
"""Объекта нет либо он недоступен вызывающему: снаружи это неразличимо."""
class Conflict(DomainError):
"""Состояние не позволяет выполнить операцию."""
class Unavailable(DomainError):
"""Обязательная внешняя зависимость временно недоступна."""
class PayloadTooLarge(DomainError):
"""Тело запроса или декодированное содержимое превышает лимит."""
class UnsupportedMediaType(DomainError):
"""Формат содержимого не поддерживается или не соответствует MIME."""
+114
View File
@@ -0,0 +1,114 @@
from dataclasses import dataclass
from datetime import datetime
from typing import Optional
from uuid import UUID
@dataclass(frozen=True, slots=True)
class User:
id: UUID
username: str
display_name: str
created_at: datetime
@dataclass(frozen=True, slots=True)
class Chat:
id: UUID
title: str
member_ids: tuple[UUID, ...]
created_at: datetime
def has_member(self, user_id: UUID) -> bool:
return user_id in self.member_ids
@dataclass(frozen=True, slots=True)
class Message:
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: tuple[UUID, ...] = ()
@dataclass(frozen=True, slots=True)
class Attachment:
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
original_key: str | None
staging_key: str | None
pipeline_version: int
content_type: str
size_bytes: int
sha256: str
width: int
height: int
status: str
error_code: str | None
created_at: datetime
updated_at: datetime
variants: tuple['AttachmentVariant', ...] = ()
@dataclass(frozen=True, slots=True)
class AttachmentVariant:
name: str
object_key: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
@dataclass(frozen=True, slots=True)
class AttachmentJob:
id: UUID
attachment_id: UUID
pipeline_version: int
state: str
attempt: int
max_attempts: int
next_attempt_at: datetime
lease_owner: str | None
lease_until: datetime | None
last_error_code: str | None
replay_count: int
created_at: datetime
updated_at: datetime
@dataclass(frozen=True, slots=True)
class OutboxEvent:
id: UUID
job_id: UUID
payload: dict
available_at: datetime
@dataclass(frozen=True, slots=True)
class Keyset:
created_at: datetime
id: UUID
@property
def order(self) -> tuple[datetime, int]:
return (self.created_at, self.id.int)
@classmethod
def from_item(cls, item) -> 'Keyset':
return cls(created_at=item.created_at, id=item.id)
def order_key(item) -> tuple[datetime, int]:
return (item.created_at, item.id.int)
@dataclass(frozen=True, slots=True)
class Page:
items: tuple
next: Optional[Keyset]
+1
View File
@@ -0,0 +1 @@
"""Durable attachment job publisher and worker."""
+56
View File
@@ -0,0 +1,56 @@
from __future__ import annotations
import asyncio
import logging
import aio_pika
from aio_pika import DeliveryMode, ExchangeType
MAIN_EXCHANGE = 'attachments'
MAIN_QUEUE = 'attachments.process'
ROUTING_KEY = 'attachment.process.v1'
DEAD_EXCHANGE = 'attachments.dead'
DEAD_QUEUE = 'attachments.failed'
logger = logging.getLogger('app.broker')
async def connect(url: str) -> aio_pika.RobustConnection:
delay = 0.5
while True:
try:
return await aio_pika.connect_robust(url, timeout=5)
except (OSError, aio_pika.exceptions.AMQPException):
logger.warning('broker is not ready; reconnecting in %.1fs', delay)
await asyncio.sleep(delay)
delay = min(delay * 2, 5.0)
async def topology(channel: aio_pika.abc.AbstractChannel):
dead_exchange = await channel.declare_exchange(
DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True
)
dead_queue = await channel.declare_queue(DEAD_QUEUE, durable=True)
await dead_queue.bind(dead_exchange, ROUTING_KEY)
exchange = await channel.declare_exchange(
MAIN_EXCHANGE, ExchangeType.DIRECT, durable=True
)
queue = await channel.declare_queue(
MAIN_QUEUE,
durable=True,
arguments={
'x-dead-letter-exchange': DEAD_EXCHANGE,
'x-dead-letter-routing-key': ROUTING_KEY,
},
)
await queue.bind(exchange, ROUTING_KEY)
return exchange, queue
def message(body: bytes, event_id: str) -> aio_pika.Message:
return aio_pika.Message(
body=body,
content_type='application/json',
delivery_mode=DeliveryMode.PERSISTENT,
message_id=event_id,
type=ROUTING_KEY,
)
+98
View File
@@ -0,0 +1,98 @@
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from io import BytesIO
from PIL import Image, ImageOps, UnidentifiedImageError
class PermanentImageError(Exception):
def __init__(self, code: str) -> None:
super().__init__(code)
self.code = code
@dataclass(frozen=True, slots=True)
class GeneratedVariant:
name: str
content_type: str
width: int
height: int
sha256: str
content: bytes
def _jpeg_image(image: Image.Image) -> Image.Image:
if image.mode in {'RGBA', 'LA'} or 'transparency' in image.info:
rgba = image.convert('RGBA')
background = Image.new('RGBA', rgba.size, 'white')
background.alpha_composite(rgba)
return background.convert('RGB')
return image.convert('RGB')
def _encode(image: Image.Image, fmt: str, **options) -> bytes:
output = BytesIO()
image.save(output, format=fmt, **options)
return output.getvalue()
def _variant(name: str, content_type: str, image: Image.Image, content: bytes) -> GeneratedVariant:
return GeneratedVariant(
name=name,
content_type=content_type,
width=image.width,
height=image.height,
sha256=hashlib.sha256(content).hexdigest(),
content=content,
)
def process_image(
source: bytes,
declared_type: str,
expected_sha256: str,
max_pixels: int,
thumbnail_quality: int,
crop_quality: int,
webp_quality: int,
) -> tuple[GeneratedVariant, ...]:
if hashlib.sha256(source).hexdigest() != expected_sha256:
raise PermanentImageError('checksum_mismatch')
try:
with Image.open(BytesIO(source)) as opened:
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(opened.format or '')
if actual != declared_type:
raise PermanentImageError('unsupported_media_type')
if opened.width * opened.height > max_pixels:
raise PermanentImageError('image_too_large')
opened.load()
oriented = ImageOps.exif_transpose(opened)
base = oriented.copy()
except PermanentImageError:
raise
except (Image.DecompressionBombError, UnidentifiedImageError, OSError, SyntaxError) as error:
raise PermanentImageError('invalid_image') from error
thumbnail = _jpeg_image(base.copy())
thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS, reducing_gap=3.0)
thumbnail_bytes = _encode(
thumbnail, 'JPEG', quality=thumbnail_quality, optimize=True, progressive=True
)
crop_source = _jpeg_image(base.copy())
crop = ImageOps.fit(
crop_source, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
)
crop_bytes = _encode(crop, 'JPEG', quality=crop_quality, optimize=True, progressive=True)
optimized = base.copy()
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS, reducing_gap=3.0)
optimized_bytes = _encode(optimized, 'WEBP', quality=webp_quality, method=6)
return (
_variant('thumbnail', 'image/jpeg', thumbnail, thumbnail_bytes),
_variant('crop', 'image/jpeg', crop, crop_bytes),
_variant('optimized', 'image/webp', optimized, optimized_bytes),
)
+64
View File
@@ -0,0 +1,64 @@
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import os
import socket
from datetime import datetime, timezone
from ..config import load_settings
from ..repositories import JobRepository, OutboxRepository
from .broker import ROUTING_KEY, connect, message, topology
from .runtime import configure_logging, dependencies
logger = logging.getLogger('app.publisher')
async def run(once: bool = False) -> None:
settings = load_settings()
engine, sessions, _storage = dependencies(settings)
outbox = OutboxRepository(sessions)
jobs = JobRepository(sessions)
owner = f'{socket.gethostname()}:{os.getpid()}'
connection = await connect(settings.broker_url)
try:
channel = await connection.channel(publisher_confirms=True)
exchange, _queue = await topology(channel)
while True:
now = datetime.now(timezone.utc)
await jobs.recover_expired(now)
events = await outbox.claim(owner, now, lease_seconds=30)
for event in events:
try:
body = json.dumps(event.payload, separators=(',', ':')).encode()
await exchange.publish(
message(body, str(event.id)), routing_key=ROUTING_KEY, mandatory=True
)
if os.getenv('FAULT_PUBLISHER_AFTER_CONFIRM') == '1':
os._exit(91)
await outbox.published(event.id, owner, datetime.now(timezone.utc))
except Exception:
logger.exception('outbox publish failed event_id=%s', event.id)
await outbox.failed(
event.id, owner, 'broker_unavailable', datetime.now(timezone.utc)
)
if once:
return
await asyncio.sleep(0.5 if events else 1.0)
finally:
await connection.close()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('--once', action='store_true')
args = parser.parse_args()
configure_logging()
asyncio.run(run(args.once))
if __name__ == '__main__':
main()
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
import logging
import sys
from ..config import Settings
from ..db import create_engine, create_session_factory
from ..storage import S3Storage
def configure_logging() -> None:
logging.basicConfig(
level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)s %(name)s %(message)s',
force=True,
)
def dependencies(settings: Settings):
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
endpoint=settings.s3_endpoint,
access_key=settings.s3_access_key_id,
secret_key=settings.s3_secret_access_key,
bucket=settings.s3_bucket,
region=settings.s3_region,
)
return engine, sessions, storage
+282
View File
@@ -0,0 +1,282 @@
from __future__ import annotations
import asyncio
import json
import logging
import multiprocessing
import os
import random
import socket
from datetime import datetime, timedelta, timezone
from uuid import UUID
import aio_pika
from ..config import Settings, load_settings
from ..domain.errors import Unavailable
from ..domain.models import AttachmentVariant
from ..repositories import JobRepository
from .broker import connect, topology
from .pipeline import GeneratedVariant, PermanentImageError, process_image
from .runtime import configure_logging, dependencies
logger = logging.getLogger('app.worker')
class ProcessingTimeout(Exception):
pass
def _pipeline_child(connection, arguments: tuple, memory_bytes: int) -> None:
try:
if memory_bytes:
try:
import resource
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
except (ImportError, OSError, ValueError):
pass
connection.send(('ok', process_image(*arguments)))
except PermanentImageError as error:
connection.send(('permanent', error.code))
except MemoryError:
connection.send(('temporary', 'processing_memory_limit'))
except BaseException:
connection.send(('temporary', 'processing_failed'))
finally:
connection.close()
def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[GeneratedVariant, ...]:
context = multiprocessing.get_context('spawn')
receiver, sender = context.Pipe(duplex=False)
process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes))
process.start()
sender.close()
if not receiver.poll(timeout):
process.terminate()
process.join(5)
receiver.close()
raise ProcessingTimeout('processing_timeout')
try:
kind, result = receiver.recv()
except EOFError as error:
process.join(5)
receiver.close()
raise ProcessingTimeout('processing_failed') from error
receiver.close()
process.join(5)
if process.is_alive():
process.terminate()
process.join(5)
raise ProcessingTimeout('processing_failed')
if kind == 'ok':
return result
if kind == 'permanent':
raise PermanentImageError(result)
raise ProcessingTimeout(result)
class Worker:
def __init__(self, settings: Settings, jobs: JobRepository, storage) -> None:
self.settings = settings
self.jobs = jobs
self.storage = storage
self.worker_id = f'{socket.gethostname()}:{os.getpid()}'
self.semaphore = asyncio.Semaphore(settings.worker_concurrency)
def _delay(self, attempt: int) -> float:
base = min(
self.settings.retry_max_seconds,
self.settings.retry_base_seconds * (2 ** max(0, attempt - 1)),
)
return base + random.uniform(0, base)
async def _handle_failure(
self,
incoming: aio_pika.IncomingMessage,
job_id: UUID,
attempt: int,
code: str,
permanent: bool,
) -> None:
now = datetime.now(timezone.utc)
available_at = now + timedelta(seconds=self._delay(attempt))
try:
state = await self.jobs.retry_or_fail(
job_id,
self.worker_id,
code,
permanent,
available_at,
now,
)
except Exception:
logger.exception('failed to persist job failure job_id=%s', job_id)
await incoming.nack(requeue=True)
return
if state == 'failed':
await incoming.reject(requeue=False)
else:
await incoming.ack()
async def handle(self, incoming: aio_pika.IncomingMessage) -> None:
async with self.semaphore:
try:
payload = json.loads(incoming.body)
if payload.get('schema_version') != 1 or payload.get('type') != 'attachment.process.v1':
await incoming.reject(requeue=False)
return
job_id = UUID(payload['job_id'])
attachment_id = UUID(payload['attachment_id'])
attempt = int(payload['attempt'])
pipeline_version = int(payload['pipeline_version'])
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
await incoming.reject(requeue=False)
return
current = await self.jobs.state(job_id)
if current is None:
await incoming.reject(requeue=False)
return
if current.attachment_id != attachment_id or current.pipeline_version != pipeline_version:
await incoming.reject(requeue=False)
return
if current.state == 'ready':
if await self.jobs.ready_effect_complete(job_id):
await incoming.ack()
else:
await incoming.reject(requeue=False)
return
if current.state == 'failed':
await incoming.reject(requeue=False)
return
now = datetime.now(timezone.utc)
claimed = await self.jobs.claim(
job_id, attempt, self.worker_id, now, self.settings.job_lease_seconds
)
if claimed is None:
await incoming.ack()
return
job, attachment = claimed
if os.getenv('FAULT_WORKER_PERMANENT') == '1':
await self._handle_failure(
incoming, job.id, job.attempt, 'fault_injected', True
)
os._exit(94)
if not attachment.staging_key:
await self._handle_failure(
incoming, job.id, job.attempt, 'staging_missing', True
)
return
try:
source = await self.storage.read(
attachment.staging_key, self.settings.max_upload_bytes
)
arguments = (
source,
attachment.content_type,
attachment.sha256,
self.settings.max_image_pixels,
self.settings.thumbnail_jpeg_quality,
self.settings.crop_jpeg_quality,
self.settings.optimized_webp_quality,
)
generated = await asyncio.to_thread(
_run_isolated,
arguments,
self.settings.job_timeout_seconds,
512 * 1024 * 1024,
)
original_key = f'originals/{attachment.id}/v{job.pipeline_version}/original'
await self.storage.put_bytes(original_key, source, attachment.content_type)
variants: list[AttachmentVariant] = []
suffixes = {'thumbnail': 'jpg', 'crop': 'jpg', 'optimized': 'webp'}
for item in generated:
key = (
f'variants/{attachment.id}/v{job.pipeline_version}/'
f'{item.name}.{suffixes[item.name]}'
)
await self.storage.put_bytes(key, item.content, item.content_type)
variants.append(
AttachmentVariant(
name=item.name,
object_key=key,
content_type=item.content_type,
size_bytes=len(item.content),
width=item.width,
height=item.height,
sha256=item.sha256,
)
)
if os.getenv('FAULT_WORKER_AFTER_OBJECTS') == '1':
os._exit(92)
completed = await self.jobs.complete(
job.id,
self.worker_id,
original_key,
tuple(variants),
datetime.now(timezone.utc),
)
if not completed:
await incoming.nack(requeue=True)
return
if os.getenv('FAULT_WORKER_AFTER_COMMIT') == '1':
os._exit(93)
try:
await self.storage.remove(attachment.staging_key)
except Unavailable:
logger.warning('staging cleanup deferred attachment_id=%s', attachment.id)
await incoming.ack()
except PermanentImageError as error:
await self._handle_failure(
incoming, job.id, job.attempt, error.code, True
)
except ProcessingTimeout as error:
await self._handle_failure(
incoming, job.id, job.attempt, str(error), False
)
except (Unavailable, OSError, ValueError):
logger.exception('temporary processing failure job_id=%s', job.id)
await self._handle_failure(
incoming, job.id, job.attempt, 'storage_unavailable', False
)
except Exception:
logger.exception('unexpected processing failure job_id=%s', job.id)
await self._handle_failure(
incoming, job.id, job.attempt, 'processing_failed', False
)
async def run() -> None:
settings = load_settings()
engine, sessions, storage = dependencies(settings)
jobs = JobRepository(sessions)
worker = Worker(settings, jobs, storage)
connection = await connect(settings.broker_url)
try:
channel = await connection.channel()
await channel.set_qos(prefetch_count=settings.worker_prefetch)
_exchange, queue = await topology(channel)
await queue.consume(worker.handle, no_ack=False)
logger.info(
'worker started id=%s concurrency=%d prefetch=%d',
worker.worker_id,
settings.worker_concurrency,
settings.worker_prefetch,
)
await asyncio.Future()
finally:
await connection.close()
await engine.dispose()
def main() -> None:
configure_logging()
asyncio.run(run())
if __name__ == '__main__':
main()
+113
View File
@@ -0,0 +1,113 @@
"""Сборка FastAPI приложения и подключение обязательных хранилищ."""
from __future__ import annotations
import logging
import os
import sys
from contextlib import asynccontextmanager
from fastapi import FastAPI
from sqlalchemy import text
from .api import errors
from .api.deps import Services
from .api.middleware import RequestContext
from .api.routers import attachments, auth, chats, health, messages, users
from .config import load_settings
from .db import RedisSessionStore, create_engine, create_redis, create_session_factory
from .repositories import AttachmentRepository, ChatRepository, MessageRepository, UserRepository
from .services.attachments import AttachmentService
from .services.chats import ChatService
from .services.messages import MessageService
from .services.users import UserService
from .storage import S3Storage
logger = logging.getLogger('app')
DESCRIPTION = 'PostgreSQL хранит задания, RabbitMQ доставляет их worker, S3 хранит изображения.'
def configure_logging() -> None:
logging.basicConfig(
level=os.getenv('LOG_LEVEL', 'info').upper(),
stream=sys.stdout,
format='%(asctime)s %(levelname)s %(name)s %(message)s',
force=True,
)
def create_app() -> FastAPI:
configure_logging()
@asynccontextmanager
async def lifespan(application: FastAPI):
application.state.ready = False
settings = load_settings()
engine = create_engine(settings.database_url)
redis = create_redis(settings.redis_url)
storage = S3Storage(
endpoint=settings.s3_endpoint,
access_key=settings.s3_access_key_id,
secret_key=settings.s3_secret_access_key,
bucket=settings.s3_bucket,
region=settings.s3_region,
)
try:
async with engine.connect() as connection:
await connection.execute(text('SELECT 1'))
session_store = RedisSessionStore(redis, settings.session_ttl_seconds)
await session_store.ping()
await storage.ping()
session_factory = create_session_factory(engine)
user_repository = UserRepository(session_factory)
chat_repository = ChatRepository(session_factory)
message_repository = MessageRepository(session_factory)
attachment_repository = AttachmentRepository(session_factory)
application.state.services = Services(
attachments=AttachmentService(
attachment_repository,
storage,
settings.max_upload_bytes,
settings.max_image_pixels,
settings.job_max_attempts,
),
users=UserService(user_repository),
chats=ChatService(chat_repository),
messages=MessageService(message_repository),
sessions=session_store,
)
application.state.engine = engine
application.state.redis = redis
application.state.storage = storage
application.state.ready = True
logger.info('startup storage=postgres+s3 redis_sessions=true async_attachments=true')
yield
finally:
application.state.ready = False
await redis.aclose()
await engine.dispose()
logger.info('shutdown complete')
api = FastAPI(
title='MTUSI Messenger. LR 5',
version='1.5.0',
description=DESCRIPTION,
lifespan=lifespan,
)
api.router.redirect_slashes = False
api.state.ready = False
api.add_middleware(RequestContext)
errors.install(api)
for router in (
health.router,
users.router,
auth.router,
chats.router,
messages.router,
attachments.router,
):
api.include_router(router)
return api
app = create_app()
+18
View File
@@ -0,0 +1,18 @@
from .attachment import AttachmentRow, AttachmentVariantRow, MessageAttachmentRow
from .chat import ChatMemberRow, ChatRow
from .message import MessageRow
from .user import UserRow
from .job import AttachmentJobRow, JobEventRow, OutboxEventRow
__all__ = [
'AttachmentRow',
'AttachmentVariantRow',
'AttachmentJobRow',
'ChatMemberRow',
'ChatRow',
'MessageAttachmentRow',
'MessageRow',
'UserRow',
'JobEventRow',
'OutboxEventRow',
]
+70
View File
@@ -0,0 +1,70 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class AttachmentRow(Base):
__tablename__ = 'attachments'
__table_args__ = (
Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'),
UniqueConstraint('original_key', name='attachments_object_key_key'),
UniqueConstraint('staging_key', name='attachments_staging_key_key'),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
filename: Mapped[str] = mapped_column(String(255), nullable=False)
original_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
width: Mapped[int] = mapped_column(Integer, nullable=False)
height: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(16), nullable=False)
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class AttachmentVariantRow(Base):
__tablename__ = 'attachment_variants'
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
)
pipeline_version: Mapped[int] = mapped_column(Integer, primary_key=True)
name: Mapped[str] = mapped_column(String(16), primary_key=True)
object_key: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
width: Mapped[int] = mapped_column(Integer, nullable=False)
height: Mapped[int] = mapped_column(Integer, nullable=False)
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class MessageAttachmentRow(Base):
__tablename__ = 'message_attachments'
__table_args__ = (
UniqueConstraint('message_id', 'position', name='message_attachments_message_position_key'),
Index('message_attachments_attachment_idx', 'attachment_id'),
)
message_id: Mapped[UUID] = mapped_column(
ForeignKey('messages.id', ondelete='CASCADE'), primary_key=True
)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
)
position: Mapped[int] = mapped_column(Integer, nullable=False)
+26
View File
@@ -0,0 +1,26 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import DateTime, ForeignKey, Index, String
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class ChatRow(Base):
__tablename__ = 'chats'
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
title: Mapped[str] = mapped_column(String(100), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ChatMemberRow(Base):
__tablename__ = 'chat_members'
__table_args__ = (Index('chat_members_user_chat_idx', 'user_id', 'chat_id'),)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
user_id: Mapped[UUID] = mapped_column(ForeignKey('users.id', ondelete='CASCADE'), primary_key=True)
+70
View File
@@ -0,0 +1,70 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class AttachmentJobRow(Base):
__tablename__ = 'attachment_jobs'
__table_args__ = (
UniqueConstraint(
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
),
Index('attachment_jobs_lease_idx', 'state', 'lease_until'),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey('attachments.id', ondelete='CASCADE'), nullable=False
)
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False)
state: Mapped[str] = mapped_column(String(16), nullable=False)
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
replay_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class OutboxEventRow(Base):
__tablename__ = 'outbox_events'
__table_args__ = (Index('outbox_events_pending_idx', 'published_at', 'available_at'),)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
job_id: Mapped[UUID] = mapped_column(
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
)
payload: Mapped[dict] = mapped_column(JSON, nullable=False)
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
claim_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
claim_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
publish_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class JobEventRow(Base):
__tablename__ = 'job_events'
__table_args__ = (Index('job_events_job_created_idx', 'job_id', 'created_at'),)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
job_id: Mapped[UUID] = mapped_column(
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
)
kind: Mapped[str] = mapped_column(String(32), nullable=False)
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import DateTime, ForeignKey, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class MessageRow(Base):
__tablename__ = 'messages'
__table_args__ = (
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
UniqueConstraint(
'sender_id',
'chat_id',
'idempotency_key',
name='messages_sender_chat_idempotency_key',
),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
sender_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
text: Mapped[str] = mapped_column(String(4000), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import DateTime, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class UserRow(Base):
__tablename__ = 'users'
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
username: Mapped[str] = mapped_column(String(32), unique=True, nullable=False)
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+10
View File
@@ -0,0 +1,10 @@
from .attachments import AttachmentRepository
from .chats import ChatRepository
from .messages import MessageRepository
from .users import UserRepository
from .jobs import JobRepository, OutboxRepository
__all__ = [
'AttachmentRepository', 'ChatRepository', 'JobRepository', 'MessageRepository',
'OutboxRepository', 'UserRepository',
]
+173
View File
@@ -0,0 +1,173 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID, uuid4
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.models import Attachment, AttachmentVariant
from ..models.attachment import AttachmentRow, AttachmentVariantRow
from ..models.chat import ChatMemberRow, ChatRow
from ..models.job import AttachmentJobRow, OutboxEventRow
def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, now: datetime) -> dict:
return {
'schema_version': 1,
'job_id': str(job_id),
'type': 'attachment.process.v1',
'attachment_id': str(attachment_id),
'pipeline_version': pipeline_version,
'attempt': attempt,
'created_at': now.isoformat().replace('+00:00', 'Z'),
}
class AttachmentRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _variant(row: AttachmentVariantRow) -> AttachmentVariant:
return AttachmentVariant(
name=row.name,
object_key=row.object_key,
content_type=row.content_type,
size_bytes=row.size_bytes,
width=row.width,
height=row.height,
sha256=row.sha256,
)
@classmethod
def _to_domain(
cls, row: AttachmentRow, variants: tuple[AttachmentVariantRow, ...] = ()
) -> Attachment:
return Attachment(
id=row.id,
chat_id=row.chat_id,
owner_id=row.owner_id,
filename=row.filename,
original_key=row.original_key,
staging_key=row.staging_key,
pipeline_version=row.pipeline_version,
content_type=row.content_type,
size_bytes=row.size_bytes,
sha256=row.sha256,
width=row.width,
height=row.height,
status=row.status,
error_code=row.error_code,
created_at=row.created_at,
updated_at=row.updated_at,
variants=tuple(cls._variant(item) for item in variants),
)
async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool:
async with self._sessions() as session:
found = await session.scalar(
select(ChatRow.id)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
)
return found is not None
async def create_queued(
self,
*,
attachment_id: UUID,
job_id: UUID,
chat_id: UUID,
owner_id: UUID,
filename: str,
staging_key: str,
content_type: str,
size_bytes: int,
sha256: str,
width: int,
height: int,
max_attempts: int,
now: datetime,
) -> Attachment:
row = AttachmentRow(
id=attachment_id,
chat_id=chat_id,
owner_id=owner_id,
filename=filename,
original_key=None,
staging_key=staging_key,
pipeline_version=1,
content_type=content_type,
size_bytes=size_bytes,
sha256=sha256,
width=width,
height=height,
status='queued',
error_code=None,
created_at=now,
updated_at=now,
)
job = AttachmentJobRow(
id=job_id,
attachment_id=attachment_id,
pipeline_version=1,
state='queued',
attempt=1,
max_attempts=max_attempts,
next_attempt_at=now,
lease_owner=None,
lease_until=None,
last_error_code=None,
replay_count=0,
created_at=now,
updated_at=now,
)
outbox = OutboxEventRow(
id=uuid4(),
event_type='attachment.process.v1',
job_id=job_id,
payload=job_payload(job_id, attachment_id, 1, 1, now),
available_at=now,
published_at=None,
claim_owner=None,
claim_until=None,
publish_attempts=0,
last_error_code=None,
created_at=now,
updated_at=now,
)
async with self._sessions() as session:
async with session.begin():
session.add_all((row, job, outbox))
await session.flush()
return self._to_domain(row)
async def _variants(
self, session: AsyncSession, attachment_id: UUID, pipeline_version: int
) -> tuple[AttachmentVariantRow, ...]:
rows = await session.scalars(
select(AttachmentVariantRow)
.where(
AttachmentVariantRow.attachment_id == attachment_id,
AttachmentVariantRow.pipeline_version == pipeline_version,
)
.order_by(AttachmentVariantRow.name)
)
return tuple(rows)
async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None:
async with self._sessions() as session:
row = await session.scalar(
select(AttachmentRow)
.join(ChatMemberRow, ChatMemberRow.chat_id == AttachmentRow.chat_id)
.where(
AttachmentRow.id == attachment_id,
ChatMemberRow.user_id == user_id,
)
)
if row is None:
return None
return self._to_domain(
row, await self._variants(session, row.id, row.pipeline_version)
)
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
from datetime import datetime
from typing import Sequence
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.errors import Invalid
from ..domain.models import Chat, Keyset
from ..models.chat import ChatMemberRow, ChatRow
from ..models.user import UserRow
class ChatRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: ChatRow, member_ids: Sequence[UUID]) -> Chat:
return Chat(id=row.id, title=row.title, member_ids=tuple(member_ids), created_at=row.created_at)
@staticmethod
async def _members(session: AsyncSession, chat_id: UUID) -> tuple[UUID, ...]:
result = await session.scalars(
select(ChatMemberRow.user_id)
.where(ChatMemberRow.chat_id == chat_id)
.order_by(ChatMemberRow.user_id)
)
return tuple(result)
async def create(self, chat_id: UUID, title: str, member_ids: Sequence[UUID], created_at: datetime) -> Chat:
async with self._sessions() as session:
async with session.begin():
found = set(await session.scalars(select(UserRow.id).where(UserRow.id.in_(member_ids))))
if len(found) != len(member_ids):
raise Invalid(f'member_ids содержит неизвестных участников: {len(member_ids) - len(found)}')
row = ChatRow(id=chat_id, title=title, created_at=created_at)
session.add(row)
await session.flush()
session.add_all(ChatMemberRow(chat_id=chat_id, user_id=user_id) for user_id in member_ids)
await session.flush()
return self._to_domain(row, member_ids)
async def get_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None:
async with self._sessions() as session:
row = await session.scalar(
select(ChatRow)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
)
if row is None:
return None
return self._to_domain(row, await self._members(session, row.id))
async def page_for_member(self, user_id: UUID, after: Keyset | None, limit: int) -> list[Chat]:
async with self._sessions() as session:
query = (
select(ChatRow)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatMemberRow.user_id == user_id)
.order_by(ChatRow.created_at, ChatRow.id)
.limit(limit)
)
if after is not None:
query = query.where(tuple_(ChatRow.created_at, ChatRow.id) > tuple_(after.created_at, after.id))
rows = list((await session.scalars(query)).all())
return [self._to_domain(row, await self._members(session, row.id)) for row in rows]
+366
View File
@@ -0,0 +1,366 @@
from __future__ import annotations
from datetime import datetime, timedelta
from uuid import UUID, uuid4
from sqlalchemy import or_, select, update
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.models import Attachment, AttachmentJob, AttachmentVariant, OutboxEvent
from ..models.attachment import AttachmentRow, AttachmentVariantRow
from ..models.job import AttachmentJobRow, JobEventRow, OutboxEventRow
from .attachments import AttachmentRepository, job_payload
class JobRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _job(row: AttachmentJobRow) -> AttachmentJob:
return AttachmentJob(
id=row.id,
attachment_id=row.attachment_id,
pipeline_version=row.pipeline_version,
state=row.state,
attempt=row.attempt,
max_attempts=row.max_attempts,
next_attempt_at=row.next_attempt_at,
lease_owner=row.lease_owner,
lease_until=row.lease_until,
last_error_code=row.last_error_code,
replay_count=row.replay_count,
created_at=row.created_at,
updated_at=row.updated_at,
)
async def claim(
self,
job_id: UUID,
attempt: int,
worker_id: str,
now: datetime,
lease_seconds: int,
) -> tuple[AttachmentJob, Attachment] | None:
async with self._sessions() as session:
async with session.begin():
row = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if row is None or row.state != 'queued' or row.attempt != attempt:
return None
attachment_row = await session.get(AttachmentRow, row.attachment_id)
if attachment_row is None:
return None
row.state = 'processing'
row.lease_owner = worker_id
row.lease_until = now + timedelta(seconds=lease_seconds)
row.updated_at = now
attachment_row.status = 'processing'
attachment_row.updated_at = now
session.add(
JobEventRow(
job_id=row.id,
kind='claimed',
attempt=row.attempt,
error_code=None,
created_at=now,
)
)
await session.flush()
return self._job(row), AttachmentRepository._to_domain(attachment_row)
async def state(self, job_id: UUID) -> AttachmentJob | None:
async with self._sessions() as session:
row = await session.get(AttachmentJobRow, job_id)
return self._job(row) if row is not None else None
async def id_for_attachment(self, attachment_id: UUID) -> UUID | None:
async with self._sessions() as session:
return await session.scalar(
select(AttachmentJobRow.id).where(
AttachmentJobRow.attachment_id == attachment_id
)
)
async def failed_staging_key(self, job_id: UUID) -> str | None:
async with self._sessions() as session:
return await session.scalar(
select(AttachmentRow.staging_key)
.join(AttachmentJobRow, AttachmentJobRow.attachment_id == AttachmentRow.id)
.where(AttachmentJobRow.id == job_id, AttachmentJobRow.state == 'failed')
)
async def ready_effect_complete(self, job_id: UUID) -> bool:
async with self._sessions() as session:
job = await session.get(AttachmentJobRow, job_id)
if job is None or job.state != 'ready':
return False
attachment = await session.get(AttachmentRow, job.attachment_id)
if attachment is None or not attachment.original_key:
return False
names = set(
await session.scalars(
select(AttachmentVariantRow.name).where(
AttachmentVariantRow.attachment_id == job.attachment_id,
AttachmentVariantRow.pipeline_version == job.pipeline_version,
)
)
)
return names == {'thumbnail', 'crop', 'optimized'}
async def complete(
self,
job_id: UUID,
worker_id: str,
original_key: str,
variants: tuple[AttachmentVariant, ...],
now: datetime,
) -> bool:
async with self._sessions() as session:
async with session.begin():
job = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if job is None or job.state == 'ready':
return job is not None
if job.state != 'processing' or job.lease_owner != worker_id:
return False
attachment = await session.get(AttachmentRow, job.attachment_id)
if attachment is None:
return False
for variant in variants:
statement = (
insert(AttachmentVariantRow)
.values(
attachment_id=attachment.id,
pipeline_version=job.pipeline_version,
name=variant.name,
object_key=variant.object_key,
content_type=variant.content_type,
size_bytes=variant.size_bytes,
width=variant.width,
height=variant.height,
sha256=variant.sha256,
created_at=now,
updated_at=now,
)
.on_conflict_do_update(
index_elements=[
AttachmentVariantRow.attachment_id,
AttachmentVariantRow.pipeline_version,
AttachmentVariantRow.name,
],
set_={
'object_key': variant.object_key,
'content_type': variant.content_type,
'size_bytes': variant.size_bytes,
'width': variant.width,
'height': variant.height,
'sha256': variant.sha256,
'updated_at': now,
},
)
)
await session.execute(statement)
attachment.original_key = original_key
attachment.status = 'ready'
attachment.error_code = None
attachment.updated_at = now
job.state = 'ready'
job.lease_owner = None
job.lease_until = None
job.last_error_code = None
job.updated_at = now
session.add(
JobEventRow(
job_id=job.id,
kind='ready',
attempt=job.attempt,
error_code=None,
created_at=now,
)
)
return True
async def retry_or_fail(
self,
job_id: UUID,
worker_id: str | None,
error_code: str,
permanent: bool,
available_at: datetime,
now: datetime,
) -> str:
async with self._sessions() as session:
async with session.begin():
job = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if job is None or job.state in {'ready', 'failed'}:
return job.state if job is not None else 'missing'
if job.state != 'processing':
return job.state
if worker_id is not None:
if job.lease_owner != worker_id:
return job.state
elif job.lease_until is None or job.lease_until >= now:
return job.state
attachment = await session.get(AttachmentRow, job.attachment_id)
exhausted = job.attempt >= job.max_attempts
if permanent or exhausted:
job.state = 'failed'
attachment.status = 'failed'
attachment.error_code = error_code if permanent else 'retry_exhausted'
kind = 'failed'
else:
job.attempt += 1
job.state = 'queued'
job.next_attempt_at = available_at
attachment.status = 'queued'
attachment.error_code = None
session.add(
OutboxEventRow(
id=uuid4(),
event_type='attachment.process.v1',
job_id=job.id,
payload=job_payload(
job.id,
job.attachment_id,
job.pipeline_version,
job.attempt,
now,
),
available_at=available_at,
published_at=None,
claim_owner=None,
claim_until=None,
publish_attempts=0,
last_error_code=None,
created_at=now,
updated_at=now,
)
)
kind = 'retry_scheduled'
job.last_error_code = error_code
job.lease_owner = None
job.lease_until = None
job.updated_at = now
attachment.updated_at = now
session.add(
JobEventRow(
job_id=job.id,
kind=kind,
attempt=job.attempt,
error_code=error_code,
created_at=now,
)
)
await session.flush()
return job.state
async def recover_expired(self, now: datetime) -> int:
async with self._sessions() as session:
ids = tuple(
await session.scalars(
select(AttachmentJobRow.id).where(
AttachmentJobRow.state == 'processing',
AttachmentJobRow.lease_until < now,
)
)
)
recovered = 0
for job_id in ids:
state = await self.retry_or_fail(
job_id, None, 'lease_expired', False, now, now
)
recovered += state in {'queued', 'failed'}
return recovered
async def replay(self, job_id: UUID, now: datetime, dry_run: bool = False) -> bool:
async with self._sessions() as session:
async with session.begin():
job = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if job is None or job.state != 'failed':
return False
attachment = await session.get(AttachmentRow, job.attachment_id)
if attachment is None or not attachment.staging_key:
return False
if dry_run:
return True
job.state = 'queued'
job.attempt += 1
job.max_attempts = job.attempt + 4
job.replay_count += 1
job.last_error_code = None
job.next_attempt_at = now
job.updated_at = now
attachment.status = 'queued'
attachment.error_code = None
attachment.updated_at = now
session.add_all(
(
OutboxEventRow(
id=uuid4(), event_type='attachment.process.v1', job_id=job.id,
payload=job_payload(job.id, job.attachment_id, job.pipeline_version, job.attempt, now),
available_at=now, published_at=None, claim_owner=None, claim_until=None,
publish_attempts=0, last_error_code=None, created_at=now, updated_at=now,
),
JobEventRow(job_id=job.id, kind='replayed', attempt=job.attempt, error_code=None, created_at=now),
)
)
return True
class OutboxRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
async def claim(self, owner: str, now: datetime, lease_seconds: int, limit: int = 20) -> tuple[OutboxEvent, ...]:
async with self._sessions() as session:
async with session.begin():
rows = tuple(
await session.scalars(
select(OutboxEventRow)
.where(
OutboxEventRow.published_at.is_(None),
OutboxEventRow.available_at <= now,
or_(OutboxEventRow.claim_until.is_(None), OutboxEventRow.claim_until < now),
)
.order_by(OutboxEventRow.available_at, OutboxEventRow.created_at)
.limit(limit)
.with_for_update(skip_locked=True)
)
)
until = now + timedelta(seconds=lease_seconds)
for row in rows:
row.claim_owner = owner
row.claim_until = until
row.publish_attempts += 1
row.updated_at = now
return tuple(OutboxEvent(row.id, row.job_id, row.payload, row.available_at) for row in rows)
async def published(self, event_id: UUID, owner: str, now: datetime) -> None:
async with self._sessions() as session:
async with session.begin():
await session.execute(
update(OutboxEventRow)
.where(
OutboxEventRow.id == event_id,
OutboxEventRow.claim_owner == owner,
OutboxEventRow.published_at.is_(None),
)
.values(published_at=now, claim_owner=None, claim_until=None, last_error_code=None, updated_at=now)
)
async def failed(self, event_id: UUID, owner: str, code: str, now: datetime) -> None:
async with self._sessions() as session:
async with session.begin():
await session.execute(
update(OutboxEventRow)
.where(OutboxEventRow.id == event_id, OutboxEventRow.claim_owner == owner)
.values(claim_owner=None, claim_until=None, last_error_code=code, updated_at=now)
)
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
from datetime import datetime
from typing import Sequence
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.errors import Conflict, NotFound
from ..domain.models import Keyset, Message
from ..models.attachment import AttachmentRow, MessageAttachmentRow
from ..models.chat import ChatMemberRow, ChatRow
from ..models.message import MessageRow
class MessageRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: MessageRow, attachment_ids: Sequence[UUID]) -> Message:
return Message(
id=row.id,
chat_id=row.chat_id,
sender_id=row.sender_id,
text=row.text,
created_at=row.created_at,
attachment_ids=tuple(attachment_ids),
)
@staticmethod
async def _attachment_ids(session: AsyncSession, message_id: UUID) -> tuple[UUID, ...]:
result = await session.scalars(
select(MessageAttachmentRow.attachment_id)
.where(MessageAttachmentRow.message_id == message_id)
.order_by(MessageAttachmentRow.position)
)
return tuple(result)
@staticmethod
async def _accessible_chat_id(session: AsyncSession, chat_id: UUID, user_id: UUID) -> UUID | None:
return await session.scalar(
select(ChatRow.id)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
)
async def create_for_member(
self,
message_id: UUID,
chat_id: UUID,
sender_id: UUID,
text: str,
attachment_ids: Sequence[UUID],
created_at: datetime,
idempotency_key: str,
) -> Message | None:
async with self._sessions() as session:
async with session.begin():
accessible_chat_id = await self._accessible_chat_id(session, chat_id, sender_id)
if accessible_chat_id is None:
return None
if attachment_ids:
found = {
attachment_id: status
for attachment_id, status in (
await session.execute(
select(AttachmentRow.id, AttachmentRow.status).where(
AttachmentRow.id.in_(attachment_ids),
AttachmentRow.chat_id == accessible_chat_id,
)
)
).all()
}
if set(found) != set(attachment_ids):
raise NotFound('вложение не найдено')
if any(status == 'failed' for status in found.values()):
raise Conflict('неуспешное вложение нельзя прикрепить')
statement = (
insert(MessageRow)
.values(
id=message_id,
chat_id=accessible_chat_id,
sender_id=sender_id,
text=text,
created_at=created_at,
idempotency_key=idempotency_key,
)
.on_conflict_do_nothing(
index_elements=[
MessageRow.sender_id,
MessageRow.chat_id,
MessageRow.idempotency_key,
]
)
.returning(MessageRow.id)
)
created_id = await session.scalar(statement)
if created_id is not None:
row = await session.get(MessageRow, created_id)
session.add_all(
MessageAttachmentRow(
message_id=created_id,
attachment_id=attachment_id,
position=position,
)
for position, attachment_id in enumerate(attachment_ids)
)
await session.flush()
result_attachment_ids = tuple(attachment_ids)
else:
row = await session.scalar(
select(MessageRow).where(
MessageRow.sender_id == sender_id,
MessageRow.chat_id == accessible_chat_id,
MessageRow.idempotency_key == idempotency_key,
)
)
if row is None:
raise RuntimeError('результат идемпотентной операции не найден')
result_attachment_ids = await self._attachment_ids(session, row.id)
if row.text != text or result_attachment_ids != tuple(attachment_ids):
raise Conflict('Idempotency-Key уже использован с другим сообщением')
return self._to_domain(row, result_attachment_ids)
async def page_for_member(
self, chat_id: UUID, user_id: UUID, after: Keyset | None, limit: int
) -> list[Message] | None:
async with self._sessions() as session:
accessible_chat_id = await self._accessible_chat_id(session, chat_id, user_id)
if accessible_chat_id is None:
return None
query = (
select(MessageRow)
.where(MessageRow.chat_id == accessible_chat_id)
.order_by(MessageRow.created_at, MessageRow.id)
.limit(limit)
)
if after is not None:
query = query.where(tuple_(MessageRow.created_at, MessageRow.id) > tuple_(after.created_at, after.id))
rows = list((await session.scalars(query)).all())
return [
self._to_domain(row, await self._attachment_ids(session, row.id))
for row in rows
]
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.errors import Conflict
from ..domain.models import User
from ..models.user import UserRow
class UserRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: UserRow) -> User:
return User(id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at)
async def register(
self, user_id: UUID, username: str, display_name: str, password_hash: str, created_at: datetime
) -> User:
row = UserRow(
id=user_id,
username=username,
display_name=display_name,
password_hash=password_hash,
created_at=created_at,
)
try:
async with self._sessions() as session:
async with session.begin():
session.add(row)
await session.flush()
except IntegrityError as exc:
raise Conflict('username уже занят') from exc
return self._to_domain(row)
async def get_by_id(self, user_id: UUID) -> User | None:
async with self._sessions() as session:
row = await session.get(UserRow, user_id)
return self._to_domain(row) if row is not None else None
async def get_credentials(self, username: str) -> UserRow | None:
async with self._sessions() as session:
return await session.scalar(select(UserRow).where(UserRow.username == username))
@classmethod
def to_domain(cls, row: UserRow) -> User:
return cls._to_domain(row)
View File
+167
View File
@@ -0,0 +1,167 @@
from __future__ import annotations
import asyncio
import hashlib
import logging
from datetime import datetime, timezone
from tempfile import SpooledTemporaryFile
from typing import AsyncIterator, BinaryIO
import unicodedata
from uuid import UUID, uuid4
from PIL import Image, UnidentifiedImageError
from ..domain.errors import (
Conflict,
Invalid,
NotFound,
PayloadTooLarge,
Unavailable,
UnsupportedMediaType,
)
from ..domain.models import Attachment, User
from ..repositories import AttachmentRepository
from ..storage import ObjectStream, S3Storage
logger = logging.getLogger('app.attachments')
MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
def _validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255:
raise Invalid('filename должен содержать от 1 до 255 символов')
if any(
character in {'/', '\\'} or unicodedata.category(character) == 'Cc'
for character in filename
):
raise Invalid('filename содержит запрещённые символы')
def _inspect_image(source: BinaryIO, declared_type: str, max_pixels: int) -> tuple[str, int, int]:
try:
source.seek(0)
with Image.open(source) as image:
actual_type = MIME_BY_FORMAT.get(image.format or '')
if actual_type is None or actual_type != declared_type:
raise UnsupportedMediaType('MIME не соответствует содержимому PNG/JPEG')
width, height = image.size
if width <= 0 or height <= 0:
raise UnsupportedMediaType('изображение имеет неверные размеры')
if width * height > max_pixels:
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей')
image.verify()
except Image.DecompressionBombError as error:
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') from error
except (UnidentifiedImageError, OSError, SyntaxError) as error:
raise UnsupportedMediaType('повреждённое или неподдерживаемое изображение') from error
finally:
source.seek(0)
return actual_type, width, height
class AttachmentService:
def __init__(
self,
repository: AttachmentRepository,
storage: S3Storage,
max_upload_bytes: int,
max_image_pixels: int,
job_max_attempts: int,
) -> None:
self._repository = repository
self._storage = storage
self._max_upload_bytes = max_upload_bytes
self._max_image_pixels = max_image_pixels
self._job_max_attempts = job_max_attempts
async def upload(
self,
actor: User,
chat_id: UUID,
filename: str,
declared_type: str,
chunks: AsyncIterator[bytes],
) -> Attachment:
_validate_filename(filename)
if declared_type not in MIME_BY_FORMAT.values():
raise UnsupportedMediaType('поддерживаются только image/png и image/jpeg')
if not await self._repository.chat_exists_for_member(chat_id, actor.id):
raise NotFound('чат не найден')
digest = hashlib.sha256()
size = 0
with SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b') as source:
async for chunk in chunks:
size += len(chunk)
if size > self._max_upload_bytes:
raise PayloadTooLarge('файл превышает лимит 10 MiB')
digest.update(chunk)
source.write(chunk)
if size == 0:
raise UnsupportedMediaType('пустой файл не является изображением')
actual_type, width, height = await asyncio.to_thread(
_inspect_image,
source,
declared_type,
self._max_image_pixels,
)
attachment_id = uuid4()
job_id = uuid4()
staging_key = f'staging/{attachment_id}/original'
await self._storage.put(staging_key, source, size, actual_type)
now = datetime.now(timezone.utc)
try:
return await self._repository.create_queued(
attachment_id=attachment_id,
job_id=job_id,
chat_id=chat_id,
owner_id=actor.id,
filename=filename,
staging_key=staging_key,
content_type=actual_type,
size_bytes=size,
sha256=digest.hexdigest(),
width=width,
height=height,
max_attempts=self._job_max_attempts,
now=now,
)
except Exception:
try:
await self._storage.remove(staging_key)
except Unavailable:
logger.warning('failed to compensate staging object key=%s', staging_key)
raise
async def get(self, actor: User, attachment_id: UUID) -> Attachment:
attachment = await self._repository.get_for_member(attachment_id, actor.id)
if attachment is None:
raise NotFound('вложение не найдено')
return attachment
async def content(
self, actor: User, attachment_id: UUID, variant: str
) -> tuple[Attachment, ObjectStream, str, int]:
attachment = await self.get(actor, attachment_id)
if attachment.status != 'ready':
raise Conflict('вложение ещё не готово')
if variant == 'original':
key = attachment.original_key
content_type = attachment.content_type
size_bytes = attachment.size_bytes
else:
found = next((item for item in attachment.variants if item.name == variant), None)
if found is None:
raise NotFound('вариант вложения не найден')
key = found.object_key
content_type = found.content_type
size_bytes = found.size_bytes
if key is None:
raise Unavailable('объект вложения отсутствует в метаданных')
try:
stream = await self._storage.get(key)
except FileNotFoundError as error:
raise Unavailable('объект вложения отсутствует в хранилище') from error
return attachment, stream, content_type, size_bytes
+35
View File
@@ -0,0 +1,35 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Optional, Sequence
from uuid import UUID, uuid4
from ..domain.errors import Invalid, NotFound
from ..domain.models import Chat, Keyset, Page, User
from ..repositories import ChatRepository
class ChatService:
def __init__(self, repository: ChatRepository) -> None:
self._repository = repository
async def create(self, actor: User, title: str, member_ids: Sequence[UUID]) -> Chat:
if actor.id not in member_ids:
raise Invalid('вызывающий должен входить в member_ids')
return await self._repository.create(
chat_id=uuid4(),
title=title,
member_ids=member_ids,
created_at=datetime.now(timezone.utc),
)
async def get(self, actor: User, chat_id: UUID) -> Chat:
chat = await self._repository.get_for_member(chat_id, actor.id)
if chat is None:
raise NotFound('чат не найден')
return chat
async def page(self, actor: User, after: Optional[Keyset], limit: int) -> Page:
rows = await self._repository.page_for_member(actor.id, after, limit + 1)
items = tuple(rows[:limit])
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Optional, Sequence
from uuid import UUID, uuid4
from ..domain.errors import NotFound
from ..domain.models import Keyset, Message, Page, User
from ..repositories import MessageRepository
class MessageService:
def __init__(self, repository: MessageRepository) -> None:
self._repository = repository
async def send(
self,
actor: User,
chat_id: UUID,
text: str,
attachment_ids: Sequence[UUID],
idempotency_key: str,
) -> Message:
message = await self._repository.create_for_member(
message_id=uuid4(),
chat_id=chat_id,
sender_id=actor.id,
text=text,
attachment_ids=attachment_ids,
created_at=datetime.now(timezone.utc),
idempotency_key=idempotency_key,
)
if message is None:
raise NotFound('чат не найден')
return message
async def history(
self, actor: User, chat_id: UUID, after: Optional[Keyset], limit: int
) -> Page:
rows = await self._repository.page_for_member(chat_id, actor.id, after, limit + 1)
if rows is None:
raise NotFound('чат не найден')
items = tuple(rows[:limit])
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
+35
View File
@@ -0,0 +1,35 @@
from __future__ import annotations
from datetime import datetime, timezone
from uuid import UUID, uuid4
from ..auth.passwords import hash_password, password_matches
from ..domain.errors import Unauthorized
from ..domain.models import User
from ..repositories import UserRepository
class UserService:
def __init__(self, repository: UserRepository) -> None:
self._repository = repository
async def register(self, username: str, display_name: str, password: str) -> User:
return await self._repository.register(
user_id=uuid4(),
username=username,
display_name=display_name,
password_hash=hash_password(password),
created_at=datetime.now(timezone.utc),
)
async def identify(self, user_id: UUID) -> User:
user = await self._repository.get_by_id(user_id)
if user is None:
raise Unauthorized('требуется действующая сессия')
return user
async def authenticate(self, username: str, password: str) -> User | None:
row = await self._repository.get_credentials(username)
if row is None or not password_matches(row.password_hash, password):
return None
return self._repository.to_domain(row)
+3
View File
@@ -0,0 +1,3 @@
from .s3 import ObjectStream, S3Storage, StoredObject
__all__ = ['ObjectStream', 'S3Storage', 'StoredObject']
+146
View File
@@ -0,0 +1,146 @@
from __future__ import annotations
import asyncio
import certifi
from dataclasses import dataclass
from datetime import datetime
from typing import BinaryIO, Iterator
from io import BytesIO
from urllib.parse import urlparse
from minio import Minio
from minio.error import InvalidResponseError, S3Error
from urllib3 import PoolManager, Retry, Timeout
from urllib3.exceptions import HTTPError
from ..domain.errors import Unavailable
def _unavailable(error: Exception) -> Unavailable:
return Unavailable('объектное хранилище недоступно')
@dataclass(slots=True)
class ObjectStream:
response: object
def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]:
try:
while chunk := self.response.read(size):
yield chunk
finally:
self.response.close()
self.response.release_conn()
@dataclass(frozen=True, slots=True)
class StoredObject:
key: str
last_modified: datetime
class S3Storage:
def __init__(
self,
endpoint: str,
access_key: str,
secret_key: str,
bucket: str,
region: str,
) -> None:
parsed = urlparse(endpoint)
if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}:
raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port')
self._client = Minio(
parsed.netloc,
access_key=access_key,
secret_key=secret_key,
secure=parsed.scheme == 'https',
region=region,
http_client=PoolManager(
maxsize=8,
timeout=Timeout(connect=2, read=5),
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
),
)
self.bucket = bucket
@staticmethod
def _translate(error: Exception) -> None:
if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}:
raise FileNotFoundError from error
if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)):
raise _unavailable(error) from error
raise error
async def ping(self) -> None:
try:
exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket)
except Exception as error:
self._translate(error)
if not exists:
raise Unavailable('bucket объектного хранилища не создан')
async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None:
source.seek(0)
try:
await asyncio.to_thread(
self._client.put_object,
self.bucket,
key,
source,
length,
content_type,
)
except Exception as error:
self._translate(error)
async def get(self, key: str) -> ObjectStream:
try:
response = await asyncio.to_thread(self._client.get_object, self.bucket, key)
except Exception as error:
self._translate(error)
return ObjectStream(response)
async def read(self, key: str, max_bytes: int) -> bytes:
stream = await self.get(key)
result = bytearray()
for chunk in stream.chunks():
result.extend(chunk)
if len(result) > max_bytes:
raise ValueError('stored object exceeds configured byte limit')
return bytes(result)
async def put_bytes(self, key: str, content: bytes, content_type: str) -> None:
await self.put(key, BytesIO(content), len(content), content_type)
async def exists(self, key: str) -> bool:
try:
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
return True
except Exception as error:
try:
self._translate(error)
except FileNotFoundError:
return False
raise
async def remove(self, key: str) -> None:
try:
await asyncio.to_thread(self._client.remove_object, self.bucket, key)
except Exception as error:
self._translate(error)
async def list(self, prefix: str = '') -> tuple[StoredObject, ...]:
def collect() -> tuple[StoredObject, ...]:
return tuple(
StoredObject(item.object_name, item.last_modified)
for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True)
)
try:
return await asyncio.to_thread(collect)
except Exception as error:
self._translate(error)
+1
View File
@@ -0,0 +1 @@
"""Операторские CLI-инструменты приложения."""
+65
View File
@@ -0,0 +1,65 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage
async def cleanup(apply: bool, grace_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
sessions = create_session_factory(engine)
async with sessions() as session:
known = set(
key
for key in await session.scalars(select(AttachmentRow.original_key))
if key is not None
)
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
removed = 0
for item in await storage.list('originals/'):
if item.key in known or item.last_modified > cutoff:
continue
action = 'DELETE' if apply else 'WOULD_DELETE'
print(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
if apply:
await storage.remove(item.key)
removed += 1
print(
f'summary mode={"apply" if apply else "dry-run"} '
f'known={len(known)} removed={removed} grace_seconds={grace_seconds}'
)
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description='Safely find old S3 objects without PostgreSQL metadata')
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--grace-seconds', type=int, default=3600)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error('--grace-seconds must be at least 60')
raise SystemExit(asyncio.run(cleanup(args.apply, args.grace_seconds)))
if __name__ == '__main__':
main()
+68
View File
@@ -0,0 +1,68 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage
async def cleanup(apply: bool, retention_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
cutoff = datetime.now(timezone.utc) - timedelta(seconds=retention_seconds)
try:
async with sessions() as session:
active = set(
key
for key in await session.scalars(
select(AttachmentRow.staging_key).where(
AttachmentRow.staging_key.is_not(None),
AttachmentRow.status.in_(('queued', 'processing')),
)
)
if key is not None
)
removed = 0
for item in await storage.list('staging/'):
if item.key in active or item.last_modified > cutoff:
continue
print(f'{"DELETE" if apply else "WOULD_DELETE"} key={item.key}')
if apply:
await storage.remove(item.key)
removed += 1
print(f'summary active={len(active)} removed={removed} mode={"apply" if apply else "dry-run"}')
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description='Clean inactive durable staging objects')
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--retention-seconds', type=int)
args = parser.parse_args()
settings = load_settings()
retention = args.retention_seconds or settings.staging_retention_seconds
if retention < 60:
parser.error('retention must be at least 60 seconds')
raise SystemExit(asyncio.run(cleanup(args.apply, retention)))
if __name__ == '__main__':
main()
+250
View File
@@ -0,0 +1,250 @@
from __future__ import annotations
import argparse
import asyncio
import json
import time
import tracemalloc
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Callable, Sequence
from uuid import UUID
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage, StoredObject
MISSING_ERROR_CODE = 'original_missing'
@dataclass(frozen=True, slots=True)
class AttachmentSnapshot:
id: UUID
object_key: str
status: str
updated_at: datetime
@dataclass(frozen=True, slots=True)
class ReconcilePlan:
orphan: tuple[StoredObject, ...]
fresh_orphan: tuple[StoredObject, ...]
missing: tuple[AttachmentSnapshot, ...]
fresh_missing: tuple[AttachmentSnapshot, ...]
correct: int
def build_plan(
attachments: Sequence[AttachmentSnapshot],
objects: Sequence[StoredObject],
cutoff: datetime,
) -> ReconcilePlan:
"""Compare immutable PostgreSQL and S3 snapshots without changing either side."""
known_keys = {attachment.object_key for attachment in attachments}
object_keys = {item.key for item in objects}
unknown_objects = [item for item in objects if item.key not in known_keys]
orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff)
fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff)
ready_without_object = [
attachment
for attachment in attachments
if attachment.status == 'ready' and attachment.object_key not in object_keys
]
missing = tuple(
attachment for attachment in ready_without_object if attachment.updated_at <= cutoff
)
fresh_missing = tuple(
attachment for attachment in ready_without_object if attachment.updated_at > cutoff
)
correct = sum(attachment.object_key in object_keys for attachment in attachments)
return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct)
async def _load_attachments(
sessions: async_sessionmaker[AsyncSession],
) -> tuple[AttachmentSnapshot, ...]:
async with sessions() as session:
rows = (
await session.execute(
select(
AttachmentRow.id,
AttachmentRow.original_key,
AttachmentRow.status,
AttachmentRow.updated_at,
)
)
).all()
return tuple(AttachmentSnapshot(*row) for row in rows)
async def _metadata_key_exists(
sessions: async_sessionmaker[AsyncSession], object_key: str
) -> bool:
async with sessions() as session:
attachment_id = await session.scalar(
select(AttachmentRow.id).where(AttachmentRow.original_key == object_key).limit(1)
)
return attachment_id is not None
async def reconcile(
sessions: async_sessionmaker[AsyncSession],
storage: S3Storage,
*,
apply: bool,
grace_seconds: int,
now: datetime | None = None,
output: Callable[[str], None] = print,
) -> dict[str, int | float | str]:
started = time.perf_counter()
tracemalloc.start()
current_time = now or datetime.now(timezone.utc)
cutoff = current_time - timedelta(seconds=grace_seconds)
attachments = await _load_attachments(sessions)
objects = await storage.list('originals/')
plan = build_plan(attachments, objects, cutoff)
for item in plan.fresh_orphan:
output(
f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}'
)
for attachment in plan.fresh_missing:
output(
'SKIP_FRESH_MISSING '
f'attachment_id={attachment.id} key={attachment.object_key} '
f'updated={attachment.updated_at.isoformat()}'
)
removed = 0
race_protected = 0
for item in plan.orphan:
action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN'
if not apply:
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
continue
# PostgreSQL and S3 cannot share a transaction. Re-read the authoritative
# metadata immediately before a destructive action so a row created after
# the initial snapshot protects its object.
if await _metadata_key_exists(sessions, item.key):
output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}')
race_protected += 1
continue
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
await storage.remove(item.key)
removed += 1
marked_failed = 0
for attachment in plan.missing:
if not apply:
output(
f'WOULD_MARK_MISSING attachment_id={attachment.id} '
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
)
continue
# The object may have been restored after list_objects(). Do not turn
# healthy metadata into failed based only on the earlier snapshot.
if await storage.exists(attachment.object_key):
output(
f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} '
f'key={attachment.object_key}'
)
race_protected += 1
continue
async with sessions() as session:
async with session.begin():
result = await session.execute(
update(AttachmentRow)
.where(
AttachmentRow.id == attachment.id,
AttachmentRow.original_key == attachment.object_key,
AttachmentRow.status == 'ready',
AttachmentRow.updated_at == attachment.updated_at,
)
.values(
status='failed',
error_code=MISSING_ERROR_CODE,
updated_at=current_time,
)
)
changed = result.rowcount or 0
if changed:
output(
f'MARK_MISSING attachment_id={attachment.id} '
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
)
marked_failed += changed
else:
output(
f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} '
f'key={attachment.object_key}'
)
race_protected += 1
_, peak_bytes = tracemalloc.get_traced_memory()
tracemalloc.stop()
summary: dict[str, int | float | str] = {
'mode': 'apply' if apply else 'dry-run',
'metadata': len(attachments),
'objects': len(objects),
'correct': plan.correct,
'orphan': len(plan.orphan),
'missing': len(plan.missing),
'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing),
'removed': removed,
'marked_failed': marked_failed,
'race_protected': race_protected,
'grace_seconds': grace_seconds,
'elapsed_ms': round((time.perf_counter() - started) * 1000, 3),
'peak_memory_bytes': peak_bytes,
}
output(f'summary {json.dumps(summary, sort_keys=True)}')
return summary
async def run(apply: bool, grace_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
await reconcile(
create_session_factory(engine),
storage,
apply=apply,
grace_seconds=grace_seconds,
)
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(
description='Reconcile attachment metadata in PostgreSQL with original objects in S3'
)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--grace-seconds', type=int, default=3600)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error('--grace-seconds must be at least 60')
raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds)))
if __name__ == '__main__':
main()
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timezone
from uuid import UUID
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..repositories import JobRepository
from ..storage import S3Storage
async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
try:
repository = JobRepository(create_session_factory(engine))
if job_id is None:
assert attachment_id is not None
job_id = await repository.id_for_attachment(attachment_id)
if job_id is None:
print(f'REJECTED attachment_id={attachment_id} reason=job_not_found')
return 2
staging_key = await repository.failed_staging_key(job_id)
if staging_key is None:
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
return 2
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
if not await storage.exists(staging_key):
print(f'REJECTED job_id={job_id} reason=staging_object_missing')
return 2
accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run)
if not accepted:
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
return 2
print(f'{"WOULD_REPLAY" if dry_run else "REPLAYED"} job_id={job_id}')
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description='Safely replay one failed attachment job')
target = parser.add_mutually_exclusive_group(required=True)
target.add_argument('--job-id', type=UUID)
target.add_argument('--attachment-id', type=UUID)
parser.add_argument('--dry-run', action='store_true')
args = parser.parse_args()
raise SystemExit(asyncio.run(replay(args.job_id, args.attachment_id, args.dry_run)))
if __name__ == '__main__':
main()
+138
View File
@@ -0,0 +1,138 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from io import BytesIO
from uuid import NAMESPACE_URL, uuid5
from sqlalchemy import delete, insert
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..models.chat import ChatMemberRow, ChatRow
from ..models.user import UserRow
from ..storage import S3Storage
USER_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:user')
CHAT_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:chat')
OBJECT_PREFIX = 'originals/reconciler-benchmark/'
WORKERS = 16
async def _put_objects(
storage: S3Storage,
keys: list[str],
) -> None:
for offset in range(0, len(keys), WORKERS):
batch = keys[offset : offset + WORKERS]
await asyncio.gather(
*(storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream') for key in batch)
)
async def prepare(size: int) -> None:
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
old = datetime.now(timezone.utc) - timedelta(hours=2)
rows = [
{
'id': uuid5(NAMESPACE_URL, f'reconciler-benchmark:attachment:{index}'),
'chat_id': CHAT_ID,
'owner_id': USER_ID,
'filename': f'benchmark-{index}.png',
'original_key': f'{OBJECT_PREFIX}{index}',
'staging_key': None,
'pipeline_version': 1,
'content_type': 'image/png',
'size_bytes': 1,
'sha256': '0' * 64,
'width': 1,
'height': 1,
'status': 'ready',
'error_code': None,
'created_at': old,
'updated_at': old,
}
for index in range(size)
]
try:
async with sessions() as session:
async with session.begin():
session.add(UserRow(
id=USER_ID,
username=f'bench-{USER_ID.hex[:16]}',
display_name='Reconciler benchmark',
password_hash='not-used',
created_at=old,
))
session.add(ChatRow(id=CHAT_ID, title='Reconciler benchmark', created_at=old))
await session.flush()
session.add(ChatMemberRow(chat_id=CHAT_ID, user_id=USER_ID))
await session.flush()
for offset in range(0, size, 500):
await session.execute(insert(AttachmentRow), rows[offset : offset + 500])
await _put_objects(storage, [row['original_key'] for row in rows])
print(f'prepared metadata={size} objects={size}')
finally:
await engine.dispose()
async def cleanup() -> None:
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
objects = await storage.list(OBJECT_PREFIX)
keys = [item.key for item in objects]
for offset in range(0, len(keys), WORKERS):
await asyncio.gather(
*(storage.remove(key) for key in keys[offset : offset + WORKERS])
)
async with sessions() as session:
async with session.begin():
await session.execute(delete(ChatRow).where(ChatRow.id == CHAT_ID))
await session.execute(delete(UserRow).where(UserRow.id == USER_ID))
print(f'cleaned objects={len(keys)}')
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(
description='Prepare and clean isolated PostgreSQL/S3 data for reconciler benchmarks'
)
subparsers = parser.add_subparsers(dest='command', required=True)
prepare_parser = subparsers.add_parser('prepare')
prepare_parser.add_argument('--size', type=int, required=True)
subparsers.add_parser('cleanup')
args = parser.parse_args()
if args.command == 'prepare':
if args.size < 1 or args.size > 10000:
parser.error('--size must be between 1 and 10000')
asyncio.run(prepare(args.size))
else:
asyncio.run(cleanup())
if __name__ == '__main__':
main()
+198
View File
@@ -0,0 +1,198 @@
services:
postgres:
image: postgres:17.4-alpine
environment:
POSTGRES_USER: messenger_test
POSTGRES_PASSWORD: messenger_test
POSTGRES_DB: messenger_test
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger_test -d messenger_test"]
interval: 2s
timeout: 2s
retries: 30
redis:
image: redis:7.4.2-alpine
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 2s
timeout: 2s
retries: 30
broker:
image: rabbitmq:4.1.4-alpine
environment:
RABBITMQ_DEFAULT_USER: messenger_test
RABBITMQ_DEFAULT_PASS: messenger_test
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 2s
timeout: 5s
retries: 30
s3:
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
user: "0"
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: test-minio-admin
MINIO_ROOT_PASSWORD: test-minio-root-password
S3_BUCKET: messenger-attachments
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 2s
timeout: 2s
retries: 30
s3-bootstrap:
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
mc mb --ignore-existing local/"$$S3_BUCKET"
mc anonymous set none local/"$$S3_BUCKET"
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
mc admin policy create local messenger-api /tmp/s3-api-policy.json
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
environment:
MINIO_ROOT_USER: test-minio-admin
MINIO_ROOT_PASSWORD: test-minio-root-password
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
volumes:
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
depends_on:
s3:
condition: service_healthy
restart: "no"
migrate:
build: .
command: ["alembic", "upgrade", "head"]
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build: .
ports:
- "127.0.0.1:18080:8080"
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
MAX_UPLOAD_BYTES: 10485760
MAX_IMAGE_PIXELS: 40000000
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
JOB_MAX_ATTEMPTS: 5
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
interval: 2s
timeout: 2s
retries: 30
publisher:
build: .
command: ["python", "-m", "app.jobs.publisher"]
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 5s
timeout: 3s
retries: 10
worker:
build: .
command: ["python", "-m", "app.jobs.worker"]
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
WORKER_CONCURRENCY: 2
WORKER_PREFETCH: 2
JOB_MAX_ATTEMPTS: 5
JOB_TIMEOUT_SECONDS: 60
JOB_LEASE_SECONDS: 90
RETRY_BASE_SECONDS: 1
RETRY_MAX_SECONDS: 4
STAGING_RETENTION_SECONDS: 86400
THUMBNAIL_JPEG_QUALITY: 85
CROP_JPEG_QUALITY: 85
OPTIMIZED_WEBP_QUALITY: 82
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 5s
timeout: 3s
retries: 10
tests:
build:
context: .
dockerfile: Dockerfile.test
environment:
BASE_URL: http://api:8080
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
depends_on:
api:
condition: service_healthy
restart: "no"
+240
View File
@@ -0,0 +1,240 @@
services:
postgres:
image: postgres:17.4-alpine
cpus: "1.0"
mem_limit: 512m
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
interval: 5s
timeout: 3s
retries: 20
redis:
image: redis:7.4.2-alpine
cpus: "0.25"
mem_limit: 128m
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 20
broker:
image: rabbitmq:4.1.4-alpine
cpus: "0.5"
mem_limit: 384m
environment:
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger}
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-change-me-rabbitmq}
volumes:
- broker_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
s3:
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
user: "0"
cpus: "0.5"
mem_limit: 256m
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
volumes:
- s3_data:/data
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 5s
timeout: 3s
retries: 20
s3-bootstrap:
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
cpus: "0.25"
mem_limit: 128m
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
mc mb --ignore-existing local/"$$S3_BUCKET"
mc anonymous set none local/"$$S3_BUCKET"
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
mc admin policy create local messenger-api /tmp/s3-api-policy.json
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
volumes:
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
depends_on:
s3:
condition: service_healthy
restart: "no"
migrate:
build: .
cpus: "0.5"
mem_limit: 256m
command: ["alembic", "upgrade", "head"]
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
BROKER_URL: ${BROKER_URL}
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build: .
cpus: "0.5"
mem_limit: 256m
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
LOG_LEVEL: ${LOG_LEVEL:-info}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
BROKER_URL: ${BROKER_URL}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
expose:
- "8080"
deploy:
replicas: 2
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
interval: 2s
timeout: 2s
retries: 5
stop_grace_period: 10s
publisher:
build: .
command: ["python", "-m", "app.jobs.publisher"]
cpus: "0.25"
mem_limit: 192m
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
BROKER_URL: ${BROKER_URL}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 10s
timeout: 3s
retries: 3
restart: unless-stopped
stop_grace_period: 10s
worker:
build: .
command: ["python", "-m", "app.jobs.worker"]
cpus: "1.0"
mem_limit: 768m
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
BROKER_URL: ${BROKER_URL}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
RETRY_BASE_SECONDS: ${RETRY_BASE_SECONDS:-2}
RETRY_MAX_SECONDS: ${RETRY_MAX_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
THUMBNAIL_JPEG_QUALITY: ${THUMBNAIL_JPEG_QUALITY:-85}
CROP_JPEG_QUALITY: ${CROP_JPEG_QUALITY:-85}
OPTIMIZED_WEBP_QUALITY: ${OPTIMIZED_WEBP_QUALITY:-82}
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 10s
timeout: 3s
retries: 3
restart: unless-stopped
stop_grace_period: 70s
lb:
image: haproxy:3.2.26-alpine
cpus: "0.25"
mem_limit: 128m
ports:
- "${APP_PORT:-8080}:8080"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"]
interval: 2s
timeout: 2s
retries: 5
stop_grace_period: 5s
volumes:
postgres_data:
redis_data:
s3_data:
broker_data:
+93
View File
@@ -0,0 +1,93 @@
# Lab 5 grade 4 — результаты проверки
Дата: 2026-11-16, дополнено 2026-11-17. Среда: macOS host, Docker Desktop Linux containers.
Секреты, session tokens и содержимое `.env` не сохранялись.
```text
$ make check
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ docker compose -f compose.tests.yaml run --rm \
-v "$PWD/app:/tests/app:ro" -v "$PWD/tests:/tests/tests:ro" \
tests python -m pytest -q
................... [100%]
19 passed in 1.22s
$ docker compose -f compose.tests.yaml run --rm --no-deps \
-v "$PWD/app:/tests/app:ro" -v "$PWD/tests:/tests/tests:ro" \
tests python -m pytest -q tests/test_pipeline.py
....... [100%]
7 passed in 0.94s
$ docker compose -f compose.tests.yaml run --rm --no-deps \
-v "$PWD/scripts:/tests/scripts:ro" \
tests python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
$ python3 tests/smoke.py --base-url http://127.0.0.1:18080
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
$ python3 tests/pipeline_smoke.py --base-url http://127.0.0.1:18080
PASS: original plus thumbnail/crop/optimized variants are valid
$ COMPOSE_FILE_PATH=compose.tests.yaml python3 tests/outbox_recovery.py \
--base-url http://127.0.0.1:18080 \
--stop-publisher-script scripts/stop-publisher.sh \
--start-publisher-script scripts/start-publisher.sh
PASS: COMMIT-before-publish recovered without a new upload
$ COMPOSE_FILE_PATH=compose.tests.yaml python3 tests/worker_outage.py \
--base-url http://127.0.0.1:18080 \
--stop-worker-script scripts/stop-worker.sh \
--restart-api-script scripts/restart-api.sh \
--start-worker-script scripts/start-worker.sh
PASS: worker stopped -> 202 -> message -> API restart -> worker -> ready
$ COMPOSE_FILE_PATH=compose.tests.yaml python3 tests/broker_persistence.py \
--base-url http://127.0.0.1:18080 \
--stop-worker-script scripts/stop-worker.sh \
--restart-broker-script scripts/restart-broker.sh \
--start-worker-script scripts/start-worker.sh
PASS: confirmed durable message survived broker restart with its volume
$ COMPOSE_FILE_PATH=compose.tests.yaml python3 tests/publisher_confirm_crash.py \
--base-url http://127.0.0.1:18080 \
--stop-publisher-script scripts/stop-publisher.sh \
--crash-publisher-script scripts/crash-publisher-after-confirm.sh
PASS: confirm-before-outbox-mark duplicate produced one logical result
$ COMPOSE_FILE_PATH=compose.tests.yaml python3 tests/worker_crash.py \
--base-url http://127.0.0.1:18080 \
--stop-worker-script scripts/stop-worker.sh \
--crash-worker-script scripts/crash-worker.sh \
--mode objects
PASS: worker crash mode=objects recovered one logical result
$ COMPOSE_FILE_PATH=compose.tests.yaml python3 tests/worker_crash.py \
--base-url http://127.0.0.1:18080 \
--stop-worker-script scripts/stop-worker.sh \
--crash-worker-script scripts/crash-worker.sh \
--mode commit
PASS: worker crash mode=commit recovered one logical result
$ COMPOSE_FILE_PATH=compose.tests.yaml python3 tests/failed_replay.py \
--base-url http://127.0.0.1:18080 \
--stop-worker-script scripts/stop-worker.sh \
--crash-worker-script scripts/crash-worker.sh \
--replay-script scripts/replay-attachment.sh
REPLAYED job_id=c9e77317-0803-4a23-bfa6-ab1113e47ed3
REJECTED job_id=c9e77317-0803-4a23-bfa6-ab1113e47ed3 reason=not_failed_or_staging_missing
PASS: permanent failure reached failed/DLQ and guarded replay reached ready
$ docker compose -f compose.tests.yaml exec -T broker rabbitmqctl list_queues name messages messages_ready
attachments.process 0 0
attachments.failed 1 1
$ git diff --check
(no output, exit 0)
```
Fault scripts управляют реальными Compose services (`api`, `publisher`, `worker`,
`broker`) и не создают новую attachment/job для восстановления уже принятой работы.
В тесте lease используется 3 секунды для быстрого воспроизведения; штатное значение —
90 секунд.
+31
View File
@@ -0,0 +1,31 @@
global
log stdout format raw local0
defaults
log global
mode http
option httplog
timeout connect 2s
timeout client 30s
timeout server 30s
timeout check 2s
resolvers docker
nameserver docker_dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
frontend public
bind :8080
default_backend api_backend
backend api_backend
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
server-template api 2 api:8080
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::BUCKET_NAME"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::BUCKET_NAME/*"]
}
]
}
+1
View File
@@ -1,2 +1,3 @@
# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
openapi-spec-validator==0.7.2
pytest==8.3.5
+10
View File
@@ -0,0 +1,10 @@
fastapi==0.115.12
uvicorn[standard]==0.34.2
SQLAlchemy[asyncio]==2.0.40
asyncpg==0.30.0
alembic==1.15.2
redis==5.2.1
argon2-cffi==25.1.0
minio==7.2.20
Pillow==13.0.0
aio-pika==9.5.5
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
set +e
docker compose -f "$compose_file" --env-file .env run --rm \
-e FAULT_PUBLISHER_AFTER_CONFIRM=1 \
publisher python -m app.jobs.publisher --once >/dev/null
status=$?
set -e
if [ "$status" -ne 91 ]; then
printf '%s\n' "publisher fault expected exit 91, got $status" >&2
exit 1
fi
docker compose -f "$compose_file" --env-file .env up -d publisher >/dev/null
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ]; then
printf '%s\n' 'usage: crash-worker.sh objects|commit|permanent' >&2
exit 2
fi
case "$1" in
objects) variable=FAULT_WORKER_AFTER_OBJECTS; expected=92 ;;
commit) variable=FAULT_WORKER_AFTER_COMMIT; expected=93 ;;
permanent) variable=FAULT_WORKER_PERMANENT; expected=94 ;;
*) printf '%s\n' 'mode must be objects, commit or permanent' >&2; exit 2 ;;
esac
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
set +e
docker compose -f "$compose_file" --env-file .env run --rm \
-e "$variable=1" -e JOB_LEASE_SECONDS=3 \
worker python -m app.jobs.worker >/dev/null
status=$?
set -e
if [ "$status" -ne "$expected" ]; then
printf '%s\n' "worker fault expected exit $expected, got $status" >&2
exit 1
fi
docker compose -f "$compose_file" --env-file .env up -d worker >/dev/null
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ]; then
printf '%s\n' 'usage: replay-attachment.sh ATTACHMENT_ID' >&2
exit 2
fi
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env run --rm \
publisher python -m app.tools.replay_failed --attachment-id "$1"
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env restart api >/dev/null
if [ "$compose_file" = "compose.yaml" ]; then
docker compose -f "$compose_file" --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null
else
docker compose -f "$compose_file" --env-file .env up -d api >/dev/null
fi
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env restart broker >/dev/null
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
# Останавливаем публичный маршрут до перезапуска общих хранилищ.
docker compose --env-file .env stop lb api publisher worker >/dev/null
docker compose --env-file .env restart postgres redis s3 broker >/dev/null
docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 broker >/dev/null
docker compose --env-file .env up -d --wait --wait-timeout 60 publisher worker api lb >/dev/null
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env up -d publisher >/dev/null
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env up -d worker >/dev/null
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
target_container=''
for container_id in $(docker compose --env-file .env ps -q api); do
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
target_container=$container_id
break
fi
done
if [ -z "$target_container" ]; then
printf '%s\n' 'API instance from TARGET_INSTANCE_ID was not found' >&2
exit 1
fi
docker stop --time 5 "$target_container" >/dev/null
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env stop publisher >/dev/null
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env stop worker >/dev/null
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""Stop the API replica that accepted upload and download through another replica."""
import hashlib
import os
from pathlib import Path
import subprocess
import time
from client import Client, ROOT, arguments, check, run
def main():
def configure(parser):
parser.add_argument('--action-script', required=True)
args = arguments('Attachment API-replica failover', configure)
script = Path(args.action_script).expanduser().resolve()
check(script.is_file() and os.access(script, os.X_OK), 'action script must be executable')
client = Client(args.base_url, args.ca_file, timeout=3)
alice, bob, _eve, chat = client.bootstrap()
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
uploaded = client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/attachments?filename=failover.png',
202,
actor=alice,
raw=content,
headers={'Content-Type': 'image/png'},
schema='Attachment',
)
ready = client.wait_ready(bob, uploaded['json'])
target = uploaded['headers']['X-Instance-Id']
env = dict(os.environ)
env['TARGET_INSTANCE_ID'] = target
started = time.monotonic()
subprocess.run([str(script)], env=env, check=True)
deadline = time.monotonic() + 15
failures = 0
while True:
try:
downloaded = client.call(
'GET',
f'/api/v1/attachments/{ready["id"]}/content',
actor=bob,
)
if downloaded['headers']['X-Instance-Id'] != target:
break
except Exception:
failures += 1
check(time.monotonic() < deadline, 'download did not recover through another replica')
time.sleep(0.2)
check(downloaded['bytes'] == content, 'downloaded file differs after replica failure')
print(
'PASS: upload_instance=%s download_instance=%s recovery_seconds=%.3f failures=%d sha256=%s'
% (
target,
downloaded['headers']['X-Instance-Id'],
time.monotonic() - started,
failures,
hashlib.sha256(content).hexdigest(),
)
)
if __name__ == '__main__':
run(main)
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""A persistent broker volume keeps a confirmed job across broker restart."""
import os
from pathlib import Path
import subprocess
import time
from client import Client, arguments, check, run
def executable(value):
path = Path(value).resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-worker-script', required=True)
parser.add_argument('--restart-broker-script', required=True)
parser.add_argument('--start-worker-script', required=True)
args = arguments('RabbitMQ persistent delivery after restart', configure)
stop_worker = executable(args.stop_worker_script)
restart_broker = executable(args.restart_broker_script)
start_worker = executable(args.start_worker_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop_worker)], check=True)
attachment, original = client.attachment(alice, chat)
time.sleep(2)
queued = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment'
)['json']
check(queued['status'] == 'queued', 'worker processed job while expected stopped')
subprocess.run([str(restart_broker)], check=True)
subprocess.run([str(start_worker)], check=True)
client.wait_ready(bob, attachment)
downloaded = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
)['bytes']
check(downloaded == original, 'job/original lost across broker restart')
print('PASS: confirmed durable message survived broker restart with its volume')
if __name__ == '__main__':
run(main)
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
"""Prove permanent failure, DLQ routing and guarded operator replay."""
import os
from pathlib import Path
import subprocess
import time
from client import Client, arguments, check, run
def executable(value):
path = Path(value).resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-worker-script', required=True)
parser.add_argument('--crash-worker-script', required=True)
parser.add_argument('--replay-script', required=True)
args = arguments('Failed registry, DLQ and guarded replay', configure)
stop = executable(args.stop_worker_script)
crash = executable(args.crash_worker_script)
replay = executable(args.replay_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop)], check=True)
attachment, original = client.attachment(alice, chat)
subprocess.run([str(crash), 'permanent'], check=True)
deadline = time.monotonic() + 20
while True:
failed = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob,
schema='Attachment',
)['json']
if failed['status'] == 'failed':
break
check(time.monotonic() < deadline, 'attachment did not reach failed')
time.sleep(0.2)
check(failed['error_code'] == 'fault_injected', 'unsafe or unexpected error code')
subprocess.run([str(replay), attachment['id']], check=True)
ready = client.wait_ready(bob, attachment)
check(len(ready['variants']) == 3, 'replay produced incomplete variants')
content = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
)['bytes']
check(content == original, 'replay changed original')
duplicate = subprocess.run([str(replay), attachment['id']], check=False)
check(duplicate.returncode != 0, 'ready job was replayed again')
print('PASS: permanent failure reached failed/DLQ and guarded replay reached ready')
if __name__ == '__main__':
run(main)
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Prove that a committed outbox survives publisher downtime."""
import os
from pathlib import Path
import subprocess
import time
from client import Client, arguments, check, run
def main():
def configure(parser):
parser.add_argument('--stop-publisher-script', required=True)
parser.add_argument('--start-publisher-script', required=True)
args = arguments('Transactional outbox recovery', configure)
stop = Path(args.stop_publisher_script).resolve()
start = Path(args.start_publisher_script).resolve()
check(all(path.is_file() and os.access(path, os.X_OK) for path in (stop, start)), 'scripts must be executable')
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop)], check=True)
attachment, _content = client.attachment(alice, chat)
time.sleep(1)
queued = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment'
)['json']
check(queued['status'] == 'queued', 'job escaped outbox while publisher was stopped')
subprocess.run([str(start)], check=True)
ready = client.wait_ready(bob, attachment)
check(ready['status'] == 'ready', 'outbox did not recover committed job')
print('PASS: COMMIT-before-publish recovered without a new upload')
if __name__ == '__main__':
run(main)
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Grade-4 black-box check for every promised image variant."""
import hashlib
from client import Client, arguments, check, run
from image_probe import jpeg_size
def main():
args = arguments('Verify thumbnail, crop and optimized variants')
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
ready = client.wait_ready(bob, attachment)
variants = {item['name']: item for item in ready['variants']}
check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'grade-4 variants missing')
expected = {
'thumbnail': ('image/jpeg', (256, 160)),
'crop': ('image/jpeg', (128, 128)),
'optimized': ('image/webp', (320, 200)),
}
for name, (content_type, dimensions) in expected.items():
response = client.call(
'GET',
f'/api/v1/attachments/{ready["id"]}/content?variant={name}',
actor=bob,
)
metadata = variants[name]
check(response['headers'].get_content_type() == content_type, name + ': MIME mismatch')
check((metadata['width'], metadata['height']) == dimensions, name + ': metadata dimensions')
check(len(response['bytes']) == metadata['size_bytes'], name + ': size mismatch')
check(hashlib.sha256(response['bytes']).hexdigest() == metadata['sha256'], name + ': hash mismatch')
if content_type == 'image/jpeg':
check(jpeg_size(response['bytes']) == dimensions, name + ': actual dimensions')
else:
check(response['bytes'][:4] == b'RIFF' and response['bytes'][8:12] == b'WEBP', 'optimized is not WebP')
downloaded = client.call(
'GET', f'/api/v1/attachments/{ready["id"]}/content', actor=bob
)['bytes']
check(downloaded == original, 'original bytes changed')
print('PASS: original plus thumbnail/crop/optimized variants are valid')
if __name__ == '__main__':
run(main)
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Crash after publisher confirm creates a harmless duplicate delivery."""
import os
from pathlib import Path
import subprocess
from client import Client, arguments, check, run
def executable(value):
path = Path(value).resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-publisher-script', required=True)
parser.add_argument('--crash-publisher-script', required=True)
args = arguments('Crash after publisher confirm', configure)
stop = executable(args.stop_publisher_script)
crash = executable(args.crash_publisher_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop)], check=True)
attachment, _original = client.attachment(alice, chat)
subprocess.run([str(crash)], check=True)
ready = client.wait_ready(bob, attachment)
names = [item['name'] for item in ready['variants']]
check(sorted(names) == ['crop', 'optimized', 'thumbnail'], 'duplicate or missing variants')
print('PASS: confirm-before-outbox-mark duplicate produced one logical result')
if __name__ == '__main__':
run(main)
+153
View File
@@ -0,0 +1,153 @@
"""Негативные интеграционные проверки вложений для уровня 4."""
import struct
from urllib.parse import quote
import uuid
import zlib
import pytest
from tests.client import Client, ROOT
@pytest.fixture(scope='module')
def scenario():
client = Client()
alice, bob, eve, chat = client.bootstrap()
return client, alice, bob, eve, chat
def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=202):
return client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/attachments?filename={quote(filename, safe="")}',
expected,
actor=actor,
raw=content,
headers={'Content-Type': content_type},
schema='Attachment' if expected == 202 else None,
)
def oversized_dimensions_png(content: bytes) -> bytes:
data = bytearray(content)
data[16:20] = struct.pack('>I', 10_000)
data[20:24] = struct.pack('>I', 5_000)
data[29:33] = struct.pack('>I', zlib.crc32(data[12:29]) & 0xFFFFFFFF)
return bytes(data)
def test_file_larger_than_10_mib_is_rejected(scenario):
"""Отклоняет файл размером больше 10 MiB."""
client, alice, _bob, _eve, chat = scenario
upload(client, alice, chat, b'x' * (10 * 1024 * 1024 + 1), expected=413)
def test_corrupted_image_is_rejected(scenario):
"""Отклоняет повреждённое изображение."""
client, alice, _bob, _eve, chat = scenario
upload(client, alice, chat, b'not-a-png', expected=415)
def test_mime_mismatch_is_rejected(scenario):
"""Отклоняет MIME, не совпадающий с содержимым."""
client, alice, _bob, _eve, chat = scenario
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
upload(client, alice, chat, content, content_type='image/jpeg', expected=415)
def test_pixel_limit_is_checked_before_decode(scenario):
"""Проверяет лимит пикселей до полной декомпрессии."""
client, alice, _bob, _eve, chat = scenario
content = oversized_dimensions_png((ROOT / 'tests/fixtures/sample.png').read_bytes())
upload(client, alice, chat, content, expected=413)
@pytest.mark.parametrize(
'filename',
['../escape.png', 'folder\\escape.png', 'bad\x00name.png', 'bad\x85name.png'],
)
def test_dangerous_filename_is_rejected(scenario, filename):
"""Отклоняет опасное отображаемое имя файла."""
client, alice, _bob, _eve, chat = scenario
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
upload(client, alice, chat, content, filename=filename, expected=400)
def test_outsider_cannot_read_attachment(scenario):
"""Скрывает метаданные и содержимое от постороннего пользователя."""
client, alice, _bob, eve, chat = scenario
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
attachment = upload(client, alice, chat, content)['json']
client.call('GET', f'/api/v1/attachments/{attachment["id"]}', 404, actor=eve)
client.call('GET', f'/api/v1/attachments/{attachment["id"]}/content', 404, actor=eve)
def test_attachment_from_another_chat_cannot_be_linked(scenario):
"""Запрещает прикреплять вложение другого чата."""
client, alice, bob, _eve, chat = scenario
other_chat = client.call(
'POST',
'/api/v1/chats',
201,
actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]},
schema='Chat',
)['json']
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
attachment = upload(client, alice, other_chat, content)['json']
client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/messages',
404,
actor=alice,
body={'text': 'чужое вложение', 'attachment_ids': [attachment['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex},
)
def test_idempotency_distinguishes_attachment_list(scenario):
"""Включает список вложений в сравнение идемпотентного запроса."""
client, alice, _bob, _eve, chat = scenario
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
attachment = upload(client, alice, chat, content)['json']
key = uuid.uuid4().hex
first = client.message(
alice,
chat,
'идемпотентное вложение',
key=key,
attachments=[attachment['id']],
)['json']
replay = client.message(
alice,
chat,
'идемпотентное вложение',
key=key,
attachments=[attachment['id']],
)['json']
assert replay == first
client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/messages',
409,
actor=alice,
body={'text': 'идемпотентное вложение', 'attachment_ids': []},
headers={'Idempotency-Key': key},
)
def test_missing_attachment_ids_equals_empty_list_for_idempotency(scenario):
"""Считает отсутствие attachment_ids эквивалентным пустому списку."""
client, alice, _bob, _eve, chat = scenario
key = uuid.uuid4().hex
first = client.message(alice, chat, 'обычный текст', key=key)['json']
replay = client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/messages',
201,
actor=alice,
body={'text': 'обычный текст', 'attachment_ids': []},
headers={'Idempotency-Key': key},
schema='Message',
)['json']
assert replay == first
+117
View File
@@ -0,0 +1,117 @@
from __future__ import annotations
import hashlib
from io import BytesIO
import random
from types import SimpleNamespace
import pytest
from PIL import Image
from app.jobs.pipeline import PermanentImageError, process_image
from app.jobs.worker import Worker, _run_isolated
def encoded(width: int, height: int, mode: str = 'RGBA') -> bytes:
image = Image.new(mode, (width, height), (20, 80, 160, 128) if mode == 'RGBA' else 'navy')
output = BytesIO()
image.save(output, 'PNG', pnginfo=None)
return output.getvalue()
def run(source: bytes):
return process_image(
source,
'image/png',
hashlib.sha256(source).hexdigest(),
40_000_000,
85,
85,
82,
)
def test_pipeline_creates_all_grade4_variants() -> None:
"""Create every grade-4 variant with the required format, size and checksum."""
variants = {item.name: item for item in run(encoded(320, 200))}
assert set(variants) == {'thumbnail', 'crop', 'optimized'}
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (256, 160)
assert variants['thumbnail'].content_type == 'image/jpeg'
assert (variants['crop'].width, variants['crop'].height) == (128, 128)
assert variants['crop'].content_type == 'image/jpeg'
assert (variants['optimized'].width, variants['optimized'].height) == (320, 200)
assert variants['optimized'].content_type == 'image/webp'
for item in variants.values():
assert hashlib.sha256(item.content).hexdigest() == item.sha256
def test_thumbnail_and_optimized_do_not_upscale_small_image() -> None:
"""Keep small images at source size except for the required 128x128 crop."""
variants = {item.name: item for item in run(encoded(40, 20, 'RGB'))}
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (40, 20)
assert (variants['optimized'].width, variants['optimized'].height) == (40, 20)
assert (variants['crop'].width, variants['crop'].height) == (128, 128)
def test_jpeg_variants_composite_alpha_on_white() -> None:
"""Flatten transparent input before encoding JPEG variants without an alpha channel."""
variants = {item.name: item for item in run(encoded(2, 2))}
with Image.open(BytesIO(variants['thumbnail'].content)) as image:
assert image.mode == 'RGB'
def test_checksum_mismatch_is_permanent() -> None:
"""Reject changed staging bytes as a permanent checksum error."""
source = encoded(20, 20)
with pytest.raises(PermanentImageError, match='checksum_mismatch'):
process_image(source, 'image/png', '0' * 64, 40_000_000, 85, 85, 82)
def test_exif_orientation_is_applied_and_removed() -> None:
"""Apply EXIF orientation before resizing and omit metadata from generated output."""
image = Image.new('RGB', (320, 200), 'red')
exif = Image.Exif()
exif[274] = 6
output = BytesIO()
image.save(output, 'JPEG', exif=exif)
source = output.getvalue()
variants = {
item.name: item
for item in process_image(
source,
'image/jpeg',
hashlib.sha256(source).hexdigest(),
40_000_000,
85,
85,
82,
)
}
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (160, 256)
with Image.open(BytesIO(variants['thumbnail'].content)) as transformed:
assert not transformed.getexif()
def test_isolated_pipeline_returns_results_larger_than_pipe_buffer() -> None:
"""Drain child output before join so a large valid result cannot deadlock the pipe."""
pixels = random.Random(0).randbytes(512 * 512 * 3)
image = Image.frombytes('RGB', (512, 512), pixels)
output = BytesIO()
image.save(output, 'PNG')
source = output.getvalue()
variants = _run_isolated(
(source, 'image/png', hashlib.sha256(source).hexdigest(), 40_000_000, 85, 85, 82),
timeout=10,
memory_bytes=512 * 1024 * 1024,
)
assert {item.name for item in variants} == {'thumbnail', 'crop', 'optimized'}
assert sum(len(item.content) for item in variants) > 64 * 1024
def test_retry_delay_uses_capped_exponential_backoff_and_jitter(monkeypatch) -> None:
"""Calculate capped exponential backoff and add the configured random jitter."""
settings = SimpleNamespace(worker_concurrency=1, retry_base_seconds=2, retry_max_seconds=8)
worker = Worker(settings, jobs=None, storage=None)
monkeypatch.setattr('app.jobs.worker.random.uniform', lambda start, end: end / 2)
assert worker._delay(1) == 3
assert worker._delay(4) == 12
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""Fault injection around S3 effects, DB commit and broker ack."""
import os
from pathlib import Path
import subprocess
from client import Client, arguments, check, run
def executable(value):
path = Path(value).resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-worker-script', required=True)
parser.add_argument('--crash-worker-script', required=True)
parser.add_argument('--mode', choices=('objects', 'commit'), required=True)
args = arguments('Worker crash around durable effects', configure)
stop = executable(args.stop_worker_script)
crash = executable(args.crash_worker_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop)], check=True)
attachment, original = client.attachment(alice, chat)
subprocess.run([str(crash), args.mode], check=True)
ready = client.wait_ready(bob, attachment)
names = sorted(item['name'] for item in ready['variants'])
check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate or missing variants')
downloaded = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
)['bytes']
check(downloaded == original, 'original changed after crash recovery')
print('PASS: worker crash mode=%s recovered one logical result' % args.mode)
if __name__ == '__main__':
run(main)
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
"""Required scenario: accepted work survives worker and API downtime."""
import os
from pathlib import Path
import subprocess
import time
from client import Client, arguments, check, run
def executable(value):
path = Path(value).expanduser().resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-worker-script', required=True)
parser.add_argument('--restart-api-script', required=True)
parser.add_argument('--start-worker-script', required=True)
args = arguments('Worker outage and durable accepted job', configure)
stop_worker = executable(args.stop_worker_script)
restart_api = executable(args.restart_api_script)
start_worker = executable(args.start_worker_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop_worker)], check=True)
attachment, original = client.attachment(alice, chat)
message = client.message(alice, chat, '', attachments=[attachment['id']])['json']
check(message['attachment_ids'] == [attachment['id']], 'queued attachment was not linkable')
time.sleep(1)
client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', 409, actor=bob
)
subprocess.run([str(restart_api)], check=True)
deadline = time.monotonic() + 60
while True:
try:
client.call('GET', '/health/ready', schema='Health')
break
except Exception:
check(time.monotonic() < deadline, 'API did not recover after restart')
time.sleep(0.3)
persisted = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment'
)['json']
check(persisted['status'] in {'queued', 'processing'}, 'accepted job disappeared after API restart')
subprocess.run([str(start_worker)], check=True)
ready = client.wait_ready(bob, attachment)
downloaded = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
)['bytes']
check(downloaded == original, 'original changed after worker recovery')
check(ready['status'] == 'ready', 'job did not become ready')
print('PASS: worker stopped -> 202 -> message -> API restart -> worker -> ready')
if __name__ == '__main__':
run(main)