diff --git a/.dockerignore b/.dockerignore index bc16e556..cf740db5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,10 @@ __pycache__ node_modules evidence .DS_Store +docs + +.agents +.aws +.codex +.pytest_cache +venv diff --git a/.env.example b/.env.example index 119384b5..78a62682 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,86 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 -APP_ENV=lab +APP_ENV=development +LOG_LEVEL=INFO + +MAX_MESSAGES=100000 +MAX_IN_FLIGHT=16 + SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +SESSION_STORE=redis + +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_DB=0 +REDIS_MAX_CONNECTIONS=20 +REDIS_POOL_TIMEOUT_SECONDS=1 +REDIS_CONNECT_TIMEOUT_SECONDS=1 +REDIS_SOCKET_TIMEOUT_SECONDS=0.5 + +DB_HOST=postgres +DB_PORT=5432 +DB_NAME=messenger +DB_USER=messenger + +DB_POOL_SIZE=5 +DB_MAX_OVERFLOW=10 +DB_POOL_TIMEOUT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 + API_REPLICAS=2 +API_IMAGE=messenger:local +API_BUILD_REVISION=development IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +SHUTDOWN_DRAIN_SECONDS=1 +GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS=5 +READINESS_TIMEOUT_SECONDS=2 + +LB_HOST_PORT=18081 +DB_HOST_PORT=25432 +REDIS_HOST_PORT=26379 + +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +S3_CONNECT_TIMEOUT_SECONDS=2 +S3_READ_TIMEOUT_SECONDS=10 +S3_UPLOAD_TIMEOUT_SECONDS=30 +S3_DOWNLOAD_TIMEOUT_SECONDS=60 +S3_IMAGE_TIMEOUT_SECONDS=10 +S3_MAX_UPLOAD_BYTES=10485760 +S3_MAX_IMAGE_PIXELS=40000000 +S3_CHUNK_BYTES=65536 +S3_MAX_CONCURRENT_UPLOADS=2 +S3_CLEANUP_GRACE_SECONDS=86400 +S3_CLEANUP_INTERVAL_SECONDS=300 + +POSTGRES_PASSWORD_FILE=/run/secrets/db_password +MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user +MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password +MINIO_BROWSER=off + WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +BROKER_HOST=broker +BROKER_PORT=5672 +BROKER_PASSWORD_FILE=/run/secrets/broker_password +BROKER_QUEUE=attachment-jobs +BROKER_USER=messenger +BROKER_VHOST=/ +JOB_LEASE_SECONDS=90 +JOB_RETRY_SECONDS=5 +PUBLISHER_POLL_SECONDS=1 + +IMAGE_JPEG_QUALITY=85 +IMAGE_WEBP_QUALITY=80 +IMAGE_WEBP_METHOD=4 +IMAGE_MEMORY_MIB=768 +IMAGE_CPU_SECONDS=20 + +JOB_RETRY_MAX_SECONDS=120 +FAILED_STAGING_RETENTION_SECONDS=604800 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..5a088797 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +* text=auto +*.py text eol=lf +*.sh text eol=lf +*.conf text eol=lf +*.yaml text eol=lf +Dockerfile text eol=lf +*.png binary diff --git a/.gitignore b/.gitignore index b5a71896..3abc7696 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -# Local/private state .env .env.* !.env.example @@ -13,7 +12,6 @@ certs/private/ volumes/ data/ backups/ -# Runtime/build artifacts .venv/ __pycache__/ *.py[cod] @@ -28,5 +26,8 @@ coverage/ .DS_Store .idea/ .vscode/ -# Keep small, sanitized evidence summaries in evidence/ under version control. -# Inspect git diff --cached: ignore rules are not a secret scanner. +venv/ +.pytest_cache/ +.ruff_cache/ + +/docs/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..24102d5b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12.15-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +RUN useradd --system --uid 10001 --no-create-home appuser + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY alembic.ini ./ +COPY migrations ./migrations +COPY app ./app + +ARG API_BUILD_REVISION +LABEL org.opencontainers.image.revision=$API_BUILD_REVISION + +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=3s --timeout=3s --start-period=10s --retries=2 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"] + +CMD ["python", "-m", "app"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..da441e93 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,81 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Антонова Ирина Андреевна, БПИ2301. +- Целевая оценка: 4 +- Commit/PR: 8bedaa75920c24e2d625956a835e3bb592d8c5a8 +- Использованные источники, библиотеки и помощь ИИ: Codex — перенос, очистка, контейнерный запуск проверок +- Стек: Python 3.12.15, FastAPI 0.143.3, SQLAlchemy 2.1.4, asyncpg 0.32.0, Pillow 13.0.0, aio-pika 10.2.2 (`requirements.txt`). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Python 3.9+ и Docker с Compose v2 (Windows: Docker Desktop, Linux containers, privileged-контейнеры, интернет). Docker Desktop у меня на WSL2, в `%USERPROFILE%\.wslconfig` заданы `memory=10GB`, `processors=6` (после этого `wsl --shutdown` и перезапуск Docker). + +Перед проверками остановите обычный стенд (`docker compose --env-file .env down`), затем: + +```sh +python scripts/test.py +``` + +Хостовый Python только управляет Docker: тесты, миграции и black выполняются в Linux-контейнере Python 3.12.15 с вложенным Docker 29.1.3. Секреты и `.env` для этого не нужны. Первый прогон без кеша долгий и молчит: вывод пишется в `evidence/lab5/`. Блок `finally` удаляет временные контейнеры, сеть, тома и образ; общий кеш сборки может остаться. + +Обычный запуск приложения: + +```sh +python scripts/init_local.py +docker compose --env-file .env up -d --build +docker compose --env-file .env up -d --wait --wait-timeout 240 nginx +``` + +`init_local.py` создаёт `.env` и `secrets/` (в репозиторий и evidence они не попадают). HTTP: http://127.0.0.1:18081, остальные сервисы наружу не опубликованы. Миграции и bootstrap MinIO запускаются автоматически. + +Остановка с сохранением данных: `docker compose --env-file .env down`. +Намеренное удаление учебных данных: `docker compose --env-file .env down --volumes --remove-orphans`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → Nginx → две API-реплики → PostgreSQL/Redis и приватный MinIO. API кладёт исходник в staging MinIO, затем в одной транзакции PostgreSQL записывает вложение и задание (и запись outbox), и только после этого отвечает `202 queued`. Источник истины — PostgreSQL. Publisher отправляет идентификатор задания в durable-очередь RabbitMQ с publisher confirm; байты и секреты в сообщение не попадают. Брокер не участвует в синхронном приёме, его отказ покрывается outbox. + +Worker отдельный. Состояния: `queued → processing → ready/failed`. Ack отправляется после фиксации результата в PostgreSQL. Блокировка, lease 90 с и фиксированные ключи объектов защищают от дублей и от записи устаревшим worker: доставка at least once, результат идемпотентен. + +Обработка: original побайтово; JPEG thumbnail ≤256×256 (quality 85), JPEG crop 128×128, WebP ≤1280×1280 (quality 80, method 4); EXIF-ориентация учитывается, EXIF/ICC/XMP удаляются. Лимиты: 10 MiB, 40 млн пикселей, 60 с, 768 MiB и 20 с CPU на дочерний процесс. Concurrency 2, prefetch 1. + +Ошибки: постоянные сразу ведут к `failed`; временные повторяются до 5 раз с backoff и jitter (`c = min(120, 5·2^(n−1))`, задержка от `c/2` до `c` с). Failed-задания и аудит replay хранятся в PostgreSQL; replay проверяет исходник, выдаёт новый бюджет попыток и отвергает уже успешную job. Cleanup удаляет ready через сутки, failed через 7 суток, staging активных заданий сохраняет. Из lab4 сохранены аутентификация, чаты, история, идемпотентность сообщений и сессии Redis; изменён upload (201 → 202). + +Операторские команды: + +```sh +docker compose run --rm --no-deps worker python3 -m app.core.job_admin list-failed +docker compose run --rm --no-deps worker python3 -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored +``` ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Итог `python scripts/test.py`: **PASS: проверки завершены, временное окружение удалено**; все шаги `00_init`…`15_cleanup` с кодом 0. Среда: Linux x86_64, Docker 29.1.3, 6 CPU, 10 433 806 336 байт RAM; образы закреплены (postgres 16.10, redis 7.4.2, rabbitmq 4.4.0, nginx 1.28.0). -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда | Результат | Файл | +| --- | --- | --- | --- | +| Формат, контракт, OpenAPI | `black --check`; `check_contract.py`; `validate_openapi.py` | 131 файл без изменений; обе проверки PASS | `01`–`03_*.txt` | +| Стенд поднимается, миграции, bootstrap | `up -d --wait nginx` | все сервисы Healthy, `migrate` и `minio-bootstrap` завершились | `04_readiness.txt` | +| Ур. 3: smoke (202 → ready, original, thumbnail) | `tests/smoke.py` | PASS | `07_smoke.txt` | +| Ур. 3–4: регрессия, лимиты, retry, replay, lease, cleanup | `python3 -m pytest -q` | **201 passed**, 111,29 с | `06_pytest.txt` | +| Ур. 3: приватность S3, отказ MinIO | `scripts.check_s3 --with-outage` | PASS, 22,2 с | `08_storage.txt` | +| Ур. 3: worker остановлен → рестарт → ready | `scripts.check_jobs` (`worker_stopped_restart_all`) | PASS | `10_jobs.txt` | +| Ур. 3: сохранность после рестарта | `resilience.py --mode persistence` | данные и сессия сохранены, 50,8 с | `11_persistence.txt` | +| Ур. 4: crop, WebP | `check_outbox_recovery` | варианты `thumbnail`, `crop`, `optimized` | `09_outbox.txt` | +| Ур. 4: crash COMMIT → publish | `scripts.check_outbox_recovery` | SIGKILL publisher: `queued` → `ready`, без повторной загрузки | `09_outbox.txt` | +| Ур. 4: crash после PUT до ack | `check_jobs` | PASS `crash-after-object`, `crash-before-ack` | `10_jobs.txt` | +| Ур. 4: retry, постоянные/временные ошибки, replay | `check_jobs` | PASS `fail-transient`, `fail-permanent` | `10_jobs.txt` | +| Отказ одной API-реплики | `resilience.py --mode failover` | 120 успешных записей, 4 неуспешные попытки, восстановление 2,707 с | `12_failover.txt` | +| Версии, удаление ресурсов | `docker image ls`, `down --volumes` | код 0, ресурсы `messenger-test_*` удалены | `13_*`, `15_cleanup.txt` | + +Что делают hooks. restart.sh: останавливает worker, publisher, cleanup; перезапускает postgres, redis, minio, broker (тома сохраняются) и ждёт их healthy; затем перезапускает api и nginx и запускает publisher, cleanup и worker (если он работал). stop-one.sh: находит одну API-реплику по TARGET_INSTANCE_ID (hostname или INSTANCE_ID) и останавливает её командой docker stop --timeout 8; вторая реплика продолжает работать. Вывод hooks в тестах подавляется, поэтому эффект подтверждён и результатами проверок. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Нагрузочные эксперименты (трек 5) не проводились. Внедрённые отказы описаны в таблице выше; потери подтверждённых данных ни в одном сценарии нет. Времена: outbox-сценарий 15,6 с, `check_jobs` 302,7 с, failover до первого успешного ответа 2,707 с. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## Ограничения -## Ограничения и следующие шаги +Один Docker-хост и одиночные PostgreSQL, MinIO и брокер: потеря хоста или диска не покрывается репликами API. Admission control очереди не реализован, pipeline фиксирован на версии 1. Времена зависят от ресурсов Docker (у меня 6 CPU, ≈9,7 ГиБ). -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Первый прогон упал на `04_build_start`, пока на хосте параллельно работал обычный стенд; точная причина не устанавливалась. После остановки стенда и увеличения памяти Docker до ≈9,7 ГиБ прогон прошёл полностью, поэтому проверки нужно запускать при остановленном стенде. \ No newline at end of file diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..f95b20ac --- /dev/null +++ b/alembic.ini @@ -0,0 +1,38 @@ +[alembic] +script_location = migrations +prepend_sys_path = . +path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 00000000..ee209955 --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,4 @@ +from app.main import main + +if __name__ == "__main__": + main() diff --git a/app/core/__init__.py b/app/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/core/background.py b/app/core/background.py new file mode 100644 index 00000000..ad87d26a --- /dev/null +++ b/app/core/background.py @@ -0,0 +1,40 @@ +"""Остановка фоновых процессов и безопасные сообщения об отказах.""" + +import asyncio +import logging +import signal + +from aio_pika.exceptions import AMQPException, ChannelInvalidStateError +from sqlalchemy.exc import InterfaceError, OperationalError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError + +TRANSIENT_JOB_ERRORS = ( + AMQPException, + ChannelInvalidStateError, + InterfaceError, + OperationalError, + PoolTimeoutError, + OSError, +) +logger = logging.getLogger(__name__) + + +def shutdown_event() -> asyncio.Event: + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop.set) + return stop + + +async def wait_or_stop(stop: asyncio.Event, seconds: float) -> None: + try: + async with asyncio.timeout(seconds): + await stop.wait() + except TimeoutError: + pass + + +def log_background_error(operation: str, error: Exception) -> None: + + logger.warning("%s: %s", operation, type(error).__name__) diff --git a/app/core/cleanup.py b/app/core/cleanup.py new file mode 100644 index 00000000..548959fa --- /dev/null +++ b/app/core/cleanup.py @@ -0,0 +1,66 @@ +"""Периодическая компенсация незавершённых загрузок; отдельный Compose-сервис.""" + +import argparse +import asyncio +from dataclasses import asdict +import json + +from app.core.config import Settings +from app.core.jobs import JobSettings +from app.core.database import create_db_engine, create_session_factory +from app.core.s3 import S3Settings +from app.domain.attachments import reconcile_attachments +from app.domain.attachment_jobs import cleanup_staging +from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error +from app.storage.s3 import create_s3_storage + + +async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool: + completed = True + try: + async with factory() as session: + result = await reconcile_attachments(session, storage, dry_run=not apply) + staging = await cleanup_staging(session, storage, jobs, apply=apply) + print( + json.dumps({**asdict(result), "terminal_staging": staging}), flush=True + ) + except RECOVERABLE_STORAGE_ERRORS as error: + log_recoverable_error("Reconciliation", error) + completed = False + return completed + + +async def run(once: bool, apply: bool) -> None: + settings = Settings() + s3_settings = S3Settings() + jobs = JobSettings() + engine = create_db_engine(settings) + factory = create_session_factory(engine) + storage = create_s3_storage(s3_settings) + try: + while True: + completed = await cleanup_once(factory, storage, apply, jobs) + if once: + if not completed: + raise SystemExit(1) + break + await asyncio.sleep(s3_settings.cleanup_interval_seconds) + finally: + await storage.aclose() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД") + parser.add_argument("--once", action="store_true") + parser.add_argument( + "--apply", + action="store_true", + help="удалять безопасные кандидаты; по умолчанию dry-run", + ) + args = parser.parse_args() + asyncio.run(run(args.once, args.apply)) + + +if __name__ == "__main__": + main() diff --git a/app/core/config.py b/app/core/config.py new file mode 100644 index 00000000..bf6dd4bb --- /dev/null +++ b/app/core/config.py @@ -0,0 +1,128 @@ +from enum import StrEnum +from pathlib import Path +from typing import Literal, Self + +from pydantic import Field, SecretStr, model_validator +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) +from sqlalchemy import URL +from app.core.s3 import S3Settings + +ASYNC_DRIVER = "postgresql+asyncpg" +SECRETS_DIR = next( + ( + path + for path in ( + Path("/run/secrets"), + Path(__file__).resolve().parents[2] / "secrets", + ) + if (path / "db_password").is_file() + ), + None, +) + + +class LogLevel(StrEnum): + DEBUG = "DEBUG" + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + CRITICAL = "CRITICAL" + + +class Settings(BaseSettings): + s3_enabled: bool = True + s3: S3Settings = Field(default_factory=S3Settings) + app_host: str + app_port: int = Field(ge=1, le=65535) + app_env: str + log_level: LogLevel = LogLevel.INFO + + max_messages: int = Field(ge=1) + max_in_flight: int = Field(ge=1) + + session_ttl_seconds: int = Field(default=3600, ge=1) + session_store: Literal["redis"] = "redis" + + redis_host: str = "localhost" + redis_port: int = Field(default=6379, ge=1, le=65535) + redis_db: int = Field(default=0, ge=0) + redis_password: SecretStr | None = None + + redis_max_connections: int = Field(default=20, ge=1) + redis_pool_timeout_seconds: float = Field(default=1.0, gt=0) + redis_connect_timeout_seconds: float = Field(default=1.0, gt=0) + redis_socket_timeout_seconds: float = Field(default=0.5, gt=0) + + database_url: SecretStr | None = None + db_host: str | None = None + db_port: int = Field(default=5432, ge=1, le=65535) + db_name: str | None = None + db_user: str | None = None + db_password: SecretStr | None = None + + db_pool_size: int = Field(default=5, ge=1) + db_max_overflow: int = Field(default=5, ge=0) + db_pool_timeout_seconds: float = Field(default=3.0, gt=0) + db_connect_timeout_seconds: int = Field(default=3, ge=1) + db_statement_timeout_ms: int = Field(default=5000, ge=1) + + instance_id: str | None = None + idempotency_ttl_seconds: int = Field(default=86400, ge=1) + shutdown_drain_seconds: float = Field(default=1, ge=0) + graceful_shutdown_timeout_seconds: int = Field(default=5, ge=1) + readiness_timeout_seconds: float = Field(default=2, gt=0, lt=2.5) + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + secrets_dir=SECRETS_DIR, + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return init_settings, file_secret_settings, env_settings, dotenv_settings + + @model_validator(mode="after") + def _require_database_source(self) -> Self: + if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8: + raise ValueError( + "Drain и graceful shutdown должны суммарно занимать менее 8 секунд" + ) + if self.database_url is not None: + if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER): + raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://") + return self + + if not (self.db_host and self.db_name and self.db_user and self.db_password): + raise ValueError( + "Нужны DATABASE_URL либо DB_HOST, DB_NAME, DB_USER и секрет db_password" + ) + return self + + def get_database_url(self) -> str: + if self.database_url is not None: + return self.database_url.get_secret_value() + + assert self.db_password is not None + url = URL.create( + ASYNC_DRIVER, + username=self.db_user, + password=self.db_password.get_secret_value(), + host=self.db_host, + port=self.db_port, + database=self.db_name, + ) + return url.render_as_string(hide_password=False) diff --git a/app/core/database.py b/app/core/database.py new file mode 100644 index 00000000..c7a65381 --- /dev/null +++ b/app/core/database.py @@ -0,0 +1,28 @@ +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from app.core.config import Settings + + +def create_db_engine(settings: Settings) -> AsyncEngine: + return create_async_engine( + settings.get_database_url(), + pool_size=settings.db_pool_size, + max_overflow=settings.db_max_overflow, + pool_timeout=settings.db_pool_timeout_seconds, + pool_pre_ping=True, + connect_args={ + "timeout": settings.db_connect_timeout_seconds, + "server_settings": { + "statement_timeout": str(settings.db_statement_timeout_ms) + }, + }, + ) + + +def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]: + return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False) diff --git a/app/core/image_processing.py b/app/core/image_processing.py new file mode 100644 index 00000000..107dc42f --- /dev/null +++ b/app/core/image_processing.py @@ -0,0 +1,16 @@ +"""Качество вариантов и пределы отдельного процесса обработки.""" + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class ImageSettings(BaseSettings): + model_config = SettingsConfigDict( + env_prefix="IMAGE_", env_file=".env", extra="ignore" + ) + + jpeg_quality: int = Field(default=85, ge=1, le=95) + webp_quality: int = Field(default=80, ge=1, le=100) + webp_method: int = Field(default=4, ge=0, le=6) + memory_mib: int = Field(default=768, ge=256, le=1024) + cpu_seconds: int = Field(default=20, ge=1, le=120) diff --git a/app/core/image_worker.py b/app/core/image_worker.py new file mode 100644 index 00000000..a74795c3 --- /dev/null +++ b/app/core/image_worker.py @@ -0,0 +1,24 @@ +"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU""" + +from pathlib import Path +import resource +import sys + +from app.domain.image_validation import decode_image +from app.errors.exceptions import IMAGE_WORKER_ERRORS, image_worker_error_code + + +def main() -> int: + resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024)) + resource.setrlimit(resource.RLIMIT_CPU, (10, 10)) + path, content_type, maximum = sys.argv[1:] + status = 0 + try: + decode_image(Path(path), content_type, int(maximum)) + except IMAGE_WORKER_ERRORS as error: + status = image_worker_error_code(error) + return status + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/core/instance.py b/app/core/instance.py new file mode 100644 index 00000000..8365b7ce --- /dev/null +++ b/app/core/instance.py @@ -0,0 +1,38 @@ +"""Идентификатор API-реплики и заголовок X-Instance-Id.""" + +import os +from uuid import uuid4 + +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.requests import Request +from starlette.responses import Response + +INSTANCE_ID_HEADER = "X-Instance-Id" + + +def resolve_instance_id(configured: str | None) -> str: + if configured and configured.strip(): + return configured.strip() + + hostname = os.getenv("HOSTNAME", "").strip() + if hostname: + return hostname + + return str(uuid4()) + + +class InstanceIdMiddleware(BaseHTTPMiddleware): + + def __init__(self, app, instance_id: str) -> None: + super().__init__(app) + self.instance_id = instance_id + + async def dispatch( + self, + request: Request, + call_next: RequestResponseEndpoint, + ) -> Response: + response = await call_next(request) + response.headers[INSTANCE_ID_HEADER] = self.instance_id + + return response diff --git a/app/core/job_admin.py b/app/core/job_admin.py new file mode 100644 index 00000000..f75efa67 --- /dev/null +++ b/app/core/job_admin.py @@ -0,0 +1,82 @@ +"""Операторский интерфейс failed registry. Доступ требует credentials стенда.""" + +import argparse +import asyncio +import json +from uuid import UUID + +from app.core.background import log_background_error +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.jobs import JobSettings +from app.domain.job_admin import ( + REPLAY_REASONS, + history, + list_failed, + replay_failed_job, + show_job, +) +from app.errors.exceptions import JobOperationError +from app.storage.s3 import create_s3_storage + + +async def execute(args): + settings = Settings() + engine = create_db_engine(settings) + storage = None + try: + factory = create_session_factory(engine) + async with factory() as session: + if args.command == "list-failed": + result = await list_failed(session, args.limit) + elif args.command == "show": + result = await show_job(session, args.job_id) + elif args.command == "history": + result = await history(session, args.job_id) + else: + storage = create_s3_storage(settings.s3) + result = await replay_failed_job( + session, + storage, + JobSettings(), + args.job_id, + request_id=args.request_id, + actor=args.actor, + reason=args.reason, + ) + print(json.dumps(result, default=str), flush=True) + finally: + if storage is not None: + await storage.aclose() + await engine.dispose() + + +def main(): + parser = argparse.ArgumentParser( + description="Реестр failed и явный replay с аудитом" + ) + commands = parser.add_subparsers(dest="command", required=True) + listing = commands.add_parser("list-failed") + listing.add_argument( + "--limit", type=int, choices=range(1, 501), default=50, metavar="1..500" + ) + for name in ("show", "history", "replay"): + command = commands.add_parser(name) + command.add_argument("job_id", type=UUID) + if name == "replay": + command.add_argument("--request-id", type=UUID, required=True) + command.add_argument("--actor", required=True) + command.add_argument("--reason", choices=REPLAY_REASONS, required=True) + args = parser.parse_args() + try: + asyncio.run(execute(args)) + except JobOperationError as error: + print(json.dumps({"error_code": error.code}), flush=True) + raise SystemExit(2) from None + except Exception as error: + log_background_error("Job administration failed", error) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/app/core/job_worker.py b/app/core/job_worker.py new file mode 100644 index 00000000..6cdb3b2d --- /dev/null +++ b/app/core/job_worker.py @@ -0,0 +1,169 @@ +"""Отдельные consumers с ограниченной конкуренцией; API не выполняет resize.""" + +import argparse +import asyncio +from dataclasses import dataclass +import json +import logging +import os +import signal +from uuid import UUID, uuid4 + +from PIL import features +from pydantic import ValidationError + +from app.core.background import ( + TRANSIENT_JOB_ERRORS, + log_background_error, + shutdown_event, + wait_or_stop, +) +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.jobs import JobSettings +from app.domain.attachment_jobs import process_attachment_job +from app.errors.exceptions import ProcessingError +from app.schemas.job import JobMessage +from app.storage.broker import JobBroker +from app.storage.s3 import create_s3_storage + + +@dataclass(frozen=True) +class Faults: + + crash_after_object: UUID | None = None + crash_before_ack: UUID | None = None + fail_transient: UUID | None = None + fail_permanent: UUID | None = None + object_name: str = "optimized" + + def kill(self, attachment_id: UUID, phase: str) -> None: + print( + json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), + flush=True, + ) + os.kill(os.getpid(), signal.SIGKILL) + + def after_object(self, attachment_id: UUID, name: str) -> None: + if attachment_id == self.crash_after_object and name == self.object_name: + self.kill(attachment_id, "after_object:" + name) + + def before_ack(self, attachment_id: UUID) -> None: + if attachment_id == self.crash_before_ack: + self.kill(attachment_id, "before_ack") + + def before_processing(self, attachment_id: UUID) -> None: + if attachment_id == self.fail_transient: + raise TimeoutError("injected transient failure") + if attachment_id == self.fail_permanent: + raise ProcessingError("injected_permanent") + + +async def consume_connected( + factory, + storage, + broker: JobBroker, + settings: JobSettings, + stop: asyncio.Event, + faults: Faults, +) -> None: + consumer = uuid4().hex + while not stop.is_set(): + try: + await broker.ensure_ready() + delivery = await broker.receive(consumer) + if delivery is None: + continue + try: + message = JobMessage.model_validate_json(delivery.payload) + except ValidationError: + await broker.reject_invalid(delivery.id) + continue + try: + async with factory() as session: + acknowledge = await process_attachment_job( + session, + storage, + message, + settings, + after_object=faults.after_object, + before_processing=faults.before_processing, + ) + except ProcessingError: + await broker.reject_invalid(delivery.id) + else: + if acknowledge: + faults.before_ack(message.attachment_id) + await broker.acknowledge_job(delivery.id) + else: + await broker.defer_job(delivery.id) + except TRANSIENT_JOB_ERRORS as error: + log_background_error("Worker delivery deferred", error) + await broker.aclose() + await wait_or_stop(stop, settings.publisher_poll_seconds) + + +async def consume(factory, storage, settings, stop, faults) -> None: + broker = JobBroker(settings) + try: + await consume_connected(factory, storage, broker, settings, stop, faults) + finally: + await broker.aclose() + + +async def run_worker(faults: Faults) -> None: + settings = Settings() + jobs = JobSettings() + logging.basicConfig(level=settings.log_level) + if not features.check("webp"): + logging.error("Pillow requires WebP support for the image worker") + raise RuntimeError("WebP codec is unavailable") + stop = shutdown_event() + engine = create_db_engine(settings) + storage = None + try: + storage = create_s3_storage(settings.s3) + factory = create_session_factory(engine) + async with asyncio.TaskGroup() as group: + tasks = [ + group.create_task(consume(factory, storage, jobs, stop, faults)) + for _ in range(jobs.worker_concurrency) + ] + await stop.wait() + + for task in tasks: + task.cancel() + finally: + if storage is not None: + await storage.aclose() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Image worker с явными fault-injection hooks" + ) + modes = parser.add_mutually_exclusive_group() + for flag in ( + "crash-after-object", + "crash-before-ack", + "fail-transient", + "fail-permanent", + ): + modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID") + parser.add_argument( + "--object-name", + choices=("original", "thumbnail", "crop", "optimized"), + default="optimized", + ) + args = parser.parse_args() + faults = Faults(**vars(args)) + try: + asyncio.run(run_worker(faults)) + except Exception as error: + log_background_error("Worker stopped", error) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/app/core/jobs.py b/app/core/jobs.py new file mode 100644 index 00000000..a67c86da --- /dev/null +++ b/app/core/jobs.py @@ -0,0 +1,37 @@ +"""Настройки очереди из .env; credentials брокера читаются отдельно из файла.""" + +from pathlib import Path +from typing import Self + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class JobSettings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + + broker_host: str = "broker" + broker_port: int = Field(default=5672, ge=1, le=65535) + broker_password_file: Path = Path("/run/secrets/broker_password") + broker_queue: str = "attachment-jobs" + broker_user: str = "messenger" + broker_vhost: str = "/" + worker_concurrency: int = Field(default=2, ge=1, le=4) + job_max_attempts: int = Field(default=5, ge=1, le=100) + job_timeout_seconds: int = Field(default=60, ge=10, le=300) + job_lease_seconds: int = Field(default=90, ge=20, le=600) + job_retry_seconds: int = Field(default=5, ge=1, le=300) + job_retry_max_seconds: int = Field(default=120, ge=1, le=3600) + failed_staging_retention_seconds: int = Field(default=604800, ge=300) + publisher_poll_seconds: float = Field(default=1, gt=0, le=10) + staging_retention_seconds: int = Field(default=86400, ge=300) + + @model_validator(mode="after") + def validate_lease(self) -> Self: + if self.job_lease_seconds < self.job_timeout_seconds + 30: + raise ValueError( + "Lease должен превышать timeout задания минимум на 30 секунд" + ) + if self.job_retry_max_seconds < self.job_retry_seconds: + raise ValueError("Retry maximum must be at least the base delay") + return self diff --git a/app/core/lifecycle.py b/app/core/lifecycle.py new file mode 100644 index 00000000..23086779 --- /dev/null +++ b/app/core/lifecycle.py @@ -0,0 +1,45 @@ +"""Готовность реплики и мягкое завершение сервера.""" + +import asyncio +from types import FrameType + +import uvicorn + + +class Lifecycle: + + def __init__(self) -> None: + self._draining = False + + @property + def draining(self) -> bool: + return self._draining + + def start_draining(self) -> None: + self._draining = True + + +class GracefulServer(uvicorn.Server): + + def __init__( + self, config: uvicorn.Config, lifecycle: Lifecycle, drain_seconds: float + ) -> None: + super().__init__(config) + self.lifecycle = lifecycle + self.drain_seconds = drain_seconds + self._drain_task: asyncio.Task[None] | None = None + + def handle_exit(self, sig: int, frame: FrameType | None) -> None: + if self._drain_task is not None: + if not self._drain_task.done(): + self._drain_task.cancel() + super().handle_exit(sig, frame) + return + self.lifecycle.start_draining() + self._drain_task = asyncio.get_running_loop().create_task( + self._finish_draining(sig, frame) + ) + + async def _finish_draining(self, sig: int, frame: FrameType | None) -> None: + await asyncio.sleep(self.drain_seconds) + super().handle_exit(sig, frame) diff --git a/app/core/outbox_publisher.py b/app/core/outbox_publisher.py new file mode 100644 index 00000000..fa0f90d4 --- /dev/null +++ b/app/core/outbox_publisher.py @@ -0,0 +1,87 @@ +"""Доставка намерений из PostgreSQL; crash hook включается только явно через CLI.""" + +import argparse +import asyncio +import json +import logging +import os +import signal +from uuid import UUID + +from app.core.background import ( + TRANSIENT_JOB_ERRORS, + log_background_error, + shutdown_event, + wait_or_stop, +) +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.jobs import JobSettings +from app.domain.attachment_jobs import publish_next_job, recover_expired_jobs +from app.storage.broker import JobBroker + + +def crash_before_publish(attachment_id: UUID) -> None: + print( + json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), + flush=True, + ) + os.kill(os.getpid(), signal.SIGKILL) + + +async def run_publisher(crash_attachment_id: UUID | None = None) -> None: + settings = Settings() + jobs = JobSettings() + logging.basicConfig(level=settings.log_level) + stop = shutdown_event() + engine = create_db_engine(settings) + broker = None + try: + broker = JobBroker(jobs) + factory = create_session_factory(engine) + while not stop.is_set(): + published = False + try: + await broker.ensure_ready() + async with factory() as session: + await recover_expired_jobs(session, jobs) + published = await publish_next_job( + session, + broker, + attachment_id=crash_attachment_id, + before_publish=( + crash_before_publish + if crash_attachment_id is not None + else None + ), + ) + except TRANSIENT_JOB_ERRORS as error: + log_background_error("Publisher deferred", error) + if not published: + await wait_or_stop(stop, jobs.publisher_poll_seconds) + finally: + if broker is not None: + await broker.aclose() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Публикация устойчивых jobs в RabbitMQ" + ) + parser.add_argument( + "--crash-before-publish", + type=UUID, + metavar="ATTACHMENT_ID", + help="аварийно завершить этот процесс перед публикацией только указанного вложения", + ) + args = parser.parse_args() + try: + asyncio.run(run_publisher(args.crash_before_publish)) + except Exception as error: + log_background_error("Publisher stopped", error) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/app/core/redis.py b/app/core/redis.py new file mode 100644 index 00000000..fb61304d --- /dev/null +++ b/app/core/redis.py @@ -0,0 +1,24 @@ +from redis.asyncio import BlockingConnectionPool, Redis +from redis.asyncio.retry import Retry +from redis.backoff import NoBackoff + +from app.core.config import Settings + + +def create_redis_client(settings: Settings) -> Redis: + password = ( + settings.redis_password.get_secret_value() if settings.redis_password else None + ) + pool = BlockingConnectionPool( + host=settings.redis_host, + port=settings.redis_port, + db=settings.redis_db, + password=password, + max_connections=settings.redis_max_connections, + timeout=settings.redis_pool_timeout_seconds, + socket_connect_timeout=settings.redis_connect_timeout_seconds, + socket_timeout=settings.redis_socket_timeout_seconds, + retry=Retry(NoBackoff(), retries=1), + decode_responses=True, + ) + return Redis.from_pool(pool) diff --git a/app/core/request_capacity.py b/app/core/request_capacity.py new file mode 100644 index 00000000..31968e70 --- /dev/null +++ b/app/core/request_capacity.py @@ -0,0 +1,42 @@ +from collections.abc import AsyncIterator + +from fastapi import Request + +from app.errors.exceptions import rate_limited, unavailable + + +class RequestCapacity: + def __init__(self, max_in_flight: int) -> None: + self._max_in_flight = max_in_flight + self._in_flight = 0 + + @property + def in_flight(self) -> int: + return self._in_flight + + def try_acquire(self) -> bool: + if self._in_flight >= self._max_in_flight: + return False + self._in_flight += 1 + return True + + def release(self) -> None: + self._in_flight -= 1 + + +def create_request_capacity(max_in_flight: int) -> RequestCapacity: + return RequestCapacity(max_in_flight) + + +async def acquire_request_slot(request: Request) -> AsyncIterator[None]: + if request.app.state.lifecycle.draining: + raise unavailable("Реплика завершает работу") + capacity: RequestCapacity = request.app.state.request_capacity + + if not capacity.try_acquire(): + raise rate_limited() + + try: + yield + finally: + capacity.release() diff --git a/app/core/request_id.py b/app/core/request_id.py new file mode 100644 index 00000000..a78a8607 --- /dev/null +++ b/app/core/request_id.py @@ -0,0 +1,22 @@ +from uuid import uuid4 + +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.requests import Request +from starlette.responses import Response + +REQUEST_ID_HEADER = "X-Request-Id" + + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch( + self, + request: Request, + call_next: RequestResponseEndpoint, + ) -> Response: + request_id = str(uuid4()) + request.state.request_id = request_id + + response = await call_next(request) + response.headers[REQUEST_ID_HEADER] = request_id + + return response diff --git a/app/core/s3.py b/app/core/s3.py new file mode 100644 index 00000000..f2de1880 --- /dev/null +++ b/app/core/s3.py @@ -0,0 +1,47 @@ +"""Конфигурация MinIO; значения credentials читаются только из файлов.""" + +from pathlib import Path +from urllib.parse import urlsplit + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class S3Settings(BaseSettings): + endpoint: str = "http://minio:9000" + region: str = "us-east-1" + bucket: str = "messenger-attachments" + access_key_id_file: Path = Path("/run/secrets/s3_access_key_id") + secret_access_key_file: Path = Path("/run/secrets/s3_secret_access_key") + connect_timeout_seconds: float = Field(default=2, gt=0, le=5) + read_timeout_seconds: float = Field(default=10, gt=0, le=15) + upload_timeout_seconds: float = Field(default=30, gt=0, le=60) + download_timeout_seconds: float = Field(default=60, gt=0, le=120) + image_timeout_seconds: float = Field(default=10, gt=0, le=15) + max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760) + max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000) + chunk_bytes: int = Field(default=65_536, ge=1024, le=262_144) + max_concurrent_uploads: int = Field(default=2, ge=1, le=2) + cleanup_grace_seconds: int = Field(default=86_400, ge=300) + cleanup_interval_seconds: int = Field(default=300, ge=1) + + model_config = SettingsConfigDict( + env_prefix="S3_", env_file=".env", extra="ignore", frozen=True + ) + + @model_validator(mode="after") + def validate_endpoint(self): + url = urlsplit(self.endpoint) + if ( + url.scheme not in {"http", "https"} + or not url.hostname + or url.username + or url.password + or url.path not in {"", "/"} + or url.query + or url.fragment + ): + raise ValueError( + "S3_ENDPOINT должен быть http(s) адресом без credentials/пути" + ) + return self diff --git a/app/core/security.py b/app/core/security.py new file mode 100644 index 00000000..5e7bb979 --- /dev/null +++ b/app/core/security.py @@ -0,0 +1,56 @@ +"""Хеширование паролей (Argon2id) и выпуск непрозрачных токенов сессий.""" + +import asyncio +import hashlib +import os +import secrets +from concurrent.futures import ThreadPoolExecutor + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError + +TOKEN_BYTES = 32 + +_hasher = PasswordHasher() +_executor = ThreadPoolExecutor( + max_workers=max(1, min(2, (os.cpu_count() or 1) // _hasher.parallelism)), + thread_name_prefix="argon2", +) + + +def _hash_password_sync(password: str) -> str: + return _hasher.hash(password) + + +def _verify_password_sync(password: str, password_hash: str) -> bool: + try: + return _hasher.verify(password_hash, password) + except (VerificationError, InvalidHashError): + return False + + +_DUMMY_HASH = _hash_password_sync(secrets.token_urlsafe(TOKEN_BYTES)) + + +async def hash_password(password: str) -> str: + loop = asyncio.get_running_loop() + return await loop.run_in_executor(_executor, _hash_password_sync, password) + + +async def verify_password(password: str, password_hash: str) -> bool: + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + _executor, _verify_password_sync, password, password_hash + ) + + +async def dummy_verify_password(password: str) -> None: + await verify_password(password, _DUMMY_HASH) + + +def generate_token() -> str: + return secrets.token_urlsafe(TOKEN_BYTES) + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest() diff --git a/app/core/thumbnail_worker.py b/app/core/thumbnail_worker.py new file mode 100644 index 00000000..2c465107 --- /dev/null +++ b/app/core/thumbnail_worker.py @@ -0,0 +1,32 @@ +"""Один pipeline изображений в Linux-процессе с лимитами памяти и CPU.""" + +from pathlib import Path +import resource +import sys + +from app.core.image_processing import ImageSettings +from app.domain.image_pipeline import create_variants +from app.errors.exceptions import IMAGE_WORKER_ERRORS, pipeline_worker_error_code + + +def main() -> int: + settings = ImageSettings() + memory = settings.memory_mib * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit( + resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds) + ) + source, directory, metadata, content_type, maximum = sys.argv[1:] + code = 0 + try: + result = create_variants( + Path(source), Path(directory), content_type, int(maximum), settings + ) + Path(metadata).write_text(result.model_dump_json()) + except IMAGE_WORKER_ERRORS as error: + code = pipeline_worker_error_code(error) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/domain/__init__.py b/app/domain/__init__.py new file mode 100644 index 00000000..93c96d54 --- /dev/null +++ b/app/domain/__init__.py @@ -0,0 +1 @@ +"""Бизнес-логика: проверки доступа, транзакции и ошибки API.""" diff --git a/app/domain/attachment_jobs.py b/app/domain/attachment_jobs.py new file mode 100644 index 00000000..35c93aaa --- /dev/null +++ b/app/domain/attachment_jobs.py @@ -0,0 +1,260 @@ +"""Приём, durable публикация, попытки и фиксация результата обработки.""" + +import asyncio +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import UUID, uuid4 + +from minio.error import MinioException +from sqlalchemy.ext.asyncio import AsyncSession +from urllib3.exceptions import HTTPError as StorageHTTPError + +from app.core.jobs import JobSettings +from app.domain.job_retry import classify_failure, retry_delay +from app.domain.job_files import fetch_original, render_variants +from app.errors.exceptions import ProcessingError +from app.schemas.attachment import VARIANT_FILENAMES +from app.schemas.job import JobMessage +from app.storage import jobs, outbox +from app.storage.broker import JobBroker +from app.storage.jobs import JobWork +from app.storage.s3 import MinioStorage + + +@dataclass(frozen=True) +class Claim: + work: JobWork | None = None + acknowledge: bool = False + + +async def enqueue_attachment_job( + session: AsyncSession, attachment_id: UUID, staging_key: str +) -> UUID: + return await jobs.create_job(session, attachment_id, staging_key) + + +async def publish_next_job( + session: AsyncSession, + broker: JobBroker, + *, + attachment_id: UUID | None = None, + before_publish: Callable[[UUID], None] | None = None, +) -> bool: + job = await outbox.next_unpublished_job(session, attachment_id=attachment_id) + published = job is not None + if job is not None: + if before_publish is not None: + before_publish(job.attachment_id) + await broker.publish_job( + JobMessage( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + attempt=job.attempt + 1, + created_at=job.created_at, + ) + ) + outbox.mark_published(job) + await session.commit() + return published + + +async def claim_job( + session: AsyncSession, message: JobMessage, settings: JobSettings +) -> Claim: + row = await jobs.lock_job(session, message.job_id) + result = Claim() + if row is None: + raise ProcessingError("unknown_job") + job, attachment = row + if ( + message.attachment_id != attachment.id + or message.pipeline_version != job.pipeline_version + ): + raise ProcessingError("invalid_job") + now = await jobs.database_time(session) + if job.attempt_limit is None: + job.attempt_limit = settings.job_max_attempts + if attachment.status in {"ready", "failed"}: + result = Claim(acknowledge=True) + elif message.attempt <= job.attempt and attachment.status == "queued": + + result = Claim(acknowledge=True) + elif message.attempt < job.attempt: + result = Claim(acknowledge=True) + elif message.attempt > job.attempt + 1: + raise ProcessingError("invalid_attempt") + elif attachment.status == "queued" and job.next_attempt_at <= now: + if ( + job.attempt >= (job.attempt_limit or settings.job_max_attempts) + or job.staging_key is None + ): + jobs.record_failure( + job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0 + ) + result = Claim(acknowledge=True) + else: + result = Claim( + work=jobs.start_job( + job, attachment, uuid4(), now, settings.job_lease_seconds + ) + ) + await session.commit() + return result + + +async def save_failure( + session: AsyncSession, + work: JobWork, + settings: JobSettings, + code: str, + *, + permanent: bool, +) -> None: + + await session.rollback() + row = await jobs.lock_job(session, work.job_id) + if row is not None: + job, attachment = row + if attachment.status == "processing" and job.lease_token == work.lease_token: + exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) + jobs.record_failure( + job, + attachment, + await jobs.database_time(session), + "retry_exhausted" if exhausted and not permanent else code, + terminal=permanent or exhausted, + retry_seconds=retry_delay( + settings, job.attempt - job.cycle_start_attempt + ), + ) + job.last_error_code = code + job.last_error_permanent = permanent + await session.commit() + + +async def build_result( + session: AsyncSession, + storage: MinioStorage, + work: JobWork, + after_object: Callable[[UUID, str], None] | None = None, +) -> bool: + with TemporaryDirectory(prefix="messenger-job-") as directory: + source = Path(directory) / "original" + await fetch_original(storage, work, source) + result = await render_variants(storage, work, source, Path(directory)) + row = await jobs.lock_job(session, work.job_id) + if row is None: + await session.rollback() + return True + job, attachment = row + now = await jobs.database_time(session) + if attachment.status != "processing" or job.lease_token != work.lease_token: + await session.rollback() + return True + if job.lease_until is None or job.lease_until <= now: + await session.rollback() + return False + + await storage.put_file( + work.original_key, + source, + size_bytes=work.size_bytes, + content_type=work.content_type, + sha256=work.sha256, + ) + if after_object is not None: + after_object(work.attachment_id, "original") + for variant in result.variants: + await storage.put_file( + work.variant_key(variant.name), + Path(directory) / VARIANT_FILENAMES[variant.name], + size_bytes=variant.size_bytes, + content_type=variant.content_type, + sha256=variant.sha256, + ) + if after_object is not None: + after_object(work.attachment_id, variant.name) + jobs.complete_job( + job, + attachment, + work, + [variant.model_dump() for variant in result.variants], + await jobs.database_time(session), + ) + await session.commit() + return True + + +async def process_attachment_job( + session: AsyncSession, + storage: MinioStorage, + message: JobMessage, + settings: JobSettings, + *, + after_object: Callable[[UUID, str], None] | None = None, + before_processing: Callable[[UUID], None] | None = None, +) -> bool: + claim = await claim_job(session, message, settings) + acknowledge = claim.acknowledge + if claim.work is not None: + work = claim.work + try: + async with asyncio.timeout(settings.job_timeout_seconds): + if before_processing is not None: + before_processing(work.attachment_id) + acknowledge = await build_result(session, storage, work, after_object) + except ( + ProcessingError, + MinioException, + StorageHTTPError, + OSError, + TimeoutError, + ) as error: + code, permanent = classify_failure(error) + await save_failure(session, work, settings, code, permanent=permanent) + acknowledge = True + + return acknowledge + + +async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> int: + rows = await jobs.expired_jobs(session) + now = await jobs.database_time(session) + for job, attachment in rows: + if job.attempt_limit is None: + job.attempt_limit = settings.job_max_attempts + exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) + jobs.record_failure( + job, + attachment, + now, + "retry_exhausted" if exhausted else "worker_timeout", + terminal=exhausted, + retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), + ) + job.last_error_code = "worker_timeout" + job.last_error_permanent = False + await session.commit() + return len(rows) + + +async def cleanup_staging( + session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool +) -> int: + rows = await jobs.terminal_staging( + session, + settings.staging_retention_seconds, + settings.failed_staging_retention_seconds, + ) + if apply: + for job, attachment in rows: + if job.staging_key != attachment.object_key: + await storage.delete_object(job.staging_key) + jobs.clear_staging(job) + await session.commit() + else: + await session.rollback() + return len(rows) diff --git a/app/domain/attachments.py b/app/domain/attachments.py new file mode 100644 index 00000000..3600ea62 --- /dev/null +++ b/app/domain/attachments.py @@ -0,0 +1,337 @@ +"""Авторизация, приём оригинала и согласованность PostgreSQL/MinIO.""" + +from collections.abc import AsyncIterator, Iterator +from contextlib import contextmanager, suppress +from dataclasses import dataclass +import hashlib +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import UUID, uuid4 + +from anyio import CancelScope +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.request_capacity import RequestCapacity +from app.domain.chats import ensure_chat_member +from app.domain.attachment_jobs import enqueue_attachment_job +from app.domain.images import validate_filename, validate_image +from app.errors.exceptions import ( + conflict, + invalid_request, + not_found, + payload_too_large, + rate_limited, + unavailable, + unsupported_media_type, +) +from app.errors.handlers import ( + RECOVERABLE_STORAGE_ERRORS, + log_recoverable_error, + operation_timeout, +) +from app.schemas.attachment import Attachment, VARIANT_FILENAMES +from app.schemas.user import User +from app.storage import attachments as repository +from app.storage.s3 import MinioStorage + + +@dataclass(frozen=True) +class CleanupResult: + examined: int + candidates: int + deleted: int + + +@dataclass(frozen=True) +class OriginalFile: + path: Path + size_bytes: int + sha256: str + + +def require_storage(storage: MinioStorage | None) -> MinioStorage: + if storage is None: + raise unavailable("Объектное хранилище не настроено") + return storage + + +def validate_declared_size(value: str | None, maximum: int) -> None: + if value is not None: + if not value.isascii() or not value.isdecimal() or len(value) > 20: + raise invalid_request("Некорректный Content-Length") + if int(value) > maximum: + raise payload_too_large() + + +@contextmanager +def upload_slot(capacity: RequestCapacity) -> Iterator[None]: + if not capacity.try_acquire(): + raise rate_limited() + try: + yield + finally: + capacity.release() + + +async def receive_attachment( + session: AsyncSession, + storage: MinioStorage | None, + user: User, + chat_id: UUID, + *, + capacity: RequestCapacity, + filename: str, + content_type: str, + declared_length: str | None, + chunks: AsyncIterator[bytes], +) -> Attachment: + storage = require_storage(storage) + validate_declared_size(declared_length, storage.settings.max_upload_bytes) + timeout_error = invalid_request("Истекло время загрузки") + with upload_slot(capacity): + async with operation_timeout( + storage.settings.upload_timeout_seconds, timeout_error + ): + result = await upload_attachment( + session, + storage, + user, + chat_id, + filename=filename, + content_type=content_type, + chunks=chunks, + ) + return result + + +async def read_original( + chunks: AsyncIterator[bytes], + path: Path, + maximum: int, + chunk_bytes: int, +) -> OriginalFile: + checksum = hashlib.sha256() + size = 0 + with path.open("wb") as target: + async for chunk in chunks: + size += len(chunk) + if size > maximum: + raise payload_too_large() + checksum.update(chunk) + view = memoryview(chunk) + for offset in range(0, len(view), chunk_bytes): + target.write(view[offset : offset + chunk_bytes]) + if size == 0: + raise unsupported_media_type("Пустой файл") + return OriginalFile(path, size, checksum.hexdigest()) + + +async def compensate_upload( + session: AsyncSession, storage: MinioStorage, id: UUID +) -> None: + await session.rollback() + intent = await repository.lock_upload(session, id) + if intent is not None: + if await repository.get_by_key(session, intent.object_key) is None: + await storage.delete_object(intent.object_key) + await repository.finish_upload(session, id) + await session.commit() + + +async def recover_upload( + session: AsyncSession, storage: MinioStorage, id: UUID +) -> None: + try: + await compensate_upload(session, storage, id) + except RECOVERABLE_STORAGE_ERRORS as error: + log_recoverable_error("Upload compensation", error) + + with suppress(*RECOVERABLE_STORAGE_ERRORS): + await session.rollback() + + +async def stage_attachment( + session: AsyncSession, + storage: MinioStorage, + user: User, + chat_id: UUID, + *, + original: OriginalFile, + filename: str, + content_type: str, +) -> Attachment: + id = uuid4() + key = f"staging/{uuid4().hex}" + + await repository.reserve_upload(session, id, key) + await session.commit() + try: + if await repository.lock_upload(session, id) is None: + raise unavailable("Загрузка отменена до записи оригинала") + await storage.put_file( + key, + original.path, + size_bytes=original.size_bytes, + content_type=content_type, + sha256=original.sha256, + ) + record = await repository.create_attachment( + session, + id=id, + chat_id=chat_id, + owner_id=user.id, + object_key=f"originals/{id.hex}/v1", + status="queued", + filename=filename, + content_type=content_type, + size_bytes=original.size_bytes, + sha256=original.sha256, + ) + await enqueue_attachment_job(session, id, key) + await repository.finish_upload(session, id) + + result = Attachment.model_validate(record) + await session.commit() + except BaseException: + + with CancelScope(shield=True): + await recover_upload(session, storage, id) + raise + return result + + +async def upload_attachment( + session: AsyncSession, + storage: MinioStorage, + user: User, + chat_id: UUID, + *, + filename: str, + content_type: str, + chunks: AsyncIterator[bytes], +) -> Attachment: + await ensure_chat_member(session, user, chat_id) + validate_filename(filename) + if content_type not in {"image/png", "image/jpeg"}: + raise unsupported_media_type("Принимаются только image/png и image/jpeg") + + await session.rollback() + settings = storage.settings + with TemporaryDirectory(prefix="messenger-upload-") as directory: + original = await read_original( + chunks, + Path(directory) / "original", + settings.max_upload_bytes, + settings.chunk_bytes, + ) + await validate_image(original.path, content_type, settings) + result = await stage_attachment( + session, + storage, + user, + chat_id, + original=original, + filename=filename, + content_type=content_type, + ) + return result + + +async def get_attachment( + session: AsyncSession, user: User, attachment_id: UUID +) -> Attachment: + record = await repository.get_attachment(session, attachment_id) + if record is None: + raise not_found() + await ensure_chat_member(session, user, record.chat_id) + return Attachment.model_validate(record) + + +@dataclass(frozen=True) +class DownloadInfo: + filename: str + content_type: str + size_bytes: int + + +async def download_attachment( + session: AsyncSession, + storage: MinioStorage | None, + user: User, + attachment_id: UUID, + *, + variant: str = "original", +) -> tuple[DownloadInfo, AsyncIterator[bytes]]: + result = await get_attachment(session, user, attachment_id) + if result.status != "ready": + raise conflict("Вложение пока недоступно для скачивания") + storage = require_storage(storage) + record = await repository.get_attachment(session, attachment_id) + if record is None: + raise not_found() + if variant == "original": + key = record.object_key + info = DownloadInfo(result.filename, result.content_type, result.size_bytes) + else: + selected = next( + (item for item in result.variants if item.name == variant), None + ) + keys = { + "thumbnail": record.thumbnail_key, + "crop": record.crop_key, + "optimized": record.optimized_key, + } + key = keys.get(variant) + if selected is None or key is None: + raise not_found() + info = DownloadInfo( + VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes + ) + await session.rollback() + stored = await storage.stat_object(key) + if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type: + raise unavailable("Файл временно недоступен") + stream = await storage.open_object(key) + return info, stream + + +async def validate_message_attachments( + session: AsyncSession, + user: User, + chat_id: UUID, + attachment_ids: list[UUID], +) -> None: + await ensure_chat_member(session, user, chat_id) + records = await repository.get_for_chat(session, chat_id, attachment_ids) + if len(records) != len(attachment_ids): + raise not_found() + if any(record.status == "failed" for record in records): + raise conflict("Обработка вложения завершилась ошибкой") + + +async def reconcile_attachments( + session: AsyncSession, + storage: MinioStorage, + *, + batch_size: int = 20, + dry_run: bool = True, +) -> CleanupResult: + if not 1 <= batch_size <= 100: + raise invalid_request("batch_size должен быть от 1 до 100") + intents = await repository.stale_uploads( + session, storage.settings.cleanup_grace_seconds, batch_size + ) + candidates = deleted = 0 + for intent in intents: + if await repository.get_by_key(session, intent.object_key) is None: + candidates += 1 + if not dry_run: + await storage.delete_object(intent.object_key) + deleted += 1 + if not dry_run: + await repository.finish_upload(session, intent.id) + if dry_run: + await session.rollback() + else: + await session.commit() + return CleanupResult(len(intents), candidates, deleted) diff --git a/app/domain/auth.py b/app/domain/auth.py new file mode 100644 index 00000000..b05320ac --- /dev/null +++ b/app/domain/auth.py @@ -0,0 +1,81 @@ +import re +from datetime import UTC, datetime, timedelta + +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import security +from app.domain.users import to_user_schema +from app.errors.exceptions import unauthorized_basic, unauthorized_bearer +from app.schemas.session import AuthSession +from app.schemas.user import ( + PASSWORD_MAX_LENGTH, + PASSWORD_MIN_LENGTH, + USERNAME_PATTERN, + User, +) +from app.storage import sessions as sessions_storage +from app.storage import users as users_storage + +USERNAME_RE = re.compile(USERNAME_PATTERN) + + +async def create_session( + session: AsyncSession, + redis: Redis, + username: str, + password: str, + ttl_seconds: int, +) -> AuthSession: + if ( + USERNAME_RE.fullmatch(username) is None + or not PASSWORD_MIN_LENGTH <= len(password) <= PASSWORD_MAX_LENGTH + or not password.strip() + ): + raise unauthorized_basic() + + user = await users_storage.get_user_by_username(session, username) + await session.commit() + + if user is None: + await security.dummy_verify_password(password) + raise unauthorized_basic() + + if not await security.verify_password(password, user.password_hash): + raise unauthorized_basic() + + token = security.generate_token() + expires_at = datetime.now(UTC) + timedelta(seconds=ttl_seconds) + await sessions_storage.create_session( + redis, + token_hash=security.hash_token(token), + user_id=user.id, + expires_at=expires_at, + ) + auth_session = AuthSession( + token=token, expires_at=expires_at, user=to_user_schema(user) + ) + return auth_session + + +async def authenticate_token(session: AsyncSession, redis: Redis, token: str) -> User: + user_id = await sessions_storage.get_session_user_id( + redis, security.hash_token(token) + ) + if user_id is None: + raise unauthorized_bearer() + + user = await users_storage.get_user_by_id(session, user_id) + if user is None: + raise unauthorized_bearer() + + current_user = to_user_schema(user) + return current_user + + +async def revoke_session(redis: Redis, token: str) -> None: + is_deleted = await sessions_storage.delete_session( + redis, security.hash_token(token) + ) + if not is_deleted: + raise unauthorized_bearer() diff --git a/app/domain/chats.py b/app/domain/chats.py new file mode 100644 index 00000000..ec884a58 --- /dev/null +++ b/app/domain/chats.py @@ -0,0 +1,98 @@ +from uuid import UUID + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.domain.pagination import ( + KIND_CHATS, + build_next_cursor, + decode_cursor, + split_page, +) +from app.errors.exceptions import invalid_request, not_found +from app.models.chat import Chat as ChatModel +from app.schemas.chat import Chat, ChatPage, CreateChat +from app.schemas.user import User +from app.storage import chats as chats_storage +from app.storage import users as users_storage +from app.storage.types import Position + +UNKNOWN_MEMBER_MESSAGE = "member_ids содержит неизвестного пользователя" + + +def to_chat_schema(chat: ChatModel) -> Chat: + chat_schema = Chat( + id=chat.id, + title=chat.title, + member_ids=[member.user_id for member in chat.members], + created_at=chat.created_at, + ) + return chat_schema + + +def validate_new_chat(creator: User, new_chat: CreateChat) -> None: + member_ids = new_chat.member_ids + + if creator.id not in member_ids: + raise invalid_request("Создатель чата должен входить в member_ids") + + if len(set(member_ids)) != len(member_ids): + raise invalid_request("member_ids содержит дубликаты") + + +async def create_chat( + session: AsyncSession, creator: User, new_chat: CreateChat +) -> Chat: + validate_new_chat(creator, new_chat) + + existing_ids = await users_storage.get_existing_user_ids( + session, new_chat.member_ids + ) + if existing_ids != set(new_chat.member_ids): + raise invalid_request(UNKNOWN_MEMBER_MESSAGE) + + try: + chat = await chats_storage.create_chat( + session, title=new_chat.title, member_ids=new_chat.member_ids + ) + await session.commit() + except IntegrityError: + await session.rollback() + raise invalid_request(UNKNOWN_MEMBER_MESSAGE) from None + + created_chat = to_chat_schema(chat) + return created_chat + + +async def get_chat(session: AsyncSession, user: User, chat_id: UUID) -> Chat: + chat = await chats_storage.get_chat_for_member(session, chat_id, user.id) + if chat is None: + raise not_found() + + found_chat = to_chat_schema(chat) + return found_chat + + +async def ensure_chat_member(session: AsyncSession, user: User, chat_id: UUID) -> None: + if not await chats_storage.is_chat_member(session, chat_id, user.id): + raise not_found() + + +async def list_chats( + session: AsyncSession, + user: User, + cursor: str | None, + limit: int, +) -> ChatPage: + after: Position | None = None + if cursor is not None: + after = decode_cursor(cursor, KIND_CHATS, user.id) + + rows = await chats_storage.list_chats_for_user(session, user.id, after, limit + 1) + page_rows, has_next = split_page(rows, limit) + items = [to_chat_schema(chat) for chat in page_rows] + page = ChatPage( + items=items, + next_cursor=build_next_cursor(KIND_CHATS, user.id, items, has_next), + ) + return page diff --git a/app/domain/health.py b/app/domain/health.py new file mode 100644 index 00000000..c5d6ac31 --- /dev/null +++ b/app/domain/health.py @@ -0,0 +1,36 @@ +"""Условия готовности реплики и ограниченная проверка зависимостей.""" + +import asyncio + +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.lifecycle import Lifecycle +from app.errors.exceptions import unavailable +from app.errors.handlers import operation_timeout +from app.schemas.base import Health +from app.storage.health import check_database, check_redis +from app.storage.s3 import MinioStorage + + +def liveness() -> Health: + return Health(status="ok") + + +async def readiness( + engine: AsyncEngine, + redis: Redis, + storage: MinioStorage | None, + lifecycle: Lifecycle, + timeout_seconds: float, +) -> Health: + if lifecycle.draining: + raise unavailable() + checks = [check_database(engine), check_redis(redis)] + if storage is not None: + checks.append(storage.check_available()) + async with operation_timeout(timeout_seconds, unavailable()): + dependencies = await asyncio.gather(*checks, return_exceptions=True) + if not all(result is True for result in dependencies): + raise unavailable() + return Health(status="ok") diff --git a/app/domain/idempotency.py b/app/domain/idempotency.py new file mode 100644 index 00000000..fc1c7f4b --- /dev/null +++ b/app/domain/idempotency.py @@ -0,0 +1,75 @@ +"""Атомарная отправка сообщения с защитой от повторов.""" + +import hashlib +import json +import re +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.domain.chats import ensure_chat_member +from app.domain.attachments import validate_message_attachments +from app.errors.exceptions import conflict, invalid_request, unavailable +from app.schemas.message import CreateMessage, Message +from app.schemas.user import User +from app.storage import idempotency as key_storage +from app.storage import messages as message_storage +from app.storage import attachments as attachment_storage + +IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +def validate_idempotency_key(key: str | None) -> str: + if key is None or IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is None: + raise invalid_request("Требуется корректный Idempotency-Key") + return key + + +def request_fingerprint(new_message: CreateMessage) -> str: + body = new_message.model_dump(mode="json") + + if not body["attachment_ids"]: + body.pop("attachment_ids") + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False) + return hashlib.sha256(canonical.encode()).hexdigest() + + +async def send_message_idempotent( + session: AsyncSession, + sender: User, + chat_id: UUID, + new_message: CreateMessage, + idempotency_key: str, + max_messages: int, + ttl_seconds: int, +) -> Message: + await ensure_chat_member(session, sender, chat_id) + scope = (sender.id, "POST", f"/api/v1/chats/{chat_id}/messages", idempotency_key) + fingerprint = request_fingerprint(new_message) + + await key_storage.lock_key(session, scope) + previous = await key_storage.get_active_key(session, scope, ttl_seconds) + if previous is not None: + if previous.request_hash != fingerprint: + raise conflict("Idempotency-Key уже использован для другого сообщения") + message = await message_storage.get_message(session, previous.message_id) + else: + await validate_message_attachments( + session, sender, chat_id, new_message.attachment_ids + ) + if not await message_storage.reserve_message_slot(session, max_messages): + raise unavailable("Достигнут лимит количества сообщений") + message = await message_storage.create_message( + session, chat_id=chat_id, sender_id=sender.id, text=new_message.text + ) + await attachment_storage.bind_attachments( + session, message.id, new_message.attachment_ids + ) + await key_storage.save_result(session, scope, fingerprint, message.id) + + links = await attachment_storage.message_attachment_ids(session, [message.id]) + result = Message.model_validate(message).model_copy( + update={"attachment_ids": links[message.id]} + ) + await session.commit() + return result diff --git a/app/domain/image_pipeline.py b/app/domain/image_pipeline.py new file mode 100644 index 00000000..dee38e69 --- /dev/null +++ b/app/domain/image_pipeline.py @@ -0,0 +1,101 @@ +"""Thumbnail, crop и WebP; один ограниченный процесс, последовательная обработка.""" + +import hashlib +from pathlib import Path + +from PIL import Image, ImageOps + +from app.core.image_processing import ImageSettings +from app.domain.image_validation import decode_image +from app.schemas.attachment import PipelineResult, Variant + + +def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]: + longest = max(size) + result = size + if longest > maximum: + result = ( + max(1, size[0] * maximum // longest), + max(1, size[1] * maximum // longest), + ) + return result + + +def describe_variant( + name: str, content_type: str, path: Path, size: tuple[int, int] +) -> Variant: + checksum = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(65536), b""): + checksum.update(chunk) + return Variant( + name=name, + content_type=content_type, + size_bytes=path.stat().st_size, + width=size[0], + height=size[1], + sha256=checksum.hexdigest(), + ) + + +def save_jpeg(image: Image.Image, destination: Path, quality: int) -> None: + with Image.new("RGB", image.size, "white") as clean: + with image.getchannel("A") as alpha: + clean.paste(image, mask=alpha) + clean.save(destination, format="JPEG", quality=quality, exif=b"") + + +def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Variant: + size = fitted_size(image.size, 256) + with image.resize(size, Image.Resampling.LANCZOS) as resized: + save_jpeg(resized, destination, quality) + return describe_variant("thumbnail", "image/jpeg", destination, size) + + +def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant: + with ImageOps.fit( + image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5) + ) as crop: + save_jpeg(crop, destination, quality) + return describe_variant("crop", "image/jpeg", destination, (128, 128)) + + +def create_optimized( + image: Image.Image, destination: Path, settings: ImageSettings +) -> Variant: + size = fitted_size(image.size, 1280) + with image.resize(size, Image.Resampling.LANCZOS) as resized: + with Image.new("RGBA", size) as clean: + clean.paste(resized) + clean.save( + destination, + format="WEBP", + lossless=False, + quality=settings.webp_quality, + method=settings.webp_method, + exif=b"", + icc_profile=b"", + xmp=b"", + ) + return describe_variant("optimized", "image/webp", destination, size) + + +def create_variants( + source: Path, + directory: Path, + content_type: str, + max_pixels: int, + settings: ImageSettings, +) -> PipelineResult: + decode_image(source, content_type, max_pixels) + with Image.open(source, formats=["PNG", "JPEG"]) as image: + ImageOps.exif_transpose(image, in_place=True) + with image.convert("RGBA") as pixels: + variants = [ + create_thumbnail( + pixels, directory / "thumbnail.jpg", settings.jpeg_quality + ), + create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality), + create_optimized(pixels, directory / "optimized.webp", settings), + ] + return PipelineResult(variants=variants) diff --git a/app/domain/image_validation.py b/app/domain/image_validation.py new file mode 100644 index 00000000..eeec06a1 --- /dev/null +++ b/app/domain/image_validation.py @@ -0,0 +1,23 @@ +"""Синхронная проверка изображения; вызывается только из ограниченного процесса.""" + +from pathlib import Path +import warnings + +from PIL import Image + +from app.errors.exceptions import payload_too_large, unsupported_media_type + + +def decode_image(path: Path, content_type: str, max_pixels: int) -> None: + Image.MAX_IMAGE_PIXELS = max_pixels + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(path, formats=["PNG", "JPEG"]) as image: + if image.width * image.height > max_pixels: + raise payload_too_large("Превышен лимит пикселей изображения") + image_format = image.format + if image_format is None or Image.MIME.get(image_format) != content_type: + raise unsupported_media_type("Содержимое не соответствует Content-Type") + image.verify() + + with Image.open(path, formats=["PNG", "JPEG"]) as image: + image.load() diff --git a/app/domain/images.py b/app/domain/images.py new file mode 100644 index 00000000..4436656f --- /dev/null +++ b/app/domain/images.py @@ -0,0 +1,48 @@ +"""Проверка имени и запуск ограниченного декодера.""" + +import asyncio +from pathlib import Path +import sys +import unicodedata + +from anyio import CancelScope + +from app.core.s3 import S3Settings +from app.errors.exceptions import ( + image_validation_error, + invalid_request, + payload_too_large, +) +from app.errors.handlers import operation_timeout + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or any( + char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"} + for char in filename + ): + raise invalid_request("Недопустимое имя файла") + + +async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None: + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "app.core.image_worker", + str(path), + content_type, + str(settings.max_image_pixels), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + error = payload_too_large("Превышен бюджет проверки изображения") + try: + async with operation_timeout(settings.image_timeout_seconds, error): + code = await process.wait() + finally: + if process.returncode is None: + process.kill() + with CancelScope(shield=True): + await process.wait() + if code != 0: + raise image_validation_error(code) diff --git a/app/domain/job_admin.py b/app/domain/job_admin.py new file mode 100644 index 00000000..15a9b351 --- /dev/null +++ b/app/domain/job_admin.py @@ -0,0 +1,157 @@ +"""Replay только failed с проверенным исходником и ограниченным новым бюджетом.""" + +import asyncio +import re +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import uuid4 + +from minio.error import S3Error + +from app.domain.job_files import fetch_original +from app.errors.exceptions import JobOperationError, ProcessingError +from app.storage import job_admin, jobs +from app.storage.jobs import JobWork + +REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review") + + +def snapshot(job, attachment): + return dict( + job_id=str(job.id), + attachment_id=str(attachment.id), + status=attachment.status, + attempt=job.attempt, + attempt_limit=job.attempt_limit, + cycle_start_attempt=job.cycle_start_attempt, + error_code=attachment.error_code, + last_error_code=job.last_error_code, + last_error_permanent=job.last_error_permanent, + staging_retained=job.staging_key is not None, + next_attempt_at=job.next_attempt_at, + lease_until=job.lease_until, + finished_at=job.finished_at, + ) + + +async def list_failed(session, limit): + return [ + snapshot(job, attachment) + for job, attachment in await job_admin.failed_jobs(session, limit) + ] + + +async def show_job(session, job_id): + row = await jobs.lock_job(session, job_id) + if row is None: + raise JobOperationError("unknown_job") + result = snapshot(*row) + await session.rollback() + return result + + +async def history(session, job_id): + return [ + dict( + request_id=str(row.id), + actor=row.actor, + reason=row.reason, + previous_attempt=row.previous_attempt, + attempt_limit=row.attempt_limit, + previous_error_code=row.previous_error_code, + created_at=row.created_at, + ) + for row in await job_admin.replay_history(session, job_id) + ] + + +def same_request(record, job_id, actor, reason): + if record.job_id != job_id or record.actor != actor or record.reason != reason: + raise JobOperationError("request_conflict") + return { + "request_id": str(record.id), + "job_id": str(job_id), + "already_applied": True, + } + + +def validate_replay_request(actor: str, reason: str) -> None: + if ( + re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None + or reason not in REPLAY_REASONS + ): + raise JobOperationError("invalid_audit_fields") + + +def validate_replay_job(job, attachment) -> None: + if attachment.status != "failed": + raise JobOperationError("job_not_failed") + if job.staging_key is None: + raise JobOperationError("source_expired") + + +async def verify_replay_source(storage, settings, job, attachment) -> None: + work = JobWork( + job.id, + attachment.id, + uuid4(), + job.pipeline_version, + job.staging_key, + attachment.content_type, + attachment.size_bytes, + attachment.sha256, + ) + try: + async with asyncio.timeout(settings.job_timeout_seconds): + with TemporaryDirectory(prefix="replay-source-") as directory: + await fetch_original(storage, work, Path(directory) / "original") + except ProcessingError as error: + raise JobOperationError(error.code) from None + except S3Error as error: + if error.code in {"NoSuchKey", "NoSuchObject"}: + raise JobOperationError("source_missing") from None + raise + + +async def replay_failed_job( + session, storage, settings, job_id, *, request_id, actor, reason +): + validate_replay_request(actor, reason) + row = await jobs.lock_job(session, job_id) + if row is None: + raise JobOperationError("unknown_job") + record = await job_admin.replay_record(session, request_id) + if record is not None: + result = same_request(record, job_id, actor, reason) + await session.rollback() + return result + job, attachment = row + validate_replay_job(job, attachment) + + await verify_replay_source(storage, settings, job, attachment) + limit = job.attempt + settings.job_max_attempts + inserted = await job_admin.insert_replay( + session, + id=request_id, + job_id=job_id, + actor=actor, + reason=reason, + previous_attempt=job.attempt, + attempt_limit=limit, + previous_error_code=job.last_error_code, + ) + if not inserted: + result = same_request( + await job_admin.replay_record(session, request_id), job_id, actor, reason + ) + await session.rollback() + return result + job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit) + await session.commit() + return { + "request_id": str(request_id), + "job_id": str(job_id), + "already_applied": False, + "status": "queued", + "attempt_limit": limit, + } diff --git a/app/domain/job_files.py b/app/domain/job_files.py new file mode 100644 index 00000000..b99bda6c --- /dev/null +++ b/app/domain/job_files.py @@ -0,0 +1,72 @@ +"""Получение исходника и запуск изолированного процесса обработки вариантов.""" + +import asyncio +import errno +import hashlib +from pathlib import Path +import sys + +from anyio import CancelScope + +from app.errors.exceptions import ProcessingError +from app.schemas.attachment import PipelineResult +from app.storage.jobs import JobWork +from app.storage.s3 import MinioStorage + + +async def fetch_original( + storage: MinioStorage, work: JobWork, destination: Path +) -> None: + if work.size_bytes > storage.settings.max_upload_bytes: + raise ProcessingError("image_limits") + checksum = hashlib.sha256() + size = 0 + stream = await storage.open_object(work.staging_key) + try: + with destination.open("wb") as target: + async for chunk in stream: + size += len(chunk) + if size > work.size_bytes: + raise ProcessingError("source_mismatch") + target.write(chunk) + checksum.update(chunk) + finally: + await stream.aclose() + if size != work.size_bytes or checksum.hexdigest() != work.sha256: + raise ProcessingError("source_mismatch") + + +async def render_variants( + storage: MinioStorage, work: JobWork, source: Path, directory: Path +) -> PipelineResult: + metadata = directory / "variants.json" + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "app.core.thumbnail_worker", + str(source), + str(directory), + str(metadata), + work.content_type, + str(storage.settings.max_image_pixels), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + code = await process.wait() + finally: + if process.returncode is None: + process.kill() + with CancelScope(shield=True): + await process.wait() + if code == 2: + raise ProcessingError("invalid_image") + if code == 3 or code < 0: + raise ProcessingError("image_limits") + if code == 4: + raise OSError(errno.EIO, "Pipeline filesystem unavailable") + if code == 5: + raise ProcessingError("processing_configuration") + if code != 0: + raise RuntimeError("Image pipeline subprocess failed") + return PipelineResult.model_validate_json(metadata.read_text()) diff --git a/app/domain/job_retry.py b/app/domain/job_retry.py new file mode 100644 index 00000000..f6a67906 --- /dev/null +++ b/app/domain/job_retry.py @@ -0,0 +1,51 @@ +"""Ограниченный exponential backoff: jitter сохраняется вместе с новой попыткой.""" + +import errno +import random + +from minio.error import S3Error + +from app.core.jobs import JobSettings +from app.errors.exceptions import ProcessingError + + +def retry_delay(settings: JobSettings, cycle_attempt: int) -> float: + ceiling = min( + settings.job_retry_max_seconds, + settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)), + ) + return random.uniform(ceiling / 2, ceiling) + + +def classify_failure(error: Exception) -> tuple[str, bool]: + if isinstance(error, ProcessingError): + return error.code, True + if isinstance(error, S3Error): + if error.code in {"NoSuchKey", "NoSuchObject"}: + return "source_missing", True + if error.code in { + "AccessDenied", + "InvalidAccessKeyId", + "SignatureDoesNotMatch", + "NoSuchBucket", + }: + return "storage_configuration", True + if error.code in { + "InternalError", + "SlowDown", + "ServiceUnavailable", + "RequestTimeout", + "Throttling", + }: + return "storage_unavailable", False + return "storage_rejected", True + if isinstance(error, TimeoutError): + return "processing_timeout", False + if isinstance(error, OSError) and error.errno in { + errno.EACCES, + errno.EPERM, + errno.EROFS, + errno.ENOENT, + }: + return "processing_configuration", True + return "processing_unavailable", False diff --git a/app/domain/messages.py b/app/domain/messages.py new file mode 100644 index 00000000..35b13aa9 --- /dev/null +++ b/app/domain/messages.py @@ -0,0 +1,47 @@ +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.domain.chats import ensure_chat_member +from app.domain.pagination import ( + KIND_MESSAGES, + build_next_cursor, + decode_cursor, + split_page, +) +from app.schemas.message import Message, MessagePage +from app.schemas.user import User +from app.storage import messages as messages_storage +from app.storage import attachments as attachments_storage +from app.storage.types import Position + + +async def list_messages( + session: AsyncSession, + user: User, + chat_id: UUID, + cursor: str | None, + limit: int, +) -> MessagePage: + after: Position | None = None + if cursor is not None: + after = decode_cursor(cursor, KIND_MESSAGES, chat_id) + + await ensure_chat_member(session, user, chat_id) + + rows = await messages_storage.list_messages(session, chat_id, after, limit + 1) + page_rows, has_next = split_page(rows, limit) + links = await attachments_storage.message_attachment_ids( + session, [message.id for message in page_rows] + ) + items = [ + Message.model_validate(message).model_copy( + update={"attachment_ids": links[message.id]}, + ) + for message in page_rows + ] + page = MessagePage( + items=items, + next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next), + ) + return page diff --git a/app/domain/pagination.py b/app/domain/pagination.py new file mode 100644 index 00000000..6c0fc3f8 --- /dev/null +++ b/app/domain/pagination.py @@ -0,0 +1,78 @@ +import base64 +import json +from collections.abc import Sequence +from datetime import datetime +from typing import Protocol, TypeVar +from uuid import UUID + +from app.errors.exceptions import invalid_request +from app.storage.types import Position + +KIND_CHATS = "chats" +KIND_MESSAGES = "messages" + +T = TypeVar("T") + + +class Positioned(Protocol): + + @property + def id(self) -> UUID: ... + + @property + def created_at(self) -> datetime: ... + + +def encode_cursor( + kind: str, + resource_id: UUID, + created_at: datetime, + item_id: UUID, +) -> str: + payload = { + "kind": kind, + "resource_id": str(resource_id), + "created_at": created_at.isoformat(), + "item_id": str(item_id), + } + raw = json.dumps(payload).encode() + return base64.urlsafe_b64encode(raw).decode() + + +def decode_cursor(cursor: str, kind: str, resource_id: UUID) -> Position: + try: + payload = json.loads(base64.urlsafe_b64decode(cursor)) + cursor_kind = payload["kind"] + cursor_resource_id = UUID(payload["resource_id"]) + created_at = datetime.fromisoformat(payload["created_at"]) + item_id = UUID(payload["item_id"]) + except (ValueError, KeyError, TypeError, AttributeError): + raise invalid_request("Некорректный cursor") from None + + if ( + cursor_kind != kind + or cursor_resource_id != resource_id + or created_at.tzinfo is None + ): + raise invalid_request("Некорректный cursor") + + return created_at, item_id + + +def split_page(rows: list[T], limit: int) -> tuple[list[T], bool]: + has_next = len(rows) > limit + items = rows[:limit] + return items, has_next + + +def build_next_cursor( + kind: str, + resource_id: UUID, + items: Sequence[Positioned], + has_next: bool, +) -> str | None: + if not has_next: + return None + + last_item = items[-1] + return encode_cursor(kind, resource_id, last_item.created_at, last_item.id) diff --git a/app/domain/users.py b/app/domain/users.py new file mode 100644 index 00000000..ec4f97a6 --- /dev/null +++ b/app/domain/users.py @@ -0,0 +1,39 @@ +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import security +from app.errors.exceptions import conflict +from app.models.user import User as UserModel +from app.schemas.user import CreateUser, User +from app.storage import users as users_storage + +USERNAME_TAKEN_MESSAGE = "Имя пользователя уже занято" + + +def to_user_schema(user: UserModel) -> User: + user_schema = User.model_validate(user) + return user_schema + + +async def register_user(session: AsyncSession, new_user: CreateUser) -> User: + existing_user = await users_storage.get_user_by_username(session, new_user.username) + await session.commit() + if existing_user is not None: + raise conflict(USERNAME_TAKEN_MESSAGE) + + password_hash = await security.hash_password(new_user.password) + + try: + user = await users_storage.create_user( + session, + username=new_user.username, + display_name=new_user.display_name, + password_hash=password_hash, + ) + await session.commit() + except IntegrityError: + await session.rollback() + raise conflict(USERNAME_TAKEN_MESSAGE) from None + + created_user = to_user_schema(user) + return created_user diff --git a/app/endpoints/__init__.py b/app/endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/endpoints/attachments.py b/app/endpoints/attachments.py new file mode 100644 index 00000000..ddeddfa9 --- /dev/null +++ b/app/endpoints/attachments.py @@ -0,0 +1,74 @@ +"""HTTP-параметры и упаковка ответов; сценарии находятся в domain.""" + +from typing import Annotated +from urllib.parse import quote +from uuid import UUID + +from fastapi import APIRouter, Query, Request +from fastapi.responses import StreamingResponse +from starlette.background import BackgroundTask + +from app.domain import attachments +from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity +from app.schemas.attachment import Attachment + +router = APIRouter(prefix="/api/v1", tags=["attachments"]) + + +@router.post("/chats/{chat_id}/attachments", status_code=202) +async def upload( + request: Request, + chat_id: UUID, + user: CurrentUser, + session: DbSession, + storage: S3Storage, + capacity: UploadCapacity, + filename: Annotated[str, Query(min_length=1, max_length=255)], +) -> Attachment: + return await attachments.receive_attachment( + session, + storage, + user, + chat_id, + capacity=capacity, + filename=filename, + content_type=request.headers.get("content-type", "").lower(), + declared_length=request.headers.get("content-length"), + chunks=request.stream(), + ) + + +@router.get("/attachments/{attachment_id}") +async def metadata( + attachment_id: UUID, user: CurrentUser, session: DbSession +) -> Attachment: + return await attachments.get_attachment(session, user, attachment_id) + + +@router.get("/attachments/{attachment_id}/content") +async def content( + attachment_id: UUID, + user: CurrentUser, + session: DbSession, + storage: S3Storage, + variant: Annotated[str, Query()] = "original", +) -> StreamingResponse: + record, stream = await attachments.download_attachment( + session, + storage, + user, + attachment_id, + variant=variant, + ) + return StreamingResponse( + stream, + media_type=record.content_type, + headers={ + "Content-Length": str(record.size_bytes), + "Content-Disposition": "attachment; filename*=UTF-8''" + + quote(record.filename, safe=""), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + }, + background=BackgroundTask(stream.aclose), + ) diff --git a/app/endpoints/auth.py b/app/endpoints/auth.py new file mode 100644 index 00000000..69f4535f --- /dev/null +++ b/app/endpoints/auth.py @@ -0,0 +1,32 @@ +from fastapi import APIRouter, Response + +from app.domain.auth import create_session, revoke_session +from app.endpoints.dependencies import ( + AppSettings, + BasicCredentials, + BearerToken, + DbSession, + RedisClient, +) +from app.schemas.session import AuthSession + +router = APIRouter(prefix="/api/v1/auth", tags=["auth"]) + + +@router.post("/sessions", status_code=201) +async def login( + credentials: BasicCredentials, + session: DbSession, + redis: RedisClient, + settings: AppSettings, +) -> AuthSession: + username, password = credentials + return await create_session( + session, redis, username, password, settings.session_ttl_seconds + ) + + +@router.delete("/sessions/current", status_code=204) +async def logout(token: BearerToken, redis: RedisClient) -> Response: + await revoke_session(redis, token) + return Response(status_code=204) diff --git a/app/endpoints/chats.py b/app/endpoints/chats.py new file mode 100644 index 00000000..91f2b211 --- /dev/null +++ b/app/endpoints/chats.py @@ -0,0 +1,32 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Query + +from app.domain.chats import create_chat, get_chat, list_chats +from app.endpoints.dependencies import CurrentUser, DbSession +from app.schemas.chat import Chat, ChatPage, CreateChat + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +@router.post("", status_code=201) +async def create_new_chat( + new_chat: CreateChat, creator: CurrentUser, session: DbSession +) -> Chat: + return await create_chat(session, creator, new_chat) + + +@router.get("") +async def list_user_chats( + user: CurrentUser, + session: DbSession, + limit: Annotated[int, Query(ge=1, le=100)] = 50, + cursor: Annotated[str | None, Query(min_length=1)] = None, +) -> ChatPage: + return await list_chats(session, user, cursor, limit) + + +@router.get("/{chat_id}") +async def get_chat_by_id(chat_id: UUID, user: CurrentUser, session: DbSession) -> Chat: + return await get_chat(session, user, chat_id) diff --git a/app/endpoints/dependencies.py b/app/endpoints/dependencies.py new file mode 100644 index 00000000..85a5bc22 --- /dev/null +++ b/app/endpoints/dependencies.py @@ -0,0 +1,109 @@ +import base64 +import binascii +from collections.abc import AsyncIterator +from typing import Annotated + +from fastapi import Depends, Request, Security +from fastapi.openapi.models import HTTPBase, HTTPBearer +from fastapi.security.base import SecurityBase +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.request_capacity import RequestCapacity +from app.domain.auth import authenticate_token +from app.errors.exceptions import unauthorized_basic, unauthorized_bearer +from app.schemas.user import User +from app.storage.s3 import MinioStorage + + +async def get_settings(request: Request) -> Settings: + settings: Settings = request.app.state.settings + return settings + + +async def get_db(request: Request) -> AsyncIterator[AsyncSession]: + session_factory = request.app.state.session_factory + async with session_factory() as session: + yield session + + +async def get_redis(request: Request) -> Redis: + redis: Redis = request.app.state.redis + return redis + + +def get_s3(request: Request) -> MinioStorage | None: + return request.app.state.s3 + + +def get_upload_capacity(request: Request) -> RequestCapacity: + return request.app.state.upload_capacity + + +class AuthorizationHeader(SecurityBase): + + def __init__(self, model: HTTPBase, scheme_name: str) -> None: + self.model = model + self.scheme_name = scheme_name + + async def __call__(self, request: Request) -> str | None: + return request.headers.get("Authorization") + + +basic_auth = AuthorizationHeader(HTTPBase(scheme="basic"), "BasicAuth") +bearer_auth = AuthorizationHeader(HTTPBearer(), "BearerAuth") + + +def _split_authorization(authorization: str | None) -> tuple[str, str] | None: + if authorization is None: + return None + scheme, _, value = authorization.partition(" ") + if not value.strip(): + return None + return scheme.lower(), value.strip() + + +async def get_bearer_token( + authorization: Annotated[str | None, Security(bearer_auth)], +) -> str: + parts = _split_authorization(authorization) + if parts is None or parts[0] != "bearer": + raise unauthorized_bearer() + return parts[1] + + +async def get_basic_credentials( + authorization: Annotated[str | None, Security(basic_auth)], +) -> tuple[str, str]: + parts = _split_authorization(authorization) + if parts is None or parts[0] != "basic": + raise unauthorized_basic() + + try: + decoded = base64.b64decode(parts[1], validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError): + raise unauthorized_basic() from None + + username, separator, password = decoded.partition(":") + if not separator or not username: + raise unauthorized_basic() + return username, password + + +DbSession = Annotated[AsyncSession, Depends(get_db)] +RedisClient = Annotated[Redis, Depends(get_redis)] +AppSettings = Annotated[Settings, Depends(get_settings)] +S3Storage = Annotated[MinioStorage | None, Depends(get_s3)] +UploadCapacity = Annotated[RequestCapacity, Depends(get_upload_capacity)] +BearerToken = Annotated[str, Depends(get_bearer_token)] +BasicCredentials = Annotated[tuple[str, str], Depends(get_basic_credentials)] + + +async def get_current_user( + token: BearerToken, session: DbSession, redis: RedisClient +) -> User: + return await authenticate_token(session, redis, token) + + +CurrentUser = Annotated[User, Depends(get_current_user)] diff --git a/app/endpoints/health.py b/app/endpoints/health.py new file mode 100644 index 00000000..2be984b6 --- /dev/null +++ b/app/endpoints/health.py @@ -0,0 +1,23 @@ +from fastapi import APIRouter, Request + +from app.domain import health +from app.schemas.base import Health + +router = APIRouter(prefix="/health", tags=["health"]) + + +@router.get("/live") +async def liveness() -> Health: + return health.liveness() + + +@router.get("/ready") +async def readiness(request: Request) -> Health: + state = request.app.state + return await health.readiness( + state.engine, + state.redis, + state.s3, + state.lifecycle, + state.settings.readiness_timeout_seconds, + ) diff --git a/app/endpoints/messages.py b/app/endpoints/messages.py new file mode 100644 index 00000000..4bfbd3f3 --- /dev/null +++ b/app/endpoints/messages.py @@ -0,0 +1,42 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Header, Query + +from app.domain.idempotency import send_message_idempotent, validate_idempotency_key +from app.domain.messages import list_messages +from app.endpoints.dependencies import AppSettings, CurrentUser, DbSession +from app.schemas.message import CreateMessage, Message, MessagePage + +router = APIRouter(prefix="/api/v1/chats", tags=["messages"]) + + +@router.post("/{chat_id}/messages", status_code=201) +async def send_new_message( + chat_id: UUID, + new_message: CreateMessage, + sender: CurrentUser, + session: DbSession, + settings: AppSettings, + idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None, +) -> Message: + return await send_message_idempotent( + session, + sender, + chat_id, + new_message, + validate_idempotency_key(idempotency_key), + settings.max_messages, + settings.idempotency_ttl_seconds, + ) + + +@router.get("/{chat_id}/messages") +async def get_history_messages( + chat_id: UUID, + user: CurrentUser, + session: DbSession, + limit: Annotated[int, Query(ge=1, le=100)] = 50, + cursor: Annotated[str | None, Query(min_length=1)] = None, +) -> MessagePage: + return await list_messages(session, user, chat_id, cursor, limit) diff --git a/app/endpoints/users.py b/app/endpoints/users.py new file mode 100644 index 00000000..c502d010 --- /dev/null +++ b/app/endpoints/users.py @@ -0,0 +1,17 @@ +from fastapi import APIRouter + +from app.domain.users import register_user +from app.endpoints.dependencies import CurrentUser, DbSession +from app.schemas.user import CreateUser, User + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) + + +@router.post("", status_code=201) +async def create_user(new_user: CreateUser, session: DbSession) -> User: + return await register_user(session, new_user) + + +@router.get("/me") +async def get_me(user: CurrentUser) -> User: + return user diff --git a/app/errors/__init__.py b/app/errors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/errors/exceptions.py b/app/errors/exceptions.py new file mode 100644 index 00000000..21897071 --- /dev/null +++ b/app/errors/exceptions.py @@ -0,0 +1,131 @@ +import errno + +from collections.abc import Mapping + +from PIL import Image, UnidentifiedImageError + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiError(Exception): + def __init__( + self, + status_code: int, + code: str, + message: str, + headers: Mapping[str, str] | None = None, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers + + +def invalid_request(message: str = "Неверный запрос") -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized( + message: str = "Нет действительной аутентификации", + www_authenticate: str | None = None, +) -> ApiError: + headers = {"WWW-Authenticate": www_authenticate} if www_authenticate else None + return ApiError(401, "unauthorized", message, headers) + + +def unauthorized_basic(message: str = "Неверные учётные данные") -> ApiError: + return unauthorized(message, BASIC_CHALLENGE) + + +def unauthorized_bearer(message: str = "Нет действительной сессии") -> ApiError: + return unauthorized(message, BEARER_CHALLENGE) + + +def not_found(message: str = "Объект не найден") -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str = "Конфликт") -> ApiError: + return ApiError(409, "conflict", message) + + +def payload_too_large(message: str = "Слишком большое тело запроса") -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type( + message: str = "Требуется Content-Type: application/json", +) -> ApiError: + return ApiError(415, "unsupported_media_type", message) + + +def rate_limited(retry_after: int = 1) -> ApiError: + return ApiError( + 429, + "rate_limited", + "Слишком много запросов", + {"Retry-After": str(max(1, retry_after))}, + ) + + +def unavailable(message: str = "Сервис временно недоступен") -> ApiError: + return ApiError(503, "unavailable", message) + + +def internal_error() -> ApiError: + return ApiError(500, "internal_error", "Внутренняя ошибка сервера") + + +IMAGE_WORKER_ERRORS = ( + ApiError, + Image.DecompressionBombError, + Image.DecompressionBombWarning, + MemoryError, + UnidentifiedImageError, + OSError, + ValueError, + SyntaxError, +) + + +def image_worker_error_code(error: Exception) -> int: + code = 2 + resource_error = isinstance( + error, + (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError), + ) + declared_limit = isinstance(error, ApiError) and error.status_code == 413 + if resource_error or declared_limit: + code = 3 + return code + + +def image_validation_error(code: int) -> ApiError: + error = unsupported_media_type("Повреждённое изображение или несовпадение MIME") + if code == 3 or code < 0: + error = payload_too_large("Превышен лимит пикселей или ресурсов изображения") + return error + + +class ProcessingError(Exception): + + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +class JobOperationError(Exception): + + def __init__(self, code: str): + self.code = code + super().__init__(code) + + +def pipeline_worker_error_code(error: Exception) -> int: + if isinstance(error, OSError) and error.errno is not None: + if error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}: + return 5 + return 4 + return image_worker_error_code(error) diff --git a/app/errors/handlers.py b/app/errors/handlers.py new file mode 100644 index 00000000..2da757c8 --- /dev/null +++ b/app/errors/handlers.py @@ -0,0 +1,234 @@ +import asyncio +import logging +from contextlib import asynccontextmanager +from collections.abc import Mapping +from uuid import uuid4 + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from minio.error import MinioException +from redis.exceptions import ConnectionError as RedisConnectionError +from redis.exceptions import TimeoutError as RedisTimeoutError +from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError, SQLAlchemyError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import ClientDisconnect +from urllib3.exceptions import HTTPError as StorageHTTPError + +from app.core.instance import INSTANCE_ID_HEADER +from app.core.request_id import REQUEST_ID_HEADER +from app.errors.exceptions import ( + ApiError, + internal_error, + invalid_request, + not_found, + unavailable, +) + +logger = logging.getLogger(__name__) + +TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57") +TRANSIENT_SQLSTATES = frozenset({"55P03"}) +RECOVERABLE_STORAGE_ERRORS = ( + ApiError, + SQLAlchemyError, + MinioException, + StorageHTTPError, + OSError, + TimeoutError, +) + + +@asynccontextmanager +async def operation_timeout(seconds: float, error: ApiError): + try: + async with asyncio.timeout(seconds): + yield + except TimeoutError: + raise error from None + + +async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse: + return await api_error_handler( + request, invalid_request("Загрузка прервана клиентом") + ) + + +def log_recoverable_error(operation: str, error: Exception) -> None: + logger.warning("%s deferred: %s", operation, type(error).__name__) + + +def _get_request_id(request: Request) -> str: + request_id = getattr(request.state, "request_id", None) + + if request_id is None: + request_id = str(uuid4()) + request.state.request_id = request_id + + return request_id + + +def _build_response( + request: Request, + status_code: int, + code: str, + message: str, + headers: Mapping[str, str] | None = None, +) -> JSONResponse: + request_id = _get_request_id(request) + + response_headers = { + **(headers or {}), + REQUEST_ID_HEADER: request_id, + } + instance_id = getattr(request.app.state, "instance_id", None) + if instance_id is not None: + response_headers[INSTANCE_ID_HEADER] = instance_id + + body = { + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + } + + return JSONResponse( + status_code=status_code, + content=body, + headers=response_headers, + ) + + +async def api_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + assert isinstance(exc, ApiError) + + return _build_response( + request, + exc.status_code, + exc.code, + exc.message, + exc.headers, + ) + + +async def routing_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + assert isinstance(exc, StarletteHTTPException) + + error = invalid_request() + if exc.status_code == 404: + error = not_found() + return _build_response( + request, + exc.status_code, + error.code, + error.message, + exc.headers, + ) + + +async def validation_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + assert isinstance(exc, RequestValidationError) + + fields = sorted({".".join(map(str, error["loc"])) for error in exc.errors()}) + message = "Неверный запрос" + + if fields: + message = f"{message}: {', '.join(fields)}" + + return _build_response( + request, + 400, + "invalid_request", + message, + ) + + +async def dependency_unavailable_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + request_id = _get_request_id(request) + logger.error( + "Dependency unavailable request_id=%s: %s", + request_id, + type(exc).__name__, + ) + + error = unavailable() + return _build_response( + request, + error.status_code, + error.code, + error.message, + {"Retry-After": "1"}, + ) + + +def _is_transient_database_error(exc: DBAPIError) -> bool: + sqlstate = getattr(exc.orig, "sqlstate", None) or "" + return ( + sqlstate.startswith(TRANSIENT_SQLSTATE_CLASSES) + or sqlstate in TRANSIENT_SQLSTATES + ) + + +async def database_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + assert isinstance(exc, DBAPIError) + + handler = unhandled_error_handler + if _is_transient_database_error(exc): + handler = dependency_unavailable_handler + return await handler(request, exc) + + +async def unhandled_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + request_id = _get_request_id(request) + + logger.error( + "Unhandled error request_id=%s", + request_id, + exc_info=exc, + ) + + error = internal_error() + + return _build_response( + request, + error.status_code, + error.code, + error.message, + ) + + +def register_exception_handlers(app: FastAPI) -> None: + app.add_exception_handler(ApiError, api_error_handler) + app.add_exception_handler(ClientDisconnect, client_disconnect_handler) + app.add_exception_handler(MinioException, dependency_unavailable_handler) + app.add_exception_handler(StorageHTTPError, dependency_unavailable_handler) + app.add_exception_handler(StarletteHTTPException, routing_error_handler) + app.add_exception_handler(RequestValidationError, validation_error_handler) + app.add_exception_handler(OperationalError, dependency_unavailable_handler) + app.add_exception_handler(InterfaceError, dependency_unavailable_handler) + app.add_exception_handler(PoolTimeoutError, dependency_unavailable_handler) + app.add_exception_handler(DBAPIError, database_error_handler) + app.add_exception_handler(RedisConnectionError, dependency_unavailable_handler) + app.add_exception_handler(RedisTimeoutError, dependency_unavailable_handler) + app.add_exception_handler(OSError, dependency_unavailable_handler) + app.add_exception_handler(Exception, unhandled_error_handler) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..21c6411f --- /dev/null +++ b/app/main.py @@ -0,0 +1,94 @@ +import logging +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +import uvicorn +from fastapi import Depends, FastAPI + +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.instance import InstanceIdMiddleware, resolve_instance_id +from app.core.lifecycle import GracefulServer, Lifecycle +from app.core.redis import create_redis_client +from app.core.request_capacity import acquire_request_slot, create_request_capacity +from app.core.request_id import RequestIdMiddleware +from app.endpoints.auth import router as auth_router +from app.endpoints.attachments import router as attachments_router +from app.endpoints.chats import router as chats_router +from app.endpoints.health import router as health_router +from app.endpoints.messages import router as messages_router +from app.endpoints.users import router as users_router +from app.errors.handlers import register_exception_handlers +from app.storage.s3 import create_s3_storage + + +def configure_logging(settings: Settings) -> None: + logging.basicConfig( + level=settings.log_level, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + + +def create_app(settings: Settings | None = None) -> FastAPI: + settings = settings or Settings() + configure_logging(settings) + + engine = create_db_engine(settings) + redis = create_redis_client(settings) + s3 = create_s3_storage(settings.s3) if settings.s3_enabled else None + + @asynccontextmanager + async def lifespan(_: FastAPI) -> AsyncGenerator[None]: + yield + if s3 is not None: + await s3.aclose() + await redis.aclose() + await engine.dispose() + + application = FastAPI(title="Messenger", version="1.5.0", lifespan=lifespan) + + application.state.settings = settings + application.state.engine = engine + application.state.session_factory = create_session_factory(engine) + application.state.redis = redis + application.state.s3 = s3 + application.state.upload_capacity = create_request_capacity( + settings.s3.max_concurrent_uploads + ) + application.state.request_capacity = create_request_capacity(settings.max_in_flight) + application.state.instance_id = resolve_instance_id(settings.instance_id) + application.state.lifecycle = Lifecycle() + application.add_middleware(RequestIdMiddleware) + application.add_middleware( + InstanceIdMiddleware, instance_id=application.state.instance_id + ) + register_exception_handlers(application) + + limited = [Depends(acquire_request_slot)] + application.include_router(health_router) + application.include_router(users_router, dependencies=limited) + application.include_router(auth_router, dependencies=limited) + application.include_router(chats_router, dependencies=limited) + application.include_router(messages_router, dependencies=limited) + application.include_router(attachments_router, dependencies=limited) + + return application + + +def main() -> None: + settings = Settings() + application = create_app(settings) + config = uvicorn.Config( + application, + host=settings.app_host, + port=settings.app_port, + log_level=settings.log_level.lower(), + timeout_graceful_shutdown=settings.graceful_shutdown_timeout_seconds, + ) + GracefulServer( + config, application.state.lifecycle, settings.shutdown_drain_seconds + ).run() + + +if __name__ == "__main__": + main() diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 00000000..d9476c43 --- /dev/null +++ b/app/models/__init__.py @@ -0,0 +1,25 @@ +from app.models.job_replay import JobReplay +from app.models.base import Base +from app.models.job import AttachmentJob +from app.models.attachment import Attachment, MessageAttachment, UploadIntent +from app.models.chat import Chat +from app.models.chat_member import ChatMember +from app.models.counter import Counter +from app.models.idempotency_key import IdempotencyKey +from app.models.message import Message +from app.models.user import User + +__all__ = [ + "JobReplay", + "AttachmentJob", + "Base", + "Attachment", + "MessageAttachment", + "UploadIntent", + "Chat", + "ChatMember", + "Counter", + "IdempotencyKey", + "Message", + "User", +] diff --git a/app/models/attachment.py b/app/models/attachment.py new file mode 100644 index 00000000..dfc12f14 --- /dev/null +++ b/app/models/attachment.py @@ -0,0 +1,70 @@ +"""Метаданные оригиналов и журнал операций до записи в S3.""" + +import uuid +from datetime import datetime + +from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class Attachment(Base): + __tablename__ = "attachments" + __table_args__ = ( + CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"), + CheckConstraint( + "content_type IN ('image/png', 'image/jpeg')", name="content_type" + ), + CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"), + CheckConstraint( + "status IN ('queued', 'processing', 'ready', 'failed')", name="status" + ), + CheckConstraint( + "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", + name="error_code", + ), + Index("ix_attachments_chat_id", "chat_id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id")) + owner_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id")) + object_key: Mapped[str] = mapped_column(String(128), unique=True) + filename: Mapped[str] = mapped_column(String(255)) + content_type: Mapped[str] = mapped_column(String(32)) + size_bytes: Mapped[int] + sha256: Mapped[str] = mapped_column(String(64)) + status: Mapped[str] = mapped_column(String(16), server_default="queued") + error_code: Mapped[str | None] = mapped_column(String(64)) + thumbnail_key: Mapped[str | None] = mapped_column(String(128)) + crop_key: Mapped[str | None] = mapped_column(String(128)) + optimized_key: Mapped[str | None] = mapped_column(String(128)) + variants: Mapped[list[dict]] = mapped_column( + JSONB, default=list, server_default=text("'[]'::jsonb") + ) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + + +class MessageAttachment(Base): + __tablename__ = "message_attachments" + + message_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True + ) + attachment_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("attachments.id"), primary_key=True + ) + position: Mapped[int] + + +class UploadIntent(Base): + + __tablename__ = "upload_intents" + __table_args__ = (Index("ix_upload_intents_created_at", "created_at"),) + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True) + object_key: Mapped[str] = mapped_column(String(128), unique=True) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/base.py b/app/models/base.py new file mode 100644 index 00000000..ff742f18 --- /dev/null +++ b/app/models/base.py @@ -0,0 +1,26 @@ +import uuid +from datetime import datetime +from typing import Any, ClassVar + +from sqlalchemy import DateTime, MetaData +from sqlalchemy.dialects.postgresql import UUID as PG_UUID +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.types import TypeEngine + +NAMING_CONVENTION = { + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + + +class Base(DeclarativeBase): + + metadata = MetaData(naming_convention=NAMING_CONVENTION) + + type_annotation_map: ClassVar[dict[Any, TypeEngine[Any]]] = { + uuid.UUID: PG_UUID(as_uuid=True), + datetime: DateTime(timezone=True), + } diff --git a/app/models/chat.py b/app/models/chat.py new file mode 100644 index 00000000..21198345 --- /dev/null +++ b/app/models/chat.py @@ -0,0 +1,31 @@ +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import String, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base + +if TYPE_CHECKING: + from app.models.chat_member import ChatMember + from app.models.message import Message + + +class Chat(Base): + + __tablename__ = "chats" + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + title: Mapped[str] = mapped_column(String(100)) + created_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + members: Mapped[list["ChatMember"]] = relationship( + back_populates="chat", + cascade="all, delete-orphan", + passive_deletes=True, + lazy="selectin", + ) + messages: Mapped[list["Message"]] = relationship( + back_populates="chat", lazy="raise" + ) diff --git a/app/models/chat_member.py b/app/models/chat_member.py new file mode 100644 index 00000000..77021f5e --- /dev/null +++ b/app/models/chat_member.py @@ -0,0 +1,25 @@ +import uuid +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey, Index +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base + +if TYPE_CHECKING: + from app.models.chat import Chat + from app.models.user import User + + +class ChatMember(Base): + + __tablename__ = "chat_members" + __table_args__ = (Index("ix_chat_members_user_id_chat_id", "user_id", "chat_id"),) + + chat_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True + ) + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True) + + chat: Mapped["Chat"] = relationship(back_populates="members", lazy="raise") + user: Mapped["User"] = relationship(back_populates="memberships", lazy="raise") diff --git a/app/models/counter.py b/app/models/counter.py new file mode 100644 index 00000000..38446c02 --- /dev/null +++ b/app/models/counter.py @@ -0,0 +1,12 @@ +from sqlalchemy import BigInteger, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class Counter(Base): + + __tablename__ = "counters" + + name: Mapped[str] = mapped_column(String(64), primary_key=True) + value: Mapped[int] = mapped_column(BigInteger) diff --git a/app/models/idempotency_key.py b/app/models/idempotency_key.py new file mode 100644 index 00000000..d39dd07b --- /dev/null +++ b/app/models/idempotency_key.py @@ -0,0 +1,23 @@ +"""Сохранённый результат отправки сообщения.""" + +import uuid +from datetime import datetime + +from sqlalchemy import ForeignKey, Index, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class IdempotencyKey(Base): + + __tablename__ = "idempotency_keys" + __table_args__ = (Index("ix_idempotency_keys_created_at", "created_at"),) + + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True) + method: Mapped[str] = mapped_column(String(8), primary_key=True) + path: Mapped[str] = mapped_column(String(255), primary_key=True) + key: Mapped[str] = mapped_column(String(128), primary_key=True) + request_hash: Mapped[str] = mapped_column(String(64)) + message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id")) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/job.py b/app/models/job.py new file mode 100644 index 00000000..01924ee8 --- /dev/null +++ b/app/models/job.py @@ -0,0 +1,40 @@ +"""Устойчивая работа и намерение публикации; один pipeline на вложение.""" + +from datetime import datetime +from uuid import UUID, uuid4 + +from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class AttachmentJob(Base): + __tablename__ = "attachment_jobs" + __table_args__ = ( + CheckConstraint("pipeline_version = 1", name="pipeline_version"), + CheckConstraint("attempt >= 0 AND published_attempt >= 0", name="attempt"), + Index("ix_attachment_jobs_due", "next_attempt_at"), + Index("ix_attachment_jobs_lease", "lease_until"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) + attachment_id: Mapped[UUID] = mapped_column( + ForeignKey("attachments.id"), + unique=True, + ) + staging_key: Mapped[str | None] = mapped_column(String(128)) + pipeline_version: Mapped[int] = mapped_column(server_default="1") + attempt: Mapped[int] = mapped_column(server_default="0") + cycle_start_attempt: Mapped[int] = mapped_column(server_default="0") + attempt_limit: Mapped[int | None] + last_error_permanent: Mapped[bool | None] + published_attempt: Mapped[int] = mapped_column(server_default="0") + next_attempt_at: Mapped[datetime] = mapped_column( + server_default=func.clock_timestamp() + ) + lease_token: Mapped[UUID | None] + lease_until: Mapped[datetime | None] + last_error_code: Mapped[str | None] = mapped_column(String(64)) + finished_at: Mapped[datetime | None] + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/job_replay.py b/app/models/job_replay.py new file mode 100644 index 00000000..7f2b27af --- /dev/null +++ b/app/models/job_replay.py @@ -0,0 +1,22 @@ +"""Аудит явного повторного запуска failed; request id обеспечивает идемпотентность.""" + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import ForeignKey, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class JobReplay(Base): + __tablename__ = "job_replays" + + id: Mapped[UUID] = mapped_column(primary_key=True) + job_id: Mapped[UUID] = mapped_column(ForeignKey("attachment_jobs.id"), index=True) + actor: Mapped[str] = mapped_column(String(64)) + reason: Mapped[str] = mapped_column(String(32)) + previous_attempt: Mapped[int] + attempt_limit: Mapped[int] + previous_error_code: Mapped[str | None] = mapped_column(String(64)) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/message.py b/app/models/message.py new file mode 100644 index 00000000..b93e289f --- /dev/null +++ b/app/models/message.py @@ -0,0 +1,29 @@ +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey, Index, Text, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base + +if TYPE_CHECKING: + from app.models.chat import Chat + from app.models.user import User + + +class Message(Base): + + __tablename__ = "messages" + __table_args__ = ( + Index("ix_messages_chat_id_created_at_id", "chat_id", "created_at", "id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id")) + sender_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id")) + text: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + + chat: Mapped["Chat"] = relationship(back_populates="messages", lazy="raise") + sender: Mapped["User"] = relationship(back_populates="messages", lazy="raise") diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 00000000..aa50e4dd --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,30 @@ +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import String, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base + +if TYPE_CHECKING: + from app.models.chat_member import ChatMember + from app.models.message import Message + + +class User(Base): + + __tablename__ = "users" + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + username: Mapped[str] = mapped_column(String(32), unique=True) + display_name: Mapped[str] = mapped_column(String(100)) + password_hash: Mapped[str] = mapped_column(String(255)) + created_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + memberships: Mapped[list["ChatMember"]] = relationship( + back_populates="user", lazy="raise" + ) + messages: Mapped[list["Message"]] = relationship( + back_populates="sender", lazy="raise" + ) diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py new file mode 100644 index 00000000..2f038940 --- /dev/null +++ b/app/schemas/__init__.py @@ -0,0 +1,6 @@ +from app.schemas.chat import Chat +from app.schemas.message import Message +from app.schemas.session import AuthSession +from app.schemas.user import User + +__all__ = ["AuthSession", "Chat", "Message", "User"] diff --git a/app/schemas/attachment.py b/app/schemas/attachment.py new file mode 100644 index 00000000..629e26b9 --- /dev/null +++ b/app/schemas/attachment.py @@ -0,0 +1,53 @@ +"""Публичные состояния и варианты; внутренние ключи S3 не сериализуются.""" + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from app.schemas.base import Base + + +class Variant(BaseModel): + model_config = ConfigDict(extra="forbid", from_attributes=True) + + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int = Field(ge=1) + width: int = Field(ge=1) + height: int = Field(ge=1) + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + + +class Attachment(Base): + chat_id: UUID + owner_id: UUID + filename: str = Field(min_length=1, max_length=255) + content_type: Literal["image/png", "image/jpeg"] + size_bytes: int = Field(ge=1, le=10_485_760) + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + status: Literal["queued", "processing", "ready", "failed"] + variants: list[Variant] = Field(default_factory=list) + error_code: str | None = None + created_at: datetime + updated_at: datetime + + +VARIANT_FILENAMES = { + "thumbnail": "thumbnail.jpg", + "crop": "crop.jpg", + "optimized": "optimized.webp", +} + + +class PipelineResult(BaseModel): + model_config = ConfigDict(extra="forbid") + variants: list[Variant] = Field(min_length=3, max_length=3) + + @field_validator("variants") + @classmethod + def require_all_variants(cls, variants: list[Variant]) -> list[Variant]: + if {item.name for item in variants} != set(VARIANT_FILENAMES): + raise ValueError("Нужны thumbnail, crop и optimized без повторов") + return variants diff --git a/app/schemas/base.py b/app/schemas/base.py new file mode 100644 index 00000000..6a18d6a8 --- /dev/null +++ b/app/schemas/base.py @@ -0,0 +1,34 @@ +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, field_validator + + +class Base(BaseModel): + model_config = ConfigDict(extra="forbid", from_attributes=True) + + id: UUID + + +class Health(BaseModel): + model_config = ConfigDict(extra="forbid") + + status: Literal["ok"] + + +class BaseCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + @field_validator("*", mode="before") + @classmethod + def reject_invalid_strings(cls, value: object) -> object: + if isinstance(value, str): + if not value.strip(): + raise ValueError("Строка не может состоять только из пробелов") + if "\x00" in value: + raise ValueError("Строка не может содержать символ NUL") + return value + + +class BasePage(BaseModel): + model_config = ConfigDict(extra="forbid") diff --git a/app/schemas/chat.py b/app/schemas/chat.py new file mode 100644 index 00000000..c0436dab --- /dev/null +++ b/app/schemas/chat.py @@ -0,0 +1,22 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import Field + +from app.schemas.base import Base, BaseCreate, BasePage + + +class Chat(Base): + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + created_at: datetime + + +class CreateChat(BaseCreate): + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + +class ChatPage(BasePage): + items: list[Chat] + next_cursor: str | None diff --git a/app/schemas/job.py b/app/schemas/job.py new file mode 100644 index 00000000..7b783709 --- /dev/null +++ b/app/schemas/job.py @@ -0,0 +1,22 @@ +"""Сообщение брокера: только идентификаторы и служебные поля контракта v1.""" + +from typing import Literal +from uuid import UUID + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field + + +class JobMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] = 1 + type: Literal["attachment.process.v1"] = "attachment.process.v1" + job_id: UUID + attachment_id: UUID + pipeline_version: int = Field(default=1, ge=1) + attempt: int = Field(ge=1) + created_at: AwareDatetime + traceparent: str | None = Field( + default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$" + ) + tracestate: str | None = Field(default=None, max_length=512) diff --git a/app/schemas/message.py b/app/schemas/message.py new file mode 100644 index 00000000..a085e03e --- /dev/null +++ b/app/schemas/message.py @@ -0,0 +1,41 @@ +from datetime import datetime +from typing import Self +from uuid import UUID + +from pydantic import Field, field_validator, model_validator + +from app.schemas.base import Base, BaseCreate, BasePage + + +class Message(Base): + chat_id: UUID + sender_id: UUID + text: str = Field(max_length=4000) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + created_at: datetime + + +class CreateMessage(BaseCreate): + text: str = Field(max_length=4000) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @field_validator("*", mode="before") + @classmethod + def reject_invalid_strings(cls, value: object) -> object: + if isinstance(value, str): + if "\x00" in value or (value != "" and not value.strip()): + raise ValueError("Некорректная строка сообщения") + return value + + @model_validator(mode="after") + def require_content(self) -> Self: + if not self.text and not self.attachment_ids: + raise ValueError("Нужен текст или вложение") + if len(self.attachment_ids) != len(set(self.attachment_ids)): + raise ValueError("attachment_ids содержит дубликаты") + return self + + +class MessagePage(BasePage): + items: list[Message] + next_cursor: str | None diff --git a/app/schemas/session.py b/app/schemas/session.py new file mode 100644 index 00000000..4b67bd67 --- /dev/null +++ b/app/schemas/session.py @@ -0,0 +1,16 @@ +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from app.schemas.user import User + + +class AuthSession(BaseModel): + + model_config = ConfigDict(extra="forbid") + + token: str = Field(min_length=32) + token_type: Literal["Bearer"] = "Bearer" + expires_at: datetime + user: User diff --git a/app/schemas/user.py b/app/schemas/user.py new file mode 100644 index 00000000..39a26dcd --- /dev/null +++ b/app/schemas/user.py @@ -0,0 +1,24 @@ +from datetime import datetime + +from pydantic import Field + +from app.schemas.base import Base, BaseCreate + +USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$" +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +class User(Base): + + username: str = Field(pattern=USERNAME_PATTERN) + display_name: str = Field(min_length=1, max_length=100) + created_at: datetime + + +class CreateUser(BaseCreate): + username: str = Field(pattern=USERNAME_PATTERN) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field( + min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH, repr=False + ) diff --git a/app/storage/__init__.py b/app/storage/__init__.py new file mode 100644 index 00000000..c881665f --- /dev/null +++ b/app/storage/__init__.py @@ -0,0 +1 @@ +"""Доступ к данным PostgreSQL и Redis.""" diff --git a/app/storage/attachments.py b/app/storage/attachments.py new file mode 100644 index 00000000..3ef1765f --- /dev/null +++ b/app/storage/attachments.py @@ -0,0 +1,99 @@ +"""SQL-запросы метаданных и журнала; commit выполняется в domain.""" + +from datetime import timedelta +from uuid import UUID + +from sqlalchemy import delete, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.attachment import Attachment, MessageAttachment, UploadIntent + + +async def create_attachment(session: AsyncSession, **values) -> Attachment: + record = Attachment(**values) + session.add(record) + await session.flush() + return record + + +async def get_attachment( + session: AsyncSession, attachment_id: UUID +) -> Attachment | None: + return await session.get(Attachment, attachment_id) + + +async def get_by_key(session: AsyncSession, key: str) -> Attachment | None: + return await session.scalar(select(Attachment).where(Attachment.object_key == key)) + + +async def get_for_chat( + session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID] +) -> list[Attachment]: + records = [] + if attachment_ids: + records = list( + await session.scalars( + select(Attachment).where( + Attachment.id.in_(attachment_ids), + Attachment.chat_id == chat_id, + ) + ) + ) + return records + + +async def bind_attachments( + session: AsyncSession, message_id: UUID, attachment_ids: list[UUID] +) -> None: + session.add_all( + MessageAttachment(message_id=message_id, attachment_id=id, position=position) + for position, id in enumerate(attachment_ids) + ) + await session.flush() + + +async def message_attachment_ids( + session: AsyncSession, message_ids: list[UUID] +) -> dict[UUID, list[UUID]]: + result: dict[UUID, list[UUID]] = {id: [] for id in message_ids} + if not message_ids: + return result + rows = await session.execute( + select(MessageAttachment) + .where(MessageAttachment.message_id.in_(message_ids)) + .order_by(MessageAttachment.position) + ) + for link in rows.scalars(): + result[link.message_id].append(link.attachment_id) + return result + + +async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None: + session.add(UploadIntent(id=id, object_key=key)) + await session.flush() + + +async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None: + return await session.scalar( + select(UploadIntent).where(UploadIntent.id == id).with_for_update() + ) + + +async def finish_upload(session: AsyncSession, id: UUID) -> None: + await session.execute(delete(UploadIntent).where(UploadIntent.id == id)) + + +async def stale_uploads( + session: AsyncSession, grace_seconds: int, limit: int +) -> list[UploadIntent]: + result = await session.scalars( + select(UploadIntent) + .where( + UploadIntent.created_at + < func.clock_timestamp() - timedelta(seconds=grace_seconds), + ) + .order_by(UploadIntent.created_at) + .limit(limit) + .with_for_update(skip_locked=True) + ) + return list(result) diff --git a/app/storage/broker.py b/app/storage/broker.py new file mode 100644 index 00000000..5b85c087 --- /dev/null +++ b/app/storage/broker.py @@ -0,0 +1,93 @@ +"""RabbitMQ: durable queue, persistent messages, publisher confirms и manual ack.""" + +import asyncio +import logging +from dataclasses import dataclass + +import aio_pika +from aio_pika.abc import AbstractIncomingMessage + +from app.core.jobs import JobSettings +from app.schemas.job import JobMessage + + +@dataclass(frozen=True) +class Delivery: + id: AbstractIncomingMessage + payload: bytes + + +class JobBroker: + + def __init__(self, settings: JobSettings) -> None: + self.settings = settings + self.connection = self.channel = self.queue = self.iterator = None + + logging.getLogger("aio_pika").setLevel(logging.CRITICAL) + logging.getLogger("aiormq").setLevel(logging.CRITICAL) + + async def ensure_ready(self) -> None: + if ( + self.connection is None + or self.connection.is_closed + or self.channel is None + or self.channel.is_closed + ): + await self.aclose() + self.connection = await aio_pika.connect( + host=self.settings.broker_host, + port=self.settings.broker_port, + login=self.settings.broker_user, + password=self.settings.broker_password_file.read_text().strip(), + virtualhost=self.settings.broker_vhost, + timeout=5, + heartbeat=30, + ) + self.channel = await self.connection.channel( + publisher_confirms=True, on_return_raises=True + ) + await self.channel.set_qos(prefetch_count=1) + self.queue = await self.channel.declare_queue( + self.settings.broker_queue, durable=True + ) + + async def publish_job(self, message: JobMessage) -> None: + await self.ensure_ready() + await self.channel.default_exchange.publish( + aio_pika.Message( + body=message.model_dump_json(exclude_none=True).encode(), + content_type="application/json", + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + message_id=f"{message.job_id}:{message.attempt}", + ), + routing_key=self.settings.broker_queue, + mandatory=True, + timeout=10, + ) + + async def receive(self, consumer: str) -> Delivery: + await self.ensure_ready() + if self.iterator is None: + self.iterator = self.queue.iterator(no_ack=False, consumer_tag=consumer) + await self.iterator.__aenter__() + message = await self.iterator.__anext__() + return Delivery(message, message.body) + + async def acknowledge_job(self, delivery_id: AbstractIncomingMessage) -> None: + await delivery_id.ack() + + async def defer_job(self, delivery_id: AbstractIncomingMessage) -> None: + + await asyncio.sleep(self.settings.publisher_poll_seconds) + await delivery_id.nack(requeue=True) + + async def reject_invalid(self, delivery_id: AbstractIncomingMessage) -> None: + + logging.getLogger(__name__).warning("Rejected invalid job envelope") + await delivery_id.reject(requeue=False) + + async def aclose(self) -> None: + connection = self.connection + self.connection = self.channel = self.queue = self.iterator = None + if connection is not None and not connection.is_closed: + await connection.close() diff --git a/app/storage/chats.py b/app/storage/chats.py new file mode 100644 index 00000000..d3130348 --- /dev/null +++ b/app/storage/chats.py @@ -0,0 +1,67 @@ +from collections.abc import Sequence +from uuid import UUID + +from sqlalchemy import exists, select, tuple_ +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.chat import Chat +from app.models.chat_member import ChatMember +from app.storage.types import Position + + +async def create_chat( + session: AsyncSession, + *, + title: str, + member_ids: Sequence[UUID], +) -> Chat: + chat = Chat( + title=title, + members=[ChatMember(user_id=user_id) for user_id in member_ids], + ) + session.add(chat) + await session.flush() + return chat + + +async def get_chat_for_member( + session: AsyncSession, + chat_id: UUID, + user_id: UUID, +) -> Chat | None: + stmt = ( + select(Chat) + .join(ChatMember, ChatMember.chat_id == Chat.id) + .where(Chat.id == chat_id, ChatMember.user_id == user_id) + ) + result = await session.execute(stmt) + chat = result.scalar_one_or_none() + return chat + + +async def is_chat_member(session: AsyncSession, chat_id: UUID, user_id: UUID) -> bool: + stmt = select( + exists().where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id) + ) + result = await session.execute(stmt) + is_member = bool(result.scalar_one()) + return is_member + + +async def list_chats_for_user( + session: AsyncSession, + user_id: UUID, + after: Position | None, + limit: int, +) -> list[Chat]: + stmt = ( + select(Chat) + .join(ChatMember, ChatMember.chat_id == Chat.id) + .where(ChatMember.user_id == user_id) + ) + if after is not None: + stmt = stmt.where(tuple_(Chat.created_at, Chat.id) > tuple_(*after)) + stmt = stmt.order_by(Chat.created_at, Chat.id).limit(limit) + result = await session.execute(stmt) + chats = list(result.scalars().all()) + return chats diff --git a/app/storage/health.py b/app/storage/health.py new file mode 100644 index 00000000..2d034b71 --- /dev/null +++ b/app/storage/health.py @@ -0,0 +1,15 @@ +"""Простые проверки соединения с общими хранилищами.""" + +from redis.asyncio import Redis +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine + + +async def check_database(engine: AsyncEngine) -> bool: + async with engine.connect() as connection: + await connection.execute(text("SELECT 1")) + return True + + +async def check_redis(client: Redis) -> bool: + return bool(await client.ping()) diff --git a/app/storage/idempotency.py b/app/storage/idempotency.py new file mode 100644 index 00000000..4f25c0e2 --- /dev/null +++ b/app/storage/idempotency.py @@ -0,0 +1,67 @@ +"""Хранение и блокировка ключей в PostgreSQL.""" + +from datetime import timedelta +import hashlib +import json +from uuid import UUID + +from sqlalchemy import BigInteger, func, literal, select, tuple_ +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.idempotency_key import IdempotencyKey + +Scope = tuple[UUID, str, str, str] + + +async def lock_key(session: AsyncSession, scope: Scope) -> None: + raw = json.dumps([str(scope[0]), *scope[1:]]).encode() + lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True) + await session.execute( + select(func.pg_advisory_xact_lock(literal(lock_id, type_=BigInteger))) + ) + + +async def get_active_key( + session: AsyncSession, scope: Scope, ttl_seconds: int +) -> IdempotencyKey | None: + stmt = select(IdempotencyKey).where( + tuple_( + IdempotencyKey.user_id, + IdempotencyKey.method, + IdempotencyKey.path, + IdempotencyKey.key, + ) + == tuple_(*scope), + IdempotencyKey.created_at + > func.clock_timestamp() - timedelta(seconds=ttl_seconds), + ) + return await session.scalar(stmt) + + +async def save_result( + session: AsyncSession, scope: Scope, request_hash: str, message_id: UUID +) -> None: + stmt = insert(IdempotencyKey).values( + user_id=scope[0], + method=scope[1], + path=scope[2], + key=scope[3], + request_hash=request_hash, + message_id=message_id, + ) + await session.execute( + stmt.on_conflict_do_update( + index_elements=[ + IdempotencyKey.user_id, + IdempotencyKey.method, + IdempotencyKey.path, + IdempotencyKey.key, + ], + set_={ + "request_hash": request_hash, + "message_id": message_id, + "created_at": func.clock_timestamp(), + }, + ) + ) diff --git a/app/storage/job_admin.py b/app/storage/job_admin.py new file mode 100644 index 00000000..6780924a --- /dev/null +++ b/app/storage/job_admin.py @@ -0,0 +1,59 @@ +"""Операторские выборки и журнал replay; транзакция принадлежит domain.""" + +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert + +from app.models.attachment import Attachment +from app.models.job import AttachmentJob +from app.models.job_replay import JobReplay + + +async def failed_jobs(session, limit: int): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(Attachment.status == "failed") + .order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id) + .limit(limit) + ) + return result.all() + + +async def replay_record(session, request_id): + return await session.get(JobReplay, request_id) + + +async def insert_replay(session, **values) -> bool: + result = await session.scalar( + insert(JobReplay) + .values(**values) + .on_conflict_do_nothing(index_elements=[JobReplay.id]) + .returning(JobReplay.id) + ) + return result is not None + + +async def replay_history(session, job_id): + return ( + await session.scalars( + select(JobReplay) + .where(JobReplay.job_id == job_id) + .order_by(JobReplay.created_at) + .limit(100) + ) + ).all() + + +def reset_failed(job, attachment, now, attempt_limit): + job.cycle_start_attempt = job.attempt + job.attempt_limit = attempt_limit + job.published_attempt = job.attempt + job.next_attempt_at = now + job.lease_token = None + job.lease_until = None + job.finished_at = None + job.last_error_code = None + job.last_error_permanent = None + attachment.status = "queued" + attachment.error_code = None + attachment.updated_at = now diff --git a/app/storage/jobs.py b/app/storage/jobs.py new file mode 100644 index 00000000..e0ca118b --- /dev/null +++ b/app/storage/jobs.py @@ -0,0 +1,160 @@ +"""SQL и изменения jobs/attachments; commit остаётся в domain.""" + +from dataclasses import dataclass +from datetime import datetime, timedelta +from uuid import UUID + +from sqlalchemy import and_, or_, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.attachment import Attachment +from app.models.job import AttachmentJob +from app.schemas.attachment import VARIANT_FILENAMES + + +@dataclass(frozen=True) +class JobWork: + job_id: UUID + attachment_id: UUID + lease_token: UUID + pipeline_version: int + staging_key: str + content_type: str + size_bytes: int + sha256: str + + @property + def original_key(self) -> str: + return f"originals/{self.attachment_id.hex}/v{self.pipeline_version}" + + def variant_key(self, name: str) -> str: + filename = VARIANT_FILENAMES[name] + return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}" + + +async def create_job( + session: AsyncSession, attachment_id: UUID, staging_key: str +) -> UUID: + job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key) + session.add(job) + await session.flush() + return job.id + + +async def lock_job(session: AsyncSession, job_id: UUID): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(AttachmentJob.id == job_id) + .with_for_update(of=AttachmentJob) + .execution_options(populate_existing=True) + ) + return result.one_or_none() + + +async def database_time(session: AsyncSession) -> datetime: + return await session.scalar(select(func.clock_timestamp())) + + +async def expired_jobs(session: AsyncSession, limit: int = 20): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where( + Attachment.status == "processing", + AttachmentJob.lease_until < func.clock_timestamp(), + ) + .order_by(AttachmentJob.lease_until) + .limit(limit) + .with_for_update(of=AttachmentJob, skip_locked=True) + ) + return result.all() + + +def start_job( + job, attachment, token: UUID, now: datetime, lease_seconds: int +) -> JobWork: + job.attempt += 1 + job.lease_token = token + job.lease_until = now + timedelta(seconds=lease_seconds) + attachment.status = "processing" + attachment.updated_at = now + return JobWork( + job.id, + attachment.id, + token, + job.pipeline_version, + job.staging_key, + attachment.content_type, + attachment.size_bytes, + attachment.sha256, + ) + + +def record_failure( + job, + attachment, + now: datetime, + error_code: str, + *, + terminal: bool, + retry_seconds: float, +) -> None: + job.lease_token = None + job.lease_until = None + job.last_error_code = error_code + job.next_attempt_at = now + timedelta(seconds=retry_seconds) + job.finished_at = now if terminal else None + attachment.status = "failed" if terminal else "queued" + attachment.error_code = error_code if terminal else None + attachment.updated_at = now + + +def complete_job( + job, attachment, work: JobWork, variants: list[dict], now: datetime +) -> None: + attachment.object_key = work.original_key + attachment.thumbnail_key = work.variant_key("thumbnail") + attachment.crop_key = work.variant_key("crop") + attachment.optimized_key = work.variant_key("optimized") + attachment.variants = variants + attachment.status = "ready" + attachment.error_code = None + attachment.updated_at = now + job.lease_token = None + job.lease_until = None + job.last_error_code = None + job.last_error_permanent = None + job.finished_at = now + + +async def terminal_staging( + session: AsyncSession, retention: int, failed_retention: int, limit: int = 20 +): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where( + AttachmentJob.staging_key.is_not(None), + or_( + and_( + Attachment.status == "ready", + AttachmentJob.finished_at + < func.clock_timestamp() - timedelta(seconds=retention), + ), + and_( + Attachment.status == "failed", + AttachmentJob.finished_at + < func.clock_timestamp() - timedelta(seconds=failed_retention), + ), + ), + ) + .order_by(AttachmentJob.finished_at) + .limit(limit) + .with_for_update(of=AttachmentJob, skip_locked=True) + ) + return result.all() + + +def clear_staging(job) -> None: + job.staging_key = None diff --git a/app/storage/messages.py b/app/storage/messages.py new file mode 100644 index 00000000..2da9ffa6 --- /dev/null +++ b/app/storage/messages.py @@ -0,0 +1,59 @@ +from uuid import UUID + +from sqlalchemy import select, tuple_ +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.counter import Counter +from app.models.message import Message +from app.storage.types import Position + +MESSAGES_COUNTER = "messages" + + +async def get_message(session: AsyncSession, message_id: UUID) -> Message | None: + return await session.get(Message, message_id) + + +async def create_message( + session: AsyncSession, + *, + chat_id: UUID, + sender_id: UUID, + text: str, +) -> Message: + message = Message(chat_id=chat_id, sender_id=sender_id, text=text) + session.add(message) + await session.flush() + return message + + +async def list_messages( + session: AsyncSession, + chat_id: UUID, + after: Position | None, + limit: int, +) -> list[Message]: + stmt = select(Message).where(Message.chat_id == chat_id) + if after is not None: + stmt = stmt.where(tuple_(Message.created_at, Message.id) > tuple_(*after)) + stmt = stmt.order_by(Message.created_at, Message.id).limit(limit) + result = await session.execute(stmt) + messages = list(result.scalars().all()) + return messages + + +async def reserve_message_slot(session: AsyncSession, max_messages: int) -> bool: + stmt = ( + insert(Counter) + .values(name=MESSAGES_COUNTER, value=1) + .on_conflict_do_update( + index_elements=[Counter.name], + set_={"value": Counter.value + 1}, + where=Counter.value < max_messages, + ) + .returning(Counter.value) + ) + result = await session.execute(stmt) + is_reserved = result.scalar_one_or_none() is not None + return is_reserved diff --git a/app/storage/outbox.py b/app/storage/outbox.py new file mode 100644 index 00000000..be50c31b --- /dev/null +++ b/app/storage/outbox.py @@ -0,0 +1,33 @@ +"""Jobs одновременно хранят намерение публикации очередной попытки.""" + +from uuid import UUID + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.attachment import Attachment +from app.models.job import AttachmentJob + + +async def next_unpublished_job( + session: AsyncSession, *, attachment_id: UUID | None = None +) -> AttachmentJob | None: + statement = ( + select(AttachmentJob) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where( + Attachment.status == "queued", + AttachmentJob.next_attempt_at <= func.clock_timestamp(), + AttachmentJob.published_attempt <= AttachmentJob.attempt, + ) + .order_by(AttachmentJob.next_attempt_at, AttachmentJob.id) + .limit(1) + .with_for_update(of=AttachmentJob, skip_locked=True) + ) + if attachment_id is not None: + statement = statement.where(AttachmentJob.attachment_id == attachment_id) + return await session.scalar(statement) + + +def mark_published(job: AttachmentJob) -> None: + job.published_attempt = job.attempt + 1 diff --git a/app/storage/s3.py b/app/storage/s3.py new file mode 100644 index 00000000..b328ea2c --- /dev/null +++ b/app/storage/s3.py @@ -0,0 +1,180 @@ +"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции.""" + +import asyncio +from collections.abc import Callable +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import TypeVar +import time +from urllib.parse import urlsplit + +from anyio import CancelScope +from minio import Minio +import urllib3 + +from app.core.s3 import S3Settings + +T = TypeVar("T") + + +async def run_sync( + operation: Callable[..., T], *args, discard_result=None, **kwargs +) -> T: + """При отмене дождаться потока: иначе PUT может догнать компенсацию удаления.""" + task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs)) + try: + result = await asyncio.shield(task) + except asyncio.CancelledError: + with CancelScope(shield=True): + await settle_cancelled_call(task, discard_result) + raise + return result + + +async def settle_cancelled_call( + task: asyncio.Task, discard_result: Callable | None +) -> None: + """wait не переносит исключение SDK; повторная отмена не прерывает компенсацию.""" + while not task.done(): + with suppress(asyncio.CancelledError): + await asyncio.wait([task]) + completed = not task.cancelled() and task.exception() is None + if completed and discard_result is not None: + discard_result(task.result()) + + +@dataclass(frozen=True) +class ObjectInfo: + """Поля S3 сохраняют необязательность SDK; согласованность проверяет domain.""" + + key: str + size_bytes: int | None + content_type: str | None + last_modified: datetime | None + + +def close_response(response) -> None: + response.close() + response.release_conn() + + +class ObjectStream: + """Закрывается даже если StreamingResponse не успел начать итерацию.""" + + def __init__(self, storage, response): + self.storage = storage + self.response = response + self.deadline = time.monotonic() + storage.settings.download_timeout_seconds + self.closed = False + + def __aiter__(self): + return self + + async def __anext__(self): + if self.closed: + raise StopAsyncIteration + try: + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + async with asyncio.timeout(remaining): + chunk = await self.storage._call( + lambda: self.response.read(self.storage.settings.chunk_bytes) + ) + if not chunk: + await self.aclose() + raise StopAsyncIteration + except BaseException: + await self.aclose() + raise + return chunk + + async def aclose(self): + if not self.closed: + self.closed = True + close_response(self.response) + + +class MinioStorage: + def __init__(self, settings: S3Settings) -> None: + self.settings = settings + + url = urlsplit(settings.endpoint) + self._http = urllib3.PoolManager( + timeout=urllib3.Timeout( + connect=settings.connect_timeout_seconds, + read=settings.read_timeout_seconds, + ), + retries=urllib3.Retry(total=0), + maxsize=32, + ) + self._health_http = urllib3.PoolManager( + timeout=urllib3.Timeout(connect=0.25, read=0.5), + retries=urllib3.Retry(total=0), + ) + self._client = Minio( + url.netloc, + secure=url.scheme == "https", + region=settings.region, + access_key=settings.access_key_id_file.read_text().strip(), + secret_key=settings.secret_access_key_file.read_text().strip(), + http_client=self._http, + ) + + async def _call(self, operation: Callable, *, discard_result=None): + return await run_sync(operation, discard_result=discard_result) + + async def put_file( + self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str + ) -> None: + def put(): + with path.open("rb") as source: + + self._client.put_object( + self.settings.bucket, + key, + source, + size_bytes, + content_type=content_type, + metadata={"sha256": sha256}, + part_size=10_485_760, + num_parallel_uploads=1, + ) + + await self._call(put) + + async def open_object(self, key: str) -> ObjectStream: + response = await self._call( + lambda: self._client.get_object(self.settings.bucket, key), + discard_result=close_response, + ) + return ObjectStream(self, response) + + async def stat_object(self, key: str) -> ObjectInfo: + def stat(): + info = self._client.stat_object(self.settings.bucket, key) + return ObjectInfo(key, info.size, info.content_type, info.last_modified) + + return await self._call(stat) + + async def delete_object(self, key: str) -> None: + await self._call(lambda: self._client.remove_object(self.settings.bucket, key)) + + async def check_available(self) -> bool: + status = await self._call( + lambda: self._health_http.request( + "GET", + self.settings.endpoint.rstrip("/") + "/minio/health/ready", + ).status + ) + return status == 200 + + async def aclose(self) -> None: + self._http.clear() + self._health_http.clear() + + +def create_s3_storage(settings: S3Settings) -> MinioStorage: + return MinioStorage(settings) diff --git a/app/storage/sessions.py b/app/storage/sessions.py new file mode 100644 index 00000000..f845562c --- /dev/null +++ b/app/storage/sessions.py @@ -0,0 +1,34 @@ +from datetime import datetime +from uuid import UUID + +from redis.asyncio import Redis + +SESSION_KEY_PREFIX = "session:" + + +def session_key(token_hash: str) -> str: + return f"{SESSION_KEY_PREFIX}{token_hash}" + + +async def create_session( + redis: Redis, + *, + token_hash: str, + user_id: UUID, + expires_at: datetime, +) -> None: + await redis.set(session_key(token_hash), str(user_id), pxat=expires_at) + + +async def get_session_user_id(redis: Redis, token_hash: str) -> UUID | None: + raw_user_id = await redis.get(session_key(token_hash)) + if isinstance(raw_user_id, bytes): + raw_user_id = raw_user_id.decode() + user_id = UUID(raw_user_id) if raw_user_id else None + return user_id + + +async def delete_session(redis: Redis, token_hash: str) -> bool: + deleted_count = await redis.delete(session_key(token_hash)) + is_deleted = deleted_count > 0 + return is_deleted diff --git a/app/storage/types.py b/app/storage/types.py new file mode 100644 index 00000000..e0357cfa --- /dev/null +++ b/app/storage/types.py @@ -0,0 +1,6 @@ +"""Общие типы слоя хранения.""" + +from datetime import datetime +from uuid import UUID + +Position = tuple[datetime, UUID] diff --git a/app/storage/users.py b/app/storage/users.py new file mode 100644 index 00000000..ae5c7195 --- /dev/null +++ b/app/storage/users.py @@ -0,0 +1,43 @@ +from collections.abc import Collection +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.user import User + + +async def create_user( + session: AsyncSession, + *, + username: str, + display_name: str, + password_hash: str, +) -> User: + user = User( + username=username, display_name=display_name, password_hash=password_hash + ) + session.add(user) + await session.flush() + return user + + +async def get_user_by_id(session: AsyncSession, user_id: UUID) -> User | None: + user = await session.get(User, user_id) + return user + + +async def get_user_by_username(session: AsyncSession, username: str) -> User | None: + stmt = select(User).where(User.username == username) + result = await session.execute(stmt) + user = result.scalar_one_or_none() + return user + + +async def get_existing_user_ids( + session: AsyncSession, user_ids: Collection[UUID] +) -> set[UUID]: + stmt = select(User.id).where(User.id.in_(user_ids)) + result = await session.execute(stmt) + existing_ids = set(result.scalars().all()) + return existing_ids diff --git a/compose.test.yaml b/compose.test.yaml new file mode 100644 index 00000000..d4c5a709 --- /dev/null +++ b/compose.test.yaml @@ -0,0 +1,7 @@ +services: + postgres: + ports: + - "127.0.0.1:25432:5432" + redis: + ports: + - "127.0.0.1:26379:6379" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..292bfb41 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,417 @@ +name: messenger + +x-api-image: &api-image + image: ${API_IMAGE:?Set API_IMAGE in .env} + pull_policy: never + +x-base-env: &base-env + APP_HOST: ${APP_HOST:?Set APP_HOST in .env} + APP_PORT: ${APP_PORT:?Set APP_PORT in .env} + APP_ENV: ${APP_ENV:?Set APP_ENV in .env} + LOG_LEVEL: ${LOG_LEVEL:?Set LOG_LEVEL in .env} + MAX_MESSAGES: ${MAX_MESSAGES:?Set MAX_MESSAGES in .env} + MAX_IN_FLIGHT: ${MAX_IN_FLIGHT:?Set MAX_IN_FLIGHT in .env} + DB_HOST: ${DB_HOST:?Set DB_HOST in .env} + DB_PORT: ${DB_PORT:?Set DB_PORT in .env} + DB_NAME: ${DB_NAME:?Set DB_NAME in .env} + DB_USER: ${DB_USER:?Set DB_USER in .env} + DB_POOL_SIZE: ${DB_POOL_SIZE:?Set DB_POOL_SIZE in .env} + DB_MAX_OVERFLOW: ${DB_MAX_OVERFLOW:?Set DB_MAX_OVERFLOW in .env} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:?Set DB_POOL_TIMEOUT_SECONDS in .env} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:?Set DB_CONNECT_TIMEOUT_SECONDS in .env} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:?Set DB_STATEMENT_TIMEOUT_MS in .env} + SHUTDOWN_DRAIN_SECONDS: ${SHUTDOWN_DRAIN_SECONDS:?Set SHUTDOWN_DRAIN_SECONDS in .env} + GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS: ${GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS:?Set GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS in .env} + READINESS_TIMEOUT_SECONDS: ${READINESS_TIMEOUT_SECONDS:?Set READINESS_TIMEOUT_SECONDS in .env} + +x-s3-env: &s3-env + S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env} + S3_REGION: ${S3_REGION:?Set S3_REGION in .env} + S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env} + S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:?Set S3_CONNECT_TIMEOUT_SECONDS in .env} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:?Set S3_READ_TIMEOUT_SECONDS in .env} + S3_UPLOAD_TIMEOUT_SECONDS: ${S3_UPLOAD_TIMEOUT_SECONDS:?Set S3_UPLOAD_TIMEOUT_SECONDS in .env} + S3_DOWNLOAD_TIMEOUT_SECONDS: ${S3_DOWNLOAD_TIMEOUT_SECONDS:?Set S3_DOWNLOAD_TIMEOUT_SECONDS in .env} + S3_IMAGE_TIMEOUT_SECONDS: ${S3_IMAGE_TIMEOUT_SECONDS:?Set S3_IMAGE_TIMEOUT_SECONDS in .env} + S3_MAX_UPLOAD_BYTES: ${S3_MAX_UPLOAD_BYTES:?Set S3_MAX_UPLOAD_BYTES in .env} + S3_MAX_IMAGE_PIXELS: ${S3_MAX_IMAGE_PIXELS:?Set S3_MAX_IMAGE_PIXELS in .env} + S3_CHUNK_BYTES: ${S3_CHUNK_BYTES:?Set S3_CHUNK_BYTES in .env} + S3_MAX_CONCURRENT_UPLOADS: ${S3_MAX_CONCURRENT_UPLOADS:?Set S3_MAX_CONCURRENT_UPLOADS in .env} + S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env} + S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env} + +x-jobs-env: &jobs-env + BROKER_HOST: ${BROKER_HOST:?Set BROKER_HOST in .env} + BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env} + BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + BROKER_QUEUE: ${BROKER_QUEUE:?Set BROKER_QUEUE in .env} + BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env} + BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:?Set WORKER_CONCURRENCY in .env} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:?Set JOB_MAX_ATTEMPTS in .env} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:?Set JOB_TIMEOUT_SECONDS in .env} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:?Set JOB_LEASE_SECONDS in .env} + JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:?Set JOB_RETRY_MAX_SECONDS in .env} + JOB_RETRY_SECONDS: ${JOB_RETRY_SECONDS:?Set JOB_RETRY_SECONDS in .env} + PUBLISHER_POLL_SECONDS: ${PUBLISHER_POLL_SECONDS:?Set PUBLISHER_POLL_SECONDS in .env} + FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env} + IMAGE_JPEG_QUALITY: ${IMAGE_JPEG_QUALITY:?Set IMAGE_JPEG_QUALITY in .env} + IMAGE_WEBP_QUALITY: ${IMAGE_WEBP_QUALITY:?Set IMAGE_WEBP_QUALITY in .env} + IMAGE_WEBP_METHOD: ${IMAGE_WEBP_METHOD:?Set IMAGE_WEBP_METHOD in .env} + IMAGE_MEMORY_MIB: ${IMAGE_MEMORY_MIB:?Set IMAGE_MEMORY_MIB in .env} + IMAGE_CPU_SECONDS: ${IMAGE_CPU_SECONDS:?Set IMAGE_CPU_SECONDS in .env} + +services: + minio: + build: + context: . + dockerfile: infra/minio/Dockerfile + target: server + image: messenger-minio:local + command: ["server", "/data", "--console-address", "127.0.0.1:9001"] + restart: unless-stopped + environment: + MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env} + MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env} + MINIO_BROWSER: ${MINIO_BROWSER:?Set MINIO_BROWSER in .env} + secrets: + - source: minio_root_user + target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env} + - source: minio_root_password + target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env} + volumes: + - miniodata:/data + networks: [objects] + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready', timeout=2)"] + interval: 3s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + minio-bootstrap: + build: + context: . + dockerfile: infra/minio/Dockerfile + target: bootstrap + image: messenger-minio-bootstrap:local + environment: + S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env} + S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env} + S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env} + MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env} + secrets: + - source: minio_root_user + target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env} + - source: minio_root_password + target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env} + - source: s3_access_key_id + target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + - source: s3_secret_access_key + target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + networks: [objects] + restart: "no" + depends_on: + minio: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + postgres: + image: postgres:16.10-bookworm + restart: unless-stopped + environment: + POSTGRES_USER: ${DB_USER:?Set DB_USER in .env} + POSTGRES_DB: ${DB_NAME:?Set DB_NAME in .env} + POSTGRES_PASSWORD_FILE: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env} + secrets: + - source: db_password + target: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 3s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + redis: + image: redis:7.4.2-bookworm + restart: unless-stopped + entrypoint: ["/bin/sh", "/usr/local/bin/start-redis.sh"] + secrets: + - redis_password + volumes: + - ./infra/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro + - ./infra/redis/start-redis.sh:/usr/local/bin/start-redis.sh:ro + - redisdata:/data + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping"] + interval: 3s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + + broker: + image: rabbitmq:4.4.0 + hostname: broker + restart: unless-stopped + entrypoint: ["/bin/sh", "/usr/local/bin/start-broker.sh"] + environment: + BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env} + BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env} + BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env} + BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + secrets: + - source: broker_password + target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + volumes: + - ./infra/broker/start-broker.sh:/usr/local/bin/start-broker.sh:ro + - rabbitmqdata:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 20s + deploy: + resources: + limits: + cpus: "1.0" + memory: 768M + + publisher: + <<: *api-image + init: true + command: ["python", "-m", "app.core.outbox_publisher"] + restart: unless-stopped + environment: + <<: [*base-env, *jobs-env] + secrets: + - db_password + - source: broker_password + target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + disable: true + stop_grace_period: 15s + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + worker: + <<: *api-image + init: true + command: ["python", "-m", "app.core.job_worker"] + restart: unless-stopped + environment: + <<: [*base-env, *s3-env, *jobs-env] + secrets: + - db_password + - source: broker_password + target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + - source: s3_access_key_id + target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + - source: s3_secret_access_key + target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + networks: [default, objects] + tmpfs: + - /tmp:rw,noexec,nosuid,size=128m,mode=1777 + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + minio: + condition: service_healthy + minio-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + healthcheck: + disable: true + stop_signal: SIGTERM + stop_grace_period: 20s + deploy: + resources: + limits: + cpus: "2.0" + memory: 2048M + + migrate: + <<: *api-image + command: ["alembic", "upgrade", "head"] + environment: *base-env + secrets: + - db_password + depends_on: + postgres: + condition: service_healthy + restart: "no" + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + api: + <<: *api-image + build: + context: . + args: + API_BUILD_REVISION: ${API_BUILD_REVISION:?Set API_BUILD_REVISION in .env} + restart: unless-stopped + environment: + <<: [*base-env, *s3-env] + REDIS_HOST: ${REDIS_HOST:?Set REDIS_HOST in .env} + REDIS_PORT: ${REDIS_PORT:?Set REDIS_PORT in .env} + REDIS_DB: ${REDIS_DB:?Set REDIS_DB in .env} + REDIS_MAX_CONNECTIONS: ${REDIS_MAX_CONNECTIONS:?Set REDIS_MAX_CONNECTIONS in .env} + REDIS_POOL_TIMEOUT_SECONDS: ${REDIS_POOL_TIMEOUT_SECONDS:?Set REDIS_POOL_TIMEOUT_SECONDS in .env} + REDIS_CONNECT_TIMEOUT_SECONDS: ${REDIS_CONNECT_TIMEOUT_SECONDS:?Set REDIS_CONNECT_TIMEOUT_SECONDS in .env} + REDIS_SOCKET_TIMEOUT_SECONDS: ${REDIS_SOCKET_TIMEOUT_SECONDS:?Set REDIS_SOCKET_TIMEOUT_SECONDS in .env} + SESSION_STORE: ${SESSION_STORE:?Set SESSION_STORE in .env} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:?Set SESSION_TTL_SECONDS in .env} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:?Set IDEMPOTENCY_TTL_SECONDS in .env} + secrets: + - db_password + - redis_password + - source: s3_access_key_id + target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + - source: s3_secret_access_key + target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + networks: [default, objects] + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m,mode=1777 + deploy: + replicas: ${API_REPLICAS:?Set API_REPLICAS in .env} + resources: + limits: + cpus: "1.0" + memory: 1536M + healthcheck: + test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"] + interval: 3s + timeout: 3s + start_period: 10s + retries: 2 + stop_signal: SIGTERM + stop_grace_period: 8s + depends_on: + minio: + condition: service_healthy + minio-bootstrap: + condition: service_completed_successfully + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + + cleanup: + <<: *api-image + command: ["python", "-m", "app.core.cleanup", "--apply"] + restart: unless-stopped + environment: + <<: [*base-env, *s3-env] + FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env} + secrets: + - db_password + - source: s3_access_key_id + target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + - source: s3_secret_access_key + target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + networks: [default, objects] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + minio-bootstrap: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + + nginx: + image: nginx:1.28.0-alpine + restart: unless-stopped + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "127.0.0.1:${LB_HOST_PORT:?Set LB_HOST_PORT in .env}:8080" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/lb-health"] + interval: 5s + timeout: 3s + retries: 5 + depends_on: + api: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + +volumes: + pgdata: + redisdata: + miniodata: + rabbitmqdata: + +networks: + default: + objects: + internal: true + +secrets: + broker_password: + file: ./secrets/broker_password + db_password: + file: ./secrets/db_password + redis_password: + file: ./secrets/redis_password + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key diff --git a/evidence/lab5/00_init.txt b/evidence/lab5/00_init.txt new file mode 100644 index 00000000..c37c55e6 --- /dev/null +++ b/evidence/lab5/00_init.txt @@ -0,0 +1,11 @@ +$ python3 scripts/init_local.py +Создан файл secrets/db_password +Создан файл secrets/redis_password +Создан файл secrets/broker_password +Создан файл secrets/minio_root_user +Создан файл secrets/minio_root_password +Создан файл secrets/s3_access_key_id +Создан файл secrets/s3_secret_access_key +Настройки готовы; существующие значения сохранены + +exit_code=0; duration_seconds=0.215 diff --git a/evidence/lab5/01_black.txt b/evidence/lab5/01_black.txt new file mode 100644 index 00000000..9f79877f --- /dev/null +++ b/evidence/lab5/01_black.txt @@ -0,0 +1,5 @@ +$ python3 -m black --check app infra migrations scripts student-tests tests +All done! ✨ 🍰 ✨ +131 files would be left unchanged. + +exit_code=0; duration_seconds=1.921 diff --git a/evidence/lab5/02_contract.txt b/evidence/lab5/02_contract.txt new file mode 100644 index 00000000..6dd19e22 --- /dev/null +++ b/evidence/lab5/02_contract.txt @@ -0,0 +1,4 @@ +$ python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +exit_code=0; duration_seconds=0.165 diff --git a/evidence/lab5/03_openapi.txt b/evidence/lab5/03_openapi.txt new file mode 100644 index 00000000..326925b6 --- /dev/null +++ b/evidence/lab5/03_openapi.txt @@ -0,0 +1,4 @@ +$ python3 scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +exit_code=0; duration_seconds=1.425 diff --git a/evidence/lab5/04_build_start.txt b/evidence/lab5/04_build_start.txt new file mode 100644 index 00000000..62f7eb8c --- /dev/null +++ b/evidence/lab5/04_build_start.txt @@ -0,0 +1,4368 @@ +$ docker compose --env-file .env up -d --build + Image postgres:16.10-bookworm Pulling + Image nginx:1.28.0-alpine Pulling + Image rabbitmq:4.4.0 Pulling + Image redis:7.4.2-bookworm Pulling + aa8e37a1147a + 517b871c26ea + 0ea2caf05f48 + cb40c4489522 + 96aae9022e0e + e34abc083292 + db1363514c7f + 674d165e7863 + afa027603ca1 + db099b936fc8 + 1adabd6b0d6b + 5e83761a8bfc + 5f7f7c5eac2d + 799548af46de + cdfd017c753d + 72ba653f834d + 7b697787d5d2 + 861643ce2817 + 961aa0039c0e + 0dac5f77c330 + db8bf9a4f43b + 856e25603442 + 974175074a8f + d78bd8cc905b + 3e300a7cb18c + 71a39d0d04b2 + f637881d1138 + b8554c5f1ad0 + c4fca37af7b3 + 8e049f0fd151 + e6918dcfd20d + bc1d7488b05e + c497c6a20f0e [========> ] 1.049MB/5.91MB + cbb99c664e48 + 8a628cdd7ccc + 3ff860482ac5 + 759939a29cb5 + 811af041d785 + 8ae1ad8ce35e + 4f4fb700ef54 + a7e27cf18de4 + c497c6a20f0e [=================> ] 2.097MB/5.91MB + c497c6a20f0e [=================> ] 2.097MB/5.91MB + c0098bc8b90c [==================================================>] 53.3kB/53.3kB + 5f7f7c5eac2d + c497c6a20f0e [==========================> ] 3.146MB/5.91MB + aa8e37a1147a + c497c6a20f0e [===================================> ] 4.194MB/5.91MB + 5e83761a8bfc + 799548af46de + c0098bc8b90c + afa027603ca1 + c497c6a20f0e [============================================> ] 5.243MB/5.91MB + 0dac5f77c330 + def3b568b79d + 674d165e7863 + cc7884611f32 [==================================================>] 19.9kB/19.9kB + 856e25603442 + d78bd8cc905b + c497c6a20f0e + 759939a29cb5 + cc7884611f32 + cb40c4489522 [=> ] 1.049MB/50.57MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [=> ] 1.049MB/50.57MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [=> ] 1.049MB/50.57MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + 96aae9022e0e + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [==> ] 2.097MB/50.57MB + 961aa0039c0e + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [==> ] 2.097MB/50.57MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [===> ] 3.146MB/50.57MB + b8554c5f1ad0 [=============================> ] 1.049MB/1.791MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [===> ] 3.146MB/50.57MB + b8554c5f1ad0 [=============================> ] 1.049MB/1.791MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [===> ] 3.146MB/50.57MB + b8554c5f1ad0 [=============================> ] 1.049MB/1.791MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [====> ] 4.194MB/50.57MB + b8554c5f1ad0 [=============================> ] 1.049MB/1.791MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [====> ] 4.194MB/50.57MB + b8554c5f1ad0 [=============================> ] 1.049MB/1.791MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + cb40c4489522 [====> ] 4.194MB/50.57MB + b8554c5f1ad0 [=============================> ] 1.049MB/1.791MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [====> ] 4.194MB/50.57MB + b8554c5f1ad0 [==================================================>] 1.791MB/1.791MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [====> ] 4.194MB/50.57MB + b8554c5f1ad0 + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [=====> ] 5.243MB/50.57MB + 23eb6c0fdeef [=======> ] 1.049MB/7.139MB + 974175074a8f [==================================================>] 1.196MB/1.196MB + 23eb6c0fdeef [==============> ] 2.097MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [======> ] 6.291MB/50.57MB + 974175074a8f + 23eb6c0fdeef [==============> ] 2.097MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [======> ] 6.291MB/50.57MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [=======> ] 7.34MB/50.57MB + 23eb6c0fdeef [==============> ] 2.097MB/7.139MB + f637881d1138 [==============> ] 1.049MB/3.643MB + 23eb6c0fdeef [==============> ] 2.097MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [=======> ] 7.34MB/50.57MB + f637881d1138 [==============> ] 1.049MB/3.643MB + 23eb6c0fdeef [==============> ] 2.097MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [=======> ] 7.34MB/50.57MB + f637881d1138 [==============> ] 1.049MB/3.643MB + 23eb6c0fdeef [==============> ] 2.097MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [=======> ] 7.34MB/50.57MB + f637881d1138 [==============> ] 1.049MB/3.643MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [=======> ] 7.34MB/50.57MB + 23eb6c0fdeef [==============> ] 2.097MB/7.139MB + f637881d1138 [==============> ] 1.049MB/3.643MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [========> ] 8.389MB/50.57MB + 23eb6c0fdeef [==============> ] 2.097MB/7.139MB + f637881d1138 [==============> ] 1.049MB/3.643MB + 23eb6c0fdeef [==============> ] 2.097MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [========> ] 8.389MB/50.57MB + f637881d1138 [==============> ] 1.049MB/3.643MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [========> ] 8.389MB/50.57MB + f637881d1138 [==============> ] 1.049MB/3.643MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [========> ] 8.389MB/50.57MB + f637881d1138 [==============> ] 1.049MB/3.643MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + cb40c4489522 [========> ] 8.389MB/50.57MB + f637881d1138 [==============> ] 1.049MB/3.643MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=========> ] 9.437MB/50.57MB + f637881d1138 [============================> ] 2.097MB/3.643MB + e6918dcfd20d + 71a39d0d04b2 + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=========> ] 9.437MB/50.57MB + f637881d1138 [============================> ] 2.097MB/3.643MB + cb40c4489522 [===========> ] 11.53MB/50.57MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + f637881d1138 [============================> ] 2.097MB/3.643MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===========> ] 11.53MB/50.57MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + f637881d1138 [============================> ] 2.097MB/3.643MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===========> ] 11.53MB/50.57MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + f637881d1138 [============================> ] 2.097MB/3.643MB + cb40c4489522 [===========> ] 11.53MB/50.57MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + f637881d1138 [============================> ] 2.097MB/3.643MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [======================> ] 3.146MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===========> ] 11.53MB/50.57MB + f637881d1138 [============================> ] 2.097MB/3.643MB + 861643ce2817 [=========================================> ] 1.049MB/1.25MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=============> ] 13.63MB/50.57MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + f637881d1138 [===========================================> ] 3.146MB/3.643MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 861643ce2817 [=========================================> ] 1.049MB/1.25MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=============> ] 13.63MB/50.57MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + f637881d1138 [===========================================> ] 3.146MB/3.643MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 861643ce2817 [=========================================> ] 1.049MB/1.25MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=============> ] 13.63MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + f637881d1138 [===========================================> ] 3.146MB/3.643MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 861643ce2817 [=========================================> ] 1.049MB/1.25MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=============> ] 13.63MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + f637881d1138 [===========================================> ] 3.146MB/3.643MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 861643ce2817 [=========================================> ] 1.049MB/1.25MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [==============> ] 14.68MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + f637881d1138 + 8e049f0fd151 + f637881d1138 1 s + cdfd017c753d [======> ] 1.049MB/8.066MB + 861643ce2817 [=========================================> ] 1.049MB/1.25MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [==============> ] 14.68MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + f637881d1138 1 s + cdfd017c753d [======> ] 1.049MB/8.066MB + 861643ce2817 [=========================================> ] 1.049MB/1.25MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [==============> ] 14.68MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + f637881d1138 + cdfd017c753d [======> ] 1.049MB/8.066MB + 861643ce2817 [=========================================> ] 1.049MB/1.25MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [==============> ] 14.68MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + b8554c5f1ad0 1 s + cdfd017c753d [======> ] 1.049MB/8.066MB + 861643ce2817 + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + b8554c5f1ad0 1 s + cdfd017c753d [======> ] 1.049MB/8.066MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + b8554c5f1ad0 + e6918dcfd20d + 71a39d0d04b2 + 8e049f0fd151 + cdfd017c753d [======> ] 1.049MB/8.066MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + cdfd017c753d [======> ] 1.049MB/8.066MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + cdfd017c753d [======> ] 1.049MB/8.066MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 7d97d856fe7f [=======> ] 1.049MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===============> ] 15.73MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 23eb6c0fdeef [=============================> ] 4.194MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [================> ] 16.78MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=> ] 1.049MB/28.23MB + 23eb6c0fdeef [====================================> ] 5.243MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=================> ] 17.83MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=> ] 1.049MB/28.23MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [=> ] 1.049MB/42.17MB + 23eb6c0fdeef [====================================> ] 5.243MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=================> ] 17.83MB/50.57MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=> ] 1.049MB/28.23MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [=> ] 1.049MB/42.17MB + 23eb6c0fdeef [====================================> ] 5.243MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=================> ] 17.83MB/50.57MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [===> ] 2.097MB/28.23MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [=> ] 1.049MB/42.17MB + 23eb6c0fdeef [====================================> ] 5.243MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [=================> ] 17.83MB/50.57MB + c4fca37af7b3 + c4fca37af7b3 + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [===> ] 2.097MB/28.23MB + cb40c4489522 [==================> ] 18.87MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [==> ] 2.097MB/42.17MB + 23eb6c0fdeef [====================================> ] 5.243MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [===> ] 2.097MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [==================> ] 18.87MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [==> ] 2.097MB/42.17MB + 23eb6c0fdeef [====================================> ] 5.243MB/7.139MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [===> ] 2.097MB/28.23MB + cb40c4489522 [==================> ] 18.87MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [==> ] 2.097MB/42.17MB + 23eb6c0fdeef [============================================> ] 6.291MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [===> ] 2.097MB/28.23MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [==> ] 2.097MB/42.17MB + 23eb6c0fdeef [============================================> ] 6.291MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [==================> ] 18.87MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [===> ] 2.097MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [==================> ] 18.87MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [==> ] 2.097MB/42.17MB + 23eb6c0fdeef [============================================> ] 6.291MB/7.139MB + bc1d7488b05e + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=====> ] 3.146MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [==================> ] 18.87MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [==> ] 2.097MB/42.17MB + 23eb6c0fdeef [============================================> ] 6.291MB/7.139MB + bc1d7488b05e + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=====> ] 3.146MB/28.23MB + 23eb6c0fdeef [============================================> ] 6.291MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===================> ] 19.92MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [===> ] 3.146MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=====> ] 3.146MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===================> ] 19.92MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [===> ] 3.146MB/42.17MB + 23eb6c0fdeef [============================================> ] 6.291MB/7.139MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=====> ] 3.146MB/28.23MB + 23eb6c0fdeef [============================================> ] 6.291MB/7.139MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===================> ] 19.92MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [===> ] 3.146MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=====> ] 3.146MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===================> ] 19.92MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [===> ] 3.146MB/42.17MB + 23eb6c0fdeef [============================================> ] 6.291MB/7.139MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=====> ] 3.146MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===================> ] 19.92MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [===> ] 3.146MB/42.17MB + 23eb6c0fdeef + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 3ff860482ac5 + 8a628cdd7ccc [=====> ] 3.146MB/28.23MB + 811af041d785 + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [===> ] 3.146MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===================> ] 19.92MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=====> ] 3.146MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===================> ] 19.92MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [===> ] 3.146MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=====> ] 3.146MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [=> ] 1.049MB/29.18MB + cb40c4489522 [===================> ] 19.92MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [=======> ] 1.049MB/7.295MB + 517b871c26ea [===> ] 3.146MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=======> ] 4.194MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [====================> ] 20.97MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [====> ] 4.194MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=======> ] 4.194MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [====================> ] 20.97MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [====> ] 4.194MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7692042732c0 + 8a628cdd7ccc [=======> ] 4.194MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [====================> ] 20.97MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [====> ] 4.194MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [=========> ] 5.243MB/28.23MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [====> ] 4.194MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=====================> ] 22.02MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [=========> ] 5.243MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=====================> ] 22.02MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [======> ] 5.243MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [=========> ] 5.243MB/28.23MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [======> ] 5.243MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=====================> ] 22.02MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=========> ] 5.243MB/28.23MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=====================> ] 22.02MB/50.57MB + 0879dfc4a629 [============> ] 1.049MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [======> ] 5.243MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [=========> ] 5.243MB/28.23MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + cb40c4489522 [=====================> ] 22.02MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [======> ] 5.243MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + a7e27cf18de4 [====================================> ] 1.049MB/1.438MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [=========> ] 5.243MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [======================> ] 23.07MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [======> ] 5.243MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [===========> ] 6.291MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + a7e27cf18de4 [====================================> ] 1.049MB/1.438MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [======> ] 5.243MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [======================> ] 23.07MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 4f4fb700ef54 + a7e27cf18de4 + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [===========> ] 6.291MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [======================> ] 23.07MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=======> ] 6.291MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 8a628cdd7ccc [===========> ] 6.291MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + cb40c4489522 [======================> ] 23.07MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=======> ] 6.291MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [======> ] 1.049MB/8.066MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [===========> ] 6.291MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [======================> ] 23.07MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=======> ] 6.291MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [=============> ] 7.34MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [======================> ] 23.07MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=======> ] 6.291MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [=============> ] 7.34MB/28.23MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=======> ] 6.291MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=======================> ] 24.12MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [=============> ] 7.34MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=======================> ] 24.12MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [========> ] 7.34MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [=============> ] 7.34MB/28.23MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=======================> ] 24.12MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [========> ] 7.34MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==============> ] 8.389MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=======================> ] 24.12MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [========> ] 7.34MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==============> ] 8.389MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [========> ] 7.34MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=======================> ] 24.12MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==============> ] 8.389MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [========> ] 7.34MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=======================> ] 24.12MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==============> ] 8.389MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [========> ] 7.34MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=======================> ] 24.12MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==============> ] 8.389MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=======================> ] 24.12MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [========> ] 7.34MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [================> ] 9.437MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [========> ] 7.34MB/42.17MB + 7d97d856fe7f [==============> ] 2.097MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=======================> ] 24.12MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [================> ] 9.437MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [========> ] 7.34MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [================> ] 9.437MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==================> ] 10.49MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [========================> ] 25.17MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [====================> ] 11.53MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 517b871c26ea [=========> ] 8.389MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=========================> ] 26.21MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [====================> ] 11.53MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=========================> ] 26.21MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [===========> ] 9.437MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [====================> ] 11.53MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 517b871c26ea [===========> ] 9.437MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=========================> ] 26.21MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [====================> ] 11.53MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=========================> ] 26.21MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [===========> ] 9.437MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [====================> ] 11.53MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=========================> ] 26.21MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [===========> ] 9.437MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [====================> ] 11.53MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [=========================> ] 26.21MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [===========> ] 9.437MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [======================> ] 12.58MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [==========================> ] 27.26MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [============> ] 10.49MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [======================> ] 12.58MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [==========================> ] 27.26MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [==============> ] 2.097MB/7.295MB + 517b871c26ea [============> ] 10.49MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [========================> ] 13.63MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [============> ] 10.49MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [==========================> ] 27.26MB/50.57MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 8a628cdd7ccc [========================> ] 13.63MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [===> ] 2.097MB/29.18MB + cb40c4489522 [==========================> ] 27.26MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [============> ] 10.49MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [========================> ] 13.63MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [==========================> ] 27.26MB/50.57MB + 0879dfc4a629 [========================> ] 2.097MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [============> ] 10.49MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [========================> ] 13.63MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===========================> ] 28.31MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=============> ] 11.53MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==========================> ] 14.68MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [=======================> ] 1.049MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===========================> ] 28.31MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=============> ] 11.53MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 6999284f6a0c [==============================================> ] 2.097MB/2.267MB + 8a628cdd7ccc [==========================> ] 14.68MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===========================> ] 28.31MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=============> ] 11.53MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 6999284f6a0c [==============================================> ] 2.097MB/2.267MB + 8a628cdd7ccc [==========================> ] 14.68MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===========================> ] 28.31MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=============> ] 11.53MB/42.17MB + 1adabd6b0d6b [=> ] 1.049MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [==========================> ] 14.68MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [==============================================> ] 2.097MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===========================> ] 28.31MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=============> ] 11.53MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [===========================> ] 15.73MB/28.23MB + 8ae1ad8ce35e [===> ] 1.049MB/15.33MB + 6999284f6a0c [==============================================> ] 2.097MB/2.267MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [=============================> ] 29.36MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=============> ] 11.53MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 6999284f6a0c [==============================================> ] 2.097MB/2.267MB + 8a628cdd7ccc [===========================> ] 15.73MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [=============================> ] 29.36MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=============> ] 11.53MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 6999284f6a0c [==============================================> ] 2.097MB/2.267MB + 8a628cdd7ccc [===========================> ] 15.73MB/28.23MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [=============================> ] 29.36MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [==============> ] 12.58MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [============> ] 2.097MB/8.066MB + 8a628cdd7ccc [===========================> ] 15.73MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 6999284f6a0c + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [=============================> ] 29.36MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [==============> ] 12.58MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=============================> ] 16.78MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [=============================> ] 29.36MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [==============> ] 12.58MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=============================> ] 16.78MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [=============================> ] 29.36MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [==============> ] 12.58MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=============================> ] 16.78MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + cb40c4489522 [==============================> ] 30.41MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [================> ] 13.63MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + 8a628cdd7ccc [===============================> ] 17.83MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [================> ] 13.63MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [==============================> ] 30.41MB/50.57MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [===============================> ] 17.83MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [==============================> ] 30.41MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [================> ] 13.63MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [===============================> ] 17.83MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [==============================> ] 30.41MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [================> ] 13.63MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [===============================> ] 17.83MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [================> ] 13.63MB/42.17MB + 7d97d856fe7f [======================> ] 3.146MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [==============================> ] 30.41MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [===============================> ] 17.83MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===============================> ] 31.46MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [================> ] 13.63MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=================================> ] 18.87MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=================> ] 14.68MB/42.17MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===============================> ] 31.46MB/50.57MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=================================> ] 18.87MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===============================> ] 31.46MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=================> ] 14.68MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=================================> ] 18.87MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===============================> ] 31.46MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=================> ] 14.68MB/42.17MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=================================> ] 18.87MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===============================> ] 31.46MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=================> ] 14.68MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 8a628cdd7ccc [=================================> ] 18.87MB/28.23MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [===============================> ] 31.46MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=================> ] 14.68MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [===================================> ] 19.92MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [================================> ] 32.51MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=================> ] 14.68MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [===================================> ] 19.92MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [================================> ] 32.51MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=================> ] 14.68MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [===================================> ] 19.92MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [=================> ] 14.68MB/42.17MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [================================> ] 32.51MB/50.57MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [===================================> ] 19.92MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + b81f71015e92 [=====================> ] 3.146MB/7.295MB + 517b871c26ea [==================> ] 15.73MB/42.17MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [================================> ] 32.51MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 8a628cdd7ccc [===================================> ] 19.92MB/28.23MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=====> ] 3.146MB/29.18MB + cb40c4489522 [================================> ] 32.51MB/50.57MB + 0879dfc4a629 [====================================> ] 3.146MB/4.272MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [==================> ] 15.73MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 8a628cdd7ccc [=====================================> ] 20.97MB/28.23MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [================================> ] 32.51MB/50.57MB + 0879dfc4a629 [=================================================> ] 4.194MB/4.272MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [==================> ] 15.73MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 8a628cdd7ccc [=====================================> ] 20.97MB/28.23MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [=================================> ] 33.55MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [==================> ] 15.73MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=====================================> ] 20.97MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + db1363514c7f + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [=================================> ] 33.55MB/50.57MB + 0879dfc4a629 + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [==================> ] 15.73MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=====================================> ] 20.97MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [=================================> ] 33.55MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [==================> ] 15.73MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=======================================> ] 22.02MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [=================================> ] 33.55MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [==================> ] 15.73MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=======================================> ] 22.02MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [==================> ] 15.73MB/42.17MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [=================================> ] 33.55MB/50.57MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=======================================> ] 22.02MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [==================> ] 15.73MB/42.17MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [=================================> ] 33.55MB/50.57MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=======================================> ] 22.02MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [=================================> ] 33.55MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [===================> ] 16.78MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [=======================================> ] 22.02MB/28.23MB + 8ae1ad8ce35e [======> ] 2.097MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [===================> ] 16.78MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [========================================> ] 23.07MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [===================> ] 16.78MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc [========================================> ] 23.07MB/28.23MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [========================================> ] 23.07MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8a628cdd7ccc [========================================> ] 23.07MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [===================> ] 3.146MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc [========================================> ] 23.07MB/28.23MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [========================================> ] 23.07MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc [========================================> ] 23.07MB/28.23MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [========================================> ] 23.07MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [==========================================> ] 24.12MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [==================================> ] 34.6MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc [==========================================> ] 24.12MB/28.23MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [===================================> ] 35.65MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [==========================================> ] 24.12MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [=============================> ] 4.194MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [===================================> ] 35.65MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [===> ] 2.097MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc [==========================================> ] 24.12MB/28.23MB + cb40c4489522 [===================================> ] 35.65MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc [==========================================> ] 24.12MB/28.23MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [===================================> ] 35.65MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [==========================================> ] 24.12MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [===================================> ] 35.65MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=====================> ] 17.83MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [===================================> ] 35.65MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [======================> ] 18.87MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [======================> ] 18.87MB/42.17MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [===================================> ] 35.65MB/50.57MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [======================> ] 18.87MB/42.17MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [===================================> ] 35.65MB/50.57MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 517b871c26ea [======================> ] 18.87MB/42.17MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [====================================> ] 36.7MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [====================================> ] 36.7MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [======================> ] 18.87MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [====================================> ] 36.7MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [======================> ] 18.87MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [====================================> ] 36.7MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=======================> ] 19.92MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [====================================> ] 36.7MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=======================> ] 19.92MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [====================================> ] 36.7MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=======================> ] 19.92MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [============================================> ] 25.17MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [====================================> ] 36.7MB/50.57MB + b81f71015e92 [============================> ] 4.194MB/7.295MB + 517b871c26ea [=======================> ] 19.92MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [==============================================> ] 26.21MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [=======================> ] 19.92MB/42.17MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [====================================> ] 36.7MB/50.57MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [==============================================> ] 26.21MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [=======> ] 4.194MB/29.18MB + cb40c4489522 [=====================================> ] 37.75MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [=======================> ] 19.92MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [================================================> ] 27.26MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [=====================================> ] 37.75MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [========================> ] 20.97MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc [================================================> ] 27.26MB/28.23MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [=====================================> ] 37.75MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [========================> ] 20.97MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc [================================================> ] 27.26MB/28.23MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [=====================================> ] 37.75MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [========================> ] 20.97MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc [================================================> ] 27.26MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + cb40c4489522 [======================================> ] 38.8MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [========================> ] 20.97MB/42.17MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [======================================> ] 38.8MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [========================> ] 20.97MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8a628cdd7ccc + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc 1 s + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [======================================> ] 38.8MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [========================> ] 20.97MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc 1 s + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [========================> ] 20.97MB/42.17MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [======================================> ] 38.8MB/50.57MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc 1 s + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [======================================> ] 38.8MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [==========================> ] 22.02MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [==========> ] 3.146MB/15.33MB + 8a628cdd7ccc 1 s + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [======================================> ] 38.8MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [==========================> ] 22.02MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 1 s + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [======================================> ] 38.8MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [==========================> ] 22.02MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 1 s + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [======================================> ] 38.8MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [==========================> ] 22.02MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 1 s + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [=======================================> ] 39.85MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [==========================> ] 22.02MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [=========================> ] 4.194MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 1 s + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [=======================================> ] 39.85MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [==========================> ] 22.02MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 1 s + cb40c4489522 [=======================================> ] 39.85MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [==========================> ] 22.02MB/42.17MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 1 s + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [=======================================> ] 39.85MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [==========================> ] 22.02MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 1 s + cb40c4489522 [=======================================> ] 39.85MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [==========================> ] 22.02MB/42.17MB + 7d97d856fe7f [====================================> ] 5.243MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 2 s + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [=======================================> ] 39.85MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [===========================> ] 23.07MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 2 s + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [=======================================> ] 39.85MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [===========================> ] 23.07MB/42.17MB + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 2 s + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [=======================================> ] 39.85MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [===========================> ] 23.07MB/42.17MB + 1adabd6b0d6b [=====> ] 3.146MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 2 s + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [===========================> ] 23.07MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 2 s + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [===========================> ] 23.07MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 2 s + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [===========================> ] 23.07MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 2 s + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [========> ] 5.243MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + b81f71015e92 [===================================> ] 5.243MB/7.295MB + 517b871c26ea [===========================> ] 23.07MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 2 s + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [===========================> ] 23.07MB/42.17MB + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 8a628cdd7ccc 2 s + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 3ff860482ac5 + 811af041d785 + a7e27cf18de4 1 s + 8a628cdd7ccc + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + a7e27cf18de4 + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [========================================> ] 40.89MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=============> ] 4.194MB/15.33MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 7d97d856fe7f [============================================> ] 6.291MB/7.141MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 7d97d856fe7f + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [================================> ] 5.243MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + db099b936fc8 + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 72ba653f834d + cbb99c664e48 + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [============================> ] 24.12MB/42.17MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [=============================> ] 25.17MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [=============================> ] 25.17MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [=============================> ] 25.17MB/42.17MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [=========================================> ] 41.94MB/50.57MB + b81f71015e92 [===========================================> ] 6.291MB/7.295MB + 517b871c26ea [=============================> ] 25.17MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + b81f71015e92 + 517b871c26ea [=============================> ] 25.17MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [==========> ] 6.291MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + 517b871c26ea [=============================> ] 25.17MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + 517b871c26ea [=============================> ] 25.17MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + 517b871c26ea [=============================> ] 25.17MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [=================> ] 5.243MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [======================================> ] 6.291MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [==========================================> ] 42.99MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 0ea2caf05f48 [============> ] 7.34MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [===============================> ] 26.21MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d [=============================================> ] 7.34MB/8.066MB + 8ae1ad8ce35e [====================> ] 6.291MB/15.33MB + 517b871c26ea [================================> ] 27.26MB/42.17MB + 0ea2caf05f48 [==============> ] 8.389MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + cdfd017c753d + 8ae1ad8ce35e [=======================> ] 7.34MB/15.33MB + 0ea2caf05f48 [==============> ] 8.389MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [================================> ] 27.26MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [=======================> ] 7.34MB/15.33MB + 0ea2caf05f48 [==============> ] 8.389MB/29.18MB + cb40c4489522 [===========================================> ] 44.04MB/50.57MB + 517b871c26ea [================================> ] 27.26MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [=======================> ] 7.34MB/15.33MB + 517b871c26ea [================================> ] 27.26MB/42.17MB + 0ea2caf05f48 [==============> ] 8.389MB/29.18MB + cb40c4489522 [============================================> ] 45.09MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [=======================> ] 7.34MB/15.33MB + 517b871c26ea [================================> ] 27.26MB/42.17MB + 0ea2caf05f48 [==============> ] 8.389MB/29.18MB + cb40c4489522 [============================================> ] 45.09MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [=======================> ] 7.34MB/15.33MB + 0ea2caf05f48 [==============> ] 8.389MB/29.18MB + cb40c4489522 [============================================> ] 45.09MB/50.57MB + 517b871c26ea [================================> ] 27.26MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [=======================> ] 7.34MB/15.33MB + 0ea2caf05f48 [================> ] 9.437MB/29.18MB + cb40c4489522 [============================================> ] 45.09MB/50.57MB + 517b871c26ea [=================================> ] 28.31MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [=======================> ] 7.34MB/15.33MB + 517b871c26ea [=================================> ] 28.31MB/42.17MB + 0ea2caf05f48 [================> ] 9.437MB/29.18MB + cb40c4489522 [============================================> ] 45.09MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [===========================> ] 8.389MB/15.33MB + 0ea2caf05f48 [================> ] 9.437MB/29.18MB + cb40c4489522 [============================================> ] 45.09MB/50.57MB + 517b871c26ea [=================================> ] 28.31MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [===========================> ] 8.389MB/15.33MB + 0ea2caf05f48 [================> ] 9.437MB/29.18MB + cb40c4489522 [============================================> ] 45.09MB/50.57MB + 517b871c26ea [=================================> ] 28.31MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [===========================> ] 8.389MB/15.33MB + 0ea2caf05f48 [================> ] 9.437MB/29.18MB + cb40c4489522 [============================================> ] 45.09MB/50.57MB + 517b871c26ea [=================================> ] 28.31MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [===========================> ] 8.389MB/15.33MB + 0ea2caf05f48 [================> ] 9.437MB/29.18MB + cb40c4489522 [============================================> ] 45.09MB/50.57MB + 517b871c26ea [=================================> ] 28.31MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [===========================> ] 8.389MB/15.33MB + 0ea2caf05f48 [================> ] 9.437MB/29.18MB + cb40c4489522 [=============================================> ] 46.14MB/50.57MB + 517b871c26ea [=================================> ] 28.31MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [===========================> ] 8.389MB/15.33MB + 517b871c26ea [=================================> ] 28.31MB/42.17MB + 0ea2caf05f48 [=================> ] 10.49MB/29.18MB + cb40c4489522 [=============================================> ] 46.14MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [===========================> ] 8.389MB/15.33MB + 0ea2caf05f48 [=================> ] 10.49MB/29.18MB + cb40c4489522 [=============================================> ] 46.14MB/50.57MB + 517b871c26ea [==================================> ] 29.36MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==============================> ] 9.437MB/15.33MB + 0ea2caf05f48 [=================> ] 10.49MB/29.18MB + cb40c4489522 [=============================================> ] 46.14MB/50.57MB + 517b871c26ea [==================================> ] 29.36MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==============================> ] 9.437MB/15.33MB + 0ea2caf05f48 [=================> ] 10.49MB/29.18MB + cb40c4489522 [=============================================> ] 46.14MB/50.57MB + 517b871c26ea [==================================> ] 29.36MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==============================> ] 9.437MB/15.33MB + 0ea2caf05f48 [=================> ] 10.49MB/29.18MB + cb40c4489522 [=============================================> ] 46.14MB/50.57MB + 517b871c26ea [==================================> ] 29.36MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==============================> ] 9.437MB/15.33MB + 0ea2caf05f48 [===================> ] 11.53MB/29.18MB + cb40c4489522 [==============================================> ] 47.19MB/50.57MB + 517b871c26ea [==================================> ] 29.36MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==============================> ] 9.437MB/15.33MB + 0ea2caf05f48 [===================> ] 11.53MB/29.18MB + cb40c4489522 [==============================================> ] 47.19MB/50.57MB + 517b871c26ea [==================================> ] 29.36MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==============================> ] 9.437MB/15.33MB + 0ea2caf05f48 [===================> ] 11.53MB/29.18MB + cb40c4489522 [==============================================> ] 47.19MB/50.57MB + 517b871c26ea [====================================> ] 30.41MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==================================> ] 10.49MB/15.33MB + 0ea2caf05f48 [===================> ] 11.53MB/29.18MB + cb40c4489522 [==============================================> ] 47.19MB/50.57MB + 517b871c26ea [====================================> ] 30.41MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==================================> ] 10.49MB/15.33MB + 0ea2caf05f48 [===================> ] 11.53MB/29.18MB + cb40c4489522 [==============================================> ] 47.19MB/50.57MB + 517b871c26ea [====================================> ] 30.41MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==================================> ] 10.49MB/15.33MB + 517b871c26ea [====================================> ] 30.41MB/42.17MB + 0ea2caf05f48 [===================> ] 11.53MB/29.18MB + cb40c4489522 [===============================================> ] 48.23MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==================================> ] 10.49MB/15.33MB + 517b871c26ea [====================================> ] 30.41MB/42.17MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [===============================================> ] 48.23MB/50.57MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==================================> ] 10.49MB/15.33MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [===============================================> ] 48.23MB/50.57MB + 517b871c26ea [====================================> ] 30.41MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==================================> ] 10.49MB/15.33MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [===============================================> ] 48.23MB/50.57MB + 517b871c26ea [====================================> ] 30.41MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==================================> ] 10.49MB/15.33MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [===============================================> ] 48.23MB/50.57MB + 517b871c26ea [====================================> ] 30.41MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==================================> ] 10.49MB/15.33MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [===============================================> ] 48.23MB/50.57MB + 517b871c26ea [====================================> ] 30.41MB/42.17MB + 1adabd6b0d6b [=======> ] 4.194MB/28.23MB + 8ae1ad8ce35e [==================================> ] 10.49MB/15.33MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [===============================================> ] 48.23MB/50.57MB + 517b871c26ea [=====================================> ] 31.46MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=====================================> ] 11.53MB/15.33MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 517b871c26ea [=====================================> ] 31.46MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=====================================> ] 11.53MB/15.33MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 517b871c26ea [=====================================> ] 31.46MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=====================================> ] 11.53MB/15.33MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 517b871c26ea [=====================================> ] 31.46MB/42.17MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=====================================> ] 11.53MB/15.33MB + 517b871c26ea [=====================================> ] 31.46MB/42.17MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=====================================> ] 11.53MB/15.33MB + 517b871c26ea [=====================================> ] 31.46MB/42.17MB + 0ea2caf05f48 [=====================> ] 12.58MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=====================================> ] 11.53MB/15.33MB + 0ea2caf05f48 [=======================> ] 13.63MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 517b871c26ea [======================================> ] 32.51MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 0ea2caf05f48 [=======================> ] 13.63MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 517b871c26ea [======================================> ] 32.51MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 0ea2caf05f48 [=======================> ] 13.63MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 517b871c26ea [======================================> ] 32.51MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 0ea2caf05f48 [=======================> ] 13.63MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 517b871c26ea [======================================> ] 32.51MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 0ea2caf05f48 [=======================> ] 13.63MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 517b871c26ea [======================================> ] 32.51MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 517b871c26ea [======================================> ] 32.51MB/42.17MB + 0ea2caf05f48 [=======================> ] 13.63MB/29.18MB + cb40c4489522 [================================================> ] 49.28MB/50.57MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 517b871c26ea [======================================> ] 32.51MB/42.17MB + 0ea2caf05f48 [=======================> ] 13.63MB/29.18MB + cb40c4489522 [==================================================>] 50.57MB/50.57MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 0ea2caf05f48 [=======================> ] 13.63MB/29.18MB + cb40c4489522 + 517b871c26ea [=======================================> ] 33.55MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 0ea2caf05f48 [=========================> ] 14.68MB/29.18MB + 517b871c26ea [=======================================> ] 33.55MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 517b871c26ea [=======================================> ] 33.55MB/42.17MB + 0ea2caf05f48 [=========================> ] 14.68MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 0ea2caf05f48 [=========================> ] 14.68MB/29.18MB + 517b871c26ea [=======================================> ] 33.55MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 0ea2caf05f48 [=========================> ] 14.68MB/29.18MB + 517b871c26ea [=======================================> ] 33.55MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [=========================================> ] 12.58MB/15.33MB + 0ea2caf05f48 [=========================> ] 14.68MB/29.18MB + 517b871c26ea [=========================================> ] 34.6MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [============================================> ] 13.63MB/15.33MB + 517b871c26ea [=========================================> ] 34.6MB/42.17MB + 0ea2caf05f48 [=========================> ] 14.68MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [============================================> ] 13.63MB/15.33MB + 0ea2caf05f48 [=========================> ] 14.68MB/29.18MB + 517b871c26ea [=========================================> ] 34.6MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [============================================> ] 13.63MB/15.33MB + 0ea2caf05f48 [=========================> ] 14.68MB/29.18MB + 517b871c26ea [=========================================> ] 34.6MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [============================================> ] 13.63MB/15.33MB + 0ea2caf05f48 [=========================> ] 14.68MB/29.18MB + 517b871c26ea [==========================================> ] 35.65MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [============================================> ] 13.63MB/15.33MB + 517b871c26ea [==========================================> ] 35.65MB/42.17MB + 0ea2caf05f48 [==========================> ] 15.73MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [============================================> ] 13.63MB/15.33MB + 517b871c26ea [==========================================> ] 35.65MB/42.17MB + 0ea2caf05f48 [==========================> ] 15.73MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [============================================> ] 13.63MB/15.33MB + 517b871c26ea [==========================================> ] 35.65MB/42.17MB + 0ea2caf05f48 [==========================> ] 15.73MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [============================================> ] 13.63MB/15.33MB + 0ea2caf05f48 [==========================> ] 15.73MB/29.18MB + 517b871c26ea [==========================================> ] 35.65MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [===============================================> ] 14.68MB/15.33MB + 0ea2caf05f48 [==========================> ] 15.73MB/29.18MB + 517b871c26ea [===========================================> ] 36.7MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [===============================================> ] 14.68MB/15.33MB + 0ea2caf05f48 [==========================> ] 15.73MB/29.18MB + 517b871c26ea [===========================================> ] 36.7MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [===============================================> ] 14.68MB/15.33MB + 0ea2caf05f48 [==========================> ] 15.73MB/29.18MB + 517b871c26ea [============================================> ] 37.75MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e [===============================================> ] 14.68MB/15.33MB + 0ea2caf05f48 [============================> ] 16.78MB/29.18MB + 517b871c26ea [============================================> ] 37.75MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e + 8ae1ad8ce35e 1 s + 517b871c26ea [============================================> ] 37.75MB/42.17MB + 0ea2caf05f48 [============================> ] 16.78MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e 1 s + 517b871c26ea [============================================> ] 37.75MB/42.17MB + 0ea2caf05f48 [============================> ] 16.78MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e 1 s + 0ea2caf05f48 [============================> ] 16.78MB/29.18MB + 517b871c26ea [==============================================> ] 38.8MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 8ae1ad8ce35e 1 s + 0ea2caf05f48 [============================> ] 16.78MB/29.18MB + 517b871c26ea [==============================================> ] 38.8MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [============================> ] 16.78MB/29.18MB + 517b871c26ea [==============================================> ] 38.8MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 759939a29cb5 + 4f4fb700ef54 + 759939a29cb5 + 4f4fb700ef54 + cbb99c664e48 + 8ae1ad8ce35e + Image redis:7.4.2-bookworm Pulled + 0ea2caf05f48 [==============================> ] 17.83MB/29.18MB + 517b871c26ea [==============================================> ] 38.8MB/42.17MB + a48e28c27c4a + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [==============================> ] 17.83MB/29.18MB + 517b871c26ea [==============================================> ] 38.8MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 517b871c26ea [==============================================> ] 38.8MB/42.17MB + 0ea2caf05f48 [==============================> ] 17.83MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [==============================> ] 17.83MB/29.18MB + 517b871c26ea [===============================================> ] 39.85MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [================================> ] 18.87MB/29.18MB + 517b871c26ea [===============================================> ] 39.85MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [================================> ] 18.87MB/29.18MB + 517b871c26ea [===============================================> ] 39.85MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [================================> ] 18.87MB/29.18MB + 517b871c26ea [===============================================> ] 39.85MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [================================> ] 18.87MB/29.18MB + 517b871c26ea [===============================================> ] 39.85MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [================================> ] 18.87MB/29.18MB + 517b871c26ea [===============================================> ] 39.85MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [================================> ] 18.87MB/29.18MB + 517b871c26ea [===============================================> ] 39.85MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [==================================> ] 19.92MB/29.18MB + 517b871c26ea [==================================================>] 42.17MB/42.17MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 517b871c26ea + 0ea2caf05f48 [===================================> ] 20.97MB/29.18MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 517b871c26ea 1 s + 0ea2caf05f48 [===================================> ] 20.97MB/29.18MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 517b871c26ea 1 s + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 0ea2caf05f48 [===================================> ] 20.97MB/29.18MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 517b871c26ea 1 s + 0ea2caf05f48 [=====================================> ] 22.02MB/29.18MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 517b871c26ea 1 s + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [=====================================> ] 22.02MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 1 s + db8bf9a4f43b [> ] 1.049MB/111.7MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + 0ea2caf05f48 [=====================================> ] 22.02MB/29.18MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 517b871c26ea 1 s + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [=======================================> ] 23.07MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 1 s + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [=======================================> ] 23.07MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 1 s + db8bf9a4f43b [> ] 1.049MB/111.7MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [=======================================> ] 23.07MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 1 s + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [=========================================> ] 24.12MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 1 s + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [=========================================> ] 24.12MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 1 s + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 0ea2caf05f48 [===========================================> ] 25.17MB/29.18MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 2 s + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 0ea2caf05f48 [===========================================> ] 25.17MB/29.18MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 2 s + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [===========================================> ] 25.17MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 2 s + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 0ea2caf05f48 [===========================================> ] 25.17MB/29.18MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 517b871c26ea 2 s + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [============================================> ] 26.21MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 2 s + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [============================================> ] 26.21MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 517b871c26ea 2 s + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [============================================> ] 26.21MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 517b871c26ea 2 s + 0ea2caf05f48 [==============================================> ] 27.26MB/29.18MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 517b871c26ea 2 s + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [==============================================> ] 27.26MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 2 s + db8bf9a4f43b [> ] 1.049MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 [================================================> ] 28.31MB/29.18MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 2 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 0ea2caf05f48 [================================================> ] 28.31MB/29.18MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 517b871c26ea 3 s + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 3 s + db8bf9a4f43b [> ] 1.049MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 0ea2caf05f48 + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 3 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 517b871c26ea 3 s + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 3 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [=========> ] 5.243MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 3 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 517b871c26ea 3 s + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 517b871c26ea 3 s + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 96aae9022e0e + 3e300a7cb18c [===> ] 1.049MB/15.55MB + cb40c4489522 1 s + 517b871c26ea + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + cb40c4489522 1 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + cb40c4489522 1 s + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + cb40c4489522 1 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + cb40c4489522 1 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + cb40c4489522 1 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + cb40c4489522 1 s + db8bf9a4f43b [> ] 1.049MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + cb40c4489522 1 s + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [===> ] 1.049MB/15.55MB + cb40c4489522 1 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 1.049MB/111.7MB + e34abc083292 [=====> ] 1.049MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 1 s + db8bf9a4f43b [> ] 1.049MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 1 s + db8bf9a4f43b [> ] 2.097MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + cb40c4489522 2 s + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 2 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 2 s + db8bf9a4f43b [> ] 2.097MB/111.7MB + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 2 s + 7b697787d5d2 [===========> ] 1.049MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 2 s + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 2 s + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 2 s + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 2 s + db8bf9a4f43b [> ] 2.097MB/111.7MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [===========> ] 6.291MB/28.23MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + cb40c4489522 + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + db8bf9a4f43b [> ] 2.097MB/111.7MB + e34abc083292 [===========> ] 2.097MB/9.101MB + 3e300a7cb18c [======> ] 2.097MB/15.55MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + 7b697787d5d2 [=======================> ] 2.097MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + 1adabd6b0d6b [=============> ] 7.34MB/28.23MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + e34abc083292 [=================> ] 3.146MB/9.101MB + 3e300a7cb18c [==========> ] 3.146MB/15.55MB + db8bf9a4f43b [=> ] 3.146MB/111.7MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + e34abc083292 [=======================> ] 4.194MB/9.101MB + 3e300a7cb18c [=============> ] 4.194MB/15.55MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + db8bf9a4f43b [=> ] 4.194MB/111.7MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + e34abc083292 [=======================> ] 4.194MB/9.101MB + 3e300a7cb18c [=============> ] 4.194MB/15.55MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + db8bf9a4f43b [=> ] 4.194MB/111.7MB + 7b697787d5d2 [==================================> ] 3.146MB/4.534MB + e34abc083292 [=======================> ] 4.194MB/9.101MB + 3e300a7cb18c [=============> ] 4.194MB/15.55MB + 7b697787d5d2 [==============================================> ] 4.194MB/4.534MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + db8bf9a4f43b [=> ] 4.194MB/111.7MB + e34abc083292 [=======================> ] 4.194MB/9.101MB + 3e300a7cb18c [=============> ] 4.194MB/15.55MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + db8bf9a4f43b [=> ] 4.194MB/111.7MB + 7b697787d5d2 [==============================================> ] 4.194MB/4.534MB + e34abc083292 [=======================> ] 4.194MB/9.101MB + 3e300a7cb18c [=============> ] 4.194MB/15.55MB + 7b697787d5d2 [==============================================> ] 4.194MB/4.534MB + 1adabd6b0d6b [==============> ] 8.389MB/28.23MB + db8bf9a4f43b [=> ] 4.194MB/111.7MB + e34abc083292 [=======================> ] 4.194MB/9.101MB + 3e300a7cb18c [=============> ] 4.194MB/15.55MB + 7b697787d5d2 + 1adabd6b0d6b [================> ] 9.437MB/28.23MB + db8bf9a4f43b [=> ] 4.194MB/111.7MB + e34abc083292 [=======================> ] 4.194MB/9.101MB + 3e300a7cb18c [=============> ] 4.194MB/15.55MB + 1adabd6b0d6b [================> ] 9.437MB/28.23MB + db8bf9a4f43b [=> ] 4.194MB/111.7MB + e34abc083292 [=======================> ] 4.194MB/9.101MB + 3e300a7cb18c [=============> ] 4.194MB/15.55MB + db8bf9a4f43b [=> ] 4.194MB/111.7MB + 1adabd6b0d6b [================> ] 9.437MB/28.23MB + e34abc083292 [=======================> ] 4.194MB/9.101MB + 3e300a7cb18c [=============> ] 4.194MB/15.55MB + db8bf9a4f43b [=> ] 4.194MB/111.7MB + 1adabd6b0d6b [================> ] 9.437MB/28.23MB + e34abc083292 [============================> ] 5.243MB/9.101MB + 3e300a7cb18c [================> ] 5.243MB/15.55MB + 1adabd6b0d6b [================> ] 9.437MB/28.23MB + db8bf9a4f43b [==> ] 5.243MB/111.7MB + e34abc083292 [============================> ] 5.243MB/9.101MB + 3e300a7cb18c [================> ] 5.243MB/15.55MB + 1adabd6b0d6b [================> ] 9.437MB/28.23MB + db8bf9a4f43b [==> ] 5.243MB/111.7MB + e34abc083292 [============================> ] 5.243MB/9.101MB + 3e300a7cb18c [================> ] 5.243MB/15.55MB + 1adabd6b0d6b [================> ] 9.437MB/28.23MB + db8bf9a4f43b [==> ] 5.243MB/111.7MB + e34abc083292 [============================> ] 5.243MB/9.101MB + 3e300a7cb18c [================> ] 5.243MB/15.55MB + 1adabd6b0d6b [================> ] 9.437MB/28.23MB + db8bf9a4f43b [==> ] 5.243MB/111.7MB + e34abc083292 [============================> ] 5.243MB/9.101MB + 3e300a7cb18c [================> ] 5.243MB/15.55MB + db8bf9a4f43b [==> ] 5.243MB/111.7MB + 1adabd6b0d6b [==================> ] 10.49MB/28.23MB + e34abc083292 [============================> ] 5.243MB/9.101MB + 3e300a7cb18c [================> ] 5.243MB/15.55MB + 1adabd6b0d6b [==================> ] 10.49MB/28.23MB + db8bf9a4f43b [==> ] 5.243MB/111.7MB + e34abc083292 [============================> ] 5.243MB/9.101MB + 3e300a7cb18c [================> ] 5.243MB/15.55MB + db8bf9a4f43b [==> ] 5.243MB/111.7MB + 1adabd6b0d6b [==================> ] 10.49MB/28.23MB + e34abc083292 [============================> ] 5.243MB/9.101MB + 3e300a7cb18c [================> ] 5.243MB/15.55MB + 1adabd6b0d6b [==================> ] 10.49MB/28.23MB + db8bf9a4f43b [==> ] 6.291MB/111.7MB + e34abc083292 [==================================> ] 6.291MB/9.101MB + 3e300a7cb18c [====================> ] 6.291MB/15.55MB + 1adabd6b0d6b [==================> ] 10.49MB/28.23MB + db8bf9a4f43b [==> ] 6.291MB/111.7MB + e34abc083292 [==================================> ] 6.291MB/9.101MB + 3e300a7cb18c [====================> ] 6.291MB/15.55MB + db8bf9a4f43b [==> ] 6.291MB/111.7MB + 1adabd6b0d6b [==================> ] 10.49MB/28.23MB + e34abc083292 [==================================> ] 6.291MB/9.101MB + 3e300a7cb18c [====================> ] 6.291MB/15.55MB + 1adabd6b0d6b [==================> ] 10.49MB/28.23MB + db8bf9a4f43b [==> ] 6.291MB/111.7MB + e34abc083292 [==================================> ] 6.291MB/9.101MB + 3e300a7cb18c [====================> ] 6.291MB/15.55MB + db8bf9a4f43b [==> ] 6.291MB/111.7MB + 1adabd6b0d6b [====================> ] 11.53MB/28.23MB + e34abc083292 [==================================> ] 6.291MB/9.101MB + 3e300a7cb18c [====================> ] 6.291MB/15.55MB + 1adabd6b0d6b [====================> ] 11.53MB/28.23MB + db8bf9a4f43b [==> ] 6.291MB/111.7MB + e34abc083292 [==================================> ] 6.291MB/9.101MB + 3e300a7cb18c [====================> ] 6.291MB/15.55MB + 1adabd6b0d6b [====================> ] 11.53MB/28.23MB + db8bf9a4f43b [==> ] 6.291MB/111.7MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [=======================> ] 7.34MB/15.55MB + db8bf9a4f43b [===> ] 7.34MB/111.7MB + 1adabd6b0d6b [====================> ] 11.53MB/28.23MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [=======================> ] 7.34MB/15.55MB + db8bf9a4f43b [===> ] 7.34MB/111.7MB + 1adabd6b0d6b [====================> ] 11.53MB/28.23MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [=======================> ] 7.34MB/15.55MB + 1adabd6b0d6b [====================> ] 11.53MB/28.23MB + db8bf9a4f43b [===> ] 7.34MB/111.7MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [=======================> ] 7.34MB/15.55MB + db8bf9a4f43b [===> ] 7.34MB/111.7MB + 1adabd6b0d6b [====================> ] 11.53MB/28.23MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [=======================> ] 7.34MB/15.55MB + 1adabd6b0d6b [====================> ] 11.53MB/28.23MB + db8bf9a4f43b [===> ] 7.34MB/111.7MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [=======================> ] 7.34MB/15.55MB + db8bf9a4f43b [===> ] 7.34MB/111.7MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [=======================> ] 7.34MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 7.34MB/111.7MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [=======================> ] 7.34MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 7.34MB/111.7MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [=======================> ] 7.34MB/15.55MB + db8bf9a4f43b [===> ] 7.34MB/111.7MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + e34abc083292 [========================================> ] 7.34MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + e34abc083292 [==============================================> ] 8.389MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + e34abc083292 [==============================================> ] 8.389MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + e34abc083292 [==============================================> ] 8.389MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + e34abc083292 [==============================================> ] 8.389MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + e34abc083292 [==============================================> ] 8.389MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + e34abc083292 [==============================================> ] 8.389MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + e34abc083292 [==============================================> ] 8.389MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + e34abc083292 [==============================================> ] 8.389MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + e34abc083292 [==============================================> ] 8.389MB/9.101MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [======================> ] 12.58MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + e34abc083292 + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + e34abc083292 1 s + 1adabd6b0d6b [========================> ] 13.63MB/28.23MB + db8bf9a4f43b [===> ] 8.389MB/111.7MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + e34abc083292 1 s + db8bf9a4f43b [===> ] 8.389MB/111.7MB + 1adabd6b0d6b [========================> ] 13.63MB/28.23MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + e34abc083292 1 s + db8bf9a4f43b [===> ] 8.389MB/111.7MB + 1adabd6b0d6b [========================> ] 13.63MB/28.23MB + 3e300a7cb18c [==========================> ] 8.389MB/15.55MB + 1adabd6b0d6b [========================> ] 13.63MB/28.23MB + db8bf9a4f43b [====> ] 9.437MB/111.7MB + e34abc083292 1 s + 3e300a7cb18c [==============================> ] 9.437MB/15.55MB + e34abc083292 1 s + 1adabd6b0d6b [========================> ] 13.63MB/28.23MB + db8bf9a4f43b [====> ] 9.437MB/111.7MB + db1363514c7f + 3e300a7cb18c [==============================> ] 9.437MB/15.55MB + 1adabd6b0d6b [========================> ] 13.63MB/28.23MB + db8bf9a4f43b [====> ] 9.437MB/111.7MB + 0ea2caf05f48 1 s + e34abc083292 + 3e300a7cb18c [==============================> ] 9.437MB/15.55MB + 0ea2caf05f48 1 s + 1adabd6b0d6b [========================> ] 13.63MB/28.23MB + db8bf9a4f43b [====> ] 9.437MB/111.7MB + 3e300a7cb18c [==============================> ] 9.437MB/15.55MB + 1adabd6b0d6b [========================> ] 13.63MB/28.23MB + db8bf9a4f43b [====> ] 9.437MB/111.7MB + 0ea2caf05f48 1 s + 3e300a7cb18c [==============================> ] 9.437MB/15.55MB + 0ea2caf05f48 1 s + 1adabd6b0d6b [==========================> ] 14.68MB/28.23MB + db8bf9a4f43b [====> ] 9.437MB/111.7MB + 3e300a7cb18c [=================================> ] 10.49MB/15.55MB + 0ea2caf05f48 1 s + db8bf9a4f43b [====> ] 10.49MB/111.7MB + 1adabd6b0d6b [==========================> ] 14.68MB/28.23MB + 3e300a7cb18c [=================================> ] 10.49MB/15.55MB + 0ea2caf05f48 1 s + 1adabd6b0d6b [==========================> ] 14.68MB/28.23MB + db8bf9a4f43b [====> ] 10.49MB/111.7MB + 3e300a7cb18c [=================================> ] 10.49MB/15.55MB + 1adabd6b0d6b [==========================> ] 14.68MB/28.23MB + db8bf9a4f43b [====> ] 10.49MB/111.7MB + 0ea2caf05f48 1 s + 3e300a7cb18c [=================================> ] 10.49MB/15.55MB + 0ea2caf05f48 1 s + db8bf9a4f43b [====> ] 10.49MB/111.7MB + 1adabd6b0d6b [===========================> ] 15.73MB/28.23MB + 3e300a7cb18c [=================================> ] 10.49MB/15.55MB + 0ea2caf05f48 1 s + 1adabd6b0d6b [===========================> ] 15.73MB/28.23MB + db8bf9a4f43b [====> ] 10.49MB/111.7MB + 3e300a7cb18c [=================================> ] 10.49MB/15.55MB + 0ea2caf05f48 1 s + 1adabd6b0d6b [===========================> ] 15.73MB/28.23MB + db8bf9a4f43b [====> ] 10.49MB/111.7MB + 3e300a7cb18c [=================================> ] 10.49MB/15.55MB + 0ea2caf05f48 1 s + db8bf9a4f43b [====> ] 10.49MB/111.7MB + 1adabd6b0d6b [===========================> ] 15.73MB/28.23MB + 3e300a7cb18c [=================================> ] 10.49MB/15.55MB + db8bf9a4f43b [====> ] 10.49MB/111.7MB + 1adabd6b0d6b [===========================> ] 15.73MB/28.23MB + 0ea2caf05f48 2 s + 3e300a7cb18c [=====================================> ] 11.53MB/15.55MB + 0ea2caf05f48 2 s + 1adabd6b0d6b [===========================> ] 15.73MB/28.23MB + db8bf9a4f43b [=====> ] 11.53MB/111.7MB + 3e300a7cb18c [=====================================> ] 11.53MB/15.55MB + 0ea2caf05f48 2 s + 1adabd6b0d6b [=============================> ] 16.78MB/28.23MB + db8bf9a4f43b [=====> ] 11.53MB/111.7MB + 3e300a7cb18c [=====================================> ] 11.53MB/15.55MB + 0ea2caf05f48 2 s + db8bf9a4f43b [=====> ] 11.53MB/111.7MB + 1adabd6b0d6b [=============================> ] 16.78MB/28.23MB + afa027603ca1 + 3e300a7cb18c [========================================> ] 12.58MB/15.55MB + 0ea2caf05f48 + 1adabd6b0d6b [=============================> ] 16.78MB/28.23MB + db8bf9a4f43b [=====> ] 11.53MB/111.7MB + aa8e37a1147a + 674d165e7863 + db099b936fc8 + 3e300a7cb18c [========================================> ] 12.58MB/15.55MB + 1adabd6b0d6b [=============================> ] 16.78MB/28.23MB + db8bf9a4f43b [=====> ] 11.53MB/111.7MB + Image rabbitmq:4.4.0 Pulled + 3e300a7cb18c [========================================> ] 12.58MB/15.55MB + 1adabd6b0d6b [=============================> ] 16.78MB/28.23MB + db8bf9a4f43b [=====> ] 11.53MB/111.7MB + 3e300a7cb18c [========================================> ] 12.58MB/15.55MB + 1adabd6b0d6b [=============================> ] 16.78MB/28.23MB + db8bf9a4f43b [=====> ] 12.58MB/111.7MB + 3e300a7cb18c [========================================> ] 12.58MB/15.55MB + 1adabd6b0d6b [=============================> ] 16.78MB/28.23MB + db8bf9a4f43b [=====> ] 12.58MB/111.7MB + 3e300a7cb18c [========================================> ] 12.58MB/15.55MB + db8bf9a4f43b [=====> ] 12.58MB/111.7MB + 1adabd6b0d6b [===============================> ] 17.83MB/28.23MB + db8bf9a4f43b [=====> ] 12.58MB/111.7MB + 1adabd6b0d6b [===============================> ] 17.83MB/28.23MB + 3e300a7cb18c [===========================================> ] 13.63MB/15.55MB + 3e300a7cb18c [===========================================> ] 13.63MB/15.55MB + db8bf9a4f43b [=====> ] 12.58MB/111.7MB + 1adabd6b0d6b [===============================> ] 17.83MB/28.23MB + 3e300a7cb18c [===========================================> ] 13.63MB/15.55MB + 1adabd6b0d6b [===============================> ] 17.83MB/28.23MB + db8bf9a4f43b [======> ] 13.63MB/111.7MB + 3e300a7cb18c [===========================================> ] 13.63MB/15.55MB + 1adabd6b0d6b [===============================> ] 17.83MB/28.23MB + db8bf9a4f43b [======> ] 13.63MB/111.7MB + 3e300a7cb18c [===========================================> ] 13.63MB/15.55MB + 1adabd6b0d6b [=================================> ] 18.87MB/28.23MB + db8bf9a4f43b [======> ] 13.63MB/111.7MB + 3e300a7cb18c [============================================> ] 13.8MB/15.55MB + 1adabd6b0d6b [=================================> ] 18.87MB/28.23MB + db8bf9a4f43b [======> ] 13.63MB/111.7MB + 3e300a7cb18c [===============================================> ] 14.68MB/15.55MB + 1adabd6b0d6b [=================================> ] 18.87MB/28.23MB + db8bf9a4f43b [======> ] 13.63MB/111.7MB + 3e300a7cb18c [===============================================> ] 14.68MB/15.55MB + db8bf9a4f43b [======> ] 14.68MB/111.7MB + 1adabd6b0d6b [=================================> ] 18.87MB/28.23MB + 3e300a7cb18c [===============================================> ] 14.68MB/15.55MB + db8bf9a4f43b [======> ] 14.68MB/111.7MB + 1adabd6b0d6b [=================================> ] 18.87MB/28.23MB + 3e300a7cb18c [==================================================>] 15.55MB/15.55MB + 1adabd6b0d6b [===================================> ] 19.92MB/28.23MB + db8bf9a4f43b [======> ] 14.68MB/111.7MB + 3e300a7cb18c + 3e300a7cb18c 1 s + 1adabd6b0d6b [===================================> ] 19.92MB/28.23MB + db8bf9a4f43b [======> ] 14.68MB/111.7MB + 3e300a7cb18c 1 s + 1adabd6b0d6b [===================================> ] 19.92MB/28.23MB + db8bf9a4f43b [=======> ] 15.73MB/111.7MB + 3e300a7cb18c 1 s + 1adabd6b0d6b [===================================> ] 19.92MB/28.23MB + db8bf9a4f43b [=======> ] 15.73MB/111.7MB + 3e300a7cb18c 1 s + db8bf9a4f43b [=======> ] 15.73MB/111.7MB + 1adabd6b0d6b [=====================================> ] 20.97MB/28.23MB + 3e300a7cb18c 1 s + 1adabd6b0d6b [=====================================> ] 20.97MB/28.23MB + db8bf9a4f43b [=======> ] 15.73MB/111.7MB + 3e300a7cb18c 1 s + 1adabd6b0d6b [=====================================> ] 20.97MB/28.23MB + db8bf9a4f43b [=======> ] 16.78MB/111.7MB + 3e300a7cb18c 1 s + db8bf9a4f43b [=======> ] 16.78MB/111.7MB + 1adabd6b0d6b [=====================================> ] 20.97MB/28.23MB + 3e300a7cb18c + 1adabd6b0d6b [=======================================> ] 22.02MB/28.23MB + db8bf9a4f43b [=======> ] 16.78MB/111.7MB + Image nginx:1.28.0-alpine Pulled + 1adabd6b0d6b [=======================================> ] 22.02MB/28.23MB + db8bf9a4f43b [=======> ] 16.78MB/111.7MB + 1adabd6b0d6b [=======================================> ] 22.02MB/28.23MB + db8bf9a4f43b [=======> ] 17.83MB/111.7MB + db8bf9a4f43b [=======> ] 17.83MB/111.7MB + 1adabd6b0d6b [=======================================> ] 22.02MB/28.23MB + 1adabd6b0d6b [=======================================> ] 22.02MB/28.23MB + db8bf9a4f43b [=======> ] 17.83MB/111.7MB + 1adabd6b0d6b [=======================================> ] 22.02MB/28.23MB + db8bf9a4f43b [=======> ] 17.83MB/111.7MB + 1adabd6b0d6b [=======================================> ] 22.02MB/28.23MB + db8bf9a4f43b [=======> ] 17.83MB/111.7MB + 1adabd6b0d6b [========================================> ] 23.07MB/28.23MB + db8bf9a4f43b [=======> ] 17.83MB/111.7MB + 1adabd6b0d6b [========================================> ] 23.07MB/28.23MB + db8bf9a4f43b [=======> ] 17.83MB/111.7MB + 1adabd6b0d6b [========================================> ] 23.07MB/28.23MB + db8bf9a4f43b [=======> ] 17.83MB/111.7MB + db8bf9a4f43b [========> ] 18.87MB/111.7MB + 1adabd6b0d6b [==========================================> ] 24.12MB/28.23MB + db8bf9a4f43b [========> ] 18.87MB/111.7MB + 1adabd6b0d6b [==========================================> ] 24.12MB/28.23MB + 1adabd6b0d6b [==========================================> ] 24.12MB/28.23MB + db8bf9a4f43b [========> ] 18.87MB/111.7MB + db8bf9a4f43b [========> ] 18.87MB/111.7MB + 1adabd6b0d6b [==========================================> ] 24.12MB/28.23MB + 1adabd6b0d6b [==========================================> ] 24.12MB/28.23MB + db8bf9a4f43b [========> ] 19.92MB/111.7MB + 1adabd6b0d6b [============================================> ] 25.17MB/28.23MB + db8bf9a4f43b [========> ] 19.92MB/111.7MB + 1adabd6b0d6b [============================================> ] 25.17MB/28.23MB + db8bf9a4f43b [========> ] 19.92MB/111.7MB + db8bf9a4f43b [========> ] 19.92MB/111.7MB + 1adabd6b0d6b [============================================> ] 25.17MB/28.23MB + 1adabd6b0d6b [============================================> ] 25.17MB/28.23MB + db8bf9a4f43b [=========> ] 20.97MB/111.7MB + 1adabd6b0d6b [==============================================> ] 26.21MB/28.23MB + db8bf9a4f43b [=========> ] 20.97MB/111.7MB + 1adabd6b0d6b [==============================================> ] 26.21MB/28.23MB + db8bf9a4f43b [=========> ] 20.97MB/111.7MB + 1adabd6b0d6b [==============================================> ] 26.21MB/28.23MB + db8bf9a4f43b [=========> ] 22.02MB/111.7MB + 1adabd6b0d6b [================================================> ] 27.26MB/28.23MB + db8bf9a4f43b [=========> ] 22.02MB/111.7MB + 1adabd6b0d6b [================================================> ] 27.26MB/28.23MB + db8bf9a4f43b [=========> ] 22.02MB/111.7MB + 1adabd6b0d6b [================================================> ] 27.26MB/28.23MB + db8bf9a4f43b [=========> ] 22.02MB/111.7MB + 1adabd6b0d6b [================================================> ] 27.26MB/28.23MB + db8bf9a4f43b [==========> ] 23.07MB/111.7MB + db8bf9a4f43b [==========> ] 23.07MB/111.7MB + 1adabd6b0d6b [================================================> ] 27.26MB/28.23MB + 1adabd6b0d6b + db8bf9a4f43b [==========> ] 24.12MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [===========> ] 25.17MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [===========> ] 25.17MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [===========> ] 26.21MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [===========> ] 26.21MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [============> ] 27.26MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [============> ] 28.31MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [============> ] 28.31MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [=============> ] 29.36MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [=============> ] 30.41MB/111.7MB + 1adabd6b0d6b 1 s + db8bf9a4f43b [=============> ] 30.41MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [==============> ] 31.46MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [==============> ] 32.51MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [==============> ] 32.51MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [===============> ] 33.55MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [===============> ] 33.55MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [===============> ] 33.55MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [===============> ] 34.6MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [===============> ] 34.6MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [===============> ] 35.65MB/111.7MB + 1adabd6b0d6b 2 s + db8bf9a4f43b [================> ] 37.26MB/111.7MB + 1adabd6b0d6b 3 s + db8bf9a4f43b [================> ] 37.75MB/111.7MB + 799548af46de + 7b697787d5d2 1 s + 1adabd6b0d6b + db8bf9a4f43b [================> ] 37.75MB/111.7MB + 7b697787d5d2 1 s + db8bf9a4f43b [=================> ] 38.8MB/111.7MB + 7b697787d5d2 1 s + db8bf9a4f43b [=================> ] 39.85MB/111.7MB + 7b697787d5d2 1 s + db8bf9a4f43b [==================> ] 40.89MB/111.7MB + 861643ce2817 1 s + 7b697787d5d2 + db8bf9a4f43b [==================> ] 41.94MB/111.7MB + 861643ce2817 + cdfd017c753d 1 s + db8bf9a4f43b [==================> ] 41.94MB/111.7MB + cdfd017c753d 1 s + db8bf9a4f43b [===================> ] 42.99MB/111.7MB + cdfd017c753d 1 s + db8bf9a4f43b [===================> ] 42.99MB/111.7MB + cdfd017c753d 1 s + db8bf9a4f43b [===================> ] 42.99MB/111.7MB + cdfd017c753d 1 s + db8bf9a4f43b [===================> ] 44.04MB/111.7MB + cdfd017c753d 1 s + db8bf9a4f43b [===================> ] 44.04MB/111.7MB + 974175074a8f 1 s + cdfd017c753d + db8bf9a4f43b [====================> ] 46.14MB/111.7MB + 0dac5f77c330 1 s + 974175074a8f + db8bf9a4f43b [====================> ] 46.14MB/111.7MB + 5e83761a8bfc + 0dac5f77c330 + db8bf9a4f43b [====================> ] 46.14MB/111.7MB + db8bf9a4f43b [====================> ] 46.14MB/111.7MB + db8bf9a4f43b [====================> ] 46.14MB/111.7MB + db8bf9a4f43b [=====================> ] 48.23MB/111.7MB + db8bf9a4f43b [======================> ] 49.28MB/111.7MB + db8bf9a4f43b [======================> ] 50.33MB/111.7MB + db8bf9a4f43b [=======================> ] 51.38MB/111.7MB + db8bf9a4f43b [=======================> ] 51.38MB/111.7MB + db8bf9a4f43b [=======================> ] 52.43MB/111.7MB + db8bf9a4f43b [=======================> ] 53.48MB/111.7MB + db8bf9a4f43b [========================> ] 54.53MB/111.7MB + db8bf9a4f43b [========================> ] 55.57MB/111.7MB + db8bf9a4f43b [========================> ] 55.57MB/111.7MB + db8bf9a4f43b [=========================> ] 56.62MB/111.7MB + db8bf9a4f43b [=========================> ] 57.67MB/111.7MB + db8bf9a4f43b [=========================> ] 57.67MB/111.7MB + db8bf9a4f43b [=========================> ] 57.67MB/111.7MB + db8bf9a4f43b [=========================> ] 57.67MB/111.7MB + db8bf9a4f43b [=========================> ] 57.67MB/111.7MB + db8bf9a4f43b [==========================> ] 58.72MB/111.7MB + db8bf9a4f43b [==========================> ] 59.77MB/111.7MB + db8bf9a4f43b [===========================> ] 61.87MB/111.7MB + db8bf9a4f43b [===========================> ] 62.52MB/111.7MB + db8bf9a4f43b [============================> ] 62.91MB/111.7MB + db8bf9a4f43b [============================> ] 63.96MB/111.7MB + db8bf9a4f43b [============================> ] 63.96MB/111.7MB + db8bf9a4f43b [=============================> ] 65.01MB/111.7MB + db8bf9a4f43b [==============================> ] 67.11MB/111.7MB + db8bf9a4f43b [==============================> ] 68.16MB/111.7MB + db8bf9a4f43b [==============================> ] 68.16MB/111.7MB + db8bf9a4f43b [==============================> ] 69.21MB/111.7MB + db8bf9a4f43b [===============================> ] 70.25MB/111.7MB + db8bf9a4f43b [===============================> ] 71.3MB/111.7MB + db8bf9a4f43b [================================> ] 72.35MB/111.7MB + db8bf9a4f43b [================================> ] 73.4MB/111.7MB + db8bf9a4f43b [=================================> ] 74.45MB/111.7MB + db8bf9a4f43b [=================================> ] 75.5MB/111.7MB + db8bf9a4f43b [==================================> ] 76.55MB/111.7MB + db8bf9a4f43b [==================================> ] 76.55MB/111.7MB + db8bf9a4f43b [==================================> ] 76.55MB/111.7MB + db8bf9a4f43b [==================================> ] 76.55MB/111.7MB + db8bf9a4f43b [==================================> ] 77.59MB/111.7MB + db8bf9a4f43b [===================================> ] 78.64MB/111.7MB + db8bf9a4f43b [===================================> ] 78.64MB/111.7MB + db8bf9a4f43b [===================================> ] 79.69MB/111.7MB + db8bf9a4f43b [===================================> ] 79.69MB/111.7MB + db8bf9a4f43b [===================================> ] 79.69MB/111.7MB + db8bf9a4f43b [===================================> ] 79.69MB/111.7MB + db8bf9a4f43b [====================================> ] 80.74MB/111.7MB + db8bf9a4f43b [====================================> ] 80.74MB/111.7MB + db8bf9a4f43b [=====================================> ] 82.84MB/111.7MB + db8bf9a4f43b [=====================================> ] 82.84MB/111.7MB + db8bf9a4f43b [=====================================> ] 83.89MB/111.7MB + db8bf9a4f43b [======================================> ] 84.93MB/111.7MB + db8bf9a4f43b [======================================> ] 84.93MB/111.7MB + db8bf9a4f43b [======================================> ] 85.98MB/111.7MB + db8bf9a4f43b [======================================> ] 87.03MB/111.7MB + db8bf9a4f43b [=======================================> ] 88.08MB/111.7MB + db8bf9a4f43b [=======================================> ] 88.08MB/111.7MB + db8bf9a4f43b [=======================================> ] 88.08MB/111.7MB + db8bf9a4f43b [=======================================> ] 89.13MB/111.7MB + db8bf9a4f43b [========================================> ] 90.18MB/111.7MB + db8bf9a4f43b [========================================> ] 91.23MB/111.7MB + db8bf9a4f43b [=========================================> ] 92.27MB/111.7MB + db8bf9a4f43b [=========================================> ] 93.32MB/111.7MB + db8bf9a4f43b [==========================================> ] 94.37MB/111.7MB + db8bf9a4f43b [==========================================> ] 95.42MB/111.7MB + db8bf9a4f43b [===========================================> ] 96.47MB/111.7MB + db8bf9a4f43b [===========================================> ] 97.52MB/111.7MB + db8bf9a4f43b [===========================================> ] 97.52MB/111.7MB + db8bf9a4f43b [===========================================> ] 97.52MB/111.7MB + db8bf9a4f43b [============================================> ] 98.57MB/111.7MB + db8bf9a4f43b [============================================> ] 99.61MB/111.7MB + db8bf9a4f43b [=============================================> ] 100.7MB/111.7MB + db8bf9a4f43b [=============================================> ] 100.7MB/111.7MB + db8bf9a4f43b [==============================================> ] 102.8MB/111.7MB + db8bf9a4f43b [==============================================> ] 103.8MB/111.7MB + db8bf9a4f43b [==============================================> ] 103.8MB/111.7MB + db8bf9a4f43b [==============================================> ] 104.9MB/111.7MB + db8bf9a4f43b [==============================================> ] 104.9MB/111.7MB + db8bf9a4f43b [===============================================> ] 105.9MB/111.7MB + db8bf9a4f43b [===============================================> ] 107MB/111.7MB + db8bf9a4f43b [================================================> ] 108MB/111.7MB + db8bf9a4f43b [================================================> ] 108MB/111.7MB + db8bf9a4f43b [================================================> ] 109.1MB/111.7MB + db8bf9a4f43b [================================================> ] 109.1MB/111.7MB + db8bf9a4f43b [=================================================> ] 111.1MB/111.7MB + db8bf9a4f43b [==================================================>] 111.7MB/111.7MB + db8bf9a4f43b + db8bf9a4f43b 1 s + db8bf9a4f43b 1 s + db8bf9a4f43b 1 s + db8bf9a4f43b 1 s + db8bf9a4f43b 1 s + db8bf9a4f43b 1 s + db8bf9a4f43b 1 s + db8bf9a4f43b 1 s + db8bf9a4f43b 1 s + db8bf9a4f43b 1 s + db8bf9a4f43b 2 s + db8bf9a4f43b 2 s + db8bf9a4f43b 2 s + db8bf9a4f43b 2 s + db8bf9a4f43b 2 s + db8bf9a4f43b 2 s + db8bf9a4f43b 2 s + db8bf9a4f43b 2 s + db8bf9a4f43b 2 s + db8bf9a4f43b 2 s + db8bf9a4f43b 3 s + db8bf9a4f43b 3 s + db8bf9a4f43b 3 s + db8bf9a4f43b 3 s + db8bf9a4f43b 3 s + db8bf9a4f43b 3 s + db8bf9a4f43b 3 s + db8bf9a4f43b 3 s + db8bf9a4f43b 3 s + db8bf9a4f43b 3 s + db8bf9a4f43b 4 s + db8bf9a4f43b 4 s + db8bf9a4f43b 4 s + db8bf9a4f43b 4 s + db8bf9a4f43b 4 s + db8bf9a4f43b 4 s + db8bf9a4f43b 4 s + db8bf9a4f43b 4 s + db8bf9a4f43b 4 s + db8bf9a4f43b 4 s + db8bf9a4f43b 5 s + db8bf9a4f43b 5 s + db8bf9a4f43b 5 s + db8bf9a4f43b 5 s + db8bf9a4f43b 5 s + db8bf9a4f43b 5 s + db8bf9a4f43b 5 s + db8bf9a4f43b 5 s + db8bf9a4f43b 5 s + db8bf9a4f43b 5 s + db8bf9a4f43b 6 s + db8bf9a4f43b 6 s + db8bf9a4f43b 6 s + db8bf9a4f43b 6 s + db8bf9a4f43b 6 s + db8bf9a4f43b 6 s + db8bf9a4f43b 6 s + db8bf9a4f43b 6 s + db8bf9a4f43b 6 s + db8bf9a4f43b 6 s + db8bf9a4f43b 7 s + db8bf9a4f43b 7 s + db8bf9a4f43b 7 s + db8bf9a4f43b 7 s + db8bf9a4f43b 7 s + db8bf9a4f43b 7 s + db8bf9a4f43b 7 s + db8bf9a4f43b 7 s + db8bf9a4f43b 7 s + db8bf9a4f43b 7 s + db8bf9a4f43b 8 s + db8bf9a4f43b 8 s + db8bf9a4f43b 8 s + db8bf9a4f43b 8 s + db8bf9a4f43b 8 s + db8bf9a4f43b 8 s + db8bf9a4f43b 8 s + db8bf9a4f43b 8 s + db8bf9a4f43b 8 s + db8bf9a4f43b 8 s + db8bf9a4f43b 9 s + db8bf9a4f43b 9 s + db8bf9a4f43b 9 s + db8bf9a4f43b 9 s + db8bf9a4f43b + 856e25603442 + d78bd8cc905b + 961aa0039c0e + 72ba653f834d + 5f7f7c5eac2d + Image postgres:16.10-bookworm Pulled + Image messenger-minio:local Building + Image messenger:local Building + Image messenger-minio-bootstrap:local Building +#1 [internal] load local bake definitions +#1 reading from stdin 1.45kB done +#1 DONE 0.0s + +#2 [minio-bootstrap internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.20kB done +#2 DONE 0.0s + +#3 [api internal] load build definition from Dockerfile +#3 transferring dockerfile: 749B done +#3 DONE 0.0s + +#4 [api internal] load metadata for docker.io/library/python:3.12.15-slim-bookworm +#4 DONE 2.3s + +#5 [minio internal] load metadata for docker.io/library/golang:1.24.8-bookworm +#5 DONE 2.3s + +#6 [minio internal] load .dockerignore +#6 transferring context: 231B done +#6 DONE 0.0s + +#7 [api runtime 1/2] FROM docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 +#7 resolve docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 +#7 ... + +#8 [api internal] load build context +#8 transferring context: 178.00kB 0.0s done +#8 DONE 0.1s + +#9 [minio-bootstrap internal] load build context +#9 transferring context: 4.07kB done +#9 DONE 0.1s + +#7 [minio-bootstrap runtime 1/2] FROM docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 +#7 resolve docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 0.1s done +#7 sha256:b52d28f2ade3944bb78458d2d77f25816d73cb5566d86dd897c9b8829a853ad8 0B / 249B 0.2s +#7 sha256:b52d28f2ade3944bb78458d2d77f25816d73cb5566d86dd897c9b8829a853ad8 249B / 249B 0.2s done +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 0B / 13.68MB 0.3s +#7 sha256:fd4548892b7d74acf5f52391380b45441b4ec55f6c2e1506030c2af26fd3e3f6 0B / 3.34MB 0.2s +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 0B / 28.25MB 0.2s +#7 sha256:fd4548892b7d74acf5f52391380b45441b4ec55f6c2e1506030c2af26fd3e3f6 1.05MB / 3.34MB 0.8s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 1.05MB / 13.68MB 1.1s +#7 sha256:fd4548892b7d74acf5f52391380b45441b4ec55f6c2e1506030c2af26fd3e3f6 2.10MB / 3.34MB 0.9s +#7 sha256:fd4548892b7d74acf5f52391380b45441b4ec55f6c2e1506030c2af26fd3e3f6 3.34MB / 3.34MB 1.2s done +#7 sha256:fd4548892b7d74acf5f52391380b45441b4ec55f6c2e1506030c2af26fd3e3f6 3.34MB / 3.34MB 1.2s done +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 2.10MB / 13.68MB 2.0s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 3.15MB / 13.68MB 2.7s +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 2.10MB / 28.25MB 3.0s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 4.19MB / 13.68MB 3.5s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 5.24MB / 13.68MB 3.9s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 6.29MB / 13.68MB 4.4s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 7.34MB / 13.68MB 5.1s +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 4.19MB / 28.25MB 5.4s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 8.39MB / 13.68MB 5.9s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 9.44MB / 13.68MB 6.3s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 10.49MB / 13.68MB 6.9s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 11.53MB / 13.68MB 7.8s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 12.58MB / 13.68MB 8.3s +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 6.29MB / 28.25MB 8.4s +#7 sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 13.68MB / 13.68MB 8.9s done +#7 ... + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 resolve docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 0.1s done +#10 sha256:b742dd3bb10ced8d5230fec380bd9ec4e0d97c09a98a4a305537124a19e053c5 126B / 126B 0.3s done +#10 sha256:176d82c372858e34cf996dfb5a88a75a43470adce3bc792ffa33d9db7893656b 42.99MB / 79.12MB 9.2s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 0B / 92.40MB 8.6s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 0B / 24.03MB 0.8s +#10 ... + +#7 [minio runtime 1/2] FROM docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 8.00MB / 28.25MB 11.6s +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 9.44MB / 28.25MB 12.8s +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 11.53MB / 28.25MB 14.4s +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 13.63MB / 28.25MB 16.5s +#7 ... + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 sha256:176d82c372858e34cf996dfb5a88a75a43470adce3bc792ffa33d9db7893656b 71.30MB / 79.12MB 16.8s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 1.05MB / 92.40MB 16.2s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 0B / 24.03MB 8.4s +#10 sha256:176d82c372858e34cf996dfb5a88a75a43470adce3bc792ffa33d9db7893656b 75.50MB / 79.12MB 18.0s +#10 sha256:176d82c372858e34cf996dfb5a88a75a43470adce3bc792ffa33d9db7893656b 79.12MB / 79.12MB 19.7s done +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 0B / 64.40MB 0.2s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 4.19MB / 64.40MB 2.1s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 2.10MB / 92.40MB 21.3s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 0B / 24.03MB 13.5s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 8.39MB / 64.40MB 3.8s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 12.58MB / 64.40MB 5.0s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 16.78MB / 64.40MB 6.0s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 20.97MB / 64.40MB 6.9s +#10 ... + +#7 [minio runtime 1/2] FROM docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 26.21MB / 28.25MB 27.3s +#7 ... + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 3.15MB / 92.40MB 26.4s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 0B / 24.03MB 18.6s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 25.17MB / 64.40MB 8.1s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 0B / 48.48MB 0.2s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 29.36MB / 64.40MB 8.7s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 33.55MB / 64.40MB 9.8s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 37.75MB / 64.40MB 10.7s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 3.15MB / 48.48MB 2.7s +#10 ... + +#7 [api runtime 1/2] FROM docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 +#7 sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 28.25MB / 28.25MB 28.4s done +#7 extracting sha256:11422d7ca1a44390a32bd8d8d9dc5946b4d053b3bbd9507d682e35584ba84a26 2.0s done +#7 extracting sha256:fd4548892b7d74acf5f52391380b45441b4ec55f6c2e1506030c2af26fd3e3f6 0.3s done +#7 extracting sha256:87463122edacdc60df8c7840549960acce4dfb91a2b2aa36f563f62e0cd991db 1.2s done +#7 extracting sha256:b52d28f2ade3944bb78458d2d77f25816d73cb5566d86dd897c9b8829a853ad8 0.0s done +#7 DONE 32.2s + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 41.94MB / 64.40MB 11.9s +#10 ... + +#11 [api 2/8] WORKDIR /app +#11 DONE 0.4s + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 ... + +#12 [minio runtime 2/2] RUN useradd --system --uid 10001 --no-create-home minio && mkdir /data && chown minio /data +#12 0.464 useradd warning: minio's uid 10001 is greater than SYS_UID_MAX 999 +#12 DONE 0.5s + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 1.05MB / 24.03MB 23.7s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 4.19MB / 92.40MB 31.5s +#10 ... + +#13 [api 3/8] RUN useradd --system --uid 10001 --no-create-home appuser +#13 0.450 useradd warning: appuser's uid 10001 is greater than SYS_UID_MAX 999 +#13 DONE 0.5s + +#14 [api 4/8] COPY requirements.txt . +#14 DONE 0.1s + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 46.14MB / 64.40MB 12.8s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 6.29MB / 48.48MB 4.7s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 50.33MB / 64.40MB 13.5s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 54.53MB / 64.40MB 14.4s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 9.44MB / 48.48MB 6.2s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 58.72MB / 64.40MB 15.3s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 62.91MB / 64.40MB 16.2s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 12.58MB / 48.48MB 8.0s +#10 sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 64.40MB / 64.40MB 16.3s done +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 3.15MB / 24.03MB 27.9s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 15.73MB / 48.48MB 8.9s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 5.24MB / 92.40MB 36.6s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 18.87MB / 48.48MB 9.6s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 22.02MB / 48.48MB 10.2s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 25.17MB / 48.48MB 10.7s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 28.31MB / 48.48MB 11.3s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 5.24MB / 24.03MB 31.2s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 31.46MB / 48.48MB 12.3s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 34.60MB / 48.48MB 13.1s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 37.75MB / 48.48MB 14.0s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 8.39MB / 92.40MB 41.7s +#10 ... + +#15 [api 5/8] RUN pip install -r requirements.txt +#15 2.966 Collecting alembic==1.20.0 (from -r requirements.txt (line 1)) +#15 ... + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 40.89MB / 48.48MB 14.7s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 44.04MB / 48.48MB 15.3s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 7.34MB / 24.03MB 35.2s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 47.19MB / 48.48MB 15.9s +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 48.48MB / 48.48MB 16.1s done +#10 sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 48.48MB / 48.48MB 16.1s done +#10 extracting sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 9.44MB / 24.03MB 36.9s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 13.63MB / 92.40MB 45.5s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 11.53MB / 24.03MB 38.0s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 13.63MB / 24.03MB 38.6s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 15.73MB / 24.03MB 39.1s +#10 extracting sha256:c6b11972fd12973831818babf60f1ffc1c4047507943d132dffc612884022858 3.5s done +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 18.87MB / 92.40MB 47.0s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 17.83MB / 24.03MB 39.6s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 19.92MB / 24.03MB 40.2s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 24.12MB / 92.40MB 48.6s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 22.02MB / 24.03MB 40.8s +#10 sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 24.03MB / 24.03MB 41.5s done +#10 extracting sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 29.36MB / 92.40MB 49.5s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 34.60MB / 92.40MB 50.4s +#10 extracting sha256:db3dba6026a3c551d6b8e98308c073fff4fd569fd2fc61f21384cb996da82c9e 1.1s done +#10 extracting sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 39.85MB / 92.40MB 51.5s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 45.09MB / 92.40MB 52.5s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 50.33MB / 92.40MB 54.3s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 55.57MB / 92.40MB 55.0s +#10 ... + +#15 [api 5/8] RUN pip install -r requirements.txt +#15 13.26 Downloading alembic-1.20.0-py3-none-any.whl.metadata (7.3 kB) +#15 13.34 Collecting annotated-doc==0.0.5 (from -r requirements.txt (line 2)) +#15 13.41 Downloading annotated_doc-0.0.5-py3-none-any.whl.metadata (6.5 kB) +#15 13.48 Collecting annotated-types==0.8.0 (from -r requirements.txt (line 3)) +#15 13.53 Downloading annotated_types-0.8.0-py3-none-any.whl.metadata (15 kB) +#15 13.63 Collecting anyio==4.15.1 (from -r requirements.txt (line 4)) +#15 13.71 Downloading anyio-4.15.1-py3-none-any.whl.metadata (4.7 kB) +#15 13.80 Collecting attrs==26.1.0 (from -r requirements.txt (line 5)) +#15 13.86 Downloading attrs-26.1.0-py3-none-any.whl.metadata (8.8 kB) +#15 13.96 Collecting certifi==2026.7.22 (from -r requirements.txt (line 6)) +#15 14.04 Downloading certifi-2026.7.22-py3-none-any.whl.metadata (2.5 kB) +#15 14.34 Collecting charset-normalizer==3.5.2 (from -r requirements.txt (line 7)) +#15 14.44 Downloading charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (46 kB) +#15 14.55 Collecting click==8.5.0 (from -r requirements.txt (line 8)) +#15 14.62 Downloading click-8.5.0-py3-none-any.whl.metadata (2.6 kB) +#15 14.83 Collecting fastapi==0.143.3 (from -r requirements.txt (line 9)) +#15 14.92 Downloading fastapi-0.143.3-py3-none-any.whl.metadata (26 kB) +#15 15.00 Collecting h11==0.16.0 (from -r requirements.txt (line 10)) +#15 15.07 Downloading h11-0.16.0-py3-none-any.whl.metadata (8.3 kB) +#15 15.15 Collecting httpcore2==2.13.1 (from -r requirements.txt (line 11)) +#15 15.26 Downloading httpcore2-2.13.1-py3-none-any.whl.metadata (26 kB) +#15 15.39 Collecting httpx2==2.13.1 (from -r requirements.txt (line 12)) +#15 15.48 Downloading httpx2-2.13.1-py3-none-any.whl.metadata (9.8 kB) +#15 15.57 Collecting idna==3.20 (from -r requirements.txt (line 13)) +#15 15.63 Downloading idna-3.20-py3-none-any.whl.metadata (7.2 kB) +#15 15.70 Collecting iniconfig==2.3.1 (from -r requirements.txt (line 14)) +#15 15.80 Downloading iniconfig-2.3.1-py3-none-any.whl.metadata (2.8 kB) +#15 15.99 Collecting jsonschema==4.26.0 (from -r requirements.txt (line 15)) +#15 16.17 Downloading jsonschema-4.26.0-py3-none-any.whl.metadata (7.6 kB) +#15 16.34 Collecting jsonschema-path==0.3.4 (from -r requirements.txt (line 16)) +#15 16.47 Downloading jsonschema_path-0.3.4-py3-none-any.whl.metadata (4.3 kB) +#15 16.68 Collecting jsonschema-specifications==2025.9.1 (from -r requirements.txt (line 17)) +#15 16.89 Downloading jsonschema_specifications-2025.9.1-py3-none-any.whl.metadata (2.9 kB) +#15 17.18 Collecting lazy-object-proxy==1.12.0 (from -r requirements.txt (line 18)) +#15 17.30 Downloading lazy_object_proxy-1.12.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl.metadata (5.1 kB) +#15 17.49 Collecting Mako==1.4.3 (from -r requirements.txt (line 19)) +#15 17.58 Downloading mako-1.4.3-py3-none-any.whl.metadata (2.9 kB) +#15 17.78 Collecting MarkupSafe==3.0.4 (from -r requirements.txt (line 20)) +#15 17.85 Downloading markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.7 kB) +#15 17.94 Collecting openapi-schema-validator==0.6.3 (from -r requirements.txt (line 21)) +#15 18.01 Downloading openapi_schema_validator-0.6.3-py3-none-any.whl.metadata (5.4 kB) +#15 18.10 Collecting openapi-spec-validator==0.7.2 (from -r requirements.txt (line 22)) +#15 18.19 Downloading openapi_spec_validator-0.7.2-py3-none-any.whl.metadata (5.7 kB) +#15 18.32 Collecting opentelemetry-api==1.45.1 (from -r requirements.txt (line 23)) +#15 18.94 Downloading opentelemetry_api-1.45.1-py3-none-any.whl.metadata (1.4 kB) +#15 19.36 Collecting packaging==26.3 (from -r requirements.txt (line 24)) +#15 19.88 Downloading packaging-26.3-py3-none-any.whl.metadata (3.5 kB) +#15 20.15 Collecting pathable==0.4.4 (from -r requirements.txt (line 25)) +#15 20.54 Downloading pathable-0.4.4-py3-none-any.whl.metadata (1.8 kB) +#15 20.98 Collecting pluggy==1.7.0 (from -r requirements.txt (line 26)) +#15 21.58 Downloading pluggy-1.7.0-py3-none-any.whl.metadata (5.0 kB) +#15 21.97 Collecting pydantic==2.14.1 (from -r requirements.txt (line 27)) +#15 22.04 Downloading pydantic-2.14.1-py3-none-any.whl.metadata (133 kB) +#15 ... + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 extracting sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 4.9s done +#10 extracting sha256:5fb1b35a6fc14463ada297f3f0605409cbfe29368b38fd5d1e41f7dcf29bb6fb 4.9s done +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 60.82MB / 92.40MB 55.9s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 67.11MB / 92.40MB 56.4s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 73.40MB / 92.40MB 57.6s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 79.69MB / 92.40MB 58.3s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 84.93MB / 92.40MB 59.1s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 90.18MB / 92.40MB 59.8s +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 92.40MB / 92.40MB 60.3s done +#10 sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 92.40MB / 92.40MB 60.3s done +#10 extracting sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb +#10 extracting sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 6.2s done +#10 extracting sha256:a73fad327b7f40cc61887bfe623dc29039b617c6ccf251d34990bc97b86f17fb 6.2s done +#10 extracting sha256:176d82c372858e34cf996dfb5a88a75a43470adce3bc792ffa33d9db7893656b +#10 ... + +#15 [api 5/8] RUN pip install -r requirements.txt +#15 25.29 Collecting pydantic_core==2.50.1 (from -r requirements.txt (line 28)) +#15 25.50 Downloading pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.5 kB) +#15 25.68 Collecting Pygments==2.21.0 (from -r requirements.txt (line 29)) +#15 25.85 Downloading pygments-2.21.0-py3-none-any.whl.metadata (2.5 kB) +#15 26.06 Collecting pytest==9.1.1 (from -r requirements.txt (line 30)) +#15 26.23 Downloading pytest-9.1.1-py3-none-any.whl.metadata (7.6 kB) +#15 26.95 Collecting PyYAML==6.0.3 (from -r requirements.txt (line 31)) +#15 27.14 Downloading pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.4 kB) +#15 27.28 Collecting referencing==0.36.2 (from -r requirements.txt (line 32)) +#15 27.39 Downloading referencing-0.36.2-py3-none-any.whl.metadata (2.8 kB) +#15 27.60 Collecting requests==2.34.2 (from -r requirements.txt (line 33)) +#15 27.67 Downloading requests-2.34.2-py3-none-any.whl.metadata (4.8 kB) +#15 27.77 Collecting rfc3339-validator==0.1.4 (from -r requirements.txt (line 34)) +#15 27.88 Downloading rfc3339_validator-0.1.4-py2.py3-none-any.whl.metadata (1.5 kB) +#15 29.05 Collecting rpds-py==2026.9.1 (from -r requirements.txt (line 35)) +#15 29.10 Downloading rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.1 kB) +#15 29.21 Collecting six==1.17.0 (from -r requirements.txt (line 36)) +#15 29.27 Downloading six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB) +#15 29.88 Collecting SQLAlchemy==2.1.4 (from -r requirements.txt (line 37)) +#15 29.94 Downloading sqlalchemy-2.1.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (9.7 kB) +#15 30.11 Collecting starlette==1.8.0 (from -r requirements.txt (line 38)) +#15 30.20 Downloading starlette-1.8.0-py3-none-any.whl.metadata (6.6 kB) +#15 30.28 Collecting truststore==0.10.5 (from -r requirements.txt (line 39)) +#15 30.36 Downloading truststore-0.10.5-py3-none-any.whl.metadata (4.5 kB) +#15 30.43 Collecting typing-inspection==0.4.4 (from -r requirements.txt (line 40)) +#15 30.50 Downloading typing_inspection-0.4.4-py3-none-any.whl.metadata (2.6 kB) +#15 30.58 Collecting typing_extensions==4.16.0 (from -r requirements.txt (line 41)) +#15 30.67 Downloading typing_extensions-4.16.0-py3-none-any.whl.metadata (3.3 kB) +#15 30.76 Collecting urllib3==2.8.0 (from -r requirements.txt (line 42)) +#15 30.85 Downloading urllib3-2.8.0-py3-none-any.whl.metadata (7.4 kB) +#15 30.96 Collecting uvicorn==0.54.0 (from -r requirements.txt (line 43)) +#15 31.05 Downloading uvicorn-0.54.0-py3-none-any.whl.metadata (6.6 kB) +#15 31.17 Collecting asyncpg==0.32.0 (from -r requirements.txt (line 44)) +#15 31.26 Downloading asyncpg-0.32.0-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (4.4 kB) +#15 31.34 Collecting argon2-cffi==25.1.0 (from -r requirements.txt (line 45)) +#15 31.41 Downloading argon2_cffi-25.1.0-py3-none-any.whl.metadata (4.1 kB) +#15 31.54 Collecting redis==8.1.0 (from -r requirements.txt (line 46)) +#15 31.60 Downloading redis-8.1.0-py3-none-any.whl.metadata (16 kB) +#15 32.02 Collecting greenlet==3.5.6 (from -r requirements.txt (line 47)) +#15 32.10 Downloading greenlet-3.5.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl.metadata (3.8 kB) +#15 32.18 Collecting pydantic-settings==2.16.0 (from -r requirements.txt (line 48)) +#15 32.26 Downloading pydantic_settings-2.16.0-py3-none-any.whl.metadata (4.0 kB) +#15 32.35 Collecting pytest-asyncio==1.4.1 (from -r requirements.txt (line 49)) +#15 32.41 Downloading pytest_asyncio-1.4.1-py3-none-any.whl.metadata (4.4 kB) +#15 32.51 Collecting httpx==0.28.1 (from -r requirements.txt (line 50)) +#15 32.57 Downloading httpx-0.28.1-py3-none-any.whl.metadata (7.1 kB) +#15 32.72 Collecting minio==7.2.20 (from -r requirements.txt (line 51)) +#15 32.80 Downloading minio-7.2.20-py3-none-any.whl.metadata (6.5 kB) +#15 33.16 Collecting Pillow==13.0.0 (from -r requirements.txt (line 52)) +#15 33.26 Downloading pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (9.1 kB) +#15 33.38 Collecting aio-pika==10.2.2 (from -r requirements.txt (line 53)) +#15 33.43 Downloading aio_pika-10.2.2-py3-none-any.whl.metadata (16 kB) +#15 33.94 Collecting argon2-cffi-bindings (from argon2-cffi==25.1.0->-r requirements.txt (line 45)) +#15 34.01 Downloading argon2_cffi_bindings-26.2.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl.metadata (7.6 kB) +#15 34.14 Collecting python-dotenv>=1.2.2 (from pydantic-settings==2.16.0->-r requirements.txt (line 48)) +#15 34.22 Downloading python_dotenv-1.2.4-py3-none-any.whl.metadata (29 kB) +#15 34.45 Collecting httpcore==1.* (from httpx==0.28.1->-r requirements.txt (line 50)) +#15 34.51 Downloading httpcore-1.0.9-py3-none-any.whl.metadata (21 kB) +#15 34.79 Collecting pycryptodome (from minio==7.2.20->-r requirements.txt (line 51)) +#15 34.88 Downloading pycryptodome-4.5.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (4.8 kB) +#15 35.00 Collecting aiormq<8,>=7.2.4 (from aio-pika==10.2.2->-r requirements.txt (line 53)) +#15 35.07 Downloading aiormq-7.2.5-py3-none-any.whl.metadata (27 kB) +#15 35.77 Collecting yarl (from aio-pika==10.2.2->-r requirements.txt (line 53)) +#15 35.84 Downloading yarl-1.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (111 kB) +#15 36.24 Collecting pamqp<6,>=5 (from aiormq<8,>=7.2.4->aio-pika==10.2.2->-r requirements.txt (line 53)) +#15 36.32 Downloading pamqp-5.0.0-py3-none-any.whl.metadata (3.0 kB) +#15 36.53 Collecting cffi>=1.0.1 (from argon2-cffi-bindings->argon2-cffi==25.1.0->-r requirements.txt (line 45)) +#15 36.61 Downloading cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (2.5 kB) +#15 37.06 Collecting multidict>=4.0 (from yarl->aio-pika==10.2.2->-r requirements.txt (line 53)) +#15 37.15 Downloading multidict-7.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (6.1 kB) +#15 37.30 Collecting propcache>=0.2.1 (from yarl->aio-pika==10.2.2->-r requirements.txt (line 53)) +#15 37.37 Downloading propcache-0.5.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (25 kB) +#15 37.49 Collecting pycparser (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi==25.1.0->-r requirements.txt (line 45)) +#15 37.55 Downloading pycparser-3.12-py3-none-any.whl.metadata (9.7 kB) +#15 37.65 Downloading alembic-1.20.0-py3-none-any.whl (268 kB) +#15 38.17 Downloading annotated_doc-0.0.5-py3-none-any.whl (5.3 kB) +#15 38.29 Downloading annotated_types-0.8.0-py3-none-any.whl (13 kB) +#15 38.38 Downloading anyio-4.15.1-py3-none-any.whl (132 kB) +#15 38.61 Downloading attrs-26.1.0-py3-none-any.whl (67 kB) +#15 38.72 Downloading certifi-2026.7.22-py3-none-any.whl (136 kB) +#15 38.95 Downloading charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (257 kB) +#15 39.27 Downloading click-8.5.0-py3-none-any.whl (125 kB) +#15 39.44 Downloading fastapi-0.143.3-py3-none-any.whl (144 kB) +#15 39.64 Downloading h11-0.16.0-py3-none-any.whl (37 kB) +#15 39.72 Downloading httpcore2-2.13.1-py3-none-any.whl (83 kB) +#15 39.84 Downloading httpx2-2.13.1-py3-none-any.whl (95 kB) +#15 39.98 Downloading idna-3.20-py3-none-any.whl (69 kB) +#15 40.08 Downloading iniconfig-2.3.1-py3-none-any.whl (7.6 kB) +#15 40.15 Downloading jsonschema-4.26.0-py3-none-any.whl (90 kB) +#15 40.25 Downloading jsonschema_path-0.3.4-py3-none-any.whl (14 kB) +#15 40.32 Downloading jsonschema_specifications-2025.9.1-py3-none-any.whl (18 kB) +#15 40.39 Downloading lazy_object_proxy-1.12.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl (71 kB) +#15 40.50 Downloading mako-1.4.3-py3-none-any.whl (80 kB) +#15 40.64 Downloading markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB) +#15 40.74 Downloading openapi_schema_validator-0.6.3-py3-none-any.whl (8.8 kB) +#15 40.80 Downloading openapi_spec_validator-0.7.2-py3-none-any.whl (39 kB) +#15 40.92 Downloading opentelemetry_api-1.45.1-py3-none-any.whl (60 kB) +#15 41.03 Downloading packaging-26.3-py3-none-any.whl (129 kB) +#15 41.17 Downloading pathable-0.4.4-py3-none-any.whl (9.6 kB) +#15 41.23 Downloading pluggy-1.7.0-py3-none-any.whl (27 kB) +#15 41.33 Downloading pydantic-2.14.1-py3-none-any.whl (488 kB) +#15 41.70 Downloading pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.2 MB) +#15 42.65 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.2/2.2 MB 2.4 MB/s eta 0:00:00 +#15 42.71 Downloading pygments-2.21.0-py3-none-any.whl (1.3 MB) +#15 ... + +#10 [minio-bootstrap compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 +#10 extracting sha256:176d82c372858e34cf996dfb5a88a75a43470adce3bc792ffa33d9db7893656b 8.0s done +#10 extracting sha256:b742dd3bb10ced8d5230fec380bd9ec4e0d97c09a98a4a305537124a19e053c5 +#10 extracting sha256:176d82c372858e34cf996dfb5a88a75a43470adce3bc792ffa33d9db7893656b 8.0s done +#10 extracting sha256:b742dd3bb10ced8d5230fec380bd9ec4e0d97c09a98a4a305537124a19e053c5 0.0s done +#10 extracting sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1 0.0s done +#10 DONE 76.0s + +#16 [minio compiler 2/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z +#16 0.523 go: downloading github.com/minio/minio v0.0.0-20251015172955-9e49d5e7a648 +#16 ... + +#15 [api 5/8] RUN pip install -r requirements.txt +#15 43.13 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 3.2 MB/s eta 0:00:00 +#15 43.19 Downloading pytest-9.1.1-py3-none-any.whl (386 kB) +#15 43.36 Downloading pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (807 kB) +#15 43.59 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 807.9/807.9 kB 3.7 MB/s eta 0:00:00 +#15 43.65 Downloading referencing-0.36.2-py3-none-any.whl (26 kB) +#15 43.72 Downloading requests-2.34.2-py3-none-any.whl (73 kB) +#15 43.79 Downloading rfc3339_validator-0.1.4-py2.py3-none-any.whl (3.5 kB) +#15 43.85 Downloading rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (371 kB) +#15 44.01 Downloading six-1.17.0-py2.py3-none-any.whl (11 kB) +#15 44.08 Downloading sqlalchemy-2.1.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (4.7 MB) +#15 45.10 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.7/4.7 MB 4.7 MB/s eta 0:00:00 +#15 45.17 Downloading starlette-1.8.0-py3-none-any.whl (79 kB) +#15 45.24 Downloading truststore-0.10.5-py3-none-any.whl (19 kB) +#15 45.31 Downloading typing_inspection-0.4.4-py3-none-any.whl (14 kB) +#15 45.37 Downloading typing_extensions-4.16.0-py3-none-any.whl (45 kB) +#15 45.43 Downloading urllib3-2.8.0-py3-none-any.whl (135 kB) +#15 45.51 Downloading uvicorn-0.54.0-py3-none-any.whl (87 kB) +#15 45.58 Downloading asyncpg-0.32.0-cp312-cp312-manylinux_2_28_x86_64.whl (3.8 MB) +#15 46.23 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.8/3.8 MB 5.9 MB/s eta 0:00:00 +#15 46.29 Downloading argon2_cffi-25.1.0-py3-none-any.whl (14 kB) +#15 46.35 Downloading redis-8.1.0-py3-none-any.whl (560 kB) +#15 46.44 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 560.6/560.6 kB 8.1 MB/s eta 0:00:00 +#15 46.51 Downloading greenlet-3.5.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (621 kB) +#15 46.59 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 621.4/621.4 kB 8.2 MB/s eta 0:00:00 +#15 46.66 Downloading pydantic_settings-2.16.0-py3-none-any.whl (80 kB) +#15 46.74 Downloading pytest_asyncio-1.4.1-py3-none-any.whl (17 kB) +#15 46.80 Downloading httpx-0.28.1-py3-none-any.whl (73 kB) +#15 46.89 Downloading minio-7.2.20-py3-none-any.whl (93 kB) +#15 46.99 Downloading pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (7.1 MB) +#15 48.24 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.1/7.1 MB 5.7 MB/s eta 0:00:00 +#15 48.29 Downloading aio_pika-10.2.2-py3-none-any.whl (61 kB) +#15 48.36 Downloading httpcore-1.0.9-py3-none-any.whl (78 kB) +#15 48.44 Downloading aiormq-7.2.5-py3-none-any.whl (40 kB) +#15 48.52 Downloading python_dotenv-1.2.4-py3-none-any.whl (23 kB) +#15 48.58 Downloading argon2_cffi_bindings-26.2.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl (26 kB) +#15 48.66 Downloading pycryptodome-4.5.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (3.1 MB) +#15 49.07 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.1/3.1 MB 7.8 MB/s eta 0:00:00 +#15 49.13 Downloading yarl-1.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (117 kB) +#15 49.21 Downloading cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (221 kB) +#15 49.30 Downloading multidict-7.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (516 kB) +#15 49.42 Downloading pamqp-5.0.0-py3-none-any.whl (32 kB) +#15 49.49 Downloading propcache-0.5.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (250 kB) +#15 49.58 Downloading pycparser-3.12-py3-none-any.whl (62 kB) +#15 50.04 Installing collected packages: urllib3, typing_extensions, truststore, six, rpds-py, redis, PyYAML, python-dotenv, Pygments, pycryptodome, pycparser, propcache, pluggy, Pillow, pathable, pamqp, packaging, multidict, MarkupSafe, lazy-object-proxy, iniconfig, idna, h11, greenlet, click, charset-normalizer, certifi, attrs, asyncpg, annotated-types, annotated-doc, yarl, uvicorn, typing-inspection, SQLAlchemy, rfc3339-validator, requests, referencing, pytest, pydantic_core, opentelemetry-api, Mako, httpcore2, httpcore, cffi, anyio, starlette, pytest-asyncio, pydantic, jsonschema-specifications, jsonschema-path, httpx2, httpx, argon2-cffi-bindings, alembic, aiormq, pydantic-settings, jsonschema, fastapi, argon2-cffi, aio-pika, openapi-schema-validator, minio, openapi-spec-validator +#15 ... + +#16 [minio compiler 2/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z +#16 13.06 go: downloading cloud.google.com/go/storage v1.52.0 +#16 13.07 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 +#16 13.08 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 +#16 13.73 go: downloading cloud.google.com/go v0.120.1 +#16 13.91 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 +#16 13.94 go: downloading github.com/cespare/xxhash/v2 v2.3.0 +#16 14.22 go: downloading github.com/cheggaaa/pb v1.0.29 +#16 14.38 go: downloading github.com/coreos/go-oidc/v3 v3.14.1 +#16 14.43 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 +#16 14.43 go: downloading github.com/cosnicolaou/pbzip2 v1.0.5 +#16 14.49 go: downloading github.com/dchest/siphash v1.2.3 +#16 15.63 go: downloading github.com/dustin/go-humanize v1.0.1 +#16 15.63 go: downloading github.com/fatih/color v1.18.0 +#16 15.79 go: downloading github.com/felixge/fgprof v0.9.5 +#16 15.94 go: downloading github.com/go-openapi/loads v0.22.0 +#16 15.99 go: downloading github.com/golang-jwt/jwt/v4 v4.5.2 +#16 16.19 go: downloading github.com/google/uuid v1.6.0 +#16 16.25 go: downloading github.com/inconshreveable/mousetrap v1.1.0 +#16 16.28 go: downloading github.com/json-iterator/go v1.1.12 +#16 16.28 go: downloading github.com/klauspost/compress v1.18.0 +#16 ... + +#15 [api 5/8] RUN pip install -r requirements.txt +#15 60.41 Successfully installed Mako-1.4.3 MarkupSafe-3.0.4 Pillow-13.0.0 PyYAML-6.0.3 Pygments-2.21.0 SQLAlchemy-2.1.4 aio-pika-10.2.2 aiormq-7.2.5 alembic-1.20.0 annotated-doc-0.0.5 annotated-types-0.8.0 anyio-4.15.1 argon2-cffi-25.1.0 argon2-cffi-bindings-26.2.0 asyncpg-0.32.0 attrs-26.1.0 certifi-2026.7.22 cffi-2.1.1 charset-normalizer-3.5.2 click-8.5.0 fastapi-0.143.3 greenlet-3.5.6 h11-0.16.0 httpcore-1.0.9 httpcore2-2.13.1 httpx-0.28.1 httpx2-2.13.1 idna-3.20 iniconfig-2.3.1 jsonschema-4.26.0 jsonschema-path-0.3.4 jsonschema-specifications-2025.9.1 lazy-object-proxy-1.12.0 minio-7.2.20 multidict-7.2.1 openapi-schema-validator-0.6.3 openapi-spec-validator-0.7.2 opentelemetry-api-1.45.1 packaging-26.3 pamqp-5.0.0 pathable-0.4.4 pluggy-1.7.0 propcache-0.5.4 pycparser-3.12 pycryptodome-4.5.0 pydantic-2.14.1 pydantic-settings-2.16.0 pydantic_core-2.50.1 pytest-9.1.1 pytest-asyncio-1.4.1 python-dotenv-1.2.4 redis-8.1.0 referencing-0.36.2 requests-2.34.2 rfc3339-validator-0.1.4 rpds-py-2026.9.1 six-1.17.0 starlette-1.8.0 truststore-0.10.5 typing-inspection-0.4.4 typing_extensions-4.16.0 urllib3-2.8.0 uvicorn-0.54.0 yarl-1.25.1 +#15 60.41 WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning. +#15 DONE 61.5s + +#17 [api 6/8] COPY alembic.ini ./ +#17 DONE 0.2s + +#16 [minio compiler 2/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z +#16 ... + +#18 [api 7/8] COPY migrations ./migrations +#18 DONE 0.1s + +#19 [api 8/8] COPY app ./app +#19 DONE 0.1s + +#20 [api] exporting to image +#20 exporting layers +#20 ... + +#16 [minio compiler 2/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z +#16 22.66 go: downloading github.com/klauspost/filepathx v1.1.1 +#16 22.66 go: downloading github.com/klauspost/pgzip v1.2.6 +#16 22.66 go: downloading github.com/klauspost/reedsolomon v1.12.4 +#16 22.66 go: downloading github.com/klauspost/readahead v1.4.0 +#16 22.98 go: downloading github.com/lithammer/shortuuid/v4 v4.2.0 +#16 22.98 go: downloading github.com/miekg/dns v1.1.65 +#16 23.01 go: downloading github.com/minio/cli v1.24.2 +#16 23.24 go: downloading github.com/minio/console v1.7.7-0.20250905210349-2017f33b26e1 +#16 26.90 go: downloading github.com/minio/dnscache v0.1.1 +#16 26.90 go: downloading github.com/minio/kms-go/kes v0.3.1 +#16 26.90 go: downloading github.com/minio/highwayhash v1.0.3 +#16 26.90 go: downloading github.com/minio/dperf v0.6.3 +#16 27.14 go: downloading github.com/minio/madmin-go/v3 v3.0.109 +#16 27.51 go: downloading github.com/minio/minio-go/v7 v7.0.91 +#16 ... + +#20 [api] exporting to image +#20 exporting layers 8.7s done +#20 exporting manifest sha256:140c64c78a7753bbbf6a92c25bf2424e87c751501508e569728efb216ffb1acf 0.0s done +#20 exporting config sha256:1201e868a613e0918885c40e95996f2b7b6ab4b414f7efce683028537d2877b2 0.0s done +#20 exporting attestation manifest sha256:df356738c4357bd8ef4e10b161ed9016180b8784fc9bdbdeb3b0da820c156cbb 0.0s done +#20 exporting manifest list sha256:ab5533cf26d3e05ddb27723090dfff83dbe8d873f213dd4eae6c92420b780c68 0.0s done +#20 naming to docker.io/library/messenger:local done +#20 unpacking to docker.io/library/messenger:local +#20 unpacking to docker.io/library/messenger:local 4.3s done +#20 DONE 13.2s + +#16 [minio compiler 2/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z +#16 27.94 go: downloading github.com/minio/mux v1.9.2 +#16 27.94 go: downloading github.com/minio/pkg/v3 v3.1.3 +#16 28.35 go: downloading github.com/minio/selfupdate v0.6.0 +#16 28.35 go: downloading github.com/minio/sio v0.4.1 +#16 28.35 go: downloading github.com/minio/xxml v0.0.3 +#16 28.35 go: downloading github.com/minio/zipindex v0.4.0 +#16 28.92 go: downloading github.com/mitchellh/go-homedir v1.1.0 +#16 28.92 go: downloading github.com/pierrec/lz4/v4 v4.1.22 +#16 28.92 go: downloading github.com/philhofer/fwd v1.2.0 +#16 29.17 go: downloading github.com/pkg/errors v0.9.1 +#16 29.19 go: downloading github.com/pkg/sftp v1.13.9 +#16 29.23 go: downloading github.com/pkg/xattr v0.4.10 +#16 29.43 go: downloading github.com/prometheus/client_golang v1.22.0 +#16 29.87 go: downloading github.com/prometheus/client_model v0.6.2 +#16 29.87 go: downloading github.com/prometheus/common v0.63.0 +#16 29.87 go: downloading github.com/prometheus/procfs v0.16.1 +#16 31.12 go: downloading github.com/puzpuzpuz/xsync/v3 v3.5.1 +#16 31.12 go: downloading github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 +#16 31.41 go: downloading github.com/rs/cors v1.11.1 +#16 ... + +#21 [api] resolving provenance for metadata file +#21 DONE 0.0s + +#16 [minio compiler 2/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z +#16 32.34 go: downloading github.com/secure-io/sio-go v0.3.1 +#16 32.34 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 +#16 33.66 go: downloading github.com/tinylib/msgp v1.4.0 +#16 33.66 go: downloading github.com/valyala/bytebufferpool v1.0.0 +#16 33.66 go: downloading github.com/zeebo/xxh3 v1.0.2 +#16 34.03 go: downloading go.etcd.io/etcd/api/v3 v3.5.21 +#16 34.03 go: downloading go.etcd.io/etcd/client/v3 v3.5.21 +#16 34.03 go: downloading go.uber.org/atomic v1.11.0 +#16 34.38 go: downloading goftp.io/server/v2 v2.0.1 +#16 34.38 go: downloading golang.org/x/crypto v0.37.0 +#16 34.78 go: downloading golang.org/x/oauth2 v0.29.0 +#16 35.00 go: downloading golang.org/x/sync v0.13.0 +#16 35.07 go: downloading golang.org/x/sys v0.32.0 +#16 35.07 go: downloading golang.org/x/term v0.31.0 +#16 35.08 go: downloading google.golang.org/api v0.230.0 +#16 35.72 go: downloading gopkg.in/yaml.v2 v2.4.0 +#16 36.24 go: downloading gopkg.in/yaml.v3 v3.0.1 +#16 38.26 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 +#16 39.44 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 +#16 40.90 go: downloading cloud.google.com/go/compute/metadata v0.6.0 +#16 41.17 go: downloading cloud.google.com/go/iam v1.5.2 +#16 41.58 go: downloading github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 +#16 41.87 go: downloading github.com/googleapis/gax-go/v2 v2.14.1 +#16 42.17 go: downloading go.opentelemetry.io/contrib/detectors/gcp v1.35.0 +#16 42.44 go: downloading go.opentelemetry.io/otel v1.35.0 +#16 43.19 go: downloading go.opentelemetry.io/otel/sdk/metric v1.35.0 +#16 43.37 go: downloading go.opentelemetry.io/otel/sdk v1.35.0 +#16 43.72 go: downloading go.opentelemetry.io/otel/trace v1.35.0 +#16 44.46 go: downloading google.golang.org/genproto v0.0.0-20250422160041-2d3770c4ea7f +#16 44.46 go: downloading google.golang.org/grpc v1.72.0 +#16 46.58 go: downloading google.golang.org/protobuf v1.36.6 +#16 47.47 go: downloading github.com/go-jose/go-jose/v4 v4.1.0 +#16 47.47 go: downloading github.com/mattn/go-runewidth v0.0.16 +#16 47.51 go: downloading github.com/mattn/go-colorable v0.1.14 +#16 47.51 go: downloading github.com/mattn/go-isatty v0.0.20 +#16 47.73 go: downloading github.com/google/pprof v0.0.0-20250422154841-e1f9c1950416 +#16 47.80 go: downloading github.com/go-openapi/analysis v0.23.0 +#16 47.88 go: downloading github.com/go-openapi/spec v0.21.0 +#16 48.36 go: downloading github.com/go-openapi/swag v0.23.1 +#16 48.55 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd +#16 48.68 go: downloading github.com/modern-go/reflect2 v1.0.2 +#16 48.72 go: downloading github.com/klauspost/cpuid/v2 v2.2.10 +#16 48.82 go: downloading golang.org/x/net v0.39.0 +#16 49.12 go: downloading github.com/go-openapi/errors v0.22.1 +#16 49.12 go: downloading github.com/go-openapi/runtime v0.28.0 +#16 49.41 go: downloading github.com/jessevdk/go-flags v1.6.1 +#16 49.44 go: downloading github.com/minio/mc v0.0.0-20250313080218-cf909e1063a9 +#16 49.44 go: downloading github.com/minio/websocket v1.6.0 +#16 49.73 go: downloading github.com/unrolled/secure v1.17.0 +#16 49.79 go: downloading github.com/go-openapi/strfmt v0.23.0 +#16 49.94 go: downloading aead.dev/mem v0.2.0 +#16 49.94 go: downloading github.com/ncw/directio v1.0.5 +#16 49.99 go: downloading github.com/prometheus/prom2json v1.4.2 +#16 50.01 go: downloading github.com/safchain/ethtool v0.5.10 +#16 50.19 go: downloading github.com/goccy/go-json v0.10.5 +#16 50.22 go: downloading github.com/minio/crc64nvme v1.0.1 +#16 50.24 go: downloading github.com/minio/md5-simd v1.1.2 +#16 50.29 go: downloading github.com/rs/xid v1.6.0 +#16 50.50 go: downloading golang.org/x/time v0.11.0 +#16 50.55 go: downloading github.com/beevik/ntp v1.4.3 +#16 50.56 go: downloading go.uber.org/zap v1.27.0 +#16 50.58 go: downloading github.com/go-ldap/ldap/v3 v3.4.11 +#16 50.76 go: downloading github.com/rabbitmq/amqp091-go v1.10.0 +#16 50.85 go: downloading github.com/IBM/sarama v1.45.1 +#16 50.89 go: downloading github.com/eclipse/paho.mqtt.golang v1.5.0 +#16 51.07 go: downloading github.com/elastic/go-elasticsearch/v7 v7.17.10 +#16 51.26 go: downloading github.com/go-sql-driver/mysql v1.9.2 +#16 51.27 go: downloading github.com/gomodule/redigo v1.9.2 +#16 51.27 go: downloading github.com/lib/pq v1.10.9 +#16 51.58 go: downloading github.com/nats-io/nats.go v1.41.2 +#16 51.59 go: downloading github.com/nats-io/stan.go v0.10.4 +#16 51.59 go: downloading github.com/nsqio/go-nsq v1.1.0 +#16 51.65 go: downloading github.com/xdg/scram v1.0.5 +#16 51.96 go: downloading github.com/gobwas/ws v1.4.0 +#16 52.04 go: downloading github.com/buger/jsonparser v1.1.1 +#16 52.09 go: downloading aead.dev/mtls v0.2.1 +#16 52.27 go: downloading github.com/minio/kms-go/kms v0.5.1-0.20250225090116-4e64ce8d0f35 +#16 52.27 go: downloading github.com/minio/simdjson-go v0.4.5 +#16 52.60 go: downloading github.com/go-ini/ini v1.67.0 +#16 52.90 go: downloading github.com/rjeczalik/notify v0.9.3 +#16 52.90 go: downloading github.com/fatih/structs v1.1.0 +#16 53.13 go: downloading github.com/lestrrat-go/jwx/v2 v2.1.4 +#16 53.20 go: downloading aead.dev/minisign v0.3.0 +#16 53.44 go: downloading github.com/kr/fs v0.1.0 +#16 53.49 go: downloading github.com/beorn7/perks v1.0.1 +#16 53.49 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 +#16 53.49 go: downloading github.com/tklauser/go-sysconf v0.3.15 +#16 53.70 go: downloading github.com/gogo/protobuf v1.3.2 +#16 54.11 go: downloading github.com/golang/protobuf v1.5.4 +#16 54.11 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.21 +#16 54.37 go: downloading github.com/kylelemons/godebug v1.1.0 +#16 54.37 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c +#16 54.44 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f +#16 54.63 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f +#16 54.63 go: downloading cloud.google.com/go/monitoring v1.24.2 +#16 54.71 go: downloading github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 +#16 54.83 go: downloading github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 +#16 55.02 go: downloading github.com/go-logr/logr v1.4.2 +#16 55.03 go: downloading go.opentelemetry.io/otel/metric v1.35.0 +#16 55.23 go: downloading cloud.google.com/go/auth v0.16.0 +#16 55.29 go: downloading cloud.google.com/go/auth/oauth2adapt v0.2.8 +#16 55.64 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 +#16 55.64 go: downloading github.com/rivo/uniseg v0.4.7 +#16 55.64 go: downloading github.com/go-openapi/jsonpointer v0.21.1 +#16 55.64 go: downloading github.com/mailru/easyjson v0.9.0 +#16 55.96 go: downloading github.com/go-openapi/jsonreference v0.21.0 +#16 55.97 go: downloading github.com/go-openapi/validate v0.24.0 +#16 55.99 go: downloading github.com/docker/go-units v0.5.0 +#16 56.11 go: downloading github.com/charmbracelet/bubbles v0.20.0 +#16 56.40 go: downloading github.com/charmbracelet/bubbletea v1.3.4 +#16 56.40 go: downloading github.com/charmbracelet/lipgloss v1.0.0 +#16 56.43 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 +#16 56.67 go: downloading github.com/jedib0t/go-pretty/v6 v6.6.7 +#16 56.68 go: downloading github.com/mattn/go-ieproxy v0.0.12 +#16 56.68 go: downloading github.com/minio/colorjson v1.0.8 +#16 57.05 go: downloading github.com/minio/filepath v1.0.0 +#16 57.05 go: downloading github.com/muesli/reflow v0.3.0 +#16 57.10 go: downloading github.com/muesli/termenv v0.16.0 +#16 57.39 go: downloading github.com/olekukonko/tablewriter v0.0.5 +#16 57.39 go: downloading github.com/posener/complete v1.2.3 +#16 57.51 go: downloading github.com/tidwall/gjson v1.18.0 +#16 57.77 go: downloading github.com/vbauerster/mpb/v8 v8.9.3 +#16 57.78 go: downloading golang.org/x/text v0.24.0 +#16 59.06 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 +#16 59.16 go: downloading github.com/mitchellh/mapstructure v1.5.0 +#16 59.16 go: downloading github.com/oklog/ulid v1.3.1 +#16 60.14 go: downloading go.mongodb.org/mongo-driver v1.17.3 +#16 61.24 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.4 +#16 61.51 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 +#16 61.77 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 +#16 61.77 go: downloading go.uber.org/multierr v1.11.0 +#16 61.77 go: downloading github.com/gorilla/websocket v1.5.3 +#16 61.86 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc +#16 62.03 go: downloading github.com/eapache/go-resiliency v1.7.0 +#16 62.03 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 +#16 62.07 go: downloading github.com/eapache/queue v1.1.0 +#16 62.20 go: downloading github.com/hashicorp/go-multierror v1.1.1 +#16 62.27 go: downloading github.com/jcmturner/gofork v1.7.6 +#16 62.31 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 +#16 62.62 go: downloading filippo.io/edwards25519 v1.1.0 +#16 62.62 go: downloading github.com/nats-io/nuid v1.0.1 +#16 62.62 go: downloading github.com/golang/snappy v1.0.0 +#16 62.62 go: downloading github.com/xdg/stringprep v1.0.3 +#16 62.80 go: downloading github.com/nats-io/nkeys v0.4.11 +#16 62.87 go: downloading github.com/gobwas/httphead v0.1.0 +#16 62.90 go: downloading github.com/gobwas/pool v0.2.1 +#16 63.20 go: downloading github.com/minio/csvparser v1.0.0 +#16 63.20 go: downloading github.com/fraugster/parquet-go v0.12.0 +#16 63.23 go: downloading github.com/alecthomas/participle v0.7.1 +#16 63.54 go: downloading github.com/lestrrat-go/option v1.0.1 +#16 63.54 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 +#16 63.56 go: downloading github.com/lestrrat-go/httprc v1.0.6 +#16 63.56 go: downloading github.com/lestrrat-go/iter v1.0.2 +#16 63.80 go: downloading github.com/tklauser/numcpus v0.10.0 +#16 63.84 go: downloading github.com/coreos/go-semver v0.3.1 +#16 63.84 go: downloading github.com/golang-jwt/jwt/v5 v5.2.2 +#16 63.85 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 +#16 64.01 go: downloading github.com/go-logr/stdr v1.2.2 +#16 64.05 go: downloading go.opentelemetry.io/auto/sdk v1.1.0 +#16 64.05 go: downloading github.com/google/s2a-go v0.1.9 +#16 64.27 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.6 +#16 64.31 go: downloading github.com/envoyproxy/go-control-plane/envoy v1.32.4 +#16 64.91 go: downloading github.com/josharian/intern v1.0.0 +#16 65.22 go: downloading github.com/felixge/httpsnoop v1.0.4 +#16 65.53 go: downloading github.com/charmbracelet/x/ansi v0.8.0 +#16 65.72 go: downloading github.com/charmbracelet/x/term v0.2.1 +#16 65.72 go: downloading github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 +#16 65.72 go: downloading github.com/muesli/cancelreader v0.2.2 +#16 65.80 go: downloading github.com/juju/ratelimit v1.0.2 +#16 65.99 go: downloading github.com/aymanbagabas/go-osc52/v2 v2.0.1 +#16 65.99 go: downloading github.com/lucasb-eyer/go-colorful v1.2.0 +#16 65.99 go: downloading github.com/tidwall/match v1.1.1 +#16 66.08 go: downloading github.com/tidwall/pretty v1.2.1 +#16 66.26 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d +#16 66.26 go: downloading github.com/VividCortex/ewma v1.2.0 +#16 66.35 go: downloading github.com/prometheus/prometheus v0.303.0 +#16 66.37 go: downloading github.com/hashicorp/errwrap v1.1.0 +#16 66.93 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 +#16 66.93 go: downloading github.com/hashicorp/go-uuid v1.0.3 +#16 67.05 go: downloading github.com/apache/thrift v0.21.0 +#16 67.65 go: downloading github.com/lestrrat-go/httpcc v1.0.1 +#16 67.65 go: downloading github.com/spiffe/go-spiffe/v2 v2.5.0 +#16 67.91 go: downloading github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f +#16 68.19 go: downloading github.com/envoyproxy/protoc-gen-validate v1.2.1 +#16 68.19 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 +#16 68.43 go: downloading github.com/zeebo/errs v1.4.0 +#16 68.46 go: downloading cel.dev/expr v0.23.1 +#16 68.46 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 +#16 DONE 210.4s + +#22 [minio compiler 3/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z +#22 0.918 go: downloading github.com/minio/mc v0.0.0-20250813083541-7394ce0dd2a8 +#22 2.763 go: downloading github.com/minio/pkg/v3 v3.1.0 +#22 2.788 go: downloading github.com/minio/madmin-go/v3 v3.0.107-0.20250415152934-4b504b82db63 +#22 2.789 go: downloading github.com/minio/minio-go/v7 v7.0.90 +#22 3.177 go: downloading github.com/prometheus/client_golang v1.21.1 +#22 3.209 go: downloading github.com/prometheus/procfs v0.16.0 +#22 3.515 go: downloading golang.org/x/net v0.42.0 +#22 3.516 go: downloading golang.org/x/sys v0.34.0 +#22 3.516 go: downloading golang.org/x/term v0.33.0 +#22 3.964 go: downloading golang.org/x/text v0.27.0 +#22 4.831 go: downloading golang.org/x/sync v0.16.0 +#22 5.064 go: downloading golang.org/x/crypto v0.40.0 +#22 5.716 go: downloading go.etcd.io/etcd/client/v3 v3.5.19 +#22 5.751 go: downloading github.com/tinylib/msgp v1.2.5 +#22 6.001 go: downloading go.etcd.io/etcd/api/v3 v3.5.19 +#22 6.206 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.19 +#22 6.211 go: downloading google.golang.org/grpc v1.71.0 +#22 6.976 go: downloading github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c +#22 6.977 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20250311173030-29e43e6258d7 +#22 6.984 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb +#22 DONE 43.6s + +#23 [minio server 1/1] COPY --from=compiler /out/minio /usr/local/bin/minio +#23 DONE 0.4s + +#24 [minio-bootstrap bootstrap 1/2] COPY --from=compiler /out/mc /usr/local/bin/mc +#24 DONE 0.3s + +#25 [minio-bootstrap bootstrap 2/2] COPY infra/minio/bootstrap.py /bootstrap.py +#25 DONE 0.1s + +#26 [minio] exporting to image +#26 exporting layers +#26 ... + +#27 [minio-bootstrap] exporting to image +#27 exporting layers 1.6s done +#27 exporting manifest sha256:a063b2a4b069f08943338b88d3a5da84e5fc059d048adcc92f7b59eb051f2e7a 0.0s done +#27 exporting config sha256:04a4348a4e722f088351a9c55b61a764d1deff14d40a85af81ee061219e9d33a 0.0s done +#27 exporting attestation manifest sha256:8ab05e7135d721ecfc924a756a3721a83f78153ef933a0e48b6202d13cab713b 0.0s done +#27 exporting manifest list sha256:74593d60d131f14b268928fb3a11be446237cb7875f46d73c9530483fea30dff 0.0s done +#27 naming to docker.io/library/messenger-minio-bootstrap:local done +#27 unpacking to docker.io/library/messenger-minio-bootstrap:local 0.4s done +#27 DONE 2.1s + +#28 [minio-bootstrap] resolving provenance for metadata file +#28 DONE 0.0s + +#26 [minio] exporting to image +#26 exporting layers 5.5s done +#26 exporting manifest sha256:a76201cbd0c3d54329b0d803cda0c96d438c8dd5aa0aa20fdd4ec7615e4230a7 0.0s done +#26 exporting config sha256:e42c72aaed36376029e3e1e4563d8b08cf94f8a6d6b6acbbaebc77f5e7b31f4e 0.0s done +#26 exporting attestation manifest sha256:b3cfce554b45ff447039c6f689ef53c48f6714642c9e43f19bee5ace2cdd9f45 0.0s done +#26 exporting manifest list sha256:efad6f1372371f4597f6fcb6c9699badd47552c70c57d42ba61e8805735237cc 0.0s done +#26 naming to docker.io/library/messenger-minio:local +#26 naming to docker.io/library/messenger-minio:local done +#26 unpacking to docker.io/library/messenger-minio:local +#26 unpacking to docker.io/library/messenger-minio:local 2.3s done +#26 DONE 8.0s + +#29 [minio] resolving provenance for metadata file +#29 DONE 0.0s + Image messenger-minio-bootstrap:local Built + Image messenger:local Built + Image messenger-minio:local Built + Network messenger-test_objects Creating + Network messenger-test_objects Created + Network messenger-test_default Creating + Network messenger-test_default Created + Volume messenger-test_rabbitmqdata Creating + Volume messenger-test_rabbitmqdata Created + Volume messenger-test_miniodata Creating + Volume messenger-test_miniodata Created + Volume messenger-test_redisdata Creating + Volume messenger-test_redisdata Created + Volume messenger-test_pgdata Creating + Volume messenger-test_pgdata Created + Container messenger-test-redis-1 Creating + Container messenger-test-minio-1 Creating + Container messenger-test-broker-1 Creating + Container messenger-test-postgres-1 Creating + Container messenger-test-postgres-1 Created + Container messenger-test-minio-1 Created + Container messenger-test-migrate-1 Creating + Container messenger-test-broker-1 Created + Container messenger-test-redis-1 Created + Container messenger-test-minio-bootstrap-1 Creating + Container messenger-test-minio-bootstrap-1 Created + Container messenger-test-migrate-1 Created + Container messenger-test-publisher-1 Creating + Container messenger-test-worker-1 Creating + Container messenger-test-cleanup-1 Creating + Container messenger-test-api-2 Creating + Container messenger-test-api-1 Creating + Container messenger-test-cleanup-1 Created + Container messenger-test-worker-1 Created + Container messenger-test-api-2 Created + Container messenger-test-publisher-1 Created + Container messenger-test-api-1 Created + Container messenger-test-nginx-1 Creating + Container messenger-test-nginx-1 Created + Container messenger-test-minio-1 Starting + Container messenger-test-postgres-1 Starting + Container messenger-test-redis-1 Starting + Container messenger-test-broker-1 Starting + Container messenger-test-broker-1 Started + Container messenger-test-postgres-1 Started + Container messenger-test-postgres-1 Waiting + Container messenger-test-minio-1 Started + Container messenger-test-minio-1 Waiting + Container messenger-test-redis-1 Started + Container messenger-test-postgres-1 Healthy + Container messenger-test-migrate-1 Starting + Container messenger-test-migrate-1 Started + Container messenger-test-postgres-1 Waiting + Container messenger-test-broker-1 Waiting + Container messenger-test-migrate-1 Waiting + Container messenger-test-minio-1 Healthy + Container messenger-test-minio-bootstrap-1 Starting + Container messenger-test-postgres-1 Healthy + Container messenger-test-minio-bootstrap-1 Started + Container messenger-test-minio-1 Waiting + Container messenger-test-minio-bootstrap-1 Waiting + Container messenger-test-postgres-1 Waiting + Container messenger-test-redis-1 Waiting + Container messenger-test-migrate-1 Waiting + Container messenger-test-migrate-1 Waiting + Container messenger-test-postgres-1 Waiting + Container messenger-test-broker-1 Waiting + Container messenger-test-minio-bootstrap-1 Waiting + Container messenger-test-minio-1 Waiting + Container messenger-test-minio-bootstrap-1 Waiting + Container messenger-test-migrate-1 Waiting + Container messenger-test-postgres-1 Healthy + Container messenger-test-minio-1 Healthy + Container messenger-test-redis-1 Healthy + Container messenger-test-minio-1 Healthy + Container messenger-test-postgres-1 Healthy + Container messenger-test-migrate-1 Exited + Container messenger-test-migrate-1 Exited + Container messenger-test-migrate-1 Exited + Container messenger-test-migrate-1 Exited + Container messenger-test-broker-1 Healthy + Container messenger-test-publisher-1 Starting + Container messenger-test-broker-1 Healthy + Container messenger-test-publisher-1 Started + Container messenger-test-minio-bootstrap-1 Exited + Container messenger-test-minio-bootstrap-1 Exited + Container messenger-test-cleanup-1 Starting + Container messenger-test-api-2 Starting + Container messenger-test-minio-bootstrap-1 Exited + Container messenger-test-worker-1 Starting + Container messenger-test-cleanup-1 Started + Container messenger-test-worker-1 Started + Container messenger-test-api-2 Started + Container messenger-test-api-1 Starting + Container messenger-test-api-1 Started + Container messenger-test-api-2 Waiting + Container messenger-test-api-1 Waiting + Container messenger-test-api-2 Healthy + Container messenger-test-api-1 Healthy + Container messenger-test-nginx-1 Starting + Container messenger-test-nginx-1 Started + +exit_code=0; duration_seconds=435.578 diff --git a/evidence/lab5/04_readiness.txt b/evidence/lab5/04_readiness.txt new file mode 100644 index 00000000..4f1c61c4 --- /dev/null +++ b/evidence/lab5/04_readiness.txt @@ -0,0 +1,47 @@ +$ docker compose --env-file .env up -d --wait --wait-timeout 240 nginx + Container messenger-test-redis-1 Running + Container messenger-test-postgres-1 Running + Container messenger-test-minio-1 Running + Container messenger-test-api-1 Running + Container messenger-test-api-2 Running + Container messenger-test-nginx-1 Running + Container messenger-test-postgres-1 Waiting + Container messenger-test-minio-1 Waiting + Container messenger-test-postgres-1 Healthy + Container messenger-test-minio-1 Healthy + Container messenger-test-minio-bootstrap-1 Starting + Container messenger-test-migrate-1 Starting + Container messenger-test-migrate-1 Started + Container messenger-test-minio-bootstrap-1 Started + Container messenger-test-postgres-1 Waiting + Container messenger-test-redis-1 Waiting + Container messenger-test-migrate-1 Waiting + Container messenger-test-minio-1 Waiting + Container messenger-test-minio-bootstrap-1 Waiting + Container messenger-test-minio-1 Healthy + Container messenger-test-postgres-1 Healthy + Container messenger-test-redis-1 Healthy + Container messenger-test-migrate-1 Exited + Container messenger-test-minio-bootstrap-1 Exited + Container messenger-test-api-2 Waiting + Container messenger-test-api-1 Waiting + Container messenger-test-api-2 Healthy + Container messenger-test-api-1 Healthy + Container messenger-test-minio-1 Waiting + Container messenger-test-postgres-1 Waiting + Container messenger-test-redis-1 Waiting + Container messenger-test-api-2 Waiting + Container messenger-test-api-1 Waiting + Container messenger-test-migrate-1 Waiting + Container messenger-test-nginx-1 Waiting + Container messenger-test-minio-bootstrap-1 Waiting + Container messenger-test-migrate-1 Exited + Container messenger-test-postgres-1 Healthy + Container messenger-test-minio-1 Healthy + Container messenger-test-minio-bootstrap-1 Exited + Container messenger-test-nginx-1 Healthy + Container messenger-test-redis-1 Healthy + Container messenger-test-api-2 Healthy + Container messenger-test-api-1 Healthy + +exit_code=0; duration_seconds=7.972 diff --git a/evidence/lab5/05_create_test_database.txt b/evidence/lab5/05_create_test_database.txt new file mode 100644 index 00000000..7cf8cd1a --- /dev/null +++ b/evidence/lab5/05_create_test_database.txt @@ -0,0 +1,3 @@ +$ docker compose --env-file .env exec -T postgres createdb -U messenger messenger_test + +exit_code=0; duration_seconds=0.266 diff --git a/evidence/lab5/06_pytest.txt b/evidence/lab5/06_pytest.txt new file mode 100644 index 00000000..376491d2 --- /dev/null +++ b/evidence/lab5/06_pytest.txt @@ -0,0 +1,7 @@ +$ python3 -m pytest -q --junitxml=evidence/pytest.xml +........................................................................ [ 35%] +........................................................................ [ 71%] +......................................................... [100%] +201 passed in 111.29s (0:01:51) + +exit_code=0; duration_seconds=115.857 diff --git a/evidence/lab5/07_smoke.txt b/evidence/lab5/07_smoke.txt new file mode 100644 index 00000000..34fdcd45 --- /dev/null +++ b/evidence/lab5/07_smoke.txt @@ -0,0 +1,4 @@ +$ python3 tests/smoke.py --base-url http://127.0.0.1:28080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +exit_code=0; duration_seconds=4.085 diff --git a/evidence/lab5/08_storage.txt b/evidence/lab5/08_storage.txt new file mode 100644 index 00000000..5884821a --- /dev/null +++ b/evidence/lab5/08_storage.txt @@ -0,0 +1,4 @@ +$ python3 -m scripts.check_s3 --base-url http://127.0.0.1:28080 --with-outage +PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console, MinIO outage/recovery + +exit_code=0; duration_seconds=22.219 diff --git a/evidence/lab5/09_outbox.txt b/evidence/lab5/09_outbox.txt new file mode 100644 index 00000000..b4e4af3d --- /dev/null +++ b/evidence/lab5/09_outbox.txt @@ -0,0 +1,4 @@ +$ python3 -m scripts.check_outbox_recovery --base-url http://127.0.0.1:28080 +{"scenario": "commit_before_publish", "attachment_id": "1ef00ad8-9c03-487b-9b8a-72e28ae5c002", "message_id": "2a2cd1d2-f1b3-4d83-9bb4-b43281240d32", "publisher_exit_code": 137, "status_after_crash": "queued", "status_after_restart": "ready", "variants": ["thumbnail", "crop", "optimized"], "reuploaded": false} + +exit_code=0; duration_seconds=15.624 diff --git a/evidence/lab5/10_jobs.txt b/evidence/lab5/10_jobs.txt new file mode 100644 index 00000000..edc676cc --- /dev/null +++ b/evidence/lab5/10_jobs.txt @@ -0,0 +1,8 @@ +$ python3 -m scripts.check_jobs --base-url http://127.0.0.1:28080 --output evidence/jobs.json +PASS: worker_stopped_restart_all +PASS: crash-after-object +PASS: crash-before-ack +PASS: fail-transient +PASS: fail-permanent + +exit_code=0; duration_seconds=302.681 diff --git a/evidence/lab5/11_persistence.txt b/evidence/lab5/11_persistence.txt new file mode 100644 index 00000000..54248ffd --- /dev/null +++ b/evidence/lab5/11_persistence.txt @@ -0,0 +1,5 @@ +$ python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:28080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +exit_code=0; duration_seconds=50.787 diff --git a/evidence/lab5/12_failover.txt b/evidence/lab5/12_failover.txt new file mode 100644 index 00000000..47809eb4 --- /dev/null +++ b/evidence/lab5/12_failover.txt @@ -0,0 +1,6 @@ +$ python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:28080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 4, "first_recovery_seconds": 2.707} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +exit_code=0; duration_seconds=19.111 diff --git a/evidence/lab5/13_dependencies.txt b/evidence/lab5/13_dependencies.txt new file mode 100644 index 00000000..d8895ae0 --- /dev/null +++ b/evidence/lab5/13_dependencies.txt @@ -0,0 +1,72 @@ +$ python3 -m pip freeze +aio-pika==10.2.2 +aiormq==7.2.5 +alembic==1.20.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.2.0 +asyncpg==0.32.0 +attrs==26.1.0 +black==25.11.0 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.2 +click==8.5.0 +fastapi==0.143.3 +greenlet==3.5.6 +h11==0.16.0 +httpcore==1.0.9 +httpcore2==2.13.1 +httpx==0.28.1 +httpx2==2.13.1 +idna==3.20 +iniconfig==2.3.1 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.12.0 +Mako==1.4.3 +MarkupSafe==3.0.4 +minio==7.2.20 +multidict==7.2.1 +mypy_extensions==1.2.0 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +opentelemetry-api==1.45.1 +packaging==26.3 +pamqp==5.0.0 +pathable==0.4.4 +pathspec==1.1.1 +pillow==13.0.0 +platformdirs==4.14.5 +pluggy==1.7.0 +propcache==0.5.4 +pycparser==3.12 +pycryptodome==4.5.0 +pydantic==2.14.1 +pydantic-settings==2.16.0 +pydantic_core==2.50.1 +Pygments==2.21.0 +pytest==9.1.1 +pytest-asyncio==1.4.1 +python-dotenv==1.2.4 +pytokens==0.4.1 +PyYAML==6.0.3 +redis==8.1.0 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +six==1.17.0 +SQLAlchemy==2.1.4 +starlette==1.8.0 +truststore==0.10.5 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.54.0 +yarl==1.25.1 + +exit_code=0; duration_seconds=1.521 diff --git a/evidence/lab5/13_environment.txt b/evidence/lab5/13_environment.txt new file mode 100644 index 00000000..ec882d1a --- /dev/null +++ b/evidence/lab5/13_environment.txt @@ -0,0 +1,4 @@ +$ docker info --format CPUs={{.NCPU}} Memory={{.MemTotal}} Server={{.ServerVersion}} Architecture={{.Architecture}} +CPUs=6 Memory=10433806336 Server=29.1.3 Architecture=x86_64 + +exit_code=0; duration_seconds=0.266 diff --git a/evidence/lab5/13_images.txt b/evidence/lab5/13_images.txt new file mode 100644 index 00000000..dc6b250a --- /dev/null +++ b/evidence/lab5/13_images.txt @@ -0,0 +1,11 @@ +$ docker image ls --digests +REPOSITORY TAG DIGEST IMAGE ID CREATED SIZE +messenger-minio-bootstrap local sha256:74593d60d131f14b268928fb3a11be446237cb7875f46d73c9530483fea30dff 74593d60d131 9 minutes ago 227MB +messenger-minio local sha256:efad6f1372371f4597f6fcb6c9699badd47552c70c57d42ba61e8805735237cc efad6f137237 9 minutes ago 336MB +messenger local sha256:ab5533cf26d3e05ddb27723090dfff83dbe8d873f213dd4eae6c92420b780c68 ab5533cf26d3 13 minutes ago 382MB +rabbitmq 4.4.0 sha256:1166559c9e77b0208b9d96bfaf265ddbdd491ce3cebb8cf990aabe9274f16d2c 1166559c9e77 4 days ago 454MB +postgres 16.10-bookworm sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74 38471f330eb8 12 months ago 621MB +nginx 1.28.0-alpine sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284 30f1c0d78e0a 19 months ago 74.5MB +redis 7.4.2-bookworm sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9 fbdbaea47b9a 22 months ago 175MB + +exit_code=0; duration_seconds=0.165 diff --git a/evidence/lab5/14_resources.txt b/evidence/lab5/14_resources.txt new file mode 100644 index 00000000..a215cf53 --- /dev/null +++ b/evidence/lab5/14_resources.txt @@ -0,0 +1,15 @@ +$ docker compose --env-file .env ps --all --format json +{"Command":"\"python -m app\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"healthy","ID":"1a3f1874981a","Image":"messenger:local","Labels":"com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.project.working_dir=/workspace,com.docker.compose.service=api,com.docker.compose.config-hash=114b05745110e93d8ba8cb248db29bffe1c3140b71325aed96272cd091d83594,com.docker.compose.oneoff=False,com.docker.compose.version=5.0.0,org.opencontainers.image.revision=development,com.docker.compose.container-number=1,com.docker.compose.depends_on=postgres:service_healthy:false,redis:service_healthy:false,migrate:service_completed_successfully:false,minio:service_healthy:false,minio-bootstrap:service_completed_successfully:false,com.docker.compose.image=sha256:ab5533cf26d3e05ddb27723090dfff83dbe8d873f213dd4eae6c92420b780c68,com.docker.compose.project=messenger-test","LocalVolumes":"0","Mounts":"/workspace/sec…,/workspace/sec…,/workspace/sec…,/workspace/sec…","Name":"messenger-test-api-1","Names":"messenger-test-api-1","Networks":"messenger-test_default,messenger-test_objects","Ports":"8080/tcp","Project":"messenger-test","Publishers":[{"URL":"","TargetPort":8080,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"9 minutes ago","Service":"api","Size":"0B","State":"running","Status":"Up 45 seconds (healthy)"} +{"Command":"\"python -m app\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"","ID":"6da79cca3f07","Image":"messenger:local","Labels":"com.docker.compose.config-hash=114b05745110e93d8ba8cb248db29bffe1c3140b71325aed96272cd091d83594,com.docker.compose.container-number=2,com.docker.compose.depends_on=minio:service_healthy:false,minio-bootstrap:service_completed_successfully:false,postgres:service_healthy:false,redis:service_healthy:false,migrate:service_completed_successfully:false,com.docker.compose.image=sha256:ab5533cf26d3e05ddb27723090dfff83dbe8d873f213dd4eae6c92420b780c68,com.docker.compose.oneoff=False,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.project.working_dir=/workspace,com.docker.compose.service=api,com.docker.compose.project=messenger-test,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.version=5.0.0,org.opencontainers.image.revision=development","LocalVolumes":"0","Mounts":"/workspace/sec…,/workspace/sec…,/workspace/sec…,/workspace/sec…","Name":"messenger-test-api-2","Names":"messenger-test-api-2","Networks":"messenger-test_default,messenger-test_objects","Ports":"","Project":"messenger-test","Publishers":[],"RunningFor":"9 minutes ago","Service":"api","Size":"0B","State":"exited","Status":"Exited (0) 15 seconds ago"} +{"Command":"\"/bin/sh /usr/local/…\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"healthy","ID":"c2cfa3448993","Image":"rabbitmq:4.4.0","Labels":"com.docker.compose.project.working_dir=/workspace,com.docker.compose.service=broker,com.docker.compose.version=5.0.0,org.opencontainers.image.version=26.04,com.docker.compose.oneoff=False,com.docker.compose.project.environment_file=/workspace/.env,org.opencontainers.image.created=2026-11-06T10:28:50.132595+00:00,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,org.opencontainers.image.description=The Ubuntu container image maintained by Canonical\n\nUbuntu is a Debian-based Linux operating system that runs from the desktop to the cloud, to all your internet connected things.\nIt is the world's most popular operating system across public clouds and OpenStack clouds.\nIt is the number one platform for containers; from Docker to Kubernetes to LXD, Ubuntu can run your containers at scale.\nFast, secure and simple, Ubuntu powers millions of PCs worldwide.\n,org.opencontainers.image.title=ubuntu,com.docker.compose.config-hash=abe87493ed68b3ddd85b0142980b9d77c3e07f26e7a40ca556df25a40fd2f6dd,com.docker.compose.depends_on=,com.docker.compose.image=sha256:1166559c9e77b0208b9d96bfaf265ddbdd491ce3cebb8cf990aabe9274f16d2c,com.docker.compose.container-number=1,com.docker.compose.project=messenger-test","LocalVolumes":"1","Mounts":"/workspace/sec…,/workspace/inf…,messenger-test…","Name":"messenger-test-broker-1","Names":"messenger-test-broker-1","Networks":"messenger-test_default","Ports":"4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp","Project":"messenger-test","Publishers":[{"URL":"","TargetPort":4369,"PublishedPort":0,"Protocol":"tcp"},{"URL":"","TargetPort":5671,"PublishedPort":0,"Protocol":"tcp"},{"URL":"","TargetPort":5672,"PublishedPort":0,"Protocol":"tcp"},{"URL":"","TargetPort":15691,"PublishedPort":0,"Protocol":"tcp"},{"URL":"","TargetPort":15692,"PublishedPort":0,"Protocol":"tcp"},{"URL":"","TargetPort":25672,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"9 minutes ago","Service":"broker","Size":"0B","State":"running","Status":"Up 56 seconds (healthy)"} +{"Command":"\"python -m app.core.…\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"","ID":"b486a7f7b261","Image":"messenger:local","Labels":"com.docker.compose.depends_on=minio-bootstrap:service_completed_successfully:false,migrate:service_completed_successfully:false,com.docker.compose.image=sha256:ab5533cf26d3e05ddb27723090dfff83dbe8d873f213dd4eae6c92420b780c68,com.docker.compose.project=messenger-test,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.version=5.0.0,com.docker.compose.config-hash=9bb81b9464f19da20919b436ce253e78fed389a30edbff843e4255a7f341d67d,com.docker.compose.oneoff=False,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.project.working_dir=/workspace,com.docker.compose.service=cleanup,org.opencontainers.image.revision=development,com.docker.compose.container-number=1","LocalVolumes":"0","Mounts":"/workspace/sec…,/workspace/sec…,/workspace/sec…","Name":"messenger-test-cleanup-1","Names":"messenger-test-cleanup-1","Networks":"messenger-test_default,messenger-test_objects","Ports":"8080/tcp","Project":"messenger-test","Publishers":[{"URL":"","TargetPort":8080,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"9 minutes ago","Service":"cleanup","Size":"0B","State":"running","Status":"Up 27 seconds"} +{"Command":"\"alembic upgrade head\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"","ID":"4974007c1ccf","Image":"messenger:local","Labels":"com.docker.compose.container-number=1,com.docker.compose.depends_on=postgres:service_healthy:false,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.project.working_dir=/workspace,com.docker.compose.service=migrate,com.docker.compose.version=5.0.0,com.docker.compose.config-hash=a788e9b10fbdac4fd868cb97d3d29cdba8392dc60d86ca5964718e5e35893832,com.docker.compose.image=sha256:ab5533cf26d3e05ddb27723090dfff83dbe8d873f213dd4eae6c92420b780c68,com.docker.compose.oneoff=False,com.docker.compose.project=messenger-test,org.opencontainers.image.revision=development","LocalVolumes":"0","Mounts":"/workspace/sec…","Name":"messenger-test-migrate-1","Names":"messenger-test-migrate-1","Networks":"messenger-test_default","Ports":"","Project":"messenger-test","Publishers":[],"RunningFor":"9 minutes ago","Service":"migrate","Size":"0B","State":"exited","Status":"Exited (0) 23 seconds ago"} +{"Command":"\"/usr/local/bin/mini…\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"healthy","ID":"7d7ada022146","Image":"messenger-minio:local","Labels":"org.opencontainers.image.source=https://github.com/minio/minio,org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z,com.docker.compose.config-hash=e79277893af54dbd88b1132c356069f2d39d5c9e7a0a08a92c797c95e022a5c9,com.docker.compose.container-number=1,com.docker.compose.image=sha256:efad6f1372371f4597f6fcb6c9699badd47552c70c57d42ba61e8805735237cc,com.docker.compose.project=messenger-test,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.project.working_dir=/workspace,com.docker.compose.service=minio,com.docker.compose.depends_on=,com.docker.compose.oneoff=False,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.version=5.0.0","LocalVolumes":"1","Mounts":"messenger-test…,/workspace/sec…,/workspace/sec…","Name":"messenger-test-minio-1","Names":"messenger-test-minio-1","Networks":"messenger-test_objects","Ports":"","Project":"messenger-test","Publishers":[],"RunningFor":"9 minutes ago","Service":"minio","Size":"0B","State":"running","Status":"Up 56 seconds (healthy)"} +{"Command":"\"python /bootstrap.py\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"","ID":"fdb0eea937f4","Image":"messenger-minio-bootstrap:local","Labels":"com.docker.compose.image=sha256:74593d60d131f14b268928fb3a11be446237cb7875f46d73c9530483fea30dff,com.docker.compose.oneoff=False,com.docker.compose.project=messenger-test,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.project.working_dir=/workspace,com.docker.compose.container-number=1,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.service=minio-bootstrap,com.docker.compose.version=5.0.0,com.docker.compose.config-hash=acef8678bb3f4bdb3a92742b27e6c2f91d3c8f44dce1c5c200f1c04d7cc1f90b,com.docker.compose.depends_on=minio:service_healthy:false","LocalVolumes":"0","Mounts":"/workspace/sec…,/workspace/sec…,/workspace/sec…,/workspace/sec…","Name":"messenger-test-minio-bootstrap-1","Names":"messenger-test-minio-bootstrap-1","Networks":"messenger-test_objects","Ports":"","Project":"messenger-test","Publishers":[],"RunningFor":"9 minutes ago","Service":"minio-bootstrap","Size":"0B","State":"exited","Status":"Exited (0) 22 seconds ago"} +{"Command":"\"/docker-entrypoint.…\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"healthy","ID":"61d735df2e2a","Image":"nginx:1.28.0-alpine","Labels":"com.docker.compose.service=nginx,maintainer=NGINX Docker Maintainers \u003cdocker-maint@nginx.com\u003e,com.docker.compose.config-hash=cd23b99b69c23e5234ff0f09160d6003ded0dcc855b0704b37476b36a843c97a,com.docker.compose.container-number=1,com.docker.compose.depends_on=api:service_healthy:false,com.docker.compose.project=messenger-test,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.version=5.0.0,com.docker.compose.image=sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.project.working_dir=/workspace","LocalVolumes":"0","Mounts":"/workspace/inf…","Name":"messenger-test-nginx-1","Names":"messenger-test-nginx-1","Networks":"messenger-test_default","Ports":"80/tcp, 127.0.0.1:28080-\u003e8080/tcp","Project":"messenger-test","Publishers":[{"URL":"","TargetPort":80,"PublishedPort":0,"Protocol":"tcp"},{"URL":"127.0.0.1","TargetPort":8080,"PublishedPort":28080,"Protocol":"tcp"}],"RunningFor":"9 minutes ago","Service":"nginx","Size":"0B","State":"running","Status":"Up 38 seconds (healthy)"} +{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"healthy","ID":"2b7da272790d","Image":"postgres:16.10-bookworm","Labels":"com.docker.compose.project=messenger-test,com.docker.compose.version=5.0.0,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.project.working_dir=/workspace,com.docker.compose.service=postgres,com.docker.compose.config-hash=c4b352a71afbe5e1107b34f473f1e5eae91ad486298c05eaca9bc50c2fae8f0e,com.docker.compose.depends_on=,com.docker.compose.image=sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74","LocalVolumes":"1","Mounts":"/workspace/sec…,messenger-test…","Name":"messenger-test-postgres-1","Names":"messenger-test-postgres-1","Networks":"messenger-test_default","Ports":"127.0.0.1:25432-\u003e5432/tcp","Project":"messenger-test","Publishers":[{"URL":"127.0.0.1","TargetPort":5432,"PublishedPort":25432,"Protocol":"tcp"}],"RunningFor":"9 minutes ago","Service":"postgres","Size":"0B","State":"running","Status":"Up 56 seconds (healthy)"} +{"Command":"\"python -m app.core.…\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"","ID":"63d95edeea59","Image":"messenger:local","Labels":"com.docker.compose.image=sha256:ab5533cf26d3e05ddb27723090dfff83dbe8d873f213dd4eae6c92420b780c68,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.project.working_dir=/workspace,com.docker.compose.container-number=1,com.docker.compose.project=messenger-test,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.service=publisher,com.docker.compose.version=5.0.0,org.opencontainers.image.revision=development,com.docker.compose.config-hash=f936f3942e96dc0e3d5b5d54dacb421cf130c701a2766f3d38e4d13468a0fd6e,com.docker.compose.depends_on=postgres:service_healthy:false,broker:service_healthy:false,migrate:service_completed_successfully:false","LocalVolumes":"0","Mounts":"/workspace/sec…,/workspace/sec…","Name":"messenger-test-publisher-1","Names":"messenger-test-publisher-1","Networks":"messenger-test_default","Ports":"8080/tcp","Project":"messenger-test","Publishers":[{"URL":"","TargetPort":8080,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"9 minutes ago","Service":"publisher","Size":"0B","State":"running","Status":"Up 28 seconds"} +{"Command":"\"/bin/sh /usr/local/…\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"healthy","ID":"6e2cd945f2a1","Image":"redis:7.4.2-bookworm","Labels":"com.docker.compose.depends_on=,com.docker.compose.image=sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9,com.docker.compose.project.working_dir=/workspace,com.docker.compose.version=5.0.0,com.docker.compose.config-hash=93e119226e15902205e89155e38a296409b1dbd31f95d5ed1b7eef2f15c5ca11,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=messenger-test,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.service=redis","LocalVolumes":"1","Mounts":"messenger-test…,/workspace/sec…,/workspace/inf…,/workspace/inf…","Name":"messenger-test-redis-1","Names":"messenger-test-redis-1","Networks":"messenger-test_default","Ports":"127.0.0.1:26379-\u003e6379/tcp","Project":"messenger-test","Publishers":[{"URL":"127.0.0.1","TargetPort":6379,"PublishedPort":26379,"Protocol":"tcp"}],"RunningFor":"9 minutes ago","Service":"redis","Size":"0B","State":"running","Status":"Up 56 seconds (healthy)"} +{"Command":"\"python -m app.core.…\"","CreatedAt":"2026-11-16 10:39:42 +0000 UTC","ExitCode":0,"Health":"","ID":"84ce32977040","Image":"messenger:local","Labels":"com.docker.compose.version=5.0.0,com.docker.compose.config-hash=6047fe46c01d4a439a6c12d2dd8553ca6502f6d22701cffe5885f3f7732e787d,com.docker.compose.image=sha256:ab5533cf26d3e05ddb27723090dfff83dbe8d873f213dd4eae6c92420b780c68,com.docker.compose.project.config_files=/workspace/compose.yaml,/workspace/compose.test.yaml,com.docker.compose.project.environment_file=/workspace/.env,com.docker.compose.project.working_dir=/workspace,org.opencontainers.image.revision=development,com.docker.compose.container-number=1,com.docker.compose.depends_on=broker:service_healthy:false,minio:service_healthy:false,minio-bootstrap:service_completed_successfully:false,migrate:service_completed_successfully:false,postgres:service_healthy:false,com.docker.compose.oneoff=False,com.docker.compose.project=messenger-test,com.docker.compose.service=worker","LocalVolumes":"0","Mounts":"/workspace/sec…,/workspace/sec…,/workspace/sec…,/workspace/sec…","Name":"messenger-test-worker-1","Names":"messenger-test-worker-1","Networks":"messenger-test_default,messenger-test_objects","Ports":"8080/tcp","Project":"messenger-test","Publishers":[{"URL":"","TargetPort":8080,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"9 minutes ago","Service":"worker","Size":"0B","State":"running","Status":"Up 21 seconds"} + +exit_code=0; duration_seconds=0.217 diff --git a/evidence/lab5/15_cleanup.txt b/evidence/lab5/15_cleanup.txt new file mode 100644 index 00000000..50325512 --- /dev/null +++ b/evidence/lab5/15_cleanup.txt @@ -0,0 +1,63 @@ +$ docker compose --env-file .env down --volumes --remove-orphans + Container messenger-test-worker-1 Stopping + Container messenger-test-publisher-1 Stopping + Container messenger-test-cleanup-1 Stopping + Container messenger-test-nginx-1 Stopping + Container messenger-test-nginx-1 Stopped + Container messenger-test-nginx-1 Removing + Container messenger-test-nginx-1 Removed + Container messenger-test-api-1 Stopping + Container messenger-test-api-2 Stopping + Container messenger-test-api-2 Stopped + Container messenger-test-api-2 Removing + Container messenger-test-api-2 Removed + Container messenger-test-publisher-1 Stopped + Container messenger-test-publisher-1 Removing + Container messenger-test-publisher-1 Removed + Container messenger-test-worker-1 Stopped + Container messenger-test-worker-1 Removing + Container messenger-test-worker-1 Removed + Container messenger-test-broker-1 Stopping + Container messenger-test-broker-1 Stopped + Container messenger-test-broker-1 Removing + Container messenger-test-broker-1 Removed + Container messenger-test-api-1 Stopped + Container messenger-test-api-1 Removing + Container messenger-test-api-1 Removed + Container messenger-test-redis-1 Stopping + Container messenger-test-redis-1 Stopped + Container messenger-test-redis-1 Removing + Container messenger-test-redis-1 Removed + Container messenger-test-cleanup-1 Stopped + Container messenger-test-cleanup-1 Removing + Container messenger-test-cleanup-1 Removed + Container messenger-test-migrate-1 Stopping + Container messenger-test-minio-bootstrap-1 Stopping + Container messenger-test-migrate-1 Stopped + Container messenger-test-migrate-1 Removing + Container messenger-test-minio-bootstrap-1 Stopped + Container messenger-test-minio-bootstrap-1 Removing + Container messenger-test-migrate-1 Removed + Container messenger-test-postgres-1 Stopping + Container messenger-test-minio-bootstrap-1 Removed + Container messenger-test-minio-1 Stopping + Container messenger-test-minio-1 Stopped + Container messenger-test-minio-1 Removing + Container messenger-test-minio-1 Removed + Container messenger-test-postgres-1 Stopped + Container messenger-test-postgres-1 Removing + Container messenger-test-postgres-1 Removed + Volume messenger-test_miniodata Removing + Volume messenger-test_pgdata Removing + Network messenger-test_objects Removing + Volume messenger-test_rabbitmqdata Removing + Network messenger-test_default Removing + Volume messenger-test_miniodata Removed + Volume messenger-test_redisdata Removing + Volume messenger-test_pgdata Removed + Volume messenger-test_rabbitmqdata Removed + Volume messenger-test_redisdata Removed + Network messenger-test_default Removed + Network messenger-test_objects Removed + +exit_code=0; duration_seconds=12.039 diff --git a/evidence/lab5/jobs.json b/evidence/lab5/jobs.json new file mode 100644 index 00000000..7a3801a7 --- /dev/null +++ b/evidence/lab5/jobs.json @@ -0,0 +1,378 @@ +{ + "status": "passed", + "results": [ + { + "scenario": "worker_stopped_restart_all", + "attachment_id": "5fe6f066-e101-4d90-9163-831c16172a02", + "queue_before": { + "name": "attachment-jobs", + "messages_unacknowledged": 0, + "consumers": 0, + "durable": true, + "messages_ready": 1 + }, + "queue_after": { + "name": "attachment-jobs", + "messages_unacknowledged": 0, + "consumers": 0, + "durable": true, + "messages_ready": 1 + }, + "after": { + "job_id": "c242b744-218e-4f72-92a1-15287200c777", + "status": "ready", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-16 10:43:46.258740+00:00", + "next_attempt_at": "2026-11-16 10:42:51.031819+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-16 10:43:46.245000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-16 10:43:46.253000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-16 10:43:46.238000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-16 10:43:46.229000+00:00" + } + ] + } + }, + { + "scenario": "crash-after-object", + "attachment_id": "1ba7f66b-4d5d-4486-bb17-bce666b9300a", + "exit_code": 137, + "before": { + "job_id": "bad5e6eb-83ec-4416-82f3-05eca251d9dc", + "status": "processing", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-16 10:43:51.644831+00:00", + "next_attempt_at": "2026-11-16 10:43:49.546018+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-16 10:43:52.425000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-16 10:43:52.431000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-16 10:43:52.418000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-16 10:43:52.410000+00:00" + } + ] + }, + "after": { + "job_id": "bad5e6eb-83ec-4416-82f3-05eca251d9dc", + "status": "ready", + "attempt": 2, + "published_attempt": 2, + "limit": 5, + "updated_at": "2026-11-16 10:45:26.285822+00:00", + "next_attempt_at": "2026-11-16 10:45:25.093727+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-16 10:45:26.271000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-16 10:45:26.279000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-16 10:45:26.263000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-16 10:45:26.252000+00:00" + } + ] + } + }, + { + "scenario": "crash-before-ack", + "attachment_id": "34afc922-5665-41a9-9fa9-c3adeb322558", + "exit_code": 137, + "before": { + "job_id": "9052f54c-ee14-42c8-88b8-1eb9ba41bfe0", + "status": "ready", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-16 10:45:32.498071+00:00", + "next_attempt_at": "2026-11-16 10:45:29.498136+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-16 10:45:32.487000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-16 10:45:32.493000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-16 10:45:32.479000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-16 10:45:32.472000+00:00" + } + ] + }, + "after": { + "job_id": "9052f54c-ee14-42c8-88b8-1eb9ba41bfe0", + "status": "ready", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-16 10:45:32.498071+00:00", + "next_attempt_at": "2026-11-16 10:45:29.498136+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-16 10:45:32.487000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-16 10:45:32.493000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-16 10:45:32.479000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-16 10:45:32.472000+00:00" + } + ] + } + }, + { + "scenario": "fail-transient", + "attachment_id": "70a818d0-be43-47c0-839f-cd97db10b3f0", + "samples": [ + { + "job_id": "354fcfb2-c7e7-4889-97eb-05e797da966e", + "status": "queued", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-16 10:45:51.222725+00:00", + "next_attempt_at": "2026-11-16 10:45:54.870940+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "354fcfb2-c7e7-4889-97eb-05e797da966e", + "status": "queued", + "attempt": 2, + "published_attempt": 2, + "limit": 5, + "updated_at": "2026-11-16 10:45:54.895421+00:00", + "next_attempt_at": "2026-11-16 10:46:03.519270+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "354fcfb2-c7e7-4889-97eb-05e797da966e", + "status": "queued", + "attempt": 3, + "published_attempt": 3, + "limit": 5, + "updated_at": "2026-11-16 10:46:03.998686+00:00", + "next_attempt_at": "2026-11-16 10:46:17.585799+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "354fcfb2-c7e7-4889-97eb-05e797da966e", + "status": "queued", + "attempt": 4, + "published_attempt": 4, + "limit": 5, + "updated_at": "2026-11-16 10:46:18.130452+00:00", + "next_attempt_at": "2026-11-16 10:46:49.925511+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "354fcfb2-c7e7-4889-97eb-05e797da966e", + "status": "failed", + "attempt": 5, + "published_attempt": 5, + "limit": 5, + "updated_at": "2026-11-16 10:46:50.445751+00:00", + "next_attempt_at": "2026-11-16 10:48:05.723411+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + } + ], + "after": { + "job_id": "354fcfb2-c7e7-4889-97eb-05e797da966e", + "status": "ready", + "attempt": 6, + "published_attempt": 6, + "limit": 10, + "updated_at": "2026-11-16 10:47:06.381957+00:00", + "next_attempt_at": "2026-11-16 10:46:58.045330+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-16 10:47:06.371000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-16 10:47:06.377000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-16 10:47:06.364000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-16 10:47:06.355000+00:00" + } + ] + }, + "replay_idempotent": true + }, + { + "scenario": "fail-permanent", + "attachment_id": "b09d3eda-e82e-413e-b1e9-086f0257192b", + "samples": [ + { + "job_id": "27d18b3e-9d01-4b13-bd1f-1adeb43fa9c0", + "status": "failed", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-16 10:47:22.177688+00:00", + "next_attempt_at": "2026-11-16 10:47:26.117060+00:00", + "last_error_code": "injected_permanent", + "permanent": true, + "objects": [] + } + ], + "after": { + "job_id": "27d18b3e-9d01-4b13-bd1f-1adeb43fa9c0", + "status": "ready", + "attempt": 2, + "published_attempt": 2, + "limit": 6, + "updated_at": "2026-11-16 10:47:38.880740+00:00", + "next_attempt_at": "2026-11-16 10:47:30.368139+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-16 10:47:38.870000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-16 10:47:38.875000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-16 10:47:38.863000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-16 10:47:38.855000+00:00" + } + ] + }, + "replay_idempotent": true + } + ] +} diff --git a/evidence/lab5/pytest.xml b/evidence/lab5/pytest.xml new file mode 100644 index 00000000..f055bc72 --- /dev/null +++ b/evidence/lab5/pytest.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/evidence/lab5/run.json b/evidence/lab5/run.json new file mode 100644 index 00000000..0b39997a --- /dev/null +++ b/evidence/lab5/run.json @@ -0,0 +1,56 @@ +{ + "status": "passed", + "project": "messenger-check-ba29fd30bf43", + "started_at": "2026-11-16T10:32:17.712357+00:00", + "cleanup": [ + { + "command": [ + "docker", + "rm", + "--force", + "messenger-check-ba29fd30bf43-runner", + "messenger-check-ba29fd30bf43-engine" + ], + "returncode": 0, + "output": "messenger-check-ba29fd30bf43-runner\nmessenger-check-ba29fd30bf43-engine" + }, + { + "command": [ + "docker", + "volume", + "rm", + "messenger-check-ba29fd30bf43-workspace", + "messenger-check-ba29fd30bf43-data" + ], + "returncode": 0, + "output": "messenger-check-ba29fd30bf43-workspace\nmessenger-check-ba29fd30bf43-data" + }, + { + "command": [ + "docker", + "network", + "rm", + "messenger-check-ba29fd30bf43-network" + ], + "returncode": 0, + "output": "messenger-check-ba29fd30bf43-network" + }, + { + "command": [ + "docker", + "image", + "rm", + "messenger-check-ba29fd30bf43:tools" + ], + "returncode": 0, + "output": "Untagged: messenger-check-ba29fd30bf43:tools\nDeleted: sha256:ff9b5f0a7af131d5616c517783c60ba0eeaa1fdc4f25dfb175b780314da597d4" + } + ], + "remaining_resources": { + "container": [], + "volume": [], + "network": [], + "image": [] + }, + "finished_at": "2026-11-16T10:49:25.044459+00:00" +} diff --git a/evidence/lab5/verification.json b/evidence/lab5/verification.json new file mode 100644 index 00000000..acf38b15 --- /dev/null +++ b/evidence/lab5/verification.json @@ -0,0 +1,7 @@ +{ + "status": "passed", + "python": "3.12.15", + "platform": "Linux-5.15.153.1-microsoft-standard-WSL2-x86_64-with-glibc2.36", + "started_at": "2026-11-16T10:32:42.404888+00:00", + "finished_at": "2026-11-16T10:49:14.501994+00:00" +} diff --git a/infra/README.md b/infra/README.md index 5a39e862..bc99c641 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,16 @@ -# Инфраструктура студента +# Инфраструктура -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +`compose.yaml`: две API-реплики за Nginx, PostgreSQL, Redis сессий, +RabbitMQ, приватный MinIO, publisher, worker и cleanup. Настройки берутся +из `.env`, секреты передаются файлами из `secrets/`. Миграции и bootstrap +завершаются до старта зависимых сервисов. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +Publisher читает outbox PostgreSQL и ждёт подтверждения RabbitMQ. +Worker фиксирует результат в MinIO/PostgreSQL до ack, prefetch=1. +Хранилища и брокер не публикуют внешние порты. -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +`python3 scripts/test.py` создаёт временные Docker daemon и тестовый клиент. +Внутри запускается тот же Compose с `compose.test.yaml`, открывающим +PostgreSQL/Redis только по loopback временной среды. После прогона +удаляются весь вложенный стенд и внешние временные ресурсы. +Результаты копируются в `evidence/lab5/` до удаления. diff --git a/infra/broker/start-broker.sh b/infra/broker/start-broker.sh new file mode 100644 index 00000000..a9e022b8 --- /dev/null +++ b/infra/broker/start-broker.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +export RABBITMQ_DEFAULT_PASS="$(cat "${BROKER_PASSWORD_FILE:?}")" +export RABBITMQ_DEFAULT_USER="${BROKER_USER:?}" +export RABBITMQ_DEFAULT_VHOST="${BROKER_VHOST:?}" +export RABBITMQ_NODE_PORT="${BROKER_PORT:?}" +exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 00000000..fd7cc06c --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.24.8-bookworm AS compiler +ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOBIN=/out GOMAXPROCS=4 +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + go install -p 4 -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z + +FROM python:3.12.15-slim-bookworm AS runtime +RUN useradd --system --uid 10001 --no-create-home minio && mkdir /data && chown minio /data +USER minio + +FROM runtime AS server +COPY --from=compiler /out/minio /usr/local/bin/minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" \ + org.opencontainers.image.version="RELEASE.2025-10-15T17-29-55Z" +VOLUME ["/data"] +ENTRYPOINT ["/usr/local/bin/minio"] + +FROM runtime AS bootstrap +COPY --from=compiler /out/mc /usr/local/bin/mc +COPY infra/minio/bootstrap.py /bootstrap.py +ENTRYPOINT ["python", "/bootstrap.py"] diff --git a/infra/minio/bootstrap.py b/infra/minio/bootstrap.py new file mode 100644 index 00000000..d4870f0a --- /dev/null +++ b/infra/minio/bootstrap.py @@ -0,0 +1,120 @@ +"""Идемпотентный private bucket и отдельная ограниченная учётная запись API.""" + +import json +import os +from pathlib import Path +import subprocess +from tempfile import TemporaryDirectory +import time +from urllib.request import urlopen + + +def secret(name): + value = Path(os.environ[name.upper() + "_FILE"]).read_text().strip() + if not value: + raise ValueError("Пустой файл секрета") + return value + + +def main(): + endpoint = os.environ["S3_ENDPOINT"] + bucket = os.environ["S3_BUCKET"] + root_user, root_password = secret("minio_root_user"), secret("minio_root_password") + access, password = secret("s3_access_key_id"), secret("s3_secret_access_key") + if access == root_user: + raise ValueError("API не должен использовать root credentials") + deadline = time.monotonic() + 60 + while True: + try: + with urlopen(endpoint + "/minio/health/ready", timeout=2) as response: + if response.status == 200: + break + except OSError: + pass + if time.monotonic() >= deadline: + raise RuntimeError("MinIO не готов за 60 секунд") + time.sleep(1) + with TemporaryDirectory(prefix="minio-bootstrap-") as directory: + config = Path(directory) + + config_file = config / "config.json" + config_file.write_text( + json.dumps( + { + "version": "10", + "aliases": { + "storage": { + "url": endpoint, + "accessKey": root_user, + "secretKey": root_password, + "api": "S3v4", + "path": "auto", + } + }, + } + ) + ) + config_file.chmod(0o600) + + def mc(*args): + result = subprocess.run( + ["mc", "--config-dir", directory, *args], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + ) + if result.returncode: + raise RuntimeError( + "Ошибка bootstrap MinIO; значения credentials скрыты" + ) + + mc("mb", "--ignore-existing", "storage/" + bucket) + mc("anonymous", "set", "none", "storage/" + bucket) + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": [f"arn:aws:s3:::{bucket}"], + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": [ + f"arn:aws:s3:::{bucket}/{prefix}/*" + for prefix in ("originals", "staging", "variants") + ], + }, + ], + } + policy_file = config / "policy.json" + policy_file.write_text(json.dumps(policy)) + mc( + "admin", + "policy", + "create", + "storage", + "messenger-attachments", + str(policy_file), + ) + + mc("admin", "user", "add", "storage", access, password) + mc( + "admin", + "policy", + "attach", + "storage", + "messenger-attachments", + "--user", + access, + ) + print("MinIO private bucket and API policy configured", flush=True) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + print("MinIO bootstrap failed: " + type(error).__name__, flush=True) + raise SystemExit(1) from None diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 00000000..04571d40 --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,124 @@ + +worker_processes 2; + +events { + worker_connections 1024; +} + +http { + map $upstream_http_x_request_id $response_request_id { + "" $request_id; + default $upstream_http_x_request_id; + } + + log_format requests escape=json + '{"time":"$time_iso8601","request_id":"$response_request_id",' + '"method":"$request_method","uri":"$uri","status":$status,' + '"bytes":$body_bytes_sent,"request_time":$request_time,' + '"upstream":"$upstream_addr","upstream_status":"$upstream_status",' + '"upstream_time":"$upstream_response_time",' + '"instance_id":"$upstream_http_x_instance_id",' + '"limit_status":"$limit_req_status"}'; + + access_log /dev/stdout requests; + error_log /dev/stderr warn; + server_tokens off; + + resolver 127.0.0.11 valid=5s ipv6=off; + resolver_timeout 1s; + + limit_req_zone $binary_remote_addr zone=per_client:10m rate=50r/s; + + gzip on; + gzip_vary on; + gzip_min_length 1024; + gzip_comp_level 3; + gzip_types application/json; + + upstream api_backend { + zone api_backend 64k; + server api:8080 resolve max_fails=1 fail_timeout=5s; + + keepalive 16; + keepalive_timeout 4s; + } + + server { + listen 8080; + + client_header_timeout 10s; + client_body_timeout 10s; + client_max_body_size 0; + client_body_buffer_size 64k; + send_timeout 10s; + keepalive_timeout 30s; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 500ms; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 2s; + + proxy_intercept_errors off; + error_page 429 = @rate_limited; + error_page 502 503 504 = @unavailable; + + location = /lb-health { + access_log off; + default_type text/plain; + return 200 "ok\n"; + } + + location /health/ { + proxy_pass http://api_backend; + } + + location / { + limit_req zone=per_client burst=100 nodelay; + limit_req_status 429; + limit_req_log_level warn; + proxy_pass http://api_backend; + } + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + limit_req zone=per_client burst=100 nodelay; + limit_req_status 429; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 65s; + proxy_send_timeout 15s; + proxy_next_upstream off; + proxy_pass http://api_backend; + } + + location ~ ^/api/v1/attachments/[^/]+/content$ { + limit_req zone=per_client burst=100 nodelay; + limit_req_status 429; + proxy_buffering off; + proxy_read_timeout 65s; + proxy_pass http://api_backend; + } + + location @rate_limited { + default_type application/json; + add_header X-Request-Id $request_id always; + add_header Retry-After 1 always; + return 429 '{"error":{"code":"rate_limited","message":"Request rate exceeded","request_id":"$request_id"}}'; + } + + location @unavailable { + default_type application/json; + add_header X-Request-Id $request_id always; + add_header Retry-After 1 always; + return 503 '{"error":{"code":"unavailable","message":"API temporarily unavailable","request_id":"$request_id"}}'; + } + } +} diff --git a/infra/redis/redis.conf b/infra/redis/redis.conf new file mode 100644 index 00000000..f2ae982f --- /dev/null +++ b/infra/redis/redis.conf @@ -0,0 +1,5 @@ +dir /data +appendonly yes +appendfsync always +maxmemory 256mb +maxmemory-policy noeviction diff --git a/infra/redis/start-redis.sh b/infra/redis/start-redis.sh new file mode 100644 index 00000000..49ee706b --- /dev/null +++ b/infra/redis/start-redis.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +password_hash=$(tr -d '\r\n' /run/redis/users.acl +chown redis:redis /run/redis/users.acl +chmod 600 /run/redis/users.acl +exec /usr/local/bin/docker-entrypoint.sh redis-server /usr/local/etc/redis/redis.conf --aclfile /run/redis/users.acl diff --git a/infra/testing/Dockerfile b/infra/testing/Dockerfile new file mode 100644 index 00000000..a1fc5396 --- /dev/null +++ b/infra/testing/Dockerfile @@ -0,0 +1,10 @@ +FROM docker:29.1.3-cli AS dockercli +FROM python:3.12.15-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 +COPY --from=dockercli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=dockercli /usr/local/libexec/docker/cli-plugins /usr/local/libexec/docker/cli-plugins +WORKDIR /workspace +COPY requirements.txt requirements-dev.txt ./ +RUN pip install -r requirements.txt -r requirements-dev.txt +COPY scripts/container_tests.py /usr/local/bin/container_tests.py +CMD ["python3", "/usr/local/bin/container_tests.py"] diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 00000000..2b7f9d9e --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,68 @@ +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from app.core.config import Settings +from app.models import Base + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name, disable_existing_loggers=False) + +target_metadata = Base.metadata + + +def _database_url() -> str: + explicit = config.get_main_option("sqlalchemy.url") + if explicit: + return explicit + return Settings().get_database_url() + + +def run_migrations_offline() -> None: + context.configure( + url=_database_url(), + target_metadata=target_metadata, + literal_binds=True, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + section = config.get_section(config.config_ini_section, {}) + section["sqlalchemy.url"] = _database_url().replace("%", "%%") + connectable = async_engine_from_config( + section, prefix="sqlalchemy.", poolclass=pool.NullPool + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 00000000..17dcba0e --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/49d2c947a3f1_add_idempotency_keys.py b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py new file mode 100644 index 00000000..153425e2 --- /dev/null +++ b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py @@ -0,0 +1,48 @@ +"""Добавляет результаты идемпотентных запросов.""" + +from alembic import op +import sqlalchemy as sa + +revision = "49d2c947a3f1" +down_revision = "abd273076068" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Создаёт таблицу ключей без изменения существующих данных.""" + op.create_table( + "idempotency_keys", + sa.Column("user_id", sa.UUID(), nullable=False), + sa.Column("method", sa.String(8), nullable=False), + sa.Column("path", sa.String(255), nullable=False), + sa.Column("key", sa.String(128), nullable=False), + sa.Column("request_hash", sa.String(64), nullable=False), + sa.Column("message_id", sa.UUID(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users") + ), + sa.ForeignKeyConstraint( + ["message_id"], + ["messages.id"], + name=op.f("fk_idempotency_keys_message_id_messages"), + ), + sa.PrimaryKeyConstraint( + "user_id", "method", "path", "key", name=op.f("pk_idempotency_keys") + ), + ) + op.create_index( + "ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"] + ) + + +def downgrade() -> None: + """Удаляет таблицу результатов запросов.""" + op.drop_index("ix_idempotency_keys_created_at", table_name="idempotency_keys") + op.drop_table("idempotency_keys") diff --git a/migrations/versions/8f31b94a2d10_add_attachments.py b/migrations/versions/8f31b94a2d10_add_attachments.py new file mode 100644 index 00000000..34d3a6d5 --- /dev/null +++ b/migrations/versions/8f31b94a2d10_add_attachments.py @@ -0,0 +1,98 @@ +"""Attachment, связи сообщений и устойчивый журнал загрузок.""" + +from alembic import op +import sqlalchemy as sa + +revision = "8f31b94a2d10" +down_revision = "49d2c947a3f1" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "attachments", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("chat_id", sa.UUID(), nullable=False), + sa.Column("owner_id", sa.UUID(), nullable=False), + sa.Column("object_key", sa.String(128), nullable=False), + sa.Column("filename", sa.String(255), nullable=False), + sa.Column("content_type", sa.String(32), nullable=False), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("sha256", sa.String(64), nullable=False), + sa.Column("status", sa.String(16), server_default="ready", nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.CheckConstraint( + "size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes") + ), + sa.CheckConstraint( + "content_type IN ('image/png', 'image/jpeg')", + name=op.f("ck_attachments_content_type"), + ), + sa.CheckConstraint( + "sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256") + ), + sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")), + sa.ForeignKeyConstraint( + ["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats") + ), + sa.ForeignKeyConstraint( + ["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")), + sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")), + ) + op.create_index("ix_attachments_chat_id", "attachments", ["chat_id"]) + op.create_table( + "message_attachments", + sa.Column("message_id", sa.UUID(), nullable=False), + sa.Column("attachment_id", sa.UUID(), nullable=False), + sa.Column("position", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["message_id"], + ["messages.id"], + ondelete="CASCADE", + name=op.f("fk_message_attachments_message_id_messages"), + ), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + name=op.f("fk_message_attachments_attachment_id_attachments"), + ), + sa.PrimaryKeyConstraint( + "message_id", "attachment_id", name=op.f("pk_message_attachments") + ), + ) + op.create_table( + "upload_intents", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("object_key", sa.String(128), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")), + sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")), + ) + op.create_index("ix_upload_intents_created_at", "upload_intents", ["created_at"]) + + +def downgrade() -> None: + op.drop_index("ix_upload_intents_created_at", table_name="upload_intents") + op.drop_table("upload_intents") + op.drop_table("message_attachments") + op.drop_index("ix_attachments_chat_id", table_name="attachments") + op.drop_table("attachments") diff --git a/migrations/versions/a62c950e18b3_rabbitmq_cutover.py b/migrations/versions/a62c950e18b3_rabbitmq_cutover.py new file mode 100644 index 00000000..c3436771 --- /dev/null +++ b/migrations/versions/a62c950e18b3_rabbitmq_cutover.py @@ -0,0 +1,23 @@ +"""Offline cutover: повторно опубликовать queued jobs в RabbitMQ.""" + +from alembic import op + +revision = "a62c950e18b3" +down_revision = "f9183d27a640" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + + op.execute( + """ + UPDATE attachment_jobs j SET published_attempt = attempt + FROM attachments a WHERE a.id = j.attachment_id AND a.status = 'queued' + """ + ) + + +def downgrade() -> None: + + upgrade() diff --git a/migrations/versions/abd273076068_add_counters.py b/migrations/versions/abd273076068_add_counters.py new file mode 100644 index 00000000..57877dd4 --- /dev/null +++ b/migrations/versions/abd273076068_add_counters.py @@ -0,0 +1,32 @@ +"""add counters + +Revision ID: abd273076068 +Revises: db06fd3126ff +Create Date: 2026-10-26 22:01:01.910396 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "abd273076068" +down_revision: Union[str, None] = "db06fd3126ff" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + + op.create_table( + "counters", + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("value", sa.BigInteger(), nullable=False), + sa.PrimaryKeyConstraint("name", name=op.f("pk_counters")), + ) + + +def downgrade() -> None: + + op.drop_table("counters") diff --git a/migrations/versions/c35e91a2f706_async_attachment_jobs.py b/migrations/versions/c35e91a2f706_async_attachment_jobs.py new file mode 100644 index 00000000..3acb20c5 --- /dev/null +++ b/migrations/versions/c35e91a2f706_async_attachment_jobs.py @@ -0,0 +1,113 @@ +"""Состояния, варианты и устойчивые jobs с намерением публикации.""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = "c35e91a2f706" +down_revision = "8f31b94a2d10" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") + op.create_check_constraint( + op.f("ck_attachments_status"), + "attachments", + "status IN ('queued', 'processing', 'ready', 'failed')", + ) + op.alter_column("attachments", "status", server_default="queued") + op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True)) + op.add_column( + "attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True) + ) + op.add_column( + "attachments", + sa.Column( + "variants", + postgresql.JSONB(), + nullable=False, + server_default=sa.text("'[]'::jsonb"), + ), + ) + op.create_check_constraint( + op.f("ck_attachments_error_code"), + "attachments", + "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", + ) + op.create_table( + "attachment_jobs", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("attachment_id", sa.UUID(), nullable=False), + sa.Column("staging_key", sa.String(128), nullable=True), + sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False), + sa.Column("attempt", sa.Integer(), server_default="0", nullable=False), + sa.Column( + "published_attempt", sa.Integer(), server_default="0", nullable=False + ), + sa.Column( + "next_attempt_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.Column("lease_token", sa.UUID(), nullable=True), + sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_error_code", sa.String(64), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + name=op.f("fk_attachment_jobs_attachment_id_attachments"), + ), + sa.UniqueConstraint( + "attachment_id", name=op.f("uq_attachment_jobs_attachment_id") + ), + sa.CheckConstraint( + "pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version") + ), + sa.CheckConstraint( + "attempt >= 0 AND published_attempt >= 0", + name=op.f("ck_attachment_jobs_attempt"), + ), + ) + op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"]) + op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"]) + + op.execute( + "INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments" + ) + op.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()" + ) + + +def downgrade() -> None: + connection = op.get_bind() + unfinished = connection.scalar( + sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'") + ) + if unfinished: + raise RuntimeError( + "Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения" + ) + op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs") + op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs") + op.drop_table("attachment_jobs") + op.drop_constraint(op.f("ck_attachments_error_code"), "attachments", type_="check") + op.drop_column("attachments", "variants") + op.drop_column("attachments", "thumbnail_key") + op.drop_column("attachments", "error_code") + op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") + op.create_check_constraint( + op.f("ck_attachments_status"), "attachments", "status = 'ready'" + ) + op.alter_column("attachments", "status", server_default="ready") diff --git a/migrations/versions/db06fd3126ff_create_initial_schema.py b/migrations/versions/db06fd3126ff_create_initial_schema.py new file mode 100644 index 00000000..77e552e1 --- /dev/null +++ b/migrations/versions/db06fd3126ff_create_initial_schema.py @@ -0,0 +1,105 @@ +"""create initial schema + +Revision ID: db06fd3126ff +Revises: +Create Date: 2026-10-26 17:09:16.675339 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "db06fd3126ff" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + + op.create_table( + "chats", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("title", sa.String(length=100), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_chats")), + ) + op.create_table( + "users", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("username", sa.String(length=32), nullable=False), + sa.Column("display_name", sa.String(length=100), nullable=False), + sa.Column("password_hash", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), + sa.UniqueConstraint("username", name=op.f("uq_users_username")), + ) + op.create_table( + "chat_members", + sa.Column("chat_id", sa.UUID(), nullable=False), + sa.Column("user_id", sa.UUID(), nullable=False), + sa.ForeignKeyConstraint( + ["chat_id"], + ["chats.id"], + name=op.f("fk_chat_members_chat_id_chats"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], ["users.id"], name=op.f("fk_chat_members_user_id_users") + ), + sa.PrimaryKeyConstraint("chat_id", "user_id", name=op.f("pk_chat_members")), + ) + op.create_index( + "ix_chat_members_user_id_chat_id", + "chat_members", + ["user_id", "chat_id"], + unique=False, + ) + op.create_table( + "messages", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("chat_id", sa.UUID(), nullable=False), + sa.Column("sender_id", sa.UUID(), nullable=False), + sa.Column("text", sa.Text(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["chat_id"], ["chats.id"], name=op.f("fk_messages_chat_id_chats") + ), + sa.ForeignKeyConstraint( + ["sender_id"], ["users.id"], name=op.f("fk_messages_sender_id_users") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), + ) + op.create_index( + "ix_messages_chat_id_created_at_id", + "messages", + ["chat_id", "created_at", "id"], + unique=False, + ) + + +def downgrade() -> None: + + op.drop_index("ix_messages_chat_id_created_at_id", table_name="messages") + op.drop_table("messages") + op.drop_index("ix_chat_members_user_id_chat_id", table_name="chat_members") + op.drop_table("chat_members") + op.drop_table("users") + op.drop_table("chats") diff --git a/migrations/versions/e7048b6da913_attachment_crop_webp.py b/migrations/versions/e7048b6da913_attachment_crop_webp.py new file mode 100644 index 00000000..153c50b8 --- /dev/null +++ b/migrations/versions/e7048b6da913_attachment_crop_webp.py @@ -0,0 +1,63 @@ +"""Ключи crop/WebP и дообработка старых ready-вложений.""" + +from alembic import op +import sqlalchemy as sa + +revision = "e7048b6da913" +down_revision = "c35e91a2f706" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True)) + op.add_column( + "attachments", sa.Column("optimized_key", sa.String(128), nullable=True) + ) + + op.execute( + """ + INSERT INTO upload_intents (id, object_key) + SELECT gen_random_uuid(), j.staging_key + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE a.status = 'ready' AND j.staging_key IS NOT NULL + AND j.staging_key <> a.object_key + ON CONFLICT (object_key) DO NOTHING + """ + ) + op.execute( + """ + UPDATE attachment_jobs j + SET staging_key = a.object_key, attempt = 0, published_attempt = 0, + next_attempt_at = clock_timestamp(), lease_token = NULL, lease_until = NULL, + last_error_code = NULL, finished_at = NULL + FROM attachments a + WHERE a.id = j.attachment_id AND a.status = 'ready' + """ + ) + op.execute( + """ + UPDATE attachments SET status = 'queued', variants = '[]'::jsonb, + updated_at = clock_timestamp() + WHERE status = 'ready' + """ + ) + + +def downgrade() -> None: + active = op.get_bind().scalar( + sa.text( + "SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')" + ) + ) + if active: + raise RuntimeError("Перед downgrade необходимо завершить активные задания") + op.execute( + """ + UPDATE attachments a SET variants = COALESCE( + (SELECT jsonb_agg(v) FROM jsonb_array_elements(a.variants) v + WHERE v->>'name' = 'thumbnail'), '[]'::jsonb) + """ + ) + op.drop_column("attachments", "optimized_key") + op.drop_column("attachments", "crop_key") diff --git a/migrations/versions/f9183d27a640_job_replay.py b/migrations/versions/f9183d27a640_job_replay.py new file mode 100644 index 00000000..0c495fbf --- /dev/null +++ b/migrations/versions/f9183d27a640_job_replay.py @@ -0,0 +1,51 @@ +"""Бюджет повторов и аудит операторского replay.""" + +from alembic import op +import sqlalchemy as sa + +revision = "f9183d27a640" +down_revision = "e7048b6da913" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "attachment_jobs", + sa.Column( + "cycle_start_attempt", sa.Integer(), nullable=False, server_default="0" + ), + ) + op.add_column( + "attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True) + ) + op.add_column( + "attachment_jobs", + sa.Column("last_error_permanent", sa.Boolean(), nullable=True), + ) + op.create_table( + "job_replays", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column( + "job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False + ), + sa.Column("actor", sa.String(64), nullable=False), + sa.Column("reason", sa.String(32), nullable=False), + sa.Column("previous_attempt", sa.Integer(), nullable=False), + sa.Column("attempt_limit", sa.Integer(), nullable=False), + sa.Column("previous_error_code", sa.String(64), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.clock_timestamp(), + ), + ) + op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"]) + + +def downgrade() -> None: + op.drop_table("job_replays") + op.drop_column("attachment_jobs", "last_error_permanent") + op.drop_column("attachment_jobs", "attempt_limit") + op.drop_column("attachment_jobs", "cycle_start_attempt") diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..b7dd8fa8 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = student-tests +python_files = test_*.py +asyncio_mode = auto +asyncio_default_fixture_loop_scope = session +asyncio_default_test_loop_scope = session diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103ddb..823485e4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ -# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 +black==25.11.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..a3f67b5b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,53 @@ +alembic==1.20.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +attrs==26.1.0 +certifi==2026.7.22 +charset-normalizer==3.5.2 +click==8.5.0 +fastapi==0.143.3 +h11==0.16.0 +httpcore2==2.13.1 +httpx2==2.13.1 +idna==3.20 +iniconfig==2.3.1 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.12.0 +Mako==1.4.3 +MarkupSafe==3.0.4 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +opentelemetry-api==1.45.1 +packaging==26.3 +pathable==0.4.4 +pluggy==1.7.0 +pydantic==2.14.1 +pydantic_core==2.50.1 +Pygments==2.21.0 +pytest==9.1.1 +PyYAML==6.0.3 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +six==1.17.0 +SQLAlchemy==2.1.4 +starlette==1.8.0 +truststore==0.10.5 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.54.0 +asyncpg==0.32.0 +argon2-cffi==25.1.0 +redis==8.1.0 +greenlet==3.5.6 +pydantic-settings==2.16.0 +pytest-asyncio==1.4.1 +httpx==0.28.1 +minio==7.2.20 +Pillow==13.0.0 +aio-pika==10.2.2 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..592ea408 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Команды управления учебным стендом.""" diff --git a/scripts/check_contract.py b/scripts/check_contract.py index dcb34dfa..e56f7c40 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Offline structural checks; full schema validation is a separate target.""" + import ast import json from pathlib import Path @@ -7,48 +8,71 @@ import re ROOT = Path(__file__).resolve().parents[1] + def main(): - lab = json.loads((ROOT/'lab.json').read_text())['number'] - spec = json.loads((ROOT/'contracts/openapi.json').read_text()) - assert spec['openapi']=='3.1.0' - assert spec['info']['version']=='1.%d.0'%lab + lab = json.loads((ROOT / "lab.json").read_text())["number"] + spec = json.loads((ROOT / "contracts/openapi.json").read_text(encoding="utf-8")) + assert spec["openapi"] == "3.1.0" + assert spec["info"]["version"] == "1.%d.0" % lab operation_ids = set() + def walk(value): - if isinstance(value,dict): - if '$ref' in value: - pointer = value['$ref'] - assert pointer.startswith('#/'), 'unexpected external reference' + if isinstance(value, dict): + if "$ref" in value: + pointer = value["$ref"] + assert pointer.startswith("#/"), "unexpected external reference" target = spec - for part in pointer[2:].split('/'): - target = target[part.replace('~1','/').replace('~0','~')] + for part in pointer[2:].split("/"): + target = target[part.replace("~1", "/").replace("~0", "~")] for v in value.values(): walk(v) - elif isinstance(value,list): + elif isinstance(value, list): for v in value: walk(v) - walk(spec) - for route,methods in spec['paths'].items(): - placeholders = set(re.findall(r'\{([^}]+)\}',route)) - for method,op in methods.items(): - assert op['operationId'] not in operation_ids, 'duplicate operationId' - operation_ids.add(op['operationId']) - params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')} - assert placeholders==params, 'path parameter mismatch: '+route - for response in op['responses'].values(): - assert 'X-Request-Id' in response['headers'] - if lab>=3: - assert 'X-Instance-Id' in response['headers'] - assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2) - assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4) - for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')): - ast.parse(file.read_text(),filename=str(file)) - for file in ROOT.rglob('*.json'): - if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): - continue - json.loads(file.read_text()) - for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'): - assert (ROOT/required).is_file(), 'missing '+required - print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.') -if __name__=='__main__': + walk(spec) + for route, methods in spec["paths"].items(): + placeholders = set(re.findall(r"\{([^}]+)\}", route)) + for method, op in methods.items(): + assert op["operationId"] not in operation_ids, "duplicate operationId" + operation_ids.add(op["operationId"]) + params = { + p["name"] + for p in op.get("parameters", []) + if p["in"] == "path" and p.get("required") + } + assert placeholders == params, "path parameter mismatch: " + route + for response in op["responses"].values(): + assert "X-Request-Id" in response["headers"] + if lab >= 3: + assert "X-Instance-Id" in response["headers"] + assert ("/api/v1/auth/sessions" in spec["paths"]) == (lab >= 2) + assert ("/api/v1/attachments/{attachment_id}" in spec["paths"]) == (lab >= 4) + for file in list((ROOT / "tests").glob("*.py")) + list( + (ROOT / "scripts").glob("*.py") + ): + ast.parse(file.read_text(), filename=str(file)) + for file in ROOT.rglob("*.json"): + if any( + part in {".git", ".venv", "node_modules", "vendor", "output"} + for part in file.parts + ): + continue + json.loads(file.read_text(encoding="utf-8")) + for required in ( + "README.md", + "COURSE.md", + "CONTRIBUTING.md", + "REPORT.md", + "contracts/README.md", + "tests/smoke.py", + "tests/README.md", + ): + assert (ROOT / required).is_file(), "missing " + required + print( + "PASS: offline skeleton structure, local refs and Python syntax; application NOT tested." + ) + + +if __name__ == "__main__": main() diff --git a/scripts/check_jobs.py b/scripts/check_jobs.py new file mode 100644 index 00000000..5ce23346 --- /dev/null +++ b/scripts/check_jobs.py @@ -0,0 +1,429 @@ +"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные.""" + +import argparse +from datetime import datetime +import hashlib +from io import BytesIO +import json +from pathlib import Path +import subprocess +import time +from uuid import UUID, uuid4 + +from PIL import Image + +from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose +from scripts.deployment import ComposeProject +from tests.client import Client, check + +PROBE = """ +import asyncio, hashlib, json +from uuid import UUID +from sqlalchemy import select +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.models.job import AttachmentJob +from app.models.attachment import Attachment +from app.storage.s3 import create_s3_storage +async def main(): + settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3) + try: + async with create_session_factory(engine)() as session: + job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one() + result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent) + objects=[] + for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'): + entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True))) + for obj in entries: + stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256() + try: + async for chunk in stream: checksum.update(chunk) + finally: await stream.aclose() + objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified)) + result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str)) + finally: await storage.aclose();await engine.dispose() +asyncio.run(main()) +""" + + +def probe(attachment_id): + code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE + result = subprocess.run( + COMPOSE + ["exec", "-T", "api", "python", "-"], + input=code, + text=True, + capture_output=True, + cwd=ROOT, + timeout=30, + ) + check(result.returncode == 0, "Probe failed; diagnostic output hidden") + return json.loads(result.stdout) + + +def admin(*arguments, expected=0): + result = compose( + "run", + "--rm", + "--no-deps", + "worker", + "python", + "-m", + "app.core.job_admin", + *arguments + ) + check(result.returncode == expected, "Unexpected operator command exit status") + return json.loads(result.stdout) + + +def verify_result(client, actor, attachment, original): + ready = client.wait_ready(actor, attachment, wait_seconds=180) + check(ready["id"] == attachment["id"], "Attachment identity changed") + path = "/api/v1/attachments/" + ready["id"] + "/content" + check( + client.call("GET", path, actor=actor)["bytes"] == original, "Original changed" + ) + check( + {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}, + "Incomplete pipeline", + ) + for variant in ready["variants"]: + data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[ + "bytes" + ] + check( + hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch" + ) + with Image.open(BytesIO(data)) as image: + check( + image.size == (variant["width"], variant["height"]), + "Metadata dimensions mismatch", + ) + check( + image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"), + "Wrong format", + ) + check( + not image.getexif() + and not image.info.get("icc_profile") + and not image.info.get("xmp"), + "Metadata leaked", + ) + if variant["name"] == "crop": + check(image.size == (128, 128), "Wrong crop size") + snapshot = probe(ready["id"]) + check( + len(snapshot["objects"]) == 4, + "Expected exactly one original and three variants", + ) + return snapshot + + +def crash(client, actor, chat, mode): + name = "messenger-worker-fault-" + uuid4().hex + try: + require_compose("stop", "worker") + attachment, original = client.attachment(actor, chat) + result = compose( + "run", + "--rm", + "--no-deps", + "--name", + name, + "worker", + "python", + "-m", + "app.core.job_worker", + "--" + mode, + attachment["id"], + ) + phase = ( + "after_object:optimized" if mode == "crash-after-object" else "before_ack" + ) + marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]}) + check( + result.returncode == 137 and marker in result.stdout, + "Targeted crash not observed", + ) + before = probe(attachment["id"]) + check( + before["status"] + == ("processing" if mode == "crash-after-object" else "ready"), + "Unexpected crash state", + ) + check(len(before["objects"]) == 4, "Crash must occur after all four PUTs") + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) + require_compose("start", "worker") + after = verify_result(client, actor, attachment, original) + if mode == "crash-before-ack": + + wait_queue_empty() + after = probe(attachment["id"]) + check(after == before, "Ready duplicate rewrote result") + else: + check( + after["attempt"] == before["attempt"] + 1, + "Lease did not recover the interrupted attempt", + ) + check( + [(x["name"], x["sha256"]) for x in before["objects"]] + == [(x["name"], x["sha256"]) for x in after["objects"]], + "Replay changed object content", + ) + return dict( + scenario=mode, + attachment_id=attachment["id"], + exit_code=137, + before=before, + after=after, + ) + + +def retry_replay(client, actor, chat, permanent): + name = "messenger-retry-fault-" + uuid4().hex + samples = [] + configuration = ComposeProject().config["services"]["worker"]["environment"] + retry_base = float(configuration["JOB_RETRY_SECONDS"]) + retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"]) + try: + require_compose("stop", "worker") + attachment, original = client.attachment(actor, chat) + mode = "fail-permanent" if permanent else "fail-transient" + require_compose( + "run", + "-d", + "--no-deps", + "--name", + name, + "worker", + "python", + "-m", + "app.core.job_worker", + "--" + mode, + attachment["id"], + ) + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + sample = probe(attachment["id"]) + if ( + sample["attempt"] + and sample["status"] in {"queued", "failed"} + and (not samples or samples[-1]["attempt"] != sample["attempt"]) + ): + samples.append(sample) + if sample["status"] == "failed": + break + time.sleep(0.5) + check(sample["status"] == "failed", "Retry never reached failed") + check( + sample["attempt"] == (1 if permanent else sample["limit"]), + "Incorrect attempt budget", + ) + check(sample["permanent"] == permanent, "Incorrect error classification") + if not permanent: + check(len(samples) == sample["limit"], "Missed retry samples") + for entry in samples: + delay = ( + datetime.fromisoformat(entry["next_attempt_at"]) + - datetime.fromisoformat(entry["updated_at"]) + ).total_seconds() + ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1)) + check( + ceiling / 2 <= delay <= ceiling, + "Retry delay outside configured jitter bounds", + ) + client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=actor, + expected=409, + ) + job_id = sample["job_id"] + request_id = str(uuid4()) + listing = admin("list-failed") + check( + any(row["job_id"] == job_id for row in listing), + "Job missing from failed registry", + ) + subprocess.run( + ["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True + ) + first = admin( + "replay", + job_id, + "--request-id", + request_id, + "--actor", + "verification", + "--reason", + "dependency_restored", + ) + check(first["status"] == "queued", "Replay did not queue job") + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) + require_compose("start", "worker") + after = verify_result(client, actor, attachment, original) + repeat = admin( + "replay", + job_id, + "--request-id", + request_id, + "--actor", + "verification", + "--reason", + "dependency_restored", + ) + check(repeat["already_applied"], "Replay request id not idempotent") + refused = admin( + "replay", + job_id, + "--request-id", + str(uuid4()), + "--actor", + "verification", + "--reason", + "manual_review", + expected=2, + ) + check(refused["error_code"] == "job_not_failed", "Successful job was replayed") + check(probe(attachment["id"]) == after, "Operator replay altered ready job") + check(len(admin("history", job_id)) == 1, "Incorrect audit count") + return dict( + scenario=mode, + attachment_id=attachment["id"], + samples=samples, + after=after, + replay_idempotent=True, + ) + + +def queue_state(): + output = require_compose( + "exec", + "-T", + "broker", + "rabbitmqctl", + "list_queues", + "--formatter=json", + "name", + "messages_ready", + "messages_unacknowledged", + "consumers", + "durable", + ) + return next(row for row in json.loads(output) if row["name"] == "attachment-jobs") + + +def wait_queue_empty(): + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + state = queue_state() + if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0: + return state + time.sleep(0.5) + check(False, "RabbitMQ did not acknowledge all completed deliveries") + + +def persistence(client, actor, chat): + try: + require_compose("stop", "worker") + attachment, original = client.attachment(actor, chat) + client.message(actor, chat, "", attachments=[attachment["id"]]) + client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=actor, + expected=409, + ) + time.sleep(2) + queued_before = queue_state() + check( + queued_before["messages_ready"] >= 1 and queued_before["durable"], + "No durable queued message before restart", + ) + result = subprocess.run( + [str(ROOT / "scripts/restart.sh")], + cwd=ROOT, + capture_output=True, + timeout=180, + ) + check(result.returncode == 0, "Restart hook failed") + check( + "worker" + not in require_compose( + "ps", "--status", "running", "--services" + ).splitlines(), + "Restart started deliberately stopped worker", + ) + queued_after = queue_state() + check( + queued_after["messages_ready"] >= 1 and queued_after["durable"], + "Broker lost queued delivery across restart", + ) + client.message(actor, chat, "session and chat survived broker/API restart") + finally: + require_compose("start", "worker") + after = verify_result(client, actor, attachment, original) + return dict( + scenario="worker_stopped_restart_all", + attachment_id=attachment["id"], + queue_before=queued_before, + queue_after=queued_after, + after=after, + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--ca-file") + parser.add_argument( + "--scenario", + choices=[ + "all", + "persistence", + "crash-after-object", + "crash-before-ack", + "fail-transient", + "fail-permanent", + ], + default="all", + ) + parser.add_argument("--output", default="evidence/jobs.json") + args = parser.parse_args() + client = Client(args.base_url, args.ca_file, timeout=40) + aliases = {r.instance_id for r in ComposeProject().replicas() if r.running} + check( + client.call("GET", "/health/live", schema="Health")["headers"].get( + "X-Instance-Id" + ) + in aliases, + "Wrong Compose project for BASE_URL", + ) + actor, _, _, chat = client.bootstrap() + results = [] + operations = { + "persistence": persistence, + "crash-after-object": lambda *x: crash(*x, "crash-after-object"), + "crash-before-ack": lambda *x: crash(*x, "crash-before-ack"), + "fail-transient": lambda *x: retry_replay(*x, False), + "fail-permanent": lambda *x: retry_replay(*x, True), + } + selected = ( + operations.values() if args.scenario == "all" else [operations[args.scenario]] + ) + for operation in selected: + result = operation(client, actor, chat) + results.append(result) + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text( + json.dumps(dict(status="in_progress", results=results), indent=2) + "\n" + ) + print("PASS: " + result["scenario"], flush=True) + Path(args.output).write_text( + json.dumps(dict(status="passed", results=results), indent=2) + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_outbox_recovery.py b/scripts/check_outbox_recovery.py new file mode 100644 index 00000000..620a4d7a --- /dev/null +++ b/scripts/check_outbox_recovery.py @@ -0,0 +1,147 @@ +"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload. + +Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API. +Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет. +""" + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +from uuid import uuid4 + +from scripts.deployment import ComposeProject +from tests.client import Client, check + +ROOT = Path(__file__).resolve().parents[1] +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess: + return subprocess.run( + COMPOSE + list(arguments), + cwd=ROOT, + text=True, + capture_output=True, + timeout=timeout, + ) + + +def require_compose(*arguments: str) -> str: + result = compose(*arguments) + check(result.returncode == 0, "Compose command failed; output hidden") + return result.stdout + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Отказ publisher после COMMIT перед RabbitMQ publish" + ) + parser.add_argument("--base-url", required=True) + parser.add_argument("--ca-file") + args = parser.parse_args() + running = set( + require_compose("ps", "--status", "running", "--services").splitlines() + ) + check( + {"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, + "Start the application stack before this scenario", + ) + client = Client(args.base_url, args.ca_file, timeout=40) + project = ComposeProject() + aliases = {replica.instance_id for replica in project.replicas() if replica.running} + live = client.call("GET", "/health/live", schema="Health") + check( + live["headers"].get("X-Instance-Id") in aliases, + "BASE_URL does not belong to the selected Compose project", + ) + alice, _, _, chat = client.bootstrap() + attachment = None + fault_container = "messenger-outbox-fault-" + uuid4().hex + try: + require_compose("stop", "publisher") + attachment, original = client.attachment(alice, chat) + message = client.message(alice, chat, "", attachments=[attachment["id"]])[ + "json" + ] + client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content", + expected=409, + actor=alice, + ) + killed = compose( + "run", + "--rm", + "--no-deps", + "--name", + fault_container, + "publisher", + "python", + "-m", + "app.core.outbox_publisher", + "--crash-before-publish", + attachment["id"], + ) + marker = json.dumps( + {"fault": "before_publish", "attachment_id": attachment["id"]} + ) + check( + killed.returncode == 137 and marker in killed.stdout, + "Expected targeted SIGKILL before publish was not observed", + ) + pending = client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}", + actor=alice, + schema="Attachment", + )["json"] + check(pending["status"] == "queued", "Crash must leave accepted job queued") + finally: + + try: + subprocess.run( + ["docker", "rm", "-f", fault_container], capture_output=True, timeout=30 + ) + finally: + require_compose("start", "publisher") + ready = client.wait_ready(alice, attachment, wait_seconds=60) + check(ready["id"] == attachment["id"], "Recovered a different attachment") + check( + {variant["name"] for variant in ready["variants"]} + == {"thumbnail", "crop", "optimized"}, + "Missing required variants", + ) + path = f"/api/v1/attachments/{attachment['id']}/content" + check( + client.call("GET", path, actor=alice)["bytes"] == original, + "Original changed after recovery", + ) + for variant in ready["variants"]: + downloaded = client.call( + "GET", path + "?variant=" + variant["name"], actor=alice + )["bytes"] + check( + len(downloaded) == variant["size_bytes"] + and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], + "Recovered variant checksum mismatch", + ) + print( + json.dumps( + { + "scenario": "commit_before_publish", + "attachment_id": ready["id"], + "message_id": message["id"], + "publisher_exit_code": killed.returncode, + "status_after_crash": pending["status"], + "status_after_restart": ready["status"], + "variants": [variant["name"] for variant in ready["variants"]], + "reuploaded": False, + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_s3.py b/scripts/check_s3.py new file mode 100644 index 00000000..60081ff7 --- /dev/null +++ b/scripts/check_s3.py @@ -0,0 +1,152 @@ +"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ.""" + +import json +import subprocess +import time + +from scripts.deployment import ComposeProject +from tests.client import Client, arguments, check, run + +PROBE = r""" +import asyncio, json, sys +from pathlib import Path +from uuid import UUID +from urllib.request import Request, urlopen +from urllib.error import HTTPError, URLError +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.storage.s3 import create_s3_storage +from app.models.attachment import Attachment +from minio.error import S3Error +async def main(): + settings = Settings() + engine = create_db_engine(settings) + storage = create_s3_storage(settings.s3) + try: + async with create_session_factory(engine)() as session: + record = await session.get(Attachment, UUID(json.load(sys.stdin)['attachment_id'])) + key = record.object_key + results = {} + for method in ('GET', 'PUT'): + request = Request(settings.s3.endpoint + '/' + settings.s3.bucket + '/' + key, + method=method, data=b'x' if method == 'PUT' else None) + try: + with urlopen(request, timeout=3) as response: + results[method] = response.status + except HTTPError as error: + results[method] = error.code + error.close() + try: + storage._client.get_bucket_policy(settings.s3.bucket) + results['admin_denied'] = False + except S3Error as error: + results['admin_denied'] = error.code == 'AccessDenied' + try: + buckets = storage._client.list_buckets() + results['other_buckets_denied'] = all(bucket.name == settings.s3.bucket for bucket in buckets) + except S3Error as error: + results['other_buckets_denied'] = error.code == 'AccessDenied' + try: + with urlopen(settings.s3.endpoint.replace(':9000', ':9001'), timeout=1): + results['console_unreachable'] = False + except HTTPError as error: + results['console_unreachable'] = False + error.close() + except URLError: + results['console_unreachable'] = True + results['temp_originals_absent'] = not any(Path('/tmp').glob('messenger-upload-*')) + results['root_files_absent'] = not any(Path('/run/secrets', name).exists() + for name in ('minio_root_user', 'minio_root_password')) + print(json.dumps(results)) + finally: + await storage.aclose() + await engine.dispose() +asyncio.run(main()) +""" + + +def main(): + def configure(parser): + parser.add_argument( + "--with-outage", + action="store_true", + help="временно остановить MinIO выбранного проекта", + ) + + args = arguments("MinIO private bucket and failure checks", configure) + project = ComposeProject() + client = Client(args.base_url, args.ca_file, timeout=40) + alice, _, _, chat = client.bootstrap() + aliases = {replica.instance_id for replica in project.replicas() if replica.running} + live = client.call("GET", "/health/live", schema="Health") + check( + live["headers"].get("X-Instance-Id") in aliases, + "BASE_URL does not belong to the selected Compose project", + ) + attachment, original = client.attachment(alice, chat) + attachment = client.wait_ready(alice, attachment) + + for _ in range(2): + project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100) + replicas = project.replicas() + check(bool(replicas), "API not running") + result = subprocess.run( + ["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE], + input=json.dumps({"attachment_id": attachment["id"]}), + text=True, + capture_output=True, + timeout=20, + ) + check(result.returncode == 0, "private bucket probe failed (details hidden)") + statuses = json.loads(result.stdout) + check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed") + check( + statuses["admin_denied"] + and statuses["other_buckets_denied"] + and statuses["root_files_absent"], + "API has excessive access", + ) + check(statuses["console_unreachable"], "console reachable from API network") + check(statuses["temp_originals_absent"], "API retained temporary originals") + check( + not project.config["services"]["minio"].get("ports"), + "MinIO publishes host ports", + ) + env = project.config["services"]["minio"]["environment"] + check(env["MINIO_BROWSER"] == "off", "browser enabled") + check( + "127.0.0.1:9001" in project.config["services"]["minio"]["command"], + "console is not bound to loopback", + ) + path = "/api/v1/attachments/" + attachment["id"] + "/content" + check( + client.call("GET", path, actor=alice)["bytes"] == original, + "bootstrap changed original", + ) + if args.with_outage: + try: + project.run("stop", "minio") + client.call("GET", path, expected=503, actor=alice) + finally: + project.run("start", "minio") + deadline = time.monotonic() + 60 + while True: + try: + client.call("GET", "/health/ready", schema="Health") + break + except Exception: + if time.monotonic() >= deadline: + raise + time.sleep(0.5) + check( + client.call("GET", path, actor=alice)["bytes"] == original, + "original lost after MinIO restart", + ) + print( + "PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console" + + (", MinIO outage/recovery" if args.with_outage else "") + ) + + +if __name__ == "__main__": + run(main) diff --git a/scripts/container_tests.py b/scripts/container_tests.py new file mode 100644 index 00000000..c99bc45e --- /dev/null +++ b/scripts/container_tests.py @@ -0,0 +1,176 @@ +"""Проверки внутри Linux-контейнера с отдельным Docker daemon.""" + +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import subprocess +import tarfile +import time + +ROOT = Path("/workspace") +EVIDENCE = ROOT / "evidence" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def step(name, command, timeout=1800): + print(name, flush=True) + started = time.monotonic() + with (EVIDENCE / (name + ".txt")).open("w") as output: + output.write("$ " + " ".join(command) + "\n") + output.flush() + result = subprocess.run( + command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT, timeout=timeout + ) + output.write( + f"\nexit_code={result.returncode}; duration_seconds={time.monotonic() - started:.3f}\n" + ) + result.check_returncode() + + +def main(): + with tarfile.open("/tmp/source.tar") as archive: + archive.extractall(ROOT, filter="data") + os.chdir(ROOT) + EVIDENCE.mkdir(exist_ok=True) + deadline = time.monotonic() + 120 + while subprocess.run(["docker", "info"], capture_output=True).returncode: + if time.monotonic() > deadline: + raise RuntimeError("Docker daemon did not start") + time.sleep(1) + os.environ.update( + COMPOSE_PROJECT_NAME="messenger-test", + COMPOSE_FILE="compose.yaml:compose.test.yaml", + LB_HOST_PORT="28080", + DOCKER_TESTS="1", + TEST_USE_MIGRATIONS="1", + TEST_REDIS_HOST="127.0.0.1", + TEST_REDIS_PORT="26379", + TEST_REDIS_DB="15", + ) + result = { + "status": "failed", + "python": platform.python_version(), + "platform": platform.platform(), + "started_at": datetime.now(timezone.utc).isoformat(), + } + try: + step("00_init", ["python3", "scripts/init_local.py"]) + secret = (ROOT / "secrets/db_password").read_text() + os.environ["TEST_DATABASE_URL"] = ( + f"postgresql+asyncpg://messenger:{secret}@127.0.0.1:25432/messenger_test" + ) + os.environ["TEST_REDIS_PASSWORD"] = ( + ROOT / "secrets/redis_password" + ).read_text() + step( + "01_black", + [ + "python3", + "-m", + "black", + "--check", + "app", + "infra", + "migrations", + "scripts", + "student-tests", + "tests", + ], + ) + step("02_contract", ["python3", "scripts/check_contract.py"]) + step("03_openapi", ["python3", "scripts/validate_openapi.py"]) + step( + "04_build_start", + COMPOSE + ["up", "-d", "--build"], + timeout=3600, + ) + step( + "04_readiness", + COMPOSE + ["up", "-d", "--wait", "--wait-timeout", "240", "nginx"], + ) + step( + "05_create_test_database", + COMPOSE + + [ + "exec", + "-T", + "postgres", + "createdb", + "-U", + "messenger", + "messenger_test", + ], + ) + step( + "06_pytest", + ["python3", "-m", "pytest", "-q", "--junitxml=evidence/pytest.xml"], + timeout=1800, + ) + base = ["--base-url", "http://127.0.0.1:28080"] + step("07_smoke", ["python3", "tests/smoke.py", *base]) + step( + "08_storage", ["python3", "-m", "scripts.check_s3", *base, "--with-outage"] + ) + step("09_outbox", ["python3", "-m", "scripts.check_outbox_recovery", *base]) + step( + "10_jobs", + [ + "python3", + "-m", + "scripts.check_jobs", + *base, + "--output", + "evidence/jobs.json", + ], + ) + step( + "11_persistence", + [ + "python3", + "tests/resilience.py", + "--mode", + "persistence", + "--action-script", + "scripts/restart.sh", + *base, + ], + ) + step( + "12_failover", + [ + "python3", + "tests/resilience.py", + "--mode", + "failover", + "--action-script", + "scripts/stop-one.sh", + *base, + ], + ) + step( + "13_environment", + [ + "docker", + "info", + "--format", + "CPUs={{.NCPU}} Memory={{.MemTotal}} Server={{.ServerVersion}} Architecture={{.Architecture}}", + ], + ) + step("13_dependencies", ["python3", "-m", "pip", "freeze"]) + step("13_images", ["docker", "image", "ls", "--digests"]) + step("14_resources", COMPOSE + ["ps", "--all", "--format", "json"]) + result["status"] = "passed" + finally: + step( + "15_cleanup", + COMPOSE + ["down", "--volumes", "--remove-orphans"], + timeout=180, + ) + result["finished_at"] = datetime.now(timezone.utc).isoformat() + (EVIDENCE / "verification.json").write_text(json.dumps(result, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/deployment.py b/scripts/deployment.py new file mode 100644 index 00000000..1f08cf74 --- /dev/null +++ b/scripts/deployment.py @@ -0,0 +1,68 @@ +"""Общие операции с репликами текущего Compose-проекта.""" + +from dataclasses import dataclass +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class Replica: + + id: str + image: str + instance_id: str + running: bool + health: str | None + + +class ComposeProject: + + def __init__(self) -> None: + self.command = ["docker", "compose", "--env-file", ".env"] + self.config = json.loads(self.run("config", "--format", "json")) + self.name = self.config["name"] + + def docker(self, *args: str, timeout: int = 90, env=None) -> str: + result = subprocess.run( + ["docker", *args], + cwd=ROOT, + text=True, + capture_output=True, + timeout=timeout, + env={**os.environ, **(env or {})}, + ) + if result.returncode: + raise RuntimeError(result.stderr.strip() or result.stdout.strip()) + return result.stdout.strip() + + def run(self, *args: str, timeout: int = 90, env=None) -> str: + return self.docker(*self.command[1:], *args, timeout=timeout, env=env) + + def replicas(self) -> list[Replica]: + ids = self.run("ps", "--all", "--quiet", "api").splitlines() + if not ids: + return [] + template = ( + '{"id":{{json .Id}},"image":{{json .Image}},' + '"hostname":{{json .Config.Hostname}},"running":{{json .State.Running}},' + '"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}' + ) + rows = self.docker("inspect", "--format", template, *ids).splitlines() + configured = str( + self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") + or "" + ).strip() + return [ + Replica( + row["id"], + row["image"], + configured or row["hostname"], + row["running"], + row["health"], + ) + for row in map(json.loads, rows) + ] diff --git a/scripts/init_local.py b/scripts/init_local.py new file mode 100644 index 00000000..7477cd09 --- /dev/null +++ b/scripts/init_local.py @@ -0,0 +1,38 @@ +"""Создаёт только отсутствующие локальные настройки и файловые секреты.""" + +import os +from pathlib import Path +import secrets + + +def main() -> None: + root = Path(__file__).resolve().parents[1] + destination = root / ".env" + if not destination.exists(): + with destination.open("x") as target: + target.write((root / ".env.example").read_text()) + destination.chmod(0o600) + directory = root / "secrets" + directory.mkdir(mode=0o700, exist_ok=True) + for name in ( + "db_password", + "redis_password", + "broker_password", + "minio_root_user", + "minio_root_password", + "s3_access_key_id", + "s3_secret_access_key", + ): + path = directory / name + try: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) + except FileExistsError: + continue + with os.fdopen(descriptor, "w") as target: + target.write(secrets.token_urlsafe(24)) + print("Создан файл " + str(path.relative_to(root))) + print("Настройки готовы; существующие значения сохранены") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..e64d5775 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,36 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +compose() { + docker compose --env-file .env "$@" +} + +wait_healthy() { + for service in "$@"; do + attempts=0 + until states=$(compose ps --format '{{.Health}}' "$service") \ + && [ -n "$states" ] \ + && [ -z "$(printf '%s\n' "$states" | grep -v '^healthy$')" ]; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 60 ]; then + echo "$service не стал healthy за 60 секунд" >&2 + exit 1 + fi + sleep 1 + done + done +} + +worker_running=$(compose ps --status running --quiet worker) +compose stop worker publisher cleanup +compose restart postgres redis minio broker +wait_healthy postgres redis minio broker +compose restart api +wait_healthy api +compose restart nginx +wait_healthy nginx +compose start publisher cleanup +if [ -n "$worker_running" ]; then + compose start worker +fi diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..0dcd4da0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +case "$TARGET_INSTANCE_ID" in + *[!a-zA-Z0-9_.:-]*|'') exit 1 ;; +esac + +target="" +for container in $(docker compose --env-file .env ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container") + configured=$(docker inspect --format '{{range .Config.Env}}{{if eq (index (split . "=") 0) "INSTANCE_ID"}}{{println .}}{{end}}{{end}}' "$container" \ + | sed -n 's/^INSTANCE_ID=//p' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$configured" ]; then + if [ -n "$target" ]; then + echo 'Instance id matches multiple API containers' >&2 + exit 1 + fi + target="$container" + fi +done + +if [ -z "$target" ]; then + echo 'Target API instance not found in this Compose project' >&2 + exit 1 +fi + +echo "Stopping API container $target" >&2 +docker stop --timeout 8 "$target" >/dev/null diff --git a/scripts/test.py b/scripts/test.py new file mode 100644 index 00000000..cb2872cc --- /dev/null +++ b/scripts/test.py @@ -0,0 +1,189 @@ +"""Запускает проверки в одноразовом Docker-окружении и сохраняет evidence.""" + +import argparse +from datetime import datetime, timezone +import io +import json +from pathlib import Path +import subprocess +import tarfile +import tempfile +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +EXCLUDED = { + ".git", + ".agents", + ".aws", + ".codex", + ".env", + "secrets", + "evidence", + ".venv", + "venv", + "__pycache__", + ".pytest_cache", + ".github", + "docs", +} + + +def docker(*args, **kwargs): + return subprocess.run(["docker", *args], cwd=ROOT, check=True, **kwargs) + + +def archive_source(destination): + with tarfile.open(destination, "w") as archive: + for path in sorted(ROOT.rglob("*")): + relative = path.relative_to(ROOT) + if not path.is_file() or any(part in EXCLUDED for part in relative.parts): + continue + content = path.read_bytes() + if path.suffix in {".py", ".sh", ".conf", ".yaml", ".txt", ".md", ".ini"}: + content = content.replace(b"\r\n", b"\n") + entry = tarfile.TarInfo(str(relative).replace("\\", "/")) + entry.size = len(content) + entry.mode = 0o755 if path.suffix == ".sh" else 0o644 + archive.addfile(entry, io.BytesIO(content)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--format", action="store_true", help="применить Black и выйти") + args = parser.parse_args() + project = "messenger-check-" + uuid4().hex[:12] + image = project + ":tools" + engine = project + "-engine" + runner = project + "-runner" + workspace = project + "-workspace" + data = project + "-data" + network = project + "-network" + output = ROOT / "evidence" / "lab5" + output.mkdir(parents=True, exist_ok=True) + summary = { + "status": "failed", + "project": project, + "started_at": datetime.now(timezone.utc).isoformat(), + } + try: + docker("build", "--tag", image, "--file", "infra/testing/Dockerfile", ".") + if args.format: + docker( + "run", + "--rm", + "--volume", + str(ROOT) + ":/workspace", + image, + "python3", + "-m", + "black", + "app", + "infra", + "migrations", + "scripts", + "student-tests", + "tests", + ) + summary["status"] = "formatted" + return + docker("network", "create", network, stdout=subprocess.DEVNULL) + for volume in (workspace, data): + docker("volume", "create", volume, stdout=subprocess.DEVNULL) + docker( + "run", + "-d", + "--privileged", + "--name", + engine, + "--network", + network, + "--env", + "DOCKER_TLS_CERTDIR=", + "--volume", + workspace + ":/workspace", + "--volume", + data + ":/var/lib/docker", + "docker:29.1.3-dind", + "dockerd", + "--host=unix:///var/run/docker.sock", + "--host=tcp://127.0.0.1:2375", + "--tls=false", + stdout=subprocess.DEVNULL, + ) + docker( + "create", + "--name", + runner, + "--network", + "container:" + engine, + "--env", + "DOCKER_HOST=tcp://127.0.0.1:2375", + "--volume", + workspace + ":/workspace", + image, + stdout=subprocess.DEVNULL, + ) + with tempfile.TemporaryDirectory(prefix="messenger-check-") as directory: + archive = Path(directory) / "source.tar" + archive_source(archive) + docker("cp", str(archive), runner + ":/tmp/source.tar") + docker("start", runner, stdout=subprocess.DEVNULL) + result = subprocess.run( + ["docker", "wait", runner], + cwd=ROOT, + text=True, + capture_output=True, + timeout=7200, + ) + result.check_returncode() + docker("logs", runner) + docker("cp", runner + ":/workspace/evidence/.", str(output)) + if int(result.stdout.strip()) != 0: + raise RuntimeError("Проверки завершились ошибкой; см. evidence/lab5") + summary["status"] = "passed" + finally: + cleanup = [] + for command in ( + ["rm", "--force", runner, engine], + ["volume", "rm", workspace, data], + ["network", "rm", network], + ["image", "rm", image], + ): + result = subprocess.run( + ["docker", *command], cwd=ROOT, text=True, capture_output=True + ) + cleanup.append( + { + "command": ["docker", *command], + "returncode": result.returncode, + "output": result.stdout.strip() + result.stderr.strip(), + } + ) + remaining = {} + for kind in ("container", "volume", "network", "image"): + result = subprocess.run( + ["docker", kind, "ls", "--format", "{{json .}}"], + cwd=ROOT, + text=True, + capture_output=True, + ) + remaining[kind] = [ + line for line in result.stdout.splitlines() if project in line + ] + if result.returncode: + remaining[kind].append("resource audit failed") + summary["cleanup"] = cleanup + summary["remaining_resources"] = remaining + summary["finished_at"] = datetime.now(timezone.utc).isoformat() + if any(remaining.values()): + summary["status"] = "failed" + (output / "run.json").write_text( + json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + if summary["status"] == "failed": + raise SystemExit(1) + print("PASS: проверки завершены, временное окружение удалено") + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_openapi.py b/scripts/validate_openapi.py index 1dfe4a98..08ee4bee 100644 --- a/scripts/validate_openapi.py +++ b/scripts/validate_openapi.py @@ -1,23 +1,26 @@ #!/usr/bin/env python3 """Full OpenAPI and JSON Schema checks; install requirements-dev.txt first.""" + import json from pathlib import Path from openapi_spec_validator import validate from jsonschema import Draft202012Validator root = Path(__file__).resolve().parents[1] -spec = json.loads((root/'contracts/openapi.json').read_text()) +spec = json.loads((root / "contracts/openapi.json").read_text(encoding="utf-8")) validate(spec) from referencing import Registry, Resource -# The base URI resolves local #/components references inside examples. -resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'}) -registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource)) -for name, example in json.loads((root/'contracts/examples.json').read_text()).items(): - schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name} + +resource = dict(spec, **{"$schema": "https://json-schema.org/draft/2020-12/schema"}) +registry = Registry().with_resource( + "urn:messenger:openapi", Resource.from_contents(resource) +) +for name, example in json.loads((root / "contracts/examples.json").read_text()).items(): + schema = {"$ref": "urn:messenger:openapi#/components/schemas/" + name} Draft202012Validator(schema, registry=registry).validate(example) -for path in (root/'contracts').glob('*.schema.json'): +for path in (root / "contracts").glob("*.schema.json"): schema = json.loads(path.read_text()) Draft202012Validator.check_schema(schema) - for example in schema.get('examples',[]): + for example in schema.get("examples", []): Draft202012Validator(schema).validate(example) -print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.') +print("PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.") diff --git a/student-tests/conftest.py b/student-tests/conftest.py new file mode 100644 index 00000000..f95eecc3 --- /dev/null +++ b/student-tests/conftest.py @@ -0,0 +1,191 @@ +"""Фикстуры на отдельных PostgreSQL и Redis тестового Docker-стенда.""" + +import asyncio +import os +from collections.abc import AsyncIterator, Callable + +import httpx +import pytest +from alembic import command +from alembic.config import Config +from fastapi import FastAPI +from redis.asyncio import Redis +from redis.exceptions import RedisError +from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from sqlalchemy.pool import NullPool + +from app.core.config import Settings +from app.core.redis import create_redis_client +from app.main import create_app +from app.models import Base + +DEFAULT_TEST_DATABASE_URL = ( + "postgresql+asyncpg://messenger:messenger@localhost:5432/messenger_test" +) + + +def _database_url() -> str: + return os.environ.get("TEST_DATABASE_URL", DEFAULT_TEST_DATABASE_URL) + + +def _redis_settings() -> dict[str, object]: + return { + "redis_host": os.environ.get("TEST_REDIS_HOST", "localhost"), + "redis_port": int(os.environ.get("TEST_REDIS_PORT", "6379")), + "redis_password": os.environ.get("TEST_REDIS_PASSWORD") or None, + "redis_db": int(os.environ.get("TEST_REDIS_DB", "15")), + } + + +def _alembic_config(url: str) -> Config: + config = Config("alembic.ini") + config.set_main_option("sqlalchemy.url", url.replace("%", "%%")) + return config + + +async def upgrade_head(url: str) -> None: + """alembic command.upgrade внутри вызывает asyncio.run, поэтому — в отдельном потоке.""" + await asyncio.to_thread(command.upgrade, _alembic_config(url), "head") + + +async def reset_schema(engine: AsyncEngine) -> None: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.drop_all) + await connection.execute(text("DROP TABLE IF EXISTS alembic_version")) + + +async def close_app(app: FastAPI) -> None: + """ASGITransport не запускает lifespan, поэтому ресурсы приложения закрываются явно.""" + await app.state.redis.aclose() + await app.state.engine.dispose() + + +@pytest.fixture(scope="session") +def database_url() -> str: + return _database_url() + + +@pytest.fixture(scope="session") +async def db_engine(database_url: str) -> AsyncIterator[AsyncEngine]: + engine = create_async_engine(database_url, poolclass=NullPool) + try: + async with engine.connect() as connection: + await connection.execute(text("SELECT 1")) + except (SQLAlchemyError, OSError) as error: + pytest.exit( + f"PostgreSQL недоступна по {engine.url.render_as_string(hide_password=True)}: " + f"{type(error).__name__}. Запускайте через `python3 scripts/test.py`.", + returncode=2, + ) + + await reset_schema(engine) + + if os.environ.get("TEST_USE_MIGRATIONS") == "1": + await upgrade_head(database_url) + else: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + yield engine + await engine.dispose() + + +@pytest.fixture(scope="session") +async def redis_client() -> AsyncIterator[Redis]: + settings = build_settings(_database_url()) + if settings.redis_db == 0: + pytest.exit("TEST_REDIS_DB=0 запрещён: тесты очищают базу Redis.", returncode=2) + + client = create_redis_client(settings) + try: + await client.ping() + except (RedisError, OSError) as error: + pytest.exit( + f"Redis недоступен: {type(error).__name__}. Запускайте через `python3 scripts/test.py`.", + returncode=2, + ) + + yield client + await client.flushdb() + await client.aclose() + + +@pytest.fixture(autouse=True) +async def clean_db(db_engine: AsyncEngine, redis_client: Redis) -> None: + names = ", ".join(f'"{table.name}"' for table in Base.metadata.sorted_tables) + if names: + async with db_engine.begin() as connection: + await connection.execute(text(f"TRUNCATE {names} RESTART IDENTITY CASCADE")) + await redis_client.flushdb() + + +def build_settings(database_url: str, **overrides: object) -> Settings: + values: dict[str, object] = { + "s3_enabled": False, + "app_host": "127.0.0.1", + "app_port": 8080, + "app_env": "test", + "max_messages": 1000, + "max_in_flight": 32, + "log_level": "CRITICAL", + "database_url": database_url, + "session_ttl_seconds": 300, + "db_pool_size": 10, + "db_max_overflow": 10, + **_redis_settings(), + } + values.update(overrides) + return Settings(_env_file=None, **values) + + +@pytest.fixture +def settings(database_url: str) -> Settings: + return build_settings(database_url) + + +@pytest.fixture +def make_settings(database_url: str) -> Callable[..., Settings]: + def factory(**overrides: object) -> Settings: + url = str(overrides.pop("database_url", database_url)) + return build_settings(url, **overrides) + + return factory + + +def make_client( + app, *, raise_app_exceptions: bool = True, timeout: float = 30 +) -> httpx.AsyncClient: + return httpx.AsyncClient( + transport=httpx.ASGITransport( + app=app, raise_app_exceptions=raise_app_exceptions + ), + base_url="http://testserver", + timeout=timeout, + ) + + +@pytest.fixture +def app(settings: Settings): + application = create_app(settings) + + @application.get("/__test__/explode", include_in_schema=False) + async def explode() -> None: + raise RuntimeError("test-secret") + + return application + + +@pytest.fixture +async def client(app) -> AsyncIterator[httpx.AsyncClient]: + async with make_client(app) as test_client: + yield test_client + await close_app(app) + + +@pytest.fixture +async def client_no_raise(app) -> AsyncIterator[httpx.AsyncClient]: + async with make_client(app, raise_app_exceptions=False) as test_client: + yield test_client + await close_app(app) diff --git a/student-tests/helpers.py b/student-tests/helpers.py new file mode 100644 index 00000000..8272d523 --- /dev/null +++ b/student-tests/helpers.py @@ -0,0 +1,85 @@ +import base64 +from uuid import uuid4 + +from httpx import AsyncClient + +PASSWORD = "Correct-horse-battery-1" + + +def basic_header(username: str, password: str) -> dict[str, str]: + raw = f"{username}:{password}".encode() + return {"Authorization": "Basic " + base64.b64encode(raw).decode()} + + +def bearer(actor: dict) -> dict[str, str]: + return {"Authorization": "Bearer " + actor["token"]} + + +async def register( + client: AsyncClient, + username: str | None = None, + display_name: str = "Тестовый пользователь", + password: str = PASSWORD, +) -> dict: + actual_username = username or "user_" + uuid4().hex[:12] + response = await client.post( + "/api/v1/users", + json={ + "username": actual_username, + "display_name": display_name, + "password": password, + }, + ) + assert response.status_code == 201, response.text + return {**response.json(), "password": password} + + +async def login(client: AsyncClient, user: dict) -> dict: + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + assert response.status_code == 201, response.text + return {**user, "token": response.json()["token"]} + + +async def create_user(client: AsyncClient, **kwargs) -> dict: + return await login(client, await register(client, **kwargs)) + + +async def create_chat( + client: AsyncClient, + user: dict, + *members: dict, + title: str = "Тестовый чат", +) -> dict: + member_ids = [user["id"], *(member["id"] for member in members)] + response = await client.post( + "/api/v1/chats", + headers=bearer(user), + json={"title": title, "member_ids": member_ids}, + ) + assert response.status_code == 201, response.text + return response.json() + + +def idempotency_header(key: str | None = None) -> dict[str, str]: + return {"Idempotency-Key": key or uuid4().hex} + + +async def send_message( + client: AsyncClient, user: dict, chat: dict, text: str, key: str | None = None +): + return await client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**bearer(user), **idempotency_header(key)}, + json={"text": text}, + ) + + +def assert_request_id(response) -> str: + header_id = response.headers.get("X-Request-Id") + assert header_id + if response.status_code >= 400: + assert response.json()["error"]["request_id"] == header_id + return header_id diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..98c2eb54 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,482 @@ +"""API и транзакции с реальной БД; байты S3 подменены управляемым адаптером.""" + +import asyncio +import hashlib +from io import BytesIO +from datetime import datetime, timedelta, timezone +from pathlib import Path +import struct +from uuid import UUID, uuid4 +import zlib + +import pytest +from PIL import Image +from sqlalchemy import func, select +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.requests import ClientDisconnect + +from app.core.s3 import S3Settings +from app.core.jobs import JobSettings +from app.domain.attachment_jobs import process_attachment_job +from app.models.job import AttachmentJob +from app.schemas.job import JobMessage +from app.domain import attachments as domain +from app.errors.exceptions import unavailable +from app.models.attachment import Attachment, UploadIntent +from app.storage import attachments as repository +from app.storage.s3 import ObjectInfo +from helpers import bearer, create_chat, create_user, idempotency_header + +PNG = Path("tests/fixtures/sample.png").read_bytes() + + +class MemoryStorage: + + def __init__(self): + self.settings = S3Settings(_env_file=None) + self.objects = {} + self.fail_put = self.fail_delete = self.down = False + + async def put_file(self, key, path, **metadata): + self.objects[key] = (path.read_bytes(), metadata) + if self.fail_put: + raise unavailable() + + async def delete_object(self, key): + if self.fail_delete: + raise unavailable() + self.objects.pop(key, None) + + async def stat_object(self, key): + if self.down: + raise unavailable() + info = None + if key in self.objects: + data, metadata = self.objects[key] + info = ObjectInfo( + key, len(data), metadata["content_type"], datetime.now(timezone.utc) + ) + return info + + async def open_object(self, key): + async def chunks(): + yield self.objects[key][0] + + return chunks() + + async def check_available(self): + return not self.down + + +@pytest.fixture +async def context(client, app): + storage = MemoryStorage() + app.state.s3 = storage + alice, bob, eve = [await create_user(client) for _ in range(3)] + chat = await create_chat(client, alice, bob) + return storage, alice, bob, eve, chat + + +async def upload( + client, actor, chat, data=PNG, filename="фото.png", content_type="image/png" +): + return await client.post( + f"/api/v1/chats/{chat['id']}/attachments", + params={"filename": filename}, + headers={**bearer(actor), "Content-Type": content_type}, + content=data, + ) + + +async def finish_attachment(client, storage, actor, attachment_id): + engine = client._transport.app.state.engine + async with AsyncSession(engine) as session: + job = await session.scalar( + select(AttachmentJob).where( + AttachmentJob.attachment_id == UUID(attachment_id) + ) + ) + message = JobMessage( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + attempt=job.attempt + 1, + created_at=job.created_at, + ) + await session.rollback() + assert await process_attachment_job( + session, storage, message, JobSettings(_env_file=None) + ) + result = await client.get( + f"/api/v1/attachments/{attachment_id}", headers=bearer(actor) + ) + assert result.json()["status"] == "ready" + return result.json() + + +async def test_upload_download_and_attachment_only_message(client, context): + storage, alice, bob, eve, chat = context + response = await upload(client, alice, chat) + assert response.status_code == 202, response.text + attachment = response.json() + assert attachment["status"] == "queued" + before = await client.get( + f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice) + ) + assert before.status_code == 409 + attachment = await finish_attachment(client, storage, alice, attachment["id"]) + assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest() + assert {v["name"] for v in attachment["variants"]} == { + "thumbnail", + "crop", + "optimized", + } + assert attachment["error_code"] is None + assert "object_key" not in attachment + id = attachment["id"] + for actor in (alice, bob): + content = await client.get( + f"/api/v1/attachments/{id}/content", headers=bearer(actor) + ) + assert content.status_code == 200 and content.content == PNG + assert content.headers["content-type"] == "image/png" + key = idempotency_header() + body = {"text": "", "attachment_ids": [id]} + path = f"/api/v1/chats/{chat['id']}/messages" + message = await client.post(path, json=body, headers={**bearer(bob), **key}) + assert message.status_code == 201, message.text + repeat = await client.post(path, json=body, headers={**bearer(bob), **key}) + assert repeat.json() == message.json() + history = await client.get(path, headers=bearer(alice)) + assert history.json()["items"][0]["attachment_ids"] == [id] + for suffix in ("", "/content"): + denied = await client.get( + f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve) + ) + assert denied.status_code == 404 + + +@pytest.mark.parametrize( + "data,content_type", + [ + (b"not an image", "image/png"), + (PNG[:30], "image/png"), + (PNG, "image/jpeg"), + (PNG, "application/octet-stream"), + ], + ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"], +) +async def test_invalid_image_and_mime_are_415(client, context, data, content_type): + storage, alice, _, _, chat = context + response = await upload(client, alice, chat, data, content_type=content_type) + assert response.status_code == 415, response.text + assert not storage.objects + + +async def test_jpeg_is_stored_without_reencoding(client, context): + storage, alice, _, _, chat = context + output = BytesIO() + Image.new("RGB", (32, 32), "blue").save(output, format="JPEG") + original = output.getvalue() + response = await upload(client, alice, chat, original, content_type="image/jpeg") + assert response.status_code == 202, response.text + id = response.json()["id"] + await finish_attachment(client, storage, alice, id) + downloaded = await client.get( + f"/api/v1/attachments/{id}/content", headers=bearer(alice) + ) + assert downloaded.content == original + + +@pytest.mark.parametrize( + "filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256] +) +async def test_unsafe_filename_is_400(client, context, filename): + storage, alice, _, _, chat = context + response = await upload(client, alice, chat, filename=filename) + assert response.status_code == 400 + assert not storage.objects + + +async def test_oversize_chunked_upload_is_413(client, context): + storage, alice, _, _, chat = context + + async def chunks(): + for _ in range(11): + yield b"x" * 1048576 + + response = await upload(client, alice, chat, chunks()) + assert response.status_code == 413 + assert not storage.objects + + +async def test_oversize_content_length_is_413(client, context): + _, alice, _, _, chat = context + response = await client.post( + f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", + headers={ + **bearer(alice), + "Content-Type": "image/png", + "Content-Length": "10485761", + }, + content=PNG, + ) + assert response.status_code == 413 + + +async def test_exact_byte_limit_accepts_valid_png(client, context): + _, alice, _, _, chat = context + maximum = 10_485_760 + + payload = bytes(maximum - len(PNG) - 12) + kind = b"npAD" + padding = ( + struct.pack("!I", len(payload)) + + kind + + payload + + struct.pack("!I", zlib.crc32(kind + payload)) + ) + image = PNG[:-12] + padding + PNG[-12:] + response = await upload(client, alice, chat, image) + assert response.status_code == 202, response.text + assert response.json()["size_bytes"] == maximum + assert response.json()["sha256"] == hashlib.sha256(image).hexdigest() + + +async def test_exact_pixel_limit_accepts_valid_png(client, context): + _, alice, _, _, chat = context + output = BytesIO() + with Image.new("RGB", (8000, 5000), "blue") as image: + image.save(output, format="PNG") + response = await upload(client, alice, chat, output.getvalue()) + assert response.status_code == 202, response.text + + +async def test_pixel_limit_before_decoding(client, context): + _, alice, _, _, chat = context + + def chunk(kind, data): + return ( + struct.pack("!I", len(data)) + + kind + + data + + struct.pack("!I", zlib.crc32(kind + data)) + ) + + header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0) + image = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(b"\0")) + + chunk(b"IEND", b"") + ) + response = await upload(client, alice, chat, image) + assert response.status_code == 413, response.text + + +async def test_unknown_and_other_chat_attachment_are_404(client, context): + _, alice, bob, _, chat = context + attachment = (await upload(client, alice, chat)).json() + other_chat = await create_chat(client, alice, bob) + for id in (attachment["id"], str(uuid4())): + response = await client.post( + f"/api/v1/chats/{other_chat['id']}/messages", + json={"text": "", "attachment_ids": [id]}, + headers={**bearer(alice), **idempotency_header()}, + ) + assert response.status_code == 404 + + +async def test_duplicate_ids_are_400(client, context): + _, alice, _, _, chat = context + id = (await upload(client, alice, chat)).json()["id"] + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "", "attachment_ids": [id, id]}, + headers={**bearer(alice), **idempotency_header()}, + ) + assert response.status_code == 400 + + +async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context): + _, alice, _, _, chat = context + headers = {**bearer(alice), **idempotency_header()} + path = f"/api/v1/chats/{chat['id']}/messages" + first = await client.post(path, json={"text": "same"}, headers=headers) + second = await client.post( + path, json={"text": "same", "attachment_ids": []}, headers=headers + ) + assert first.status_code == second.status_code == 201 + assert first.json() == second.json() + + +async def test_s3_outage_is_503(client, context): + storage, alice, _, _, chat = context + id = (await upload(client, alice, chat)).json()["id"] + await finish_attachment(client, storage, alice, id) + storage.down = True + response = await client.get( + f"/api/v1/attachments/{id}/content", headers=bearer(alice) + ) + assert response.status_code == 503 + assert (await client.get("/health/ready")).status_code == 503 + + +async def test_uncertain_put_is_compensated(client, context, db_engine): + storage, alice, _, _, chat = context + storage.fail_put = True + response = await upload(client, alice, chat) + assert response.status_code == 503 + assert not storage.objects + async with AsyncSession(db_engine) as session: + assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0 + + +async def test_failure_after_s3_write_compensates(client, context, monkeypatch): + storage, alice, _, _, chat = context + + async def fail(*args, **kwargs): + raise OperationalError("injected", {}, None) + + monkeypatch.setattr(repository, "create_attachment", fail) + response = await upload(client, alice, chat) + assert response.status_code == 503 + assert not storage.objects + + +async def test_ambiguous_successful_commit_keeps_original( + client, context, db_engine, monkeypatch +): + storage, alice, _, _, chat = context + original = AsyncSession.commit + count = 0 + + async def uncertain(session): + nonlocal count + count += 1 + await original(session) + if count == 2: + raise OperationalError("lost commit acknowledgement", {}, None) + + monkeypatch.setattr(AsyncSession, "commit", uncertain) + response = await upload(client, alice, chat) + assert response.status_code == 503 + assert len(storage.objects) == 1 + async with AsyncSession(db_engine) as session: + assert await session.scalar(select(func.count()).select_from(Attachment)) == 1 + assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0 + + +async def test_failed_compensation_then_dry_run_and_cleanup(client, context, db_engine): + storage, alice, _, _, chat = context + storage.fail_put = storage.fail_delete = True + assert (await upload(client, alice, chat)).status_code == 503 + assert len(storage.objects) == 1 + async with AsyncSession(db_engine) as session: + intent = await session.scalar(select(UploadIntent)) + intent.created_at = datetime.now(timezone.utc) - timedelta(days=2) + await session.commit() + storage.fail_put = storage.fail_delete = False + async with AsyncSession(db_engine) as session: + dry_run = await domain.reconcile_attachments(session, storage) + assert dry_run.candidates == 1 and dry_run.deleted == 0 + assert storage.objects + applied = await domain.reconcile_attachments(session, storage, dry_run=False) + assert applied.deleted == 1 and not storage.objects + repeated = await domain.reconcile_attachments(session, storage, dry_run=False) + assert repeated.deleted == 0 + + +async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_engine): + storage, _, _, _, _ = context + old_id, fresh_id = uuid4(), uuid4() + async with AsyncSession(db_engine) as session: + session.add_all( + [ + UploadIntent( + id=old_id, + object_key="originals/old", + created_at=datetime.now(timezone.utc) - timedelta(days=2), + ), + UploadIntent(id=fresh_id, object_key="originals/fresh"), + ] + ) + await session.commit() + async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner: + await repository.lock_upload(uploader, old_id) + result = await domain.reconcile_attachments(cleaner, storage, dry_run=False) + assert result.examined == 0 + await uploader.rollback() + result = await domain.reconcile_attachments(cleaner, storage, dry_run=False) + assert result.examined == 1 + assert await cleaner.get(UploadIntent, fresh_id) is not None + + +async def test_client_disconnect_before_put_has_no_side_effects(client, context): + storage, alice, _, _, chat = context + + async def interrupted(): + yield PNG[:16] + raise ClientDisconnect() + + response = await upload(client, alice, chat, interrupted()) + assert response.status_code == 400 + assert not storage.objects + + +async def test_upload_deadline_cancels_validation(client, context, monkeypatch): + storage, alice, _, _, chat = context + storage.settings = storage.settings.model_copy( + update={"upload_timeout_seconds": 0.02} + ) + + async def stalled(*args): + await asyncio.sleep(10) + + monkeypatch.setattr(domain, "validate_image", stalled) + response = await upload(client, alice, chat) + assert response.status_code == 400 + assert not storage.objects + + +async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypatch): + storage, alice, _, _, chat = context + entered = asyncio.Event() + original = storage.put_file + + async def interrupted(*args, **kwargs): + await original(*args, **kwargs) + entered.set() + await asyncio.sleep(10) + + monkeypatch.setattr(storage, "put_file", interrupted) + task = asyncio.create_task(upload(client, alice, chat)) + await asyncio.wait_for(entered.wait(), timeout=10) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert not storage.objects + async with AsyncSession(db_engine) as session: + assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0 + + +async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine): + storage, alice, _, _, chat = context + id = (await upload(client, alice, chat)).json()["id"] + await finish_attachment(client, storage, alice, id) + async with AsyncSession(db_engine) as session: + attachment = await session.get(Attachment, UUID(id)) + key = attachment.object_key + + session.add( + UploadIntent( + id=uuid4(), + object_key=key, + created_at=datetime.now(timezone.utc) - timedelta(days=2), + ) + ) + await session.commit() + result = await domain.reconcile_attachments(session, storage, dry_run=False) + assert result.examined == 1 and result.deleted == 0 + assert key in storage.objects diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..6746320f --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,298 @@ +import asyncio +import hashlib +from datetime import datetime + +from httpx import AsyncClient + +from app.main import create_app +from conftest import close_app, make_client +from helpers import ( + PASSWORD, + assert_request_id, + basic_header, + bearer, + create_user, + login, + register, +) + + +async def test_login_returns_opaque_bearer_session(client: AsyncClient): + user = await register(client) + + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + + assert response.status_code == 201 + body = response.json() + assert set(body) == {"token", "token_type", "expires_at", "user"} + assert body["token_type"] == "Bearer" + assert len(body["token"]) >= 43 + assert body["token"] not in (user["id"], user["username"]) + assert body["user"]["id"] == user["id"] + assert body["expires_at"].endswith("Z") + assert datetime.fromisoformat(body["expires_at"]) + assert_request_id(response) + + +async def test_each_login_issues_a_different_token(client: AsyncClient): + user = await register(client) + first = await login(client, user) + second = await login(client, user) + + assert first["token"] != second["token"] + + +async def test_password_with_colon_works(client: AsyncClient): + user = await register(client, password="pass:with:colons-123") + + assert (await login(client, user))["token"] + + +async def test_unicode_password_works(client: AsyncClient): + user = await register(client, password="Пароль-с-юникодом-🔑-123") + + assert (await login(client, user))["token"] + + +async def test_wrong_password_and_unknown_user_look_identical(client: AsyncClient): + user = await register(client) + + wrong_password = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], "wrong-password-123"), + ) + unknown_user = await client.post( + "/api/v1/auth/sessions", + headers=basic_header("no_such_user", "wrong-password-123"), + ) + + for response in (wrong_password, unknown_user): + assert response.status_code == 401 + assert response.json()["error"]["code"] == "unauthorized" + assert response.headers["WWW-Authenticate"].startswith( + 'Basic realm="messenger"' + ) + assert_request_id(response) + assert ( + wrong_password.json()["error"]["message"] + == unknown_user.json()["error"]["message"] + ) + + +async def test_login_requires_basic_scheme(client: AsyncClient): + user = await register(client) + session = await login(client, user) + + for headers in ({}, bearer(session), {"Authorization": "Basic !!!not-base64"}): + response = await client.post("/api/v1/auth/sessions", headers=headers) + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"].startswith("Basic") + + +async def test_business_endpoints_require_bearer(client: AsyncClient): + user = await register(client) + + for headers in ( + {}, + basic_header(user["username"], user["password"]), + {"Authorization": "Bearer "}, + {"Authorization": "Bearer unknown-token-value"}, + {"X-User-Id": user["id"]}, + ): + response = await client.get("/api/v1/users/me", headers=headers) + assert response.status_code == 401, headers + assert response.json()["error"]["code"] == "unauthorized" + assert response.headers["WWW-Authenticate"].startswith( + 'Bearer realm="messenger"' + ) + assert_request_id(response) + + +async def test_x_user_id_cannot_override_bearer_identity(client: AsyncClient): + alice = await create_user(client) + eve = await create_user(client) + + response = await client.get( + "/api/v1/users/me", + headers={**bearer(alice), "X-User-Id": eve["id"]}, + ) + + assert response.status_code == 200 + assert response.json()["id"] == alice["id"] + + +async def test_logout_revokes_only_current_session(client: AsyncClient): + user = await register(client) + first = await login(client, user) + second = await login(client, user) + + assert ( + await client.delete("/api/v1/auth/sessions/current", headers=bearer(first)) + ).status_code == 204 + + assert ( + await client.get("/api/v1/users/me", headers=bearer(first)) + ).status_code == 401 + assert ( + await client.get("/api/v1/users/me", headers=bearer(second)) + ).status_code == 200 + + +async def test_logout_twice_returns_401(client: AsyncClient): + session = await create_user(client) + + first = await client.delete( + "/api/v1/auth/sessions/current", headers=bearer(session) + ) + again = await client.delete( + "/api/v1/auth/sessions/current", headers=bearer(session) + ) + + assert first.status_code == 204 + assert first.content == b"" + assert again.status_code == 401 + + +async def test_session_expires_after_ttl(make_settings): + app = create_app(make_settings(session_ttl_seconds=2)) + async with make_client(app) as client: + session = await create_user(client) + assert ( + await client.get("/api/v1/users/me", headers=bearer(session)) + ).status_code == 200 + + await asyncio.sleep(3) + + response = await client.get("/api/v1/users/me", headers=bearer(session)) + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"].startswith("Bearer") + await close_app(app) + + +async def test_token_is_stored_only_as_sha256_key(client: AsyncClient, redis_client): + session = await create_user(client) + + keys = [key async for key in redis_client.scan_iter("session:*")] + token_hash = hashlib.sha256(session["token"].encode()).hexdigest() + + assert keys == [f"session:{token_hash}"] + assert await redis_client.get(keys[0]) == session["id"] + assert all(session["token"] not in key for key in keys) + + +async def test_password_is_not_stored_in_plaintext(client: AsyncClient, db_engine): + from sqlalchemy import text + + await register(client, username="hash_check") + + async with db_engine.connect() as connection: + result = await connection.execute( + text("SELECT password_hash FROM users WHERE username = 'hash_check'") + ) + stored = result.scalar_one() + + assert PASSWORD not in stored + assert stored.startswith("$argon2id$") + + +async def test_login_rejects_out_of_range_password_like_wrong_password( + client: AsyncClient, +): + user = await register(client) + reference = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], "wrong-password-123"), + ) + + for password in ("x" * 11, "x" * 129, ""): + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], password), + ) + assert response.status_code == 401 + assert ( + response.json()["error"]["message"] == reference.json()["error"]["message"] + ) + assert ( + response.headers["WWW-Authenticate"] + == reference.headers["WWW-Authenticate"] + ) + + +async def test_malformed_basic_credentials_return_401(client: AsyncClient): + import base64 + + def raw(value: bytes) -> dict[str, str]: + return {"Authorization": "Basic " + base64.b64encode(value).decode()} + + for headers in ( + raw(b"no-colon-here"), + raw(b":password-without-user"), + raw(b"\xff\xfe:invalid-utf8"), + {"Authorization": "Basic"}, + {"Authorization": "Digest abc"}, + ): + response = await client.post("/api/v1/auth/sessions", headers=headers) + assert response.status_code == 401, headers + assert response.headers["WWW-Authenticate"].startswith("Basic") + + +async def test_unknown_user_takes_comparable_time_to_wrong_password( + client: AsyncClient, +): + import statistics + import time + + user = await register(client) + + async def measure(username: str) -> float: + samples = [] + for _ in range(5): + started = time.perf_counter() + await client.post( + "/api/v1/auth/sessions", + headers=basic_header(username, "wrong-password-123"), + ) + samples.append(time.perf_counter() - started) + return statistics.median(samples) + + wrong_password = await measure(user["username"]) + unknown_user = await measure("no_such_user") + + assert unknown_user > wrong_password * 0.5 + + +async def test_invalid_basic_username_returns_401_not_500(client: AsyncClient): + user = await register(client) + reference = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], "wrong-password-123"), + ) + + for username in ("bad\u0000name", "UPPER", "no", "x" * 33, "имя", "a b c"): + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(username, "wrong-password-123"), + ) + assert response.status_code == 401, username + assert ( + response.json()["error"]["message"] == reference.json()["error"]["message"] + ) + assert ( + response.headers["WWW-Authenticate"] + == reference.headers["WWW-Authenticate"] + ) + + +async def test_whitespace_only_password_at_login_returns_401(client: AsyncClient): + user = await register(client) + + response = await client.post( + "/api/v1/auth/sessions", headers=basic_header(user["username"], " " * 12) + ) + + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"].startswith("Basic") diff --git a/student-tests/test_chats.py b/student-tests/test_chats.py new file mode 100644 index 00000000..b5f4fa98 --- /dev/null +++ b/student-tests/test_chats.py @@ -0,0 +1,208 @@ +from urllib.parse import quote +from uuid import uuid4 + +import pytest +from httpx import AsyncClient + +from helpers import assert_request_id, bearer, create_chat, create_user + + +async def test_create_and_get_chat(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + chat = await create_chat(client, alice, bob) + + response = await client.get( + f"/api/v1/chats/{chat['id']}", + headers=bearer(bob), + ) + + assert response.status_code == 200 + assert response.json() == chat + assert_request_id(response) + + +async def test_chat_is_not_visible_to_outsider(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + eve = await create_user(client) + chat = await create_chat(client, alice, bob) + + response = await client.get( + f"/api/v1/chats/{chat['id']}", + headers=bearer(eve), + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + assert_request_id(response) + + +async def test_chat_creation_requires_creator_in_members(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "Чат", "member_ids": [bob["id"]]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.parametrize( + "payload", + [ + {"title": "", "member_ids": []}, + {"title": " ", "member_ids": []}, + ], +) +async def test_chat_payload_boundaries(client: AsyncClient, payload: dict): + alice = await create_user(client) + payload["member_ids"] = [alice["id"]] + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json=payload, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_duplicate_members_are_rejected(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "Чат", "member_ids": [alice["id"], bob["id"], bob["id"]]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_unknown_member_is_rejected(client: AsyncClient): + alice = await create_user(client) + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "Чат", "member_ids": [alice["id"], str(uuid4())]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_chat_exact_title_limit_is_accepted(client: AsyncClient): + alice = await create_user(client) + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "x" * 100, "member_ids": [alice["id"]]}, + ) + + assert response.status_code == 201 + assert len(response.json()["title"]) == 100 + + +async def test_chat_list_hides_chats_without_membership(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + chat_for_bob = await create_chat(client, bob, title="Only Bob") + chat_for_alice = await create_chat(client, alice, title="Only Alice") + + response = await client.get( + "/api/v1/chats?limit=100", + headers=bearer(alice), + ) + + assert response.status_code == 200 + ids = {item["id"] for item in response.json()["items"]} + assert chat_for_alice["id"] in ids + assert chat_for_bob["id"] not in ids + + +async def test_nonexistent_chat_returns_404(client: AsyncClient): + alice = await create_user(client) + + response = await client.get( + "/api/v1/chats/00000000-0000-4000-8000-000000000000", + headers=bearer(alice), + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + assert_request_id(response) + + +async def test_failed_chat_creation_leaves_no_partial_data( + client: AsyncClient, db_engine +): + from sqlalchemy import text + + alice = await create_user(client) + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "Чат", "member_ids": [alice["id"], str(uuid4())]}, + ) + assert response.status_code == 400 + + async with db_engine.connect() as connection: + chats = ( + await connection.execute(text("SELECT count(*) FROM chats")) + ).scalar_one() + members = ( + await connection.execute(text("SELECT count(*) FROM chat_members")) + ).scalar_one() + assert chats == 0 and members == 0 + + +async def test_chat_list_pagination_and_order(client: AsyncClient): + alice = await create_user(client) + chats = [await create_chat(client, alice, title=f"chat-{i}") for i in range(5)] + + page1 = (await client.get("/api/v1/chats?limit=2", headers=bearer(alice))).json() + page2 = ( + await client.get( + f"/api/v1/chats?limit=2&cursor={quote(page1['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + page3 = ( + await client.get( + f"/api/v1/chats?limit=2&cursor={quote(page2['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + + ids = [c["id"] for page in (page1, page2, page3) for c in page["items"]] + assert ids == [c["id"] for c in chats] + assert page3["next_cursor"] is None + + +async def test_chat_endpoints_require_bearer(client: AsyncClient): + assert (await client.get("/api/v1/chats")).status_code == 401 + assert ( + await client.post("/api/v1/chats", json={"title": "x", "member_ids": []}) + ).status_code in (401, 400) + + +async def test_nul_in_chat_title_is_rejected(client: AsyncClient): + alice = await create_user(client) + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "chat\u0000title", "member_ids": [alice["id"]]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..bf07e6f8 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,273 @@ +"""Конкурентность на реальной БД: гонка регистрации и параллельная запись.""" + +import asyncio +from collections import Counter +from uuid import uuid4 + +from sqlalchemy import text + +from app.main import create_app +from conftest import close_app, make_client +from helpers import ( + PASSWORD, + basic_header, + bearer, + create_user, + idempotency_header, + register, +) + + +async def test_parallel_registration_of_same_username_creates_exactly_one_user( + make_settings, +): + app = create_app( + make_settings(max_in_flight=64, db_pool_size=20, db_max_overflow=20) + ) + + async with make_client(app) as client: + responses = await asyncio.gather( + *( + client.post( + "/api/v1/users", + json={ + "username": "race_user", + "display_name": "Race", + "password": PASSWORD, + }, + ) + for _ in range(16) + ) + ) + await close_app(app) + + statuses = Counter(response.status_code for response in responses) + assert statuses == Counter({201: 1, 409: 15}), statuses + + +async def test_100_messages_from_10_clients(make_settings): + app = create_app( + make_settings( + max_messages=1000, max_in_flight=100, db_pool_size=20, db_max_overflow=20 + ) + ) + + async with make_client(app) as client: + users = [ + await create_user(client, username=f"load_{i}_{uuid4().hex[:6]}") + for i in range(10) + ] + created = await client.post( + "/api/v1/chats", + headers=bearer(users[0]), + json={"title": "Concurrency", "member_ids": [u["id"] for u in users]}, + ) + chat = created.json() + + async def worker(index: int, user: dict) -> list[dict]: + responses = await asyncio.gather( + *( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**bearer(user), **idempotency_header()}, + json={"text": f"client-{index}-message-{n}"}, + ) + for n in range(10) + ) + ) + assert all(r.status_code == 201 for r in responses), [ + r.text for r in responses if r.status_code != 201 + ] + return [r.json() for r in responses] + + results = await asyncio.gather(*(worker(i, u) for i, u in enumerate(users))) + messages = [m for batch in results for m in batch] + + assert len(messages) == 100 + assert len({m["id"] for m in messages}) == 100 + assert Counter(m["sender_id"] for m in messages) == Counter( + {u["id"]: 10 for u in users} + ) + + history = await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=100", headers=bearer(users[0]) + ) + items = history.json()["items"] + assert len(items) == 100 + assert len({m["id"] for m in items}) == 100 + await close_app(app) + + +async def test_chat_creation_in_parallel_is_atomic(make_settings, db_engine): + app = create_app( + make_settings(max_in_flight=32, db_pool_size=20, db_max_overflow=20) + ) + + async with make_client(app) as client: + alice = await create_user(client) + bob = await create_user(client) + responses = await asyncio.gather( + *( + client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": f"c{i}", "member_ids": [alice["id"], bob["id"]]}, + ) + for i in range(10) + ) + ) + assert all(r.status_code == 201 for r in responses) + await close_app(app) + + async with db_engine.connect() as connection: + chats = ( + await connection.execute(text("SELECT count(*) FROM chats")) + ).scalar_one() + members = ( + await connection.execute(text("SELECT count(*) FROM chat_members")) + ).scalar_one() + assert chats == 10 and members == 20 + + +async def test_parallel_registration_of_distinct_usernames_all_succeed(make_settings): + app = create_app(make_settings(max_in_flight=64)) + async with make_client(app) as client: + responses = await asyncio.gather( + *( + client.post( + "/api/v1/users", + json={ + "username": f"distinct_{i}", + "display_name": "D", + "password": PASSWORD, + }, + ) + for i in range(12) + ) + ) + await close_app(app) + + assert [r.status_code for r in responses] == [201] * 12 + assert len({r.json()["id"] for r in responses}) == 12 + + +async def test_parallel_logins_issue_distinct_valid_tokens(make_settings): + app = create_app(make_settings(max_in_flight=64)) + async with make_client(app) as client: + user = await register(client) + responses = await asyncio.gather( + *( + client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + for _ in range(10) + ) + ) + tokens = [r.json()["token"] for r in responses] + checks = await asyncio.gather( + *( + client.get( + "/api/v1/users/me", headers={"Authorization": f"Bearer {token}"} + ) + for token in tokens + ) + ) + await close_app(app) + + assert all(r.status_code == 201 for r in responses) + assert len(set(tokens)) == 10 + assert all(r.status_code == 200 for r in checks) + + +async def test_parallel_logout_of_same_token_succeeds_once(make_settings): + app = create_app(make_settings(max_in_flight=64)) + async with make_client(app) as client: + session = await create_user(client) + responses = await asyncio.gather( + *( + client.delete("/api/v1/auth/sessions/current", headers=bearer(session)) + for _ in range(10) + ) + ) + await close_app(app) + + assert Counter(r.status_code for r in responses) == Counter({204: 1, 401: 9}) + + +async def test_message_limit_is_exact_under_concurrency(make_settings, db_engine): + app = create_app( + make_settings( + max_messages=5, max_in_flight=64, db_pool_size=20, db_max_overflow=20 + ) + ) + async with make_client(app) as client: + user = await create_user(client) + chat = ( + await client.post( + "/api/v1/chats", + headers=bearer(user), + json={"title": "limit", "member_ids": [user["id"]]}, + ) + ).json() + responses = await asyncio.gather( + *( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**bearer(user), **idempotency_header()}, + json={"text": f"m{i}"}, + ) + for i in range(20) + ) + ) + await close_app(app) + + assert Counter(r.status_code for r in responses) == Counter({201: 5, 503: 15}) + async with db_engine.connect() as connection: + stored = ( + await connection.execute(text("SELECT count(*) FROM messages")) + ).scalar_one() + counter = ( + await connection.execute( + text("SELECT value FROM counters WHERE name = 'messages'") + ) + ).scalar_one() + assert stored == counter == 5 + + +async def test_failed_chat_insert_rolls_back_whole_transaction( + make_settings, db_engine, monkeypatch +): + import app.domain.chats as chats_domain + + async def accept_everyone(_session, user_ids): + return set(user_ids) + + app = create_app(make_settings()) + async with make_client(app) as client: + alice = await create_user(client) + monkeypatch.setattr( + chats_domain.users_storage, "get_existing_user_ids", accept_everyone + ) + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "broken", "member_ids": [alice["id"], str(uuid4())]}, + ) + monkeypatch.undo() + after = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "ok", "member_ids": [alice["id"]]}, + ) + await close_app(app) + + assert response.status_code == 400 + assert after.status_code == 201 + async with db_engine.connect() as connection: + titles = (await connection.execute(text("SELECT title FROM chats"))).scalars() + members = ( + await connection.execute(text("SELECT count(*) FROM chat_members")) + ).scalar_one() + assert list(titles) == ["ok"] + assert members == 1 diff --git a/student-tests/test_constraints.py b/student-tests/test_constraints.py new file mode 100644 index 00000000..11aa1f3e --- /dev/null +++ b/student-tests/test_constraints.py @@ -0,0 +1,46 @@ +"""Ограничения на уровне БД: приложение — не единственный защитник данных.""" + +import pytest +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError + +from helpers import create_chat, create_user + + +async def test_duplicate_username_is_rejected_by_database(client, db_engine): + await create_user(client, username="unique_name") + + with pytest.raises(IntegrityError): + async with db_engine.begin() as connection: + await connection.execute( + text( + "INSERT INTO users (id, username, display_name, password_hash) " + "VALUES (gen_random_uuid(), 'unique_name', 'Other', 'hash')" + ) + ) + + +async def test_message_requires_existing_chat(client, db_engine): + alice = await create_user(client) + + with pytest.raises(IntegrityError): + async with db_engine.begin() as connection: + await connection.execute( + text( + "INSERT INTO messages (id, chat_id, sender_id, text) " + "VALUES (gen_random_uuid(), gen_random_uuid(), :sender, 'x')" + ), + {"sender": alice["id"]}, + ) + + +async def test_duplicate_membership_is_rejected_by_database(client, db_engine): + alice = await create_user(client) + chat = await create_chat(client, alice) + + with pytest.raises(IntegrityError): + async with db_engine.begin() as connection: + await connection.execute( + text("INSERT INTO chat_members (chat_id, user_id) VALUES (:c, :u)"), + {"c": chat["id"], "u": alice["id"]}, + ) diff --git a/student-tests/test_errors.py b/student-tests/test_errors.py new file mode 100644 index 00000000..3fe5e939 --- /dev/null +++ b/student-tests/test_errors.py @@ -0,0 +1,76 @@ +import json +import logging + +from httpx import AsyncClient + +from helpers import assert_request_id + + +async def test_missing_token_returns_401(client: AsyncClient): + response = await client.get("/api/v1/users/me") + + assert response.status_code == 401 + assert response.json()["error"]["code"] == "unauthorized" + assert_request_id(response) + + +async def test_garbage_token_returns_401(client: AsyncClient): + response = await client.get( + "/api/v1/users/me", headers={"Authorization": "Bearer garbage"} + ) + + assert response.status_code == 401 + assert response.json()["error"]["code"] == "unauthorized" + assert_request_id(response) + + +async def test_malformed_json_returns_400(client: AsyncClient): + response = await client.post( + "/api/v1/users", + content='{"username":"alice",', + headers={"Content-Type": "application/json"}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_not_found_returns_404(client: AsyncClient): + response = await client.get("/no-such-path") + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + assert_request_id(response) + + +async def test_method_not_allowed_preserves_405(client: AsyncClient): + response = await client.delete("/api/v1/users") + + assert response.status_code == 405 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_unhandled_exception_is_hidden_from_client( + client_no_raise: AsyncClient, caplog +): + with caplog.at_level(logging.ERROR, logger="app.errors.handlers"): + response = await client_no_raise.get("/__test__/explode") + + assert response.status_code == 500 + payload = response.json() + assert payload["error"]["code"] == "internal_error" + assert payload["error"]["message"] == "Внутренняя ошибка сервера" + assert "test-secret" not in json.dumps(payload, ensure_ascii=False) + assert "Traceback" not in json.dumps(payload, ensure_ascii=False) + assert "Unhandled error request_id=" in caplog.text + assert "test-secret" in caplog.text + assert_request_id(response) + + +async def test_success_response_contains_request_id(client: AsyncClient): + response = await client.get("/health/live") + + assert response.status_code == 200 + assert response.headers.get("X-Request-Id") diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 00000000..9338c3b4 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,138 @@ +"""Недоступность Redis и PostgreSQL: ограниченный 503, без анонимного доступа.""" + +import time + +from app.main import create_app +from conftest import close_app, make_client +from helpers import PASSWORD, assert_request_id, basic_header, bearer, create_user + +ME = "/api/v1/users/me" +DEAD_DATABASE_URL = "postgresql+asyncpg://nobody:secretpass@127.0.0.1:1/none" + + +def assert_unavailable(response) -> None: + assert response.status_code == 503, response.text + assert response.json()["error"]["code"] == "unavailable" + assert "secretpass" not in response.text + assert "Traceback" not in response.text + assert_request_id(response) + + +async def test_redis_down_gives_503_for_login_and_protected_endpoints(make_settings): + alive = create_app(make_settings()) + async with make_client(alive) as client: + session = await create_user(client) + await close_app(alive) + + app = create_app(make_settings(redis_port=1, redis_connect_timeout_seconds=0.5)) + async with make_client(app, raise_app_exceptions=False) as client: + started = time.perf_counter() + protected = await client.get(ME, headers=bearer(session)) + elapsed = time.perf_counter() - started + + login = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(session["username"], session["password"]), + ) + logout = await client.delete( + "/api/v1/auth/sessions/current", headers=bearer(session) + ) + ready = await client.get("/health/ready") + live = await client.get("/health/live") + + await close_app(app) + + for response in (protected, login, logout, ready): + assert_unavailable(response) + assert live.status_code == 200 + assert elapsed < 3 + + +async def test_redis_down_keeps_registration_working(make_settings): + app = create_app(make_settings(redis_port=1, redis_connect_timeout_seconds=0.5)) + async with make_client(app, raise_app_exceptions=False) as client: + response = await client.post( + "/api/v1/users", + json={"username": "no_redis", "display_name": "N", "password": PASSWORD}, + ) + await close_app(app) + + assert response.status_code == 201 + + +async def test_database_down_with_valid_session_gives_503(make_settings): + alive = create_app(make_settings()) + async with make_client(alive) as client: + session = await create_user(client) + await close_app(alive) + + app = create_app( + make_settings(database_url=DEAD_DATABASE_URL, db_connect_timeout_seconds=1) + ) + async with make_client(app, raise_app_exceptions=False) as client: + protected = await client.get(ME, headers=bearer(session)) + chats = await client.get("/api/v1/chats", headers=bearer(session)) + login = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(session["username"], session["password"]), + ) + await close_app(app) + + for response in (protected, chats, login): + assert_unavailable(response) + + +async def test_dependency_failure_never_authenticates_anonymous_requests(make_settings): + app = create_app(make_settings(redis_port=1, redis_connect_timeout_seconds=0.5)) + async with make_client(app, raise_app_exceptions=False) as client: + no_token = await client.get(ME) + garbage = await client.get(ME, headers={"Authorization": "Bearer garbage"}) + await close_app(app) + + assert no_token.status_code == 401 + assert_unavailable(garbage) + + +async def test_statement_timeout_gives_503(make_settings, monkeypatch): + from sqlalchemy import text + + import app.domain.users as users_domain + + async def slow_lookup(session, _username): + await session.execute(text("SELECT pg_sleep(1)")) + + app = create_app(make_settings(db_statement_timeout_ms=100)) + monkeypatch.setattr(users_domain.users_storage, "get_user_by_username", slow_lookup) + async with make_client(app, raise_app_exceptions=False) as client: + response = await client.post( + "/api/v1/users", + json={"username": "slow_user", "display_name": "S", "password": PASSWORD}, + ) + await close_app(app) + + assert_unavailable(response) + assert response.headers["Retry-After"] == "1" + + +async def test_non_transient_database_error_stays_500(make_settings, monkeypatch): + from sqlalchemy import text + + import app.domain.users as users_domain + + async def broken_lookup(session, _username): + await session.execute(text("SELECT 1 / 0")) + + app = create_app(make_settings()) + monkeypatch.setattr( + users_domain.users_storage, "get_user_by_username", broken_lookup + ) + async with make_client(app, raise_app_exceptions=False) as client: + response = await client.post( + "/api/v1/users", + json={"username": "broken_user", "display_name": "B", "password": PASSWORD}, + ) + await close_app(app) + + assert response.status_code == 500 + assert response.json()["error"]["code"] == "internal_error" + assert "division" not in response.text diff --git a/student-tests/test_health.py b/student-tests/test_health.py new file mode 100644 index 00000000..26d15896 --- /dev/null +++ b/student-tests/test_health.py @@ -0,0 +1,62 @@ +from httpx import AsyncClient + +from app.main import create_app +from conftest import close_app, make_client +from helpers import assert_request_id + + +async def test_liveness(client: AsyncClient): + response = await client.get("/health/live") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + assert_request_id(response) + + +async def test_readiness(client: AsyncClient): + response = await client.get("/health/ready") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + assert_request_id(response) + + +async def test_readiness_is_503_when_database_is_down(make_settings): + app = create_app( + make_settings( + database_url="postgresql+asyncpg://nobody:nopass@127.0.0.1:1/none", + db_connect_timeout_seconds=1, + ) + ) + async with make_client(app) as client: + assert (await client.get("/health/live")).status_code == 200 + + ready = await client.get("/health/ready") + assert ready.status_code == 503 + assert ready.json()["error"]["code"] == "unavailable" + assert_request_id(ready) + await close_app(app) + + +async def test_business_request_is_503_when_database_is_down(make_settings): + app = create_app( + make_settings( + database_url="postgresql+asyncpg://nobody:secretpass@127.0.0.1:1/none", + db_connect_timeout_seconds=1, + ) + ) + async with make_client(app, raise_app_exceptions=False) as client: + response = await client.post( + "/api/v1/users", + json={ + "username": "alice", + "display_name": "Alice", + "password": "Correct-horse-battery-1", + }, + ) + await close_app(app) + + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert "secretpass" not in response.text + assert_request_id(response) diff --git a/student-tests/test_history_plan.py b/student-tests/test_history_plan.py new file mode 100644 index 00000000..704c9ccd --- /dev/null +++ b/student-tests/test_history_plan.py @@ -0,0 +1,78 @@ +"""План запроса истории на >= 10 000 сообщений: индекс без Sort и полного скана.""" + +import json +from datetime import datetime +from uuid import UUID + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine + +from helpers import bearer, create_chat, create_user + +HISTORY_QUERY = """ +SELECT * FROM messages +WHERE chat_id = :chat_id AND (created_at, id) > (:after_created_at, :after_id) +ORDER BY created_at, id +LIMIT 51 +""" + + +def walk(node: dict) -> list[dict]: + nodes = [node] + for child in node.get("Plans", []): + nodes.extend(walk(child)) + return nodes + + +async def seed_messages( + engine: AsyncEngine, chat_id: str, sender_id: str, count: int +) -> None: + async with engine.begin() as connection: + await connection.execute( + text( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), :chat_id, :sender_id, 'message ' || g, + now() - make_interval(secs => :count - g) + FROM generate_series(1, :count) AS g + """ + ), + {"chat_id": chat_id, "sender_id": sender_id, "count": count}, + ) + await connection.execute(text("ANALYZE messages")) + + +async def test_history_page_uses_composite_index(client, db_engine: AsyncEngine): + alice = await create_user(client) + target = await create_chat(client, alice, title="target") + noise = await create_chat(client, alice, title="noise") + await seed_messages(db_engine, target["id"], alice["id"], 10_000) + await seed_messages(db_engine, noise["id"], alice["id"], 10_000) + + middle = ( + await client.get( + f"/api/v1/chats/{target['id']}/messages?limit=100", headers=bearer(alice) + ) + ).json()["items"][-1] + + async with db_engine.connect() as connection: + raw_plan = ( + await connection.execute( + text(f"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) {HISTORY_QUERY}"), + { + "chat_id": UUID(target["id"]), + "after_created_at": datetime.fromisoformat(middle["created_at"]), + "after_id": UUID(middle["id"]), + }, + ) + ).scalar_one() + + plan = (raw_plan if isinstance(raw_plan, list) else json.loads(raw_plan))[0]["Plan"] + nodes = walk(plan) + node_types = {node["Node Type"] for node in nodes} + index_names = {node.get("Index Name") for node in nodes} + + assert "ix_messages_chat_id_created_at_id" in index_names + assert "Sort" not in node_types + assert "Seq Scan" not in node_types + assert all(node.get("Actual Rows", 0) <= 51 for node in nodes) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..7c3aa49d --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,190 @@ +"""Повторы сообщений через приложения с общими хранилищами.""" + +import asyncio +from contextlib import AsyncExitStack + +import pytest +from sqlalchemy import text + +from app.errors.exceptions import unavailable +from app.main import create_app +from app.storage import idempotency as key_storage +from conftest import close_app, make_client +from helpers import bearer, create_chat, create_user, idempotency_header, send_message + + +@pytest.fixture +async def replicas(request, make_settings): + overrides = getattr(request, "param", {}) + apps = [ + create_app(make_settings(instance_id=name, **overrides)) + for name in ("api-a", "api-b") + ] + try: + async with AsyncExitStack() as stack: + clients = [ + await stack.enter_async_context(make_client(app)) for app in apps + ] + yield clients + finally: + for app in apps: + await close_app(app) + + +@pytest.mark.parametrize("key", [None, "", "a b", "a/b", "x" * 129]) +async def test_invalid_key_returns_400(client, key): + user = await create_user(client) + chat = await create_chat(client, user) + headers = bearer(user) + if key is not None: + headers["Idempotency-Key"] = key + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", headers=headers, json={"text": "hello"} + ) + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.parametrize("key", ["a", "x" * 128, "key_1.2:3-4"]) +async def test_valid_key_boundaries(client, key): + user = await create_user(client) + chat = await create_chat(client, user) + assert (await send_message(client, user, chat, "hello", key)).status_code == 201 + + +async def test_replay_and_conflict_across_replicas(replicas, db_engine): + first, second = replicas + user = await create_user(first) + chat = await create_chat(first, user) + original = await send_message(first, user, chat, "Привет", "same-key") + repeated = await second.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={ + **bearer(user), + **idempotency_header("same-key"), + "Content-Type": "application/json", + }, + content='{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }', + ) + conflict = await send_message(second, user, chat, "другой текст", "same-key") + assert original.status_code == repeated.status_code == 201 + assert original.json() == repeated.json() + assert original.headers["X-Instance-Id"] != repeated.headers["X-Instance-Id"] + assert conflict.status_code == 409 + async with db_engine.connect() as connection: + assert ( + await connection.execute(text("SELECT count(*) FROM messages")) + ).scalar_one() == 1 + assert ( + await connection.execute( + text("SELECT value FROM counters WHERE name='messages'") + ) + ).scalar_one() == 1 + + +async def test_parallel_replays_create_one_message(replicas, db_engine): + user = await create_user(replicas[0]) + chat = await create_chat(replicas[0], user) + responses = await asyncio.gather( + *( + send_message(replicas[index % 2], user, chat, "parallel", "one-key") + for index in range(24) + ) + ) + assert all(response.status_code == 201 for response in responses) + assert len({response.json()["id"] for response in responses}) == 1 + assert {response.headers["X-Instance-Id"] for response in responses} == { + "api-a", + "api-b", + } + async with db_engine.connect() as connection: + assert ( + await connection.execute(text("SELECT count(*) FROM messages")) + ).scalar_one() == 1 + assert ( + await connection.execute(text("SELECT count(*) FROM idempotency_keys")) + ).scalar_one() == 1 + + +async def test_key_scope_separates_users_and_chats(client): + alice = await create_user(client) + bob = await create_user(client) + shared = await create_chat(client, alice, bob) + other = await create_chat(client, alice) + responses = [ + await send_message(client, alice, shared, "a", "shared-key"), + await send_message(client, bob, shared, "b", "shared-key"), + await send_message(client, alice, other, "c", "shared-key"), + ] + assert all(response.status_code == 201 for response in responses) + assert len({response.json()["id"] for response in responses}) == 3 + + +async def test_expired_key_can_be_reused_without_removing_message(client, db_engine): + user = await create_user(client) + chat = await create_chat(client, user) + first = await send_message(client, user, chat, "old", "expiring-key") + async with db_engine.begin() as connection: + await connection.execute( + text( + "UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'" + ) + ) + second = await send_message(client, user, chat, "new", "expiring-key") + assert first.status_code == second.status_code == 201 + assert first.json()["id"] != second.json()["id"] + history = await client.get( + f"/api/v1/chats/{chat['id']}/messages", headers=bearer(user) + ) + assert [item["text"] for item in history.json()["items"]] == ["old", "new"] + + +async def test_failed_transaction_does_not_poison_key(client, db_engine, monkeypatch): + user = await create_user(client) + chat = await create_chat(client, user) + original = key_storage.save_result + + async def fail_after_insert(*args): + await original(*args) + raise unavailable() + + monkeypatch.setattr(key_storage, "save_result", fail_after_insert) + failed = await send_message(client, user, chat, "retry", "retry-key") + assert failed.status_code == 503 + async with db_engine.connect() as connection: + for table in ("messages", "idempotency_keys", "counters"): + assert ( + await connection.execute(text(f"SELECT count(*) FROM {table}")) + ).scalar_one() == 0 + monkeypatch.setattr(key_storage, "save_result", original) + recovered = await send_message(client, user, chat, "retry", "retry-key") + assert recovered.status_code == 201 + + +@pytest.mark.parametrize("replicas", [{"max_messages": 1}], indirect=True) +async def test_replay_works_when_message_limit_is_full(replicas): + user = await create_user(replicas[0]) + chat = await create_chat(replicas[0], user) + first = await send_message(replicas[0], user, chat, "last", "last-key") + replay = await send_message(replicas[1], user, chat, "last", "last-key") + rejected = await send_message(replicas[1], user, chat, "extra", "other-key") + assert first.status_code == replay.status_code == 201 + assert first.json() == replay.json() + assert rejected.status_code == 503 + + +async def test_replay_checks_session_and_membership(client, db_engine): + user = await create_user(client) + chat = await create_chat(client, user) + assert ( + await send_message(client, user, chat, "private", "private-key") + ).status_code == 201 + async with db_engine.begin() as connection: + await connection.execute(text("DELETE FROM chat_members")) + assert ( + await send_message(client, user, chat, "private", "private-key") + ).status_code == 404 + await client.delete("/api/v1/auth/sessions/current", headers=bearer(user)) + assert ( + await send_message(client, user, chat, "private", "private-key") + ).status_code == 401 diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py new file mode 100644 index 00000000..f317fa59 --- /dev/null +++ b/student-tests/test_image_pipeline.py @@ -0,0 +1,113 @@ +"""Реальные преобразования и OS-лимиты дочернего процесса worker.""" + +import asyncio +from io import BytesIO +import os +from pathlib import Path +import subprocess +import sys +from uuid import uuid4 + +from PIL import Image +import pytest + +from app.core.image_processing import ImageSettings +from app.domain.image_pipeline import create_variants +from app.domain.job_files import render_variants +from app.storage.jobs import JobWork +from test_attachments import MemoryStorage + + +@pytest.mark.parametrize("size,transparent", [((1600, 800), False), ((24, 12), True)]) +async def test_variant_geometry_metadata_and_alpha(tmp_path, size, transparent): + source = tmp_path / "source.png" + with Image.new("RGBA", size, (255, 0, 0, 0 if transparent else 255)) as image: + exif = Image.Exif() + exif[270] = "private description" + image.save(source, format="PNG", exif=exif) + result = create_variants( + source, tmp_path, "image/png", 40_000_000, ImageSettings(_env_file=None) + ) + variants = {v.name: v for v in result.variants} + assert (variants["crop"].width, variants["crop"].height) == (128, 128) + assert (variants["thumbnail"].width, variants["thumbnail"].height) == ( + (256, 128) if size[0] > 256 else size + ) + assert (variants["optimized"].width, variants["optimized"].height) == ( + (1280, 640) if size[0] > 1280 else size + ) + for name in ("thumbnail.jpg", "crop.jpg", "optimized.webp"): + with Image.open(tmp_path / name) as image: + assert ( + not image.getexif() + and not image.info.get("icc_profile") + and not image.info.get("xmp") + ) + if transparent: + if name.endswith("webp"): + assert image.getpixel((0, 0))[3] == 0 + else: + assert min(image.getpixel((0, 0))) >= 250 + + +async def test_exif_orientation_applied(tmp_path): + source = tmp_path / "source.jpg" + with Image.new("RGB", (400, 200), "blue") as image: + exif = Image.Exif() + exif[274] = 6 + image.save(source, exif=exif) + result = create_variants( + source, tmp_path, "image/jpeg", 40_000_000, ImageSettings(_env_file=None) + ) + assert (result.variants[0].width, result.variants[0].height) == (128, 256) + + +@pytest.mark.parametrize("failure", ["memory", "cpu"]) +async def test_worker_os_limits(failure, tmp_path): + body = ( + "return bytearray(2 * 1024 ** 3)" if failure == "memory" else "while True: pass" + ) + code = ( + "from app.core import thumbnail_worker as w\ndef fail(*args):\n " + + body + + "\nw.create_variants=fail\nraise SystemExit(w.main())" + ) + result = await asyncio.to_thread( + subprocess.run, + [ + sys.executable, + "-c", + code, + "source", + str(tmp_path), + str(tmp_path / "metadata"), + "image/png", + "40000000", + ], + env={**os.environ, "IMAGE_MEMORY_MIB": "256", "IMAGE_CPU_SECONDS": "1"}, + capture_output=True, + timeout=15, + ) + assert result.returncode == (3 if failure == "memory" else -9) + assert not (tmp_path / "metadata").exists() + + +async def test_render_timeout_kills_child(monkeypatch, tmp_path): + create = asyncio.create_subprocess_exec + children = [] + + async def slow_child(*args, **kwargs): + child = await create( + sys.executable, "-c", "import time;time.sleep(60)", **kwargs + ) + children.append(child) + return child + + monkeypatch.setattr(asyncio, "create_subprocess_exec", slow_child) + work = JobWork( + uuid4(), uuid4(), uuid4(), 1, "staging/test", "image/png", 1, "0" * 64 + ) + with pytest.raises(TimeoutError): + async with asyncio.timeout(0.2): + await render_variants(MemoryStorage(), work, tmp_path / "source", tmp_path) + assert len(children) == 1 and children[0].returncode is not None diff --git a/student-tests/test_instance.py b/student-tests/test_instance.py new file mode 100644 index 00000000..1f262474 --- /dev/null +++ b/student-tests/test_instance.py @@ -0,0 +1,110 @@ +"""Идентификатор реплики и заголовки ответов без PostgreSQL, Redis и Docker.""" + +from uuid import UUID + +import pytest +from fastapi import Request + +from app.core.instance import resolve_instance_id +from app.errors.exceptions import unavailable +from app.main import create_app +from conftest import close_app, make_client + + +@pytest.fixture(autouse=True) +def clean_db() -> None: + """Заменяет очистку БД из conftest: эти тесты не обращаются к хранилищам.""" + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ("api-explicit", "api-explicit"), + (" api-explicit ", "api-explicit"), + (None, "container-123"), + ("", "container-123"), + (" ", "container-123"), + ], +) +def test_resolve_prefers_configured_id_then_hostname(monkeypatch, configured, expected): + monkeypatch.setenv("HOSTNAME", " container-123 ") + + assert resolve_instance_id(configured) == expected + + +@pytest.mark.parametrize("hostname", [None, "", " "]) +def test_resolve_without_config_or_hostname_returns_uuid(monkeypatch, hostname): + if hostname is None: + monkeypatch.delenv("HOSTNAME", raising=False) + else: + monkeypatch.setenv("HOSTNAME", hostname) + + assert UUID(resolve_instance_id(None)).version == 4 + + +@pytest.fixture +async def instance_client(make_settings): + app = create_app( + make_settings( + instance_id="api-test", + database_url="postgresql+asyncpg://unused@127.0.0.1:1/unused", + redis_port=1, + ) + ) + + @app.get("/__test__/request-id", include_in_schema=False) + async def request_id(request: Request) -> dict[str, str]: + return {"request_id": request.state.request_id} + + @app.get("/__test__/explode", include_in_schema=False) + async def explode() -> None: + raise RuntimeError("test error") + + @app.get("/__test__/unavailable", include_in_schema=False) + async def dependency_failure() -> None: + raise unavailable() + + try: + async with make_client(app, raise_app_exceptions=False) as client: + yield client + finally: + await close_app(app) + + +@pytest.mark.parametrize( + ("method", "path", "status"), + [ + ("GET", "/health/live", 200), + ("POST", "/api/v1/users", 400), + ("GET", "/api/v1/users/me", 401), + ("GET", "/no-such-path", 404), + ("DELETE", "/health/live", 405), + ("GET", "/__test__/explode", 500), + ("GET", "/__test__/unavailable", 503), + ], +) +async def test_instance_header_is_present_on_success_and_errors( + instance_client, method, path, status +): + response = await instance_client.request(method, path) + + assert response.status_code == status + assert response.headers["X-Instance-Id"] == "api-test" + assert response.headers["X-Request-Id"] + if status >= 400: + assert ( + response.json()["error"]["request_id"] == response.headers["X-Request-Id"] + ) + + +async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_client): + first = await instance_client.get("/__test__/request-id") + second = await instance_client.get("/__test__/request-id") + + assert first.status_code == second.status_code == 200 + assert ( + first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test" + ) + assert first.headers["X-Request-Id"] != second.headers["X-Request-Id"] + for response in (first, second): + assert response.json()["request_id"] == response.headers["X-Request-Id"] diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py new file mode 100644 index 00000000..3c6b5d6c --- /dev/null +++ b/student-tests/test_jobs.py @@ -0,0 +1,267 @@ +"""Retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL.""" + +import asyncio +from datetime import timedelta +import errno +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.jobs import JobSettings +from app.domain import attachment_jobs +from app.domain.attachment_jobs import ( + claim_job, + process_attachment_job, + cleanup_staging, + build_result, +) +from app.domain.job_admin import replay_failed_job +from app.domain.job_retry import classify_failure, retry_delay +from app.errors.exceptions import JobOperationError, ProcessingError +from app.models.attachment import Attachment +from app.models.job import AttachmentJob +from app.models.job_replay import JobReplay +from app.schemas.job import JobMessage +from app.storage import jobs +from test_attachments import context, upload, finish_attachment + + +async def envelope(session, attachment_id): + job = await session.scalar( + select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)) + ) + return JobMessage( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=1, + attempt=job.attempt + 1, + created_at=job.created_at, + ) + + +async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + settings = JobSettings(_env_file=None, job_max_attempts=3) + + def fail(_): + raise TimeoutError + + async with AsyncSession(db_engine, expire_on_commit=False) as session: + first = await envelope(session, attachment["id"]) + for attempt in range(1, 4): + message = await envelope(session, attachment["id"]) + assert await process_attachment_job( + session, storage, message, settings, before_processing=fail + ) + job, item = await jobs.lock_job(session, first.job_id) + assert job.attempt == attempt + delay = (job.next_attempt_at - item.updated_at).total_seconds() + assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1) + assert item.status == ("failed" if attempt == 3 else "queued") + assert job.last_error_permanent is False + job.next_attempt_at = await jobs.database_time(session) - timedelta( + seconds=1 + ) + await session.commit() + assert item.error_code == "retry_exhausted" + request_id = uuid4() + result = await replay_failed_job( + session, + storage, + settings, + first.job_id, + request_id=request_id, + actor="test", + reason="dependency_restored", + ) + assert result["attempt_limit"] == 6 + + assert await process_attachment_job(session, storage, first, settings) + job, item = await jobs.lock_job(session, first.job_id) + assert job.attempt == 3 and item.status == "queued" + await session.commit() + message = await envelope(session, attachment["id"]) + assert await process_attachment_job(session, storage, message, settings) + before = dict(storage.objects) + assert await process_attachment_job(session, storage, message, settings) + assert storage.objects == before + repeat = await replay_failed_job( + session, + storage, + settings, + first.job_id, + request_id=request_id, + actor="test", + reason="dependency_restored", + ) + assert repeat["already_applied"] + with pytest.raises(JobOperationError, match="job_not_failed"): + await replay_failed_job( + session, + storage, + settings, + first.job_id, + request_id=uuid4(), + actor="test", + reason="manual_review", + ) + await session.rollback() + assert len((await session.scalars(select(JobReplay))).all()) == 1 + + +async def test_permanent_failure_and_missing_source_replay(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + + def fail(_): + raise ProcessingError("image_limits") + + settings = JobSettings(_env_file=None) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment["id"]) + await process_attachment_job( + session, storage, message, settings, before_processing=fail + ) + job, item = await jobs.lock_job(session, message.job_id) + assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent + job.staging_key = None + await session.commit() + with pytest.raises(JobOperationError, match="source_expired"): + await replay_failed_job( + session, + storage, + settings, + job.id, + request_id=uuid4(), + actor="test", + reason="manual_review", + ) + + +async def test_stale_worker_cannot_write_and_cleanup_keeps_active( + client, context, db_engine +): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + settings = JobSettings(_env_file=None) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment["id"]) + claim = await claim_job(session, message, settings) + job, item = await jobs.lock_job(session, message.job_id) + job.lease_token = uuid4() + job.finished_at = await jobs.database_time(session) - timedelta(days=10) + await session.commit() + assert await cleanup_staging(session, storage, settings, apply=True) == 0 + + before = dict(storage.objects) + assert await build_result(session, storage, claim.work) + assert storage.objects == before + + +async def test_cleanup_ready_preserves_final_objects(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + await finish_attachment(client, storage, alice, attachment["id"]) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment["id"]) + job, item = await jobs.lock_job(session, message.job_id) + staging = job.staging_key + job.finished_at = await jobs.database_time(session) - timedelta(days=2) + await session.commit() + assert ( + await cleanup_staging( + session, storage, JobSettings(_env_file=None), apply=False + ) + == 1 + ) + assert staging in storage.objects + assert ( + await cleanup_staging( + session, storage, JobSettings(_env_file=None), apply=True + ) + == 1 + ) + assert staging not in storage.objects and len(storage.objects) == 4 + assert ( + await cleanup_staging( + session, storage, JobSettings(_env_file=None), apply=True + ) + == 0 + ) + + +async def test_retry_classification_and_cap(): + settings = JobSettings(_env_file=None) + for attempt in (1, 2, 6, 100): + cap = min(120, 5 * 2 ** min(30, attempt - 1)) + assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100)) + assert classify_failure(TimeoutError()) == ("processing_timeout", False) + assert classify_failure(OSError(errno.EACCES, "denied"))[1] + assert not classify_failure(OSError(errno.ENOSPC, "full"))[1] + + +async def test_timeout_records_retry_and_clears_lease( + client, context, db_engine, monkeypatch +): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + + async def stalled(*args, **kwargs): + await asyncio.sleep(60) + + monkeypatch.setattr(attachment_jobs, "build_result", stalled) + settings = JobSettings(_env_file=None).model_copy( + update={"job_timeout_seconds": 0.05} + ) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment["id"]) + assert await process_attachment_job(session, storage, message, settings) + job, item = await jobs.lock_job(session, message.job_id) + assert item.status == "queued" and job.last_error_code == "processing_timeout" + assert job.lease_token is None and job.lease_until is None + + +async def test_failed_retention_and_source_checksum_protect_replay( + client, context, db_engine +): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + + def fail(_): + raise ProcessingError("invalid_image") + + settings = JobSettings(_env_file=None) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment["id"]) + await process_attachment_job( + session, storage, message, settings, before_processing=fail + ) + job, item = await jobs.lock_job(session, message.job_id) + key = job.staging_key + job.finished_at = await jobs.database_time(session) - timedelta(days=2) + await session.commit() + assert await cleanup_staging(session, storage, settings, apply=True) == 0 + data, metadata = storage.objects[key] + storage.objects[key] = (b"x" * len(data), metadata) + with pytest.raises(JobOperationError, match="source_mismatch"): + await replay_failed_job( + session, + storage, + settings, + message.job_id, + request_id=uuid4(), + actor="test", + reason="manual_review", + ) + await session.rollback() + assert not (await session.scalars(select(JobReplay))).all() + job, item = await jobs.lock_job(session, message.job_id) + job.finished_at = await jobs.database_time(session) - timedelta(days=8) + await session.commit() + assert await cleanup_staging(session, storage, settings, apply=True) == 1 + assert key not in storage.objects diff --git a/student-tests/test_lifecycle.py b/student-tests/test_lifecycle.py new file mode 100644 index 00000000..11e4dcb1 --- /dev/null +++ b/student-tests/test_lifecycle.py @@ -0,0 +1,50 @@ +"""Готовность при завершении и ограничение времени проверки зависимостей.""" + +import asyncio +import time + +import pytest + +from app.domain import health +from app.main import create_app +from conftest import close_app, make_client + + +@pytest.fixture(autouse=True) +def clean_db() -> None: + """Эти проверки не обращаются к хранилищам.""" + + +async def test_draining_keeps_liveness_but_rejects_new_work(make_settings): + app = create_app(make_settings()) + app.state.lifecycle.start_draining() + try: + async with make_client(app) as client: + assert (await client.get("/health/live")).status_code == 200 + assert (await client.get("/health/ready")).status_code == 503 + assert (await client.get("/api/v1/users/me")).status_code == 503 + finally: + await close_app(app) + + +async def test_readiness_timeout_cancels_dependency_checks(make_settings, monkeypatch): + cancelled = [] + + async def stalled(_): + try: + await asyncio.sleep(30) + finally: + cancelled.append(True) + + monkeypatch.setattr(health, "check_database", stalled) + monkeypatch.setattr(health, "check_redis", stalled) + app = create_app(make_settings(readiness_timeout_seconds=0.03)) + try: + async with make_client(app) as client: + started = time.monotonic() + response = await client.get("/health/ready") + assert response.status_code == 503 + assert time.monotonic() - started < 1 + assert len(cancelled) == 2 + finally: + await close_app(app) diff --git a/student-tests/test_limits.py b/student-tests/test_limits.py new file mode 100644 index 00000000..9cfca1d3 --- /dev/null +++ b/student-tests/test_limits.py @@ -0,0 +1,122 @@ +"""Ограничения нагрузки: MAX_IN_FLIGHT -> 429, MAX_MESSAGES -> 503.""" + +import asyncio + +import httpx + +from app.main import create_app +from conftest import close_app, make_client +from helpers import ( + assert_request_id, + bearer, + create_chat, + create_user, + idempotency_header, +) + + +def _assert_error(response: httpx.Response, status: int, code: str) -> None: + assert response.status_code == status + body = response.json() + assert body["error"]["code"] == code + assert response.headers.get("X-Request-Id") == body["error"]["request_id"] + + +async def test_message_limit_rejects_without_deleting_confirmed_messages(make_settings): + app = create_app(make_settings(max_messages=2)) + async with make_client(app) as client: + user = await create_user(client) + chat = await create_chat(client, user) + url = f"/api/v1/chats/{chat['id']}/messages" + + for index in range(2): + ok = await client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": f"confirmed-{index}"}, + ) + assert ok.status_code == 201 + + rejected = await client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": "rejected"}, + ) + _assert_error(rejected, 503, "unavailable") + + history = await client.get(f"{url}?limit=100", headers=bearer(user)) + assert history.status_code == 200 + assert [m["text"] for m in history.json()["items"]] == [ + "confirmed-0", + "confirmed-1", + ] + await close_app(app) + + +async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypatch): + import app.endpoints.messages as message_endpoint + + app = create_app(make_settings(max_in_flight=2)) + + original_send_message = message_endpoint.send_message_idempotent + accepted_count = 0 + all_slots_taken = asyncio.Event() + release = asyncio.Event() + + async def blocking_send_message(*args, **kwargs): + nonlocal accepted_count + accepted_count += 1 + if accepted_count == 2: + all_slots_taken.set() + await asyncio.wait_for(release.wait(), timeout=5) + return await original_send_message(*args, **kwargs) + + async with make_client(app, timeout=10) as client: + user = await create_user(client) + chat = await create_chat(client, user) + + monkeypatch.setattr( + message_endpoint, "send_message_idempotent", blocking_send_message + ) + url = f"/api/v1/chats/{chat['id']}/messages" + + held = [ + asyncio.create_task( + client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": f"held-{i}"}, + ) + ) + for i in range(2) + ] + + await asyncio.wait_for(all_slots_taken.wait(), timeout=2) + + rejected = await asyncio.gather( + *( + client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": f"rejected-{i}"}, + ) + for i in range(4) + ) + ) + for response in rejected: + _assert_error(response, 429, "rate_limited") + assert response.headers.get("Retry-After") == "1" + + assert (await client.get("/health/live")).status_code == 200 + + release.set() + assert all(r.status_code == 201 for r in await asyncio.gather(*held)) + + recovered = await client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": "after"}, + ) + assert recovered.status_code == 201 + assert_request_id(recovered) + await close_app(app) diff --git a/student-tests/test_messages.py b/student-tests/test_messages.py new file mode 100644 index 00000000..fecdac2f --- /dev/null +++ b/student-tests/test_messages.py @@ -0,0 +1,237 @@ +from urllib.parse import quote + +import pytest +from httpx import AsyncClient + +from helpers import ( + assert_request_id, + bearer, + create_chat, + create_user, + idempotency_header, + send_message, +) + + +async def _message(client: AsyncClient, user: dict, chat: dict, text: str): + return await send_message(client, user, chat, text) + + +async def test_send_and_list_message(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + chat = await create_chat(client, alice, bob) + + created = await _message(client, alice, chat, "Привет, мир 👋") + + assert created.status_code == 201 + message = created.json() + assert message["sender_id"] == alice["id"] + assert message["chat_id"] == chat["id"] + assert_request_id(created) + + listed = await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=1", + headers=bearer(bob), + ) + + assert listed.status_code == 200 + assert listed.json()["items"] == [message] + assert listed.json()["next_cursor"] is None + assert_request_id(listed) + + +@pytest.mark.parametrize("text", ["", " ", "x" * 4001]) +async def test_invalid_message_text(client: AsyncClient, text: str): + alice = await create_user(client) + chat = await create_chat(client, alice) + + response = await _message(client, alice, chat, text) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_extra_message_field_is_rejected(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**bearer(alice), **idempotency_header()}, + json={"text": "hello", "sender_id": alice["id"]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_messages_are_private_to_chat_members(client: AsyncClient): + alice = await create_user(client) + eve = await create_user(client) + chat = await create_chat(client, alice) + await _message(client, alice, chat, "секрет") + + response = await client.get( + f"/api/v1/chats/{chat['id']}/messages", + headers=bearer(eve), + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + + +async def test_pagination_has_no_duplicates(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + + created = [ + (await _message(client, alice, chat, f"message-{index}")).json() + for index in range(5) + ] + + page1 = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=2", + headers=bearer(alice), + ) + ).json() + page2 = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=2&cursor={quote(page1['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + page3 = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=2&cursor={quote(page2['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + + ids = [item["id"] for page in (page1, page2, page3) for item in page["items"]] + assert ids == [item["id"] for item in created] + assert len(ids) == len(set(ids)) == 5 + assert len(page1["items"]) == 2 + assert len(page2["items"]) == 2 + assert len(page3["items"]) == 1 + assert page3["next_cursor"] is None + + +async def test_cursor_cannot_be_used_for_another_chat(client: AsyncClient): + alice = await create_user(client) + chat_a = await create_chat(client, alice, title="A") + chat_b = await create_chat(client, alice, title="B") + + await _message(client, alice, chat_a, "a1") + await _message(client, alice, chat_a, "a2") + await _message(client, alice, chat_b, "b1") + await _message(client, alice, chat_b, "b2") + + page = ( + await client.get( + f"/api/v1/chats/{chat_a['id']}/messages?limit=1", + headers=bearer(alice), + ) + ).json() + + response = await client.get( + f"/api/v1/chats/{chat_b['id']}/messages?limit=1&cursor={quote(page['next_cursor'], safe='')}", + headers=bearer(alice), + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.parametrize( + "limit, expected_status", [(0, 400), (1, 200), (100, 200), (101, 400)] +) +async def test_limit_boundaries(client: AsyncClient, limit: int, expected_status: int): + alice = await create_user(client) + chat = await create_chat(client, alice) + await _message(client, alice, chat, "one") + + response = await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit={limit}", + headers=bearer(alice), + ) + + assert response.status_code == expected_status + assert_request_id(response) + + +async def test_message_exact_max_length_is_accepted(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + + response = await _message(client, alice, chat, "x" * 4000) + + assert response.status_code == 201 + assert len(response.json()["text"]) == 4000 + + +async def test_nonexistent_chat_messages_return_404(client: AsyncClient): + alice = await create_user(client) + + response = await client.get( + "/api/v1/chats/00000000-0000-4000-8000-000000000000/messages", + headers=bearer(alice), + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + assert_request_id(response) + + +async def test_unauthenticated_requests_are_rejected(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + url = f"/api/v1/chats/{chat['id']}/messages" + + assert (await client.get(url)).status_code == 401 + assert (await client.post(url, json={"text": "hi"})).status_code == 401 + + +async def test_messages_survive_new_login(client: AsyncClient): + from helpers import login + + alice = await create_user(client) + chat = await create_chat(client, alice) + message = (await _message(client, alice, chat, "до выхода")).json() + await client.delete("/api/v1/auth/sessions/current", headers=bearer(alice)) + + alice_again = await login(client, alice) + listed = await client.get( + f"/api/v1/chats/{chat['id']}/messages", headers=bearer(alice_again) + ) + + assert listed.json()["items"] == [message] + + +async def test_history_order_is_created_at_then_id(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + created = [(await _message(client, alice, chat, f"m{i}")).json() for i in range(10)] + + listed = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=100", headers=bearer(alice) + ) + ).json()["items"] + + assert [m["id"] for m in listed] == [m["id"] for m in created] + keys = [(m["created_at"], m["id"]) for m in listed] + assert keys == sorted(keys) + + +async def test_nul_in_message_text_is_rejected(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + + response = await _message(client, alice, chat, "hello\u0000world") + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" diff --git a/student-tests/test_migrations.py b/student-tests/test_migrations.py new file mode 100644 index 00000000..c515ade1 --- /dev/null +++ b/student-tests/test_migrations.py @@ -0,0 +1,79 @@ +"""Миграции: применение с нуля, идемпотентный повтор, совпадение со схемой моделей.""" + +import pytest +from alembic.autogenerate import compare_metadata +from alembic.migration import MigrationContext +from sqlalchemy import inspect, text + +from app.models import Base +from conftest import reset_schema, upgrade_head + +EXPECTED_TABLES = { + "users", + "chats", + "chat_members", + "messages", + "counters", + "idempotency_keys", + "attachments", + "message_attachments", + "upload_intents", + "attachment_jobs", + "job_replays", +} + + +@pytest.fixture +async def migrated_engine(database_url, db_engine): + await reset_schema(db_engine) + await upgrade_head(database_url) + yield db_engine + + await reset_schema(db_engine) + async with db_engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + +async def test_upgrade_creates_all_tables(migrated_engine): + async with migrated_engine.connect() as connection: + tables = await connection.run_sync( + lambda sync: set(inspect(sync).get_table_names()) + ) + assert EXPECTED_TABLES <= tables + + +async def test_upgrade_twice_is_idempotent_and_keeps_data( + migrated_engine, database_url +): + async with migrated_engine.begin() as connection: + await connection.execute( + text( + "INSERT INTO users (id, username, display_name, password_hash) " + "VALUES (gen_random_uuid(), 'keep_me', 'Keep', 'hash')" + ) + ) + + await upgrade_head(database_url) + + async with migrated_engine.connect() as connection: + count = ( + await connection.execute( + text("SELECT count(*) FROM users WHERE username = 'keep_me'") + ) + ).scalar_one() + assert count == 1 + + +async def test_models_and_migrations_are_in_sync(migrated_engine): + def compare(sync_connection): + context = MigrationContext.configure( + sync_connection, opts={"compare_type": True} + ) + return compare_metadata(context, Base.metadata) + + async with migrated_engine.connect() as connection: + diff = await connection.run_sync(compare) + + assert ( + diff == [] + ), f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}" diff --git a/student-tests/test_pagination.py b/student-tests/test_pagination.py new file mode 100644 index 00000000..e45e5b1b --- /dev/null +++ b/student-tests/test_pagination.py @@ -0,0 +1,137 @@ +"""Курсоры: непрозрачность, привязка к ресурсу и устойчивость к подделке.""" + +import base64 +import json +from urllib.parse import quote + +import pytest +from httpx import AsyncClient + +from helpers import bearer, create_chat, create_user, send_message + + +def forged_cursor(**payload: str) -> str: + return base64.urlsafe_b64encode(json.dumps(payload).encode()).decode() + + +@pytest.mark.parametrize( + "cursor", + [ + "not-base64-%%%", + base64.urlsafe_b64encode(b"not json").decode(), + base64.urlsafe_b64encode(b"[1, 2, 3]").decode(), + forged_cursor(kind="messages"), + ], +) +async def test_garbage_cursor_returns_400(client: AsyncClient, cursor: str): + alice = await create_user(client) + chat = await create_chat(client, alice) + + for url in ( + f"/api/v1/chats?cursor={quote(cursor, safe='')}", + f"/api/v1/chats/{chat['id']}/messages?cursor={quote(cursor, safe='')}", + ): + response = await client.get(url, headers=bearer(alice)) + assert response.status_code == 400, url + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_cursor_without_timezone_is_rejected(client: AsyncClient): + alice = await create_user(client) + cursor = forged_cursor( + kind="chats", + resource_id=alice["id"], + created_at="2026-01-01T00:00:00", + item_id="00000000-0000-4000-8000-000000000000", + ) + + response = await client.get( + f"/api/v1/chats?cursor={quote(cursor, safe='')}", headers=bearer(alice) + ) + + assert response.status_code == 400 + + +async def test_chat_list_cursor_is_bound_to_its_owner(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + for index in range(2): + await create_chat(client, alice, bob, title=f"shared-{index}") + + page = (await client.get("/api/v1/chats?limit=1", headers=bearer(alice))).json() + response = await client.get( + f"/api/v1/chats?limit=1&cursor={quote(page['next_cursor'], safe='')}", + headers=bearer(bob), + ) + + assert response.status_code == 400 + + +async def test_messages_cursor_is_not_accepted_by_chat_list(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + for index in range(2): + await send_message(client, alice, chat, f"m{index}") + + page = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=bearer(alice) + ) + ).json() + response = await client.get( + f"/api/v1/chats?cursor={quote(page['next_cursor'], safe='')}", + headers=bearer(alice), + ) + + assert response.status_code == 400 + + +async def test_limit_can_change_between_pages(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + sent = [ + (await send_message(client, alice, chat, f"m{index}")).json() + for index in range(6) + ] + + first = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=bearer(alice) + ) + ).json() + rest = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=100" + f"&cursor={quote(first['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + + assert [m["id"] for m in first["items"] + rest["items"]] == [m["id"] for m in sent] + assert rest["next_cursor"] is None + + +async def test_message_sent_after_reading_last_page_is_seen_with_cursor( + client: AsyncClient, +): + alice = await create_user(client) + chat = await create_chat(client, alice) + await send_message(client, alice, chat, "first") + await send_message(client, alice, chat, "second") + + page = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=bearer(alice) + ) + ).json() + new = (await send_message(client, alice, chat, "third")).json() + rest = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=100" + f"&cursor={quote(page['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + + assert [m["text"] for m in rest["items"]] == ["second", "third"] + assert rest["items"][-1] == new diff --git a/student-tests/test_s3_stream.py b/student-tests/test_s3_stream.py new file mode 100644 index 00000000..67fc364b --- /dev/null +++ b/student-tests/test_s3_stream.py @@ -0,0 +1,63 @@ +"""Освобождение HTTP-соединения и порядок завершения SDK при отмене.""" + +import asyncio +from threading import Event +from types import SimpleNamespace + +from app.core.s3 import S3Settings +from app.storage.s3 import ObjectStream, run_sync + + +async def test_sdk_thread_finishes_before_cancellation_returns(): + entered, release, finished = Event(), Event(), Event() + + def operation(): + entered.set() + release.wait(timeout=3) + finished.set() + + task = asyncio.create_task(run_sync(operation)) + await asyncio.to_thread(entered.wait, 2) + task.cancel() + await asyncio.sleep(0) + assert not task.done() + release.set() + try: + await task + except asyncio.CancelledError: + pass + assert finished.is_set() + + +async def test_response_discarded_when_get_is_cancelled(): + entered, release = Event(), Event() + discarded = [] + response = object() + + def operation(): + entered.set() + release.wait(timeout=3) + return response + + task = asyncio.create_task(run_sync(operation, discard_result=discarded.append)) + await asyncio.to_thread(entered.wait, 2) + task.cancel() + release.set() + try: + await task + except asyncio.CancelledError: + pass + assert discarded == [response] + + +async def test_stream_closes_even_before_first_iteration(): + calls = [] + response = SimpleNamespace( + close=lambda: calls.append("close"), + release_conn=lambda: calls.append("release"), + ) + storage = SimpleNamespace(settings=S3Settings(_env_file=None)) + stream = ObjectStream(storage, response) + await stream.aclose() + await stream.aclose() + assert calls == ["close", "release"] diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 00000000..e239e227 --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,173 @@ +"""Сессии в Redis: TTL, абсолютный срок, отзыв, потеря и перезапуск хранилища.""" + +import asyncio +import hashlib +import os +import time +from datetime import datetime + +import pytest +from httpx import AsyncClient +from redis.asyncio import Redis + +from app.main import create_app +from conftest import close_app, make_client +from helpers import basic_header, bearer, create_user, register + +ME = "/api/v1/users/me" +LOGOUT = "/api/v1/auth/sessions/current" + + +def session_key(token: str) -> str: + return "session:" + hashlib.sha256(token.encode()).hexdigest() + + +async def test_session_key_ttl_matches_expires_at( + client: AsyncClient, redis_client: Redis +): + user = await register(client) + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + body = response.json() + + expires_at = datetime.fromisoformat(body["expires_at"]) + expire_time_ms = await redis_client.pexpiretime(session_key(body["token"])) + + assert abs(expire_time_ms - expires_at.timestamp() * 1000) < 2 + assert 299_000 < await redis_client.pttl(session_key(body["token"])) <= 300_000 + + +async def test_session_ttl_is_absolute_and_not_extended_by_activity( + client: AsyncClient, redis_client: Redis +): + session = await create_user(client) + key = session_key(session["token"]) + expire_before = await redis_client.pexpiretime(key) + + for _ in range(5): + assert (await client.get(ME, headers=bearer(session))).status_code == 200 + + assert await redis_client.pexpiretime(key) == expire_before + + +async def test_logout_removes_session_key(client: AsyncClient, redis_client: Redis): + session = await create_user(client) + key = session_key(session["token"]) + assert await redis_client.exists(key) == 1 + + assert (await client.delete(LOGOUT, headers=bearer(session))).status_code == 204 + + assert await redis_client.exists(key) == 0 + assert (await client.get(ME, headers=bearer(session))).status_code == 401 + + +async def test_expired_session_is_rejected_everywhere(make_settings): + app = create_app(make_settings(session_ttl_seconds=1)) + async with make_client(app) as client: + session = await create_user(client) + assert (await client.get(ME, headers=bearer(session))).status_code == 200 + + await asyncio.sleep(1.5) + + for method, url in (("GET", ME), ("GET", "/api/v1/chats"), ("DELETE", LOGOUT)): + response = await client.request(method, url, headers=bearer(session)) + assert response.status_code == 401, (method, url) + assert response.headers["WWW-Authenticate"].startswith("Bearer") + await close_app(app) + + +async def test_relogin_after_expiry_issues_new_working_token(make_settings): + app = create_app(make_settings(session_ttl_seconds=1)) + async with make_client(app) as client: + user = await register(client) + first = ( + await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + ).json() + await asyncio.sleep(1.5) + second = ( + await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + ).json() + + assert first["token"] != second["token"] + assert (await client.get(ME, headers=bearer(first))).status_code == 401 + assert (await client.get(ME, headers=bearer(second))).status_code == 200 + await close_app(app) + + +async def test_tampered_token_is_rejected(client: AsyncClient): + session = await create_user(client) + token = session["token"] + tampered = token[:-1] + ("A" if token[-1] != "A" else "B") + + response = await client.get(ME, headers={"Authorization": f"Bearer {tampered}"}) + + assert response.status_code == 401 + + +async def test_session_of_deleted_user_is_rejected(client: AsyncClient, db_engine): + from sqlalchemy import text + + session = await create_user(client) + async with db_engine.begin() as connection: + await connection.execute( + text("DELETE FROM users WHERE id = :id"), {"id": session["id"]} + ) + + response = await client.get(ME, headers=bearer(session)) + + assert response.status_code == 401 + + +async def test_redis_flush_invalidates_all_sessions( + client: AsyncClient, redis_client: Redis +): + alice = await create_user(client) + bob = await create_user(client) + + await redis_client.flushdb() + + for session in (alice, bob): + assert (await client.get(ME, headers=bearer(session))).status_code == 401 + + +@pytest.mark.skipif( + os.environ.get("DOCKER_TESTS") != "1", + reason="перезапускает контейнер redis из compose; включается DOCKER_TESTS=1", +) +async def test_sessions_survive_redis_restart(client: AsyncClient, redis_client: Redis): + active = await create_user(client) + revoked = await create_user(client) + assert (await client.delete(LOGOUT, headers=bearer(revoked))).status_code == 204 + + process = await asyncio.create_subprocess_exec( + "docker", + "compose", + "restart", + "redis", + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + assert await process.wait() == 0 + + deadline = time.monotonic() + 30 + while True: + try: + await redis_client.ping() + break + except Exception: + if time.monotonic() > deadline: + raise + await asyncio.sleep(0.5) + + response = await client.get(ME, headers=bearer(active)) + assert response.status_code == 200 + assert response.json()["id"] == active["id"] + assert (await client.get(ME, headers=bearer(revoked))).status_code == 401 diff --git a/student-tests/test_users.py b/student-tests/test_users.py new file mode 100644 index 00000000..554cf30d --- /dev/null +++ b/student-tests/test_users.py @@ -0,0 +1,128 @@ +import pytest +from httpx import AsyncClient + +from helpers import PASSWORD, assert_request_id, bearer, create_user, register + + +async def test_register_does_not_leak_password(client: AsyncClient): + response = await client.post( + "/api/v1/users", + json={"username": "alice", "display_name": "Алиса 👋", "password": PASSWORD}, + ) + + assert response.status_code == 201 + body = response.json() + assert set(body) == {"id", "username", "display_name", "created_at"} + assert PASSWORD not in response.text + assert_request_id(response) + + +async def test_get_current_user(client: AsyncClient): + user = await create_user(client, display_name="Алиса 👋") + + response = await client.get("/api/v1/users/me", headers=bearer(user)) + + assert response.status_code == 200 + assert response.json()["id"] == user["id"] + assert response.json()["username"] == user["username"] + assert_request_id(response) + + +@pytest.mark.parametrize( + "body", + [ + {"username": "alice", "display_name": "", "password": PASSWORD}, + {"username": "alice", "display_name": " ", "password": PASSWORD}, + {"username": "alice", "display_name": "a" * 101, "password": PASSWORD}, + {"username": "ab", "display_name": "Alice", "password": PASSWORD}, + {"username": "alice-123", "display_name": "Alice", "password": PASSWORD}, + {"username": "AlicE", "display_name": "Alice", "password": PASSWORD}, + {"username": "a" * 33, "display_name": "Alice", "password": PASSWORD}, + {"username": "alice", "display_name": "Alice"}, + {"username": "alice", "display_name": "Alice", "password": "x" * 11}, + {"username": "alice", "display_name": "Alice", "password": "x" * 129}, + ], +) +async def test_invalid_user_payloads(client: AsyncClient, body: dict): + response = await client.post("/api/v1/users", json=body) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_password_length_boundaries_are_accepted(client: AsyncClient): + for index, password in enumerate( + ["x" * 12, "y" * 128, "п" * 12, " spaces inside "] + ): + response = await client.post( + "/api/v1/users", + json={ + "username": f"pw_user_{index}", + "display_name": "Alice", + "password": password, + }, + ) + assert response.status_code == 201, (index, response.text) + + +async def test_extra_user_field_is_rejected(client: AsyncClient): + response = await client.post( + "/api/v1/users", + json={ + "username": "alice", + "display_name": "Alice", + "password": PASSWORD, + "extra": True, + }, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_duplicate_username_returns_conflict(client: AsyncClient): + user = await register(client, username="duplicate_user") + + response = await client.post( + "/api/v1/users", + json={ + "username": user["username"], + "display_name": "Другой", + "password": PASSWORD, + }, + ) + + assert response.status_code == 409 + assert response.json()["error"]["code"] == "conflict" + assert_request_id(response) + + +async def test_user_exact_length_boundaries_are_accepted(client: AsyncClient): + for length in (3, 32): + user = await register(client, username=("a" * length), display_name="d" * 100) + assert len(user["username"]) == length + assert len(user["display_name"]) == 100 + + +@pytest.mark.parametrize("password", [" " * 12, "\t" * 12, " \n" * 6]) +async def test_whitespace_only_password_is_rejected(client: AsyncClient, password: str): + response = await client.post( + "/api/v1/users", + json={"username": "blank_pw", "display_name": "Blank", "password": password}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +@pytest.mark.parametrize("field", ["display_name", "password"]) +async def test_nul_in_user_fields_is_rejected(client: AsyncClient, field: str): + body = {"username": "nul_user", "display_name": "Nul", "password": PASSWORD} + body[field] = body[field] + "\u0000tail" + + response = await client.post("/api/v1/users", json=body) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" diff --git a/tests/README.md b/tests/README.md index 0dfa3f89..43c2d4fa 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,57 +1,22 @@ -# Публичные проверки +# Проверки -Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv. +Полный изолированный прогон, включая публичные сценарии: ```sh -make check -make test BASE_URL=http://localhost:8080 -# без make: -python3 tests/smoke.py --base-url http://localhost:8080 +python3 scripts/test.py ``` -`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite. +Нужны Python 3.9+ и Docker с Compose. Все зависимости и сами тесты выполняются +в Linux-контейнерах. Результаты: `evidence/lab5/`. Тестовый стенд удаляется +после завершения, включая ошибку. Обычный стенд приложения не используется. -Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются. - -## Проверка сохранности, lab2+ - -Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные. +Для запущенного приложения публичный smoke можно вызвать отдельно +из окружения с зависимостями проекта: ```sh -chmod +x scripts/restart.sh -make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/smoke.py --base-url http://127.0.0.1:18081 ``` -Тест создаёт данные и сессию, запускает указанный файл без shell interpolation, ожидает готовности и проверяет те же данные **с прежним токеном**. В lab4+ также проверяется исходный файл. Hook может работать до 90 секунд, готовность затем ожидается до 90 секунд. TTL сессии для этого теста — минимум 300 секунд. Успех не доказывает, что hook действительно перезапустил БД: приложите команды/состояния контейнеров до и после. - -## Проверка отказа, lab3+ - -Реализуйте `scripts/stop-one.sh` на основе примера. `TARGET_INSTANCE_ID` в окружении содержит alias обслужившей тест API-реплики: сопоставьте его своему контейнеру/Swarm task и остановите именно его. Скрипт должен быстро вернуть 0 и **оставить эту реплику остановленной**; восстановление выполняйте отдельно. Swarm может создать новую task с другим ID — это допустимо. - -```sh -chmod +x scripts/stop-one.sh -make test-failover ACTION_SCRIPT=scripts/stop-one.sh -``` - -Перед отказом тест наблюдает ≥2 API-реплики; 15 секунд отправляет сообщения параллельно выполнению hook, повторяет неопределённый POST с тем же ключом, проверяет восстановление записи за ≤10 секунд, отсутствие остановленной реплики в последних ответах, сохранность подтверждённых сообщений и отсутствие дублей. Ошибки переходного периода учитываются. Затем восстановите реплику и повторите для другой. Скрипт — проверка одного отказа API, не гарантия HA БД/LB/host. Вывод hook скрыт, чтобы не утекли секреты; отлаживайте его отдельно с безопасным выводом. - -## Нагрузка - -```sh -python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json -``` - -Это простой **closed-loop** генератор POST сообщений; он ограничен производительностью клиента и страдает coordinated omission. Он выдаёт successful RPS, ошибки и p50/p95/p99 **только успешных запросов**. Во время нагрузки нет прозрачных retry; timeout мог скрыть совершённую запись, поэтому это throughput HTTP-подтверждений, не точный счётчик COMMIT. Изменение размера истории входит в профиль. Для серьёзного исследования используйте k6/Locust/wrk либо свой обоснованный генератор, а не выводите максимальную пропускную способность из одного запуска этого скрипта. Для нагрузки pipeline изображений нужен отдельный сценарий студента. - -## HTTPS, lab7 - -```sh -make test BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt -make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt -``` - -Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. **Внутреннее mTLS, identity/authorization, OpenBao policy и hardening проверяют отдельные тесты студента** по trust matrix. Клиентский сертификат внешнему учебному REST-клиенту не требуется: mTLS находится на внутренних связях. - -## Что ещё остаётся доказать - -Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production. +Smoke проверяет HTTP-контракт, асинхронную загрузку, неизменность original +и JPEG thumbnail. Полный прогон дополнительно проверяет PostgreSQL/Redis, +миграции, изображения, ошибки, outbox, очередь, повторную доставку и отказы. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/client.py b/tests/client.py index f0499a29..d266a668 100644 --- a/tests/client.py +++ b/tests/client.py @@ -1,4 +1,5 @@ """HTTP helpers for public black-box tests; Python 3.9+, standard library only.""" + import base64 import datetime as dt import hashlib @@ -13,205 +14,336 @@ import urllib.request import uuid ROOT = Path(__file__).resolve().parents[1] -LAB = json.loads((ROOT / 'lab.json').read_text())['number'] -SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +SPEC = json.loads((ROOT / "contracts/openapi.json").read_text()) + class Failure(AssertionError): pass + def check(condition, message): if not condition: raise Failure(message) -def timestamp(value): - check(isinstance(value, str) and value.endswith('Z'), 'timestamp must be UTC ending in Z') - try: - return dt.datetime.fromisoformat(value[:-1] + '+00:00') - except ValueError as exc: - raise Failure('invalid RFC3339 timestamp') from exc -def validate(value, schema, where='response'): +def timestamp(value): + check( + isinstance(value, str) and value.endswith("Z"), + "timestamp must be UTC ending in Z", + ) + try: + return dt.datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as exc: + raise Failure("invalid RFC3339 timestamp") from exc + + +def validate(value, schema, where="response"): """Validate the JSON Schema subset used by the supplied contract, not arbitrary OpenAPI.""" - if '$ref' in schema: + if "$ref" in schema: target = SPEC - for part in schema['$ref'].split('/')[1:]: + for part in schema["$ref"].split("/")[1:]: target = target[part] return validate(value, target, where) - kinds = schema.get('type', []) + kinds = schema.get("type", []) if isinstance(kinds, str): kinds = [kinds] - matches = {'null':value is None, 'boolean':isinstance(value,bool), 'integer':isinstance(value,int) and not isinstance(value,bool), 'number':isinstance(value,(int,float)) and not isinstance(value,bool), 'string':isinstance(value,str), 'array':isinstance(value,list), 'object':isinstance(value,dict)} + matches = { + "null": value is None, + "boolean": isinstance(value, bool), + "integer": isinstance(value, int) and not isinstance(value, bool), + "number": isinstance(value, (int, float)) and not isinstance(value, bool), + "string": isinstance(value, str), + "array": isinstance(value, list), + "object": isinstance(value, dict), + } if kinds: - check(any(matches.get(k,False) for k in kinds), where + ': incorrect type') - if 'enum' in schema: - check(value in schema['enum'], where + ': invalid enum') - if 'const' in schema: - check(value == schema['const'], where + ': invalid const') - if isinstance(value,dict): - props = schema.get('properties',{}) - check(all(k in value for k in schema.get('required',[])), where + ': missing required fields') - if schema.get('additionalProperties') is False: - check(not set(value).difference(props), where + ': unexpected fields') + check(any(matches.get(k, False) for k in kinds), where + ": incorrect type") + if "enum" in schema: + check(value in schema["enum"], where + ": invalid enum") + if "const" in schema: + check(value == schema["const"], where + ": invalid const") + if isinstance(value, dict): + props = schema.get("properties", {}) + check( + all(k in value for k in schema.get("required", [])), + where + ": missing required fields", + ) + if schema.get("additionalProperties") is False: + check(not set(value).difference(props), where + ": unexpected fields") for key, sub in props.items(): if key in value: - validate(value[key], sub, where + '.' + key) - if isinstance(value,list): - check(len(value) >= schema.get('minItems',0), where + ': too few items') - check(len(value) <= schema.get('maxItems',float('inf')), where + ': too many items') - if schema.get('uniqueItems'): - encoded = [json.dumps(x,sort_keys=True) for x in value] - check(len(encoded)==len(set(encoded)), where + ': duplicate items') - if 'items' in schema: + validate(value[key], sub, where + "." + key) + if isinstance(value, list): + check(len(value) >= schema.get("minItems", 0), where + ": too few items") + check( + len(value) <= schema.get("maxItems", float("inf")), + where + ": too many items", + ) + if schema.get("uniqueItems"): + encoded = [json.dumps(x, sort_keys=True) for x in value] + check(len(encoded) == len(set(encoded)), where + ": duplicate items") + if "items" in schema: for item in value: - validate(item, schema['items'], where + '[]') - if isinstance(value,str): - check(len(value) >= schema.get('minLength',0), where + ': too short') - check(len(value) <= schema.get('maxLength',float('inf')), where + ': too long') - if 'pattern' in schema: - check(re.search(schema['pattern'], value) is not None, where + ': pattern mismatch') - if schema.get('format') == 'uuid': + validate(item, schema["items"], where + "[]") + if isinstance(value, str): + check(len(value) >= schema.get("minLength", 0), where + ": too short") + check(len(value) <= schema.get("maxLength", float("inf")), where + ": too long") + if "pattern" in schema: + check( + re.search(schema["pattern"], value) is not None, + where + ": pattern mismatch", + ) + if schema.get("format") == "uuid": try: uuid.UUID(value) except ValueError as exc: - raise Failure(where + ': invalid UUID') from exc - if schema.get('format') == 'date-time': + raise Failure(where + ": invalid UUID") from exc + if schema.get("format") == "date-time": timestamp(value) - if isinstance(value,(int,float)) and not isinstance(value,bool): - check(value >= schema.get('minimum',-float('inf')), where + ': below minimum') - check(value <= schema.get('maximum',float('inf')), where + ': above maximum') - if 'anyOf' in schema: - for alternative in schema['anyOf']: + if isinstance(value, (int, float)) and not isinstance(value, bool): + check(value >= schema.get("minimum", -float("inf")), where + ": below minimum") + check(value <= schema.get("maximum", float("inf")), where + ": above maximum") + if "anyOf" in schema: + for alternative in schema["anyOf"]: try: validate(value, alternative, where) break except Failure: pass else: - raise Failure(where + ': no anyOf branch matches') + raise Failure(where + ": no anyOf branch matches") + def shape(value, name): - validate(value, SPEC['components']['schemas'][name]) + validate(value, SPEC["components"]["schemas"][name]) return value + class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): return None + class Client: def __init__(self, base_url=None, ca_file=None, timeout=10): - self.base = (base_url or os.environ.get('BASE_URL') or ('https://localhost:8443' if LAB==7 else 'http://localhost:8080')).rstrip('/') - check(self.base.startswith(('http://','https://')), 'BASE_URL must be HTTP(S)') - check(LAB != 7 or self.base.startswith('https://'), 'lab7 requires HTTPS') + self.base = ( + base_url + or os.environ.get("BASE_URL") + or ("https://localhost:8443" if LAB == 7 else "http://localhost:8080") + ).rstrip("/") + check(self.base.startswith(("http://", "https://")), "BASE_URL must be HTTP(S)") + check(LAB != 7 or self.base.startswith("https://"), "lab7 requires HTTPS") self.timeout = timeout - context = ssl.create_default_context(cafile=ca_file or os.environ.get('CA_FILE') or None) - self.opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context)) + context = ssl.create_default_context( + cafile=ca_file or os.environ.get("CA_FILE") or None + ) + self.opener = urllib.request.build_opener( + NoRedirect(), urllib.request.HTTPSHandler(context=context) + ) - def call(self, method, path, expected=200, body=None, actor=None, headers=None, raw=None, schema=None): - h = {'Accept':'application/json'} + def call( + self, + method, + path, + expected=200, + body=None, + actor=None, + headers=None, + raw=None, + schema=None, + ): + h = {"Accept": "application/json"} if actor: - h['X-User-Id' if LAB==1 else 'Authorization'] = actor['id'] if LAB==1 else 'Bearer ' + actor['token'] + h["X-User-Id" if LAB == 1 else "Authorization"] = ( + actor["id"] if LAB == 1 else "Bearer " + actor["token"] + ) if body is not None: - raw = json.dumps(body,ensure_ascii=False).encode('utf-8') - h['Content-Type'] = 'application/json' + raw = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" h.update(headers or {}) - req = urllib.request.Request(self.base + path, data=raw, headers=h, method=method) + req = urllib.request.Request( + self.base + path, data=raw, headers=h, method=method + ) start = time.monotonic() try: response = self.opener.open(req, timeout=self.timeout) except urllib.error.HTTPError as error: response = error - except (urllib.error.URLError,TimeoutError,OSError) as exc: - # Do not print URL query, headers, request body, or credentials. - raise Failure(method + ' ' + path.split('?')[0] + ': connection/TLS failure (' + type(exc).__name__ + ')') from None + except (urllib.error.URLError, TimeoutError, OSError) as exc: + + raise Failure( + method + + " " + + path.split("?")[0] + + ": connection/TLS failure (" + + type(exc).__name__ + + ")" + ) from None with response: status = response.code rh = response.headers - data = response.read(16*1024*1024 + 1) - check(len(data) <= 16*1024*1024, 'response exceeds test safety limit') - allowed = [expected] if isinstance(expected,int) else expected - check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) - check(bool(rh.get('X-Request-Id')), 'missing X-Request-Id') - if LAB>=3: - check(bool(rh.get('X-Instance-Id')), 'missing X-Instance-Id') + data = response.read(16 * 1024 * 1024 + 1) + check(len(data) <= 16 * 1024 * 1024, "response exceeds test safety limit") + allowed = [expected] if isinstance(expected, int) else expected + check( + status in allowed, + method + + " " + + path.split("?")[0] + + ": expected " + + str(allowed) + + ", received " + + str(status), + ) + check(bool(rh.get("X-Request-Id")), "missing X-Request-Id") + if LAB >= 3: + check(bool(rh.get("X-Instance-Id")), "missing X-Instance-Id") payload = None - if schema or status>=400: - check(rh.get_content_type()=='application/json', 'expected application/json') + if schema or status >= 400: + check( + rh.get_content_type() == "application/json", "expected application/json" + ) try: payload = json.loads(data) - except (ValueError,UnicodeError): - raise Failure('invalid response JSON') from None - shape(payload, 'Error' if status>=400 else schema) - if status>=400: - check(payload['error']['request_id']==rh.get('X-Request-Id'),'error request_id differs from header') - if status==401 and LAB>=2: - scheme = 'Basic' if path=='/api/v1/auth/sessions' else 'Bearer' - check(rh.get('WWW-Authenticate','').lower().startswith(scheme.lower()), 'incorrect WWW-Authenticate challenge') - if status==204: - check(not data, '204 must not contain body') - return {'status':status,'headers':rh,'json':payload,'bytes':data,'elapsed':time.monotonic()-start} + except (ValueError, UnicodeError): + raise Failure("invalid response JSON") from None + shape(payload, "Error" if status >= 400 else schema) + if status >= 400: + check( + payload["error"]["request_id"] == rh.get("X-Request-Id"), + "error request_id differs from header", + ) + if status == 401 and LAB >= 2: + scheme = "Basic" if path == "/api/v1/auth/sessions" else "Bearer" + check( + rh.get("WWW-Authenticate", "").lower().startswith(scheme.lower()), + "incorrect WWW-Authenticate challenge", + ) + if status == 204: + check(not data, "204 must not contain body") + return { + "status": status, + "headers": rh, + "json": payload, + "bytes": data, + "elapsed": time.monotonic() - start, + } def user(self, label): - username = 'u_' + label + '_' + uuid.uuid4().hex[:12] - password = 'T3st-' + uuid.uuid4().hex - body = {'username':username,'display_name':'Студент ' + label} - if LAB>=2: - body['password'] = password - user = self.call('POST','/api/v1/users',201,body=body,schema='User')['json'] - check(user['username']==username,'username changed') - actor = {'id':user['id'],'user':user,'registration':body} - if LAB>=2: - credential = base64.b64encode((username+':'+password).encode()).decode() - session = self.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+credential},schema='Session')['json'] - check(session['user']==user,'session user mismatch') - check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),'session already expired') - actor['token'] = session['token'] + username = "u_" + label + "_" + uuid.uuid4().hex[:12] + password = "T3st-" + uuid.uuid4().hex + body = {"username": username, "display_name": "Студент " + label} + if LAB >= 2: + body["password"] = password + user = self.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + check(user["username"] == username, "username changed") + actor = {"id": user["id"], "user": user, "registration": body} + if LAB >= 2: + credential = base64.b64encode((username + ":" + password).encode()).decode() + session = self.call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "Basic " + credential}, + schema="Session", + )["json"] + check(session["user"] == user, "session user mismatch") + check( + timestamp(session["expires_at"]) > dt.datetime.now(dt.timezone.utc), + "session already expired", + ) + actor["token"] = session["token"] return actor def bootstrap(self): - self.call('GET','/health/live',schema='Health') - self.call('GET','/health/ready',schema='Health') - alice, bob, eve = (self.user(label) for label in ('alice','bob','eve')) - chat = self.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Контрактный тест','member_ids':[alice['id'],bob['id']]},schema='Chat')['json'] - check(set(chat['member_ids'])=={alice['id'],bob['id']},'chat member mismatch') - return alice,bob,eve,chat + self.call("GET", "/health/live", schema="Health") + self.call("GET", "/health/ready", schema="Health") + alice, bob, eve = (self.user(label) for label in ("alice", "bob", "eve")) + chat = self.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Контрактный тест", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + check( + set(chat["member_ids"]) == {alice["id"], bob["id"]}, "chat member mismatch" + ) + return alice, bob, eve, chat def message(self, actor, chat, text, key=None, attachments=None): - body = {'text':text} + body = {"text": text} if attachments is not None: - body['attachment_ids'] = attachments - headers = {'Idempotency-Key':key or uuid.uuid4().hex} if LAB>=3 else {} - return self.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=actor,body=body,headers=headers,schema='Message') + body["attachment_ids"] = attachments + headers = {"Idempotency-Key": key or uuid.uuid4().hex} if LAB >= 3 else {} + return self.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 201, + actor=actor, + body=body, + headers=headers, + schema="Message", + ) def attachment(self, actor, chat): - content = (ROOT / 'tests/fixtures/sample.png').read_bytes() - att = self.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',201 if LAB==4 else 202,actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')['json'] - check(att['status']==('ready' if LAB==4 else 'queued'),'incorrect upload status') - check(att['chat_id']==chat['id'] and att['owner_id']==actor['id'],'attachment ownership mismatch') - check(att['sha256']==hashlib.sha256(content).hexdigest() and att['size_bytes']==len(content),'original checksum/size mismatch') - return att,content + content = (ROOT / "tests/fixtures/sample.png").read_bytes() + att = self.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png", + 201 if LAB == 4 else 202, + actor=actor, + raw=content, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + check( + att["status"] == ("ready" if LAB == 4 else "queued"), + "incorrect upload status", + ) + check( + att["chat_id"] == chat["id"] and att["owner_id"] == actor["id"], + "attachment ownership mismatch", + ) + check( + att["sha256"] == hashlib.sha256(content).hexdigest() + and att["size_bytes"] == len(content), + "original checksum/size mismatch", + ) + return att, content def wait_ready(self, actor, att, wait_seconds=60): - deadline = time.monotonic()+wait_seconds - while time.monotonic()=2 and i+length<=len(data),'invalid JPEG segment length') + check(i + 2 <= len(data), "truncated JPEG segment") + length = int.from_bytes(data[i : i + 2], "big") + check(length >= 2 and i + length <= len(data), "invalid JPEG segment length") if marker in sof: - check(length>=8,'invalid JPEG SOF') - return (int.from_bytes(data[i+5:i+7],'big'),int.from_bytes(data[i+3:i+5],'big')) + check(length >= 8, "invalid JPEG SOF") + return ( + int.from_bytes(data[i + 5 : i + 7], "big"), + int.from_bytes(data[i + 3 : i + 5], "big"), + ) i += length - raise Failure('JPEG dimensions not found') + raise Failure("JPEG dimensions not found") diff --git a/tests/load.py b/tests/load.py index ad81d596..d1e7f01c 100644 --- a/tests/load.py +++ b/tests/load.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Small closed-loop HTTP write benchmark; not an open-loop capacity proof.""" + from concurrent.futures import ThreadPoolExecutor import json import math @@ -11,48 +12,77 @@ from client import Client, Failure, LAB, arguments, check, run def main(): def configure(p): - p.add_argument('--duration',type=float,default=30) - p.add_argument('--concurrency',type=int,default=4) - p.add_argument('--output',default=None,help='Optional JSON metrics, no session tokens') - args = arguments('Synthetic POST message load, closed loop; creates users/messages',configure) - check(1<=args.duration<=3600,'duration must be 1..3600 seconds') - check(1<=args.concurrency<=128,'concurrency must be 1..128') - c = Client(args.base_url,args.ca_file) - alice,bob,eve,chat = c.bootstrap() + p.add_argument("--duration", type=float, default=30) + p.add_argument("--concurrency", type=int, default=4) + p.add_argument( + "--output", default=None, help="Optional JSON metrics, no session tokens" + ) + + args = arguments( + "Synthetic POST message load, closed loop; creates users/messages", configure + ) + check(1 <= args.duration <= 3600, "duration must be 1..3600 seconds") + check(1 <= args.concurrency <= 128, "concurrency must be 1..128") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() barrier = threading.Barrier(args.concurrency) + def worker(worker_id): - local = Client(args.base_url,args.ca_file,timeout=5) + local = Client(args.base_url, args.ca_file, timeout=5) barrier.wait() - deadline = time.monotonic()+args.duration - latencies,errors = [],0 + deadline = time.monotonic() + args.duration + latencies, errors = [], 0 index = 0 - while time.monotonic()=2,'persistence requires lab2+') - check(args.mode!='failover' or LAB>=3,'failover requires lab3+') + p.add_argument("--mode", choices=["persistence", "failover"], required=True) + p.add_argument( + "--action-script", + required=True, + help="Executable script authored by student; restarts stack or stops ONE API replica", + ) + + args = arguments( + "Explicit restart/failover scenario; modifies only synthetic API data", + configure, + ) + check(LAB >= 2, "persistence requires lab2+") + check(args.mode != "failover" or LAB >= 3, "failover requires lab3+") script = Path(args.action_script).expanduser().resolve() - check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)') - check('.example.' not in script.name,'copy and implement the example hook first') - c = Client(args.base_url,args.ca_file,timeout=2) - alice,bob,eve,chat = c.bootstrap() - sent = c.message(alice,chat,'Сохранить при отказе') - original = sent['json'] - target = sent['headers'].get('X-Instance-Id','') + check( + script.is_file() and os.access(script, os.X_OK), + "action script must exist and be executable (chmod +x)", + ) + check(".example." not in script.name, "copy and implement the example hook first") + c = Client(args.base_url, args.ca_file, timeout=2) + alice, bob, eve, chat = c.bootstrap() + sent = c.message(alice, chat, "Сохранить при отказе") + original = sent["json"] + target = sent["headers"].get("X-Instance-Id", "") attachment = content = None - if LAB>=4: - attachment,content = c.attachment(alice,chat) - if LAB>=5: - attachment = c.wait_ready(alice,attachment) + if LAB >= 4: + attachment, content = c.attachment(alice, chat) + if LAB >= 5: + attachment = c.wait_ready(alice, attachment) env = dict(os.environ) - env['TARGET_INSTANCE_ID'] = target + env["TARGET_INSTANCE_ID"] = target observed = set() - if args.mode=='failover': + if args.mode == "failover": for _ in range(40): - observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id']) - check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity') - print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.') - # No shell interpolation and no Docker commands in this test. - proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + observed.add( + c.call("GET", "/api/v1/users/me", actor=alice, schema="User")[ + "headers" + ]["X-Instance-Id"] + ) + check( + len(observed) >= 2, + "less than two API instances observed before failure; verify LB/session affinity", + ) + print( + "Executing explicit " + + args.mode + + " hook; hook output suppressed to avoid leaking secrets." + ) + + proc = subprocess.Popen( + [str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) started = time.monotonic() - successes,failures,after_instances = [],0,[] + successes, failures, after_instances = [], 0, [] try: - if args.mode=='persistence': + if args.mode == "persistence": try: code = proc.wait(timeout=90) except subprocess.TimeoutExpired: - raise Failure('restart hook exceeded 90 seconds') from None - check(code==0,'restart hook failed (inspect your script locally)') - deadline = time.monotonic()+90 + raise Failure("restart hook exceeded 90 seconds") from None + check(code == 0, "restart hook failed (inspect your script locally)") + deadline = time.monotonic() + 90 while True: try: - c.call('GET','/health/ready',schema='Health') - c.call('GET','/api/v1/users/me',actor=alice,schema='User') + c.call("GET", "/health/ready", schema="Health") + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") break except Failure: - check(time.monotonic()=5,'too few successful writes after stop') - check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped') - print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False)) - # Existing token, no re-login. Read all confirmations and reject duplicates. - messages = history(c,bob,chat) - ids = [m['id'] for m in messages] - check(len(ids)==len(set(ids)),'duplicated message IDs in history') - check(any(m==original for m in messages),'confirmed pre-failure message lost or altered') + check(proc.poll() == 0, "stop-one hook did not finish successfully") + check( + first_recovery is not None and first_recovery <= 10, + "writes did not recover within 10 seconds from hook start", + ) + check(len(after_instances) >= 5, "too few successful writes after stop") + check( + target not in after_instances[-5:], + "target replica still serves requests; hook must leave it stopped", + ) + print( + json.dumps( + { + "observed_instances_before": len(observed), + "successful_writes": len(successes), + "failed_attempts": failures, + "first_recovery_seconds": round(first_recovery, 3), + }, + ensure_ascii=False, + ) + ) + + messages = history(c, bob, chat) + ids = [m["id"] for m in messages] + check(len(ids) == len(set(ids)), "duplicated message IDs in history") + check( + any(m == original for m in messages), + "confirmed pre-failure message lost or altered", + ) for msg in successes: - check(any(m==msg for m in messages),'confirmed in-flight message lost or altered') - # Bodies are unique in this scenario: an uncertain request may exist once, never twice. - texts = [m['text'] for m in messages] - check(len(texts)==len(set(texts)),'uncertain retry created duplicate message') + check( + any(m == msg for m in messages), + "confirmed in-flight message lost or altered", + ) + + texts = [m["text"] for m in messages] + check( + len(texts) == len(set(texts)), "uncertain retry created duplicate message" + ) if attachment: - c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment') - downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob) - check(downloaded['bytes']==content,'attachment lost or changed across failure') - print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.') + c.call( + "GET", + "/api/v1/attachments/" + attachment["id"], + actor=bob, + schema="Attachment", + ) + downloaded = c.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob + ) + check( + downloaded["bytes"] == content, + "attachment lost or changed across failure", + ) + print( + "PASS: " + + args.mode + + " preserved confirmed data and existing session. Hook scope needs operator evidence." + ) finally: if proc.poll() is None: proc.terminate() @@ -125,5 +202,6 @@ def main(): proc.kill() proc.wait() -if __name__=='__main__': + +if __name__ == "__main__": run(main) diff --git a/tests/smoke.py b/tests/smoke.py index 4cbfa558..97d20a02 100644 --- a/tests/smoke.py +++ b/tests/smoke.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """One complete public acceptance scenario, cumulative by lab number.""" + import base64 import hashlib import uuid @@ -7,76 +8,165 @@ from client import Client, LAB, arguments, check, run, timestamp def main(): - args = arguments('Messenger public smoke; creates synthetic users and messages') - c = Client(args.base_url,args.ca_file) - alice,bob,eve,chat = c.bootstrap() - path = '/api/v1/chats/'+chat['id'] - c.call('POST','/api/v1/users',409,body=alice['registration']) - c.call('GET','/api/v1/users/me',401) - c.call('GET',path,404,actor=eve) - me = c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json'] - check(me==alice['user'],'current user mismatch') - listed = c.call('GET','/api/v1/chats?limit=100',actor=bob,schema='ChatPage')['json'] - check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat') - hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json'] - check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider') + args = arguments("Messenger public smoke; creates synthetic users and messages") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + c.call("POST", "/api/v1/users", 409, body=alice["registration"]) + c.call("GET", "/api/v1/users/me", 401) + c.call("GET", path, 404, actor=eve) + me = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"] + check(me == alice["user"], "current user mismatch") + listed = c.call("GET", "/api/v1/chats?limit=100", actor=bob, schema="ChatPage")[ + "json" + ] + check( + any(x["id"] == chat["id"] for x in listed["items"]), "member cannot list chat" + ) + hidden = c.call("GET", "/api/v1/chats?limit=100", actor=eve, schema="ChatPage")[ + "json" + ] + check(all(x["id"] != chat["id"] for x in hidden["items"]), "chat leaks to outsider") key = uuid.uuid4().hex - first = c.message(alice,chat,'Привет, мир 👋',key=key)['json'] - second = c.message(bob,chat,'Сообщение Боба')['json'] - check(first['sender_id']==alice['id'] and second['sender_id']==bob['id'],'sender was not derived from actor') - check(first['chat_id']==second['chat_id']==chat['id'],'wrong chat in message') - check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed') - page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json'] - check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor') + first = c.message(alice, chat, "Привет, мир 👋", key=key)["json"] + second = c.message(bob, chat, "Сообщение Боба")["json"] + check( + first["sender_id"] == alice["id"] and second["sender_id"] == bob["id"], + "sender was not derived from actor", + ) + check(first["chat_id"] == second["chat_id"] == chat["id"], "wrong chat in message") + check( + first["text"] == "Привет, мир 👋" and second["text"] == "Сообщение Боба", + "message text changed", + ) + page1 = c.call("GET", path + "/messages?limit=1", actor=bob, schema="MessagePage")[ + "json" + ] + check( + len(page1["items"]) == 1 and page1["next_cursor"], + "first page must have one item and cursor", + ) from urllib.parse import quote - page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json'] - check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history') - ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int)) - check(page1['items']+page2['items']==ordered,'pagination lost, duplicated or reordered a message') - h = {'Idempotency-Key':uuid.uuid4().hex} if LAB>=3 else {} - c.call('POST',path+'/messages',400,actor=alice,body={'text':''},headers=h) - c.call('POST',path+'/messages',404,actor=eve,body={'text':'чужое'},headers=h) - c.call('GET',path+'/messages?limit=0',400,actor=alice) - if LAB>=2: - bad = base64.b64encode((alice['registration']['username']+':wrong-password').encode()).decode() - c.call('POST','/api/v1/auth/sessions',401,headers={'Authorization':'Basic '+bad}) - good = base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode() - c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Basic '+good}) - c.call('GET','/api/v1/users/me',401,headers={'X-User-Id':alice['id']}) - me = c.call('GET','/api/v1/users/me',actor=alice,headers={'X-User-Id':eve['id']},schema='User')['json'] - check(me['id']==alice['id'],'X-User-Id overrode bearer identity') - if LAB>=3: - replay = c.message(alice,chat,first['text'],key=key)['json'] - check(replay==first,'idempotency replay differs') - c.call('POST',path+'/messages',409,actor=alice,body={'text':'другой body'},headers={'Idempotency-Key':key}) - c.call('POST',path+'/messages',400,actor=alice,body={'text':'без ключа'}) - if LAB>=4: - att,content = c.attachment(alice,chat) - # A queued attachment must already be linkable to its own chat. - attached = c.message(alice,chat,'',attachments=[att['id']])['json'] - check(attached['attachment_ids']==[att['id']],'message lost attachment') - ready = c.wait_ready(bob,att) if LAB>=5 else att - check(ready['error_code'] is None,'ready attachment has an error') - download = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) - check(download['bytes']==content,'download differs from original') - check(download['headers'].get_content_type()=='image/png','original content type mismatch') - for suffix in ('','/content'): - c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve) - if LAB>=5: - variants = [x for x in ready['variants'] if x['name']=='thumbnail'] - check(len(variants)==1,'expected exactly one thumbnail') - v = variants[0] - check((v['width'],v['height'])==(256,160),'thumbnail metadata has wrong dimensions') - thumb = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=thumbnail',actor=bob) - check(thumb['headers'].get_content_type()=='image/jpeg','thumbnail content type mismatch') - check(len(thumb['bytes'])==v['size_bytes'] and hashlib.sha256(thumb['bytes']).hexdigest()==v['sha256'],'thumbnail checksum mismatch') - from image_probe import jpeg_size - check(jpeg_size(thumb['bytes'])==(256,160),'actual JPEG dimensions mismatch') - if LAB>=2: - c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice) - c.call('GET','/api/v1/users/me',401,actor=alice) - c.call('DELETE','/api/v1/auth/sessions/current',401,actor=alice) - print('PASS: lab%d public API scenario; infrastructure and grading evidence remain separate.' % LAB) -if __name__=='__main__': + page2 = c.call( + "GET", + path + "/messages?limit=1&cursor=" + quote(page1["next_cursor"], safe=""), + actor=alice, + schema="MessagePage", + )["json"] + check( + len(page2["items"]) == 1 and page2["next_cursor"] is None, + "second page must finish history", + ) + ordered = sorted( + [first, second], + key=lambda x: (timestamp(x["created_at"]), uuid.UUID(x["id"]).int), + ) + check( + page1["items"] + page2["items"] == ordered, + "pagination lost, duplicated or reordered a message", + ) + h = {"Idempotency-Key": uuid.uuid4().hex} if LAB >= 3 else {} + c.call("POST", path + "/messages", 400, actor=alice, body={"text": ""}, headers=h) + c.call( + "POST", path + "/messages", 404, actor=eve, body={"text": "чужое"}, headers=h + ) + c.call("GET", path + "/messages?limit=0", 400, actor=alice) + if LAB >= 2: + bad = base64.b64encode( + (alice["registration"]["username"] + ":wrong-password").encode() + ).decode() + c.call( + "POST", + "/api/v1/auth/sessions", + 401, + headers={"Authorization": "Basic " + bad}, + ) + good = base64.b64encode( + ( + alice["registration"]["username"] + + ":" + + alice["registration"]["password"] + ).encode() + ).decode() + c.call( + "GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good} + ) + c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]}) + me = c.call( + "GET", + "/api/v1/users/me", + actor=alice, + headers={"X-User-Id": eve["id"]}, + schema="User", + )["json"] + check(me["id"] == alice["id"], "X-User-Id overrode bearer identity") + if LAB >= 3: + replay = c.message(alice, chat, first["text"], key=key)["json"] + check(replay == first, "idempotency replay differs") + c.call( + "POST", + path + "/messages", + 409, + actor=alice, + body={"text": "другой body"}, + headers={"Idempotency-Key": key}, + ) + c.call("POST", path + "/messages", 400, actor=alice, body={"text": "без ключа"}) + if LAB >= 4: + att, content = c.attachment(alice, chat) + + attached = c.message(alice, chat, "", attachments=[att["id"]])["json"] + check(attached["attachment_ids"] == [att["id"]], "message lost attachment") + ready = c.wait_ready(bob, att) if LAB >= 5 else att + check(ready["error_code"] is None, "ready attachment has an error") + download = c.call( + "GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob + ) + check(download["bytes"] == content, "download differs from original") + check( + download["headers"].get_content_type() == "image/png", + "original content type mismatch", + ) + for suffix in ("", "/content"): + c.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve) + if LAB >= 5: + variants = [x for x in ready["variants"] if x["name"] == "thumbnail"] + check(len(variants) == 1, "expected exactly one thumbnail") + v = variants[0] + check( + (v["width"], v["height"]) == (256, 160), + "thumbnail metadata has wrong dimensions", + ) + thumb = c.call( + "GET", + "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", + actor=bob, + ) + check( + thumb["headers"].get_content_type() == "image/jpeg", + "thumbnail content type mismatch", + ) + check( + len(thumb["bytes"]) == v["size_bytes"] + and hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"], + "thumbnail checksum mismatch", + ) + from image_probe import jpeg_size + + check( + jpeg_size(thumb["bytes"]) == (256, 160), + "actual JPEG dimensions mismatch", + ) + if LAB >= 2: + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + c.call("GET", "/api/v1/users/me", 401, actor=alice) + c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=alice) + print( + "PASS: lab%d public API scenario; infrastructure and grading evidence remain separate." + % LAB + ) + + +if __name__ == "__main__": run(main)