52 changed files with 2654 additions and 48 deletions
+5 -24
View File
@@ -1,25 +1,6 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_PORT=8085
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
SESSION_TTL_SECONDS=3600
APP_ENV=lab
APP_VERSION=lab5-v1
MAX_ATTEMPTS=5
+1
View File
@@ -0,0 +1 @@
*.sh text eol=lf
+27
View File
@@ -0,0 +1,27 @@
FROM docker:28.5.1-cli AS docker_cli
FROM python:3.13.15-slim-bookworm AS runtime
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \
APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt \
&& useradd --uid 10001 --create-home messenger
COPY app ./app
COPY migrations ./migrations
COPY contracts/openapi.json ./contracts/openapi.json
USER messenger
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=5 \
CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2)"
CMD ["python", "-m", "app"]
# Optional test operator. The API image does not contain the Docker client/socket.
FROM runtime AS checks
USER root
COPY --from=docker_cli /usr/local/bin/docker /usr/local/bin/docker
COPY --from=docker_cli /usr/local/libexec/docker/cli-plugins/docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose
RUN apt-get update && apt-get install -y --no-install-recommends make \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /checks
CMD ["make", "test-student"]
+104 -24
View File
@@ -1,37 +1,117 @@
# Отчёт — лабораторная 5
# Лабораторная 5 — очереди и обработка изображений
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Саламатин, БПИ-2304. Целевая оценка — 4, включая минимум на 3. Треки на 5 не выполнялись.
## Запуск из чистого clone
## Запуск
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Desktop с Linux containers и Compose. Из каталога лабораторной в PowerShell:
## Архитектура и решения
```powershell
./start.ps1
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Скрипт создаёт случайные локальные secrets и .env, собирает API, checks и MinIO, применяет миграции и запускает стенд. Повторный запуск сохраняет пароли. Не менять secrets при сохранённых volumes без процедуры смены пароля. HTTP доступен по http://127.0.0.1:8085. API_PORT меняет порт, API_REPLICAS — число API (по умолчанию 2), MAX_ATTEMPTS — предел попыток (5), SESSION_TTL_SECONDS — срок сессии (3600). APP_ENV и APP_VERSION — метки окружения и версии.
## Доказательства критериев
Ручной запуск с Python 3.9+:
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
```powershell
python scripts/init_local.py
docker compose build api checks minio
docker compose up -d --no-build --wait --wait-timeout 180
```
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Остановить с сохранением данных: `docker compose stop` или `docker compose down`. Команда намеренного удаления учебных данных — `docker compose down --volumes`; тесты её не вызывают.
## Эксперименты и отказы
Версии закреплены в Dockerfile, requirements.txt и compose.yaml: Python 3.13.15, PostgreSQL 17.6, Redis 7.4.5, HAProxy 3.0.12. MinIO и mc собираются из официальных исходных релизов RELEASE.2025-09-07T16-13-09Z и RELEASE.2025-08-13T08-35-41Z. При первой сборке нужен доступ к реестрам и исходникам; она может занять несколько минут. Собранный ранее локальный MinIO тоже подходит. Минимальная машина — Docker с достаточной памятью для всех контейнеров; проверялось на локальном Docker Desktop.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
## Что изменилось после lab4
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Перенесена собственная lab4: FastAPI, PostgreSQL, scrypt, Redis-сессии с TTL, история с cursor, идемпотентная отправка сообщений, две API-реплики, HAProxy и приватный MinIO. Контракт и публичные tests текущей лабы сохранены.
## Ограничения и следующие шаги
Раньше API возвращал готовое вложение с кодом 201. Теперь он принимает файл, проверяет его и возвращает 202 queued после устойчивого handoff. Клиент может сразу отправить сообщение с queued-вложением; скачивание до ready отвечает 409. GET метаданных доступен во всех четырёх состояниях. Failed нельзя прикрепить к новому сообщению.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Путь работы: клиент → HAProxy → API → staging MinIO и PostgreSQL jobs → publisher → Redis Streams → worker → конечные объекты MinIO и атомарный результат PostgreSQL. Брокер отделён от Redis сессий, имеет свой volume и AOF с appendfsync always. PostgreSQL, S3, сессии и брокер используют разные именованные volumes. API и worker не публикуют свои порты; снаружи доступен только LB на loopback.
## Граница 202 и outbox
API ограничивает тело 10 MiB, принимает его по частям во временный файл, проверяет PNG/JPEG и предел 40 млн пикселей до распаковки. MIME определяется содержимым, filename используется только для отображения. Затем отдельно фиксирует pending-строку, которая учитывает будущий staging key. Под блокировкой этой строки записывает полный оригинал в staging/{attachment_id}, меняет статус на queued и создаёт jobs в одной SQL-транзакции. Ответ 202 формируется из снимка queued и уходит после COMMIT. Временный файл API после запроса не нужен.
Таблица jobs одновременно служит transactional outbox и реестром выполнения. Publisher выбирает готовые к отправке queued-строки через FOR UPDATE SKIP LOCKED, публикует job и отмечает время отправки. До конечного состояния намерение остаётся в PostgreSQL; если задача не захвачена, публикация повторяется раз в 5 секунд. Это сознательно допускает дубли и восстанавливает намерение даже при потере записи в брокере. Уже готовые и failed задания не публикуются.
Разрыв после staging до queued оставляет pending, который cleanup удаляет через час под блокировкой строки. Разрыв после SQL COMMIT до publish не теряет принятую задачу: publisher найдёт ту же jobs-строку после восстановления. Сбой после XADD до SQL COMMIT приведёт к повторной публикации с тем же job_id. Redis Streams с AOF always отвечает после записи AOF. Это граница приёма брокером; она отличается от consumer ack.
Сообщение соответствует contracts/jobs.schema.json: schema_version, job_id, type, attachment_id, pipeline_version=1, attempt и created_at. В нём нет байтов, storage keys, URL, filename и credentials. job_id постоянен при redelivery. БД определяет реальное число попыток; старое значение attempt в дубле не сбрасывает счётчик.
## Worker и идемпотентность
Worker читает Streams consumer group по одному сообщению; concurrency=1 и count/prefetch=1. Захват сохраняет processing, номер попытки и lease на 90 секунд. Затем отдельная транзакция держит блокировку jobs-строки на время эффекта. Она не даёт другому worker или восстановителю одновременно опубликовать результат той же работы. После смерти процесса соединение закрывается, а просроченный lease позволяет publisher вернуть задачу в queued. Зависший декодер принудительно завершается по таймауту.
Оригинал копируется из staging в original/{id} без перекодирования. Варианты записываются в variants/{id}/v1/{name}. Ключ эффекта — attachment_id, pipeline_version=1, variant. После всех S3 PUT одна SQL-транзакция публикует variants с размерами и SHA-256 и ставит attachment/job в ready. Только затем отправляется XACK и удаляется запись Streams. Дубль ready — no-op; он не увеличивает attempts и не возвращает объект в processing. При временной ошибке до ack устойчиво фиксируется намерение retry с available_at. При failed — устойчивый реестр ошибки. Если COMMIT не состоялся, ack не выполняется.
Если процесс погиб после S3 PUT до SQL COMMIT, повтор пишет в те же ключи. Сбой после COMMIT результата до ack безопасен: дубль увидит ready. Обещается at least once доставка с идемпотентным эффектом; exactly once доставка не заявляется. Версия pipeline только 1; смена версий относится к невыполненному треку на 5.
## Варианты и ограничения
Thumbnail — JPEG quality 85, вписан в 256×256 без увеличения, размеры округляются вниз, минимум 1 px. Для исходника 320×200 получается 256×160. Crop — центрированный квадрат JPEG 128×128, quality 85; маленький исходник можно увеличить. Optimized — WebP quality 80, вписан в 1280×1280 без увеличения. EXIF orientation применяется до расчёта размеров; alpha накладывается на белый. Новое RGB-изображение убирает EXIF, ICC и текстовые метаданные. WebP не обязан быть меньше каждого исходника.
Worker запускает декодирование в отдельном дочернем процессе с таймаутом 30 s. Лимит контейнера — 512 MiB и 1 CPU, одновременно декодируется один файл. Эти ограничения защищают хост, но предельно большие допустимые по пикселям файлы могут не уложиться в память и завершиться failed после ограниченных попыток. API тоже ограничен 512 MiB, одним декодированием на реплику, 10 s ожидания слота, 20 s чтения тела и общим HTTP-бюджетом 35 s. S3 connect/read timeout — 2/3 s.
Постоянный сбой содержимого даёт invalid_image и сразу failed. Ошибка S3 или таймаут обработки временные: backoff min(60, 2^attempt) секунд плюс jitter 0–1 s. Предел по умолчанию 5 попыток, затем retry_exhausted. Зависшая последняя попытка также переходит в failed после lease. Ошибки наружу не содержат SQL, traceback или секреты.
Вместо отдельной DLQ используется jobs + attachments со state/status failed. Оператор видит их в БД. Явный replay:
```powershell
docker compose exec worker python -m app.pipeline replay JOB_UUID "Причина повторной обработки"
```
Команда принимает только failed, блокирует строку, создаёт replay_audit и начинает новый ограниченный цикл попыток того же job_id. Ready повторно запустить этой командой нельзя. Staging ready удаляет publisher после безопасного COMMIT; неуспешный исходник хранится 7 дней для replay, затем очищается. Реестр сохраняется, replay после очистки запрещён. Queued/processing очистка не трогает. Неподтверждённые pending очищает отдельный cleanup после часа. Конечные orphan-объекты частично выполненного failed задания автоматически не удаляются; их число ограничено фиксированными ключами.
## Проверки и воспроизведение
Полный прогон с Python на хосте:
```powershell
python scripts/verify.py
```
На этом компьютере доступен `../.tools/python/python.exe scripts/verify.py`. Проверки явно останавливают и перезапускают только контейнеры этого Compose-проекта, сохраняют volumes и восстанавливают API. Не запускать два сценария отказа одновременно.
Отдельные команды:
```powershell
docker compose run --rm --no-deps checks make check test
docker compose run --rm --no-deps -e PYTHONPATH=/app checks python -m unittest discover -s student-tests -p test_images.py -v
docker compose run --rm --no-deps -e PYTHONPATH=/app checks python student-tests/pipeline_checks.py
docker compose run --rm --no-deps checks sh -ec 'chmod +x scripts/restart.sh; make test-persistence ACTION_SCRIPT=scripts/restart.sh'
docker compose run --rm --no-deps checks sh -ec 'chmod +x scripts/stop-one.sh; make test-failover ACTION_SCRIPT=scripts/stop-one.sh'
docker compose start api
```
Проверки выполнены 21 ноября 2026 года. Исходный skeleton — 276dbdad8a34da4c5b57fbf1879ca169b581d75b, ветка solution/bpi-2304-salamatin. Результаты полного прогона — evidence/verification.json, подробности — pipeline.json и текстовые логи. before-restart.json и after-restart.json показывают реальные контейнеры, время старта и сохранённые volumes.
| Проверка | Что доказывает | Файл |
| --- | --- | --- |
| Неизменённый публичный smoke | 202, ready, авторизация, оригинал и настоящий JPEG thumbnail | smoke.txt, smoke-final.txt |
| Два теста изображения | EXIF, белый фон, отсутствие увеличения, удаление metadata | images.txt |
| Worker остановлен, API/broker перезапущены | Сообщение с queued-вложением создано, задание сохранено и завершено | pipeline.json |
| API killed между COMMIT и publish при выключенном publisher | Outbox доставляет прежнее задание автоматически | pipeline.json |
| Одноразовый os._exit после копирования original | Восстановление lease, тот же результат, ровно три объекта вариантов | pipeline.json |
| Три дубля ready-job | Attempts и checksums не меняются | pipeline.json |
| Повреждённый staging и replay | invalid_image, аудит, восстановление того же job_id | pipeline.json |
| Выключенный MinIO | Пять попыток с backoff, итоговый retry_exhausted | pipeline.json |
| Негативные загрузки | Размер, повреждение, MIME, filename возвращают 4xx | pipeline.json |
| Persistence и failover | Сохранность прежней сессии и данных; работа через оставшуюся API | persistence.txt, failover.txt |
Для crash после объекта тест создаёт одноразовый /tmp/crash_after_object в worker. Это тестовый fault injection внутри локального контейнера, через HTTP он недоступен. Первый отладочный запуск выявил ошибку тестового окружения: Compose поднимал MinIO как зависимость worker и мешал проверить outage. Исправленный сценарий запускает существующий worker через docker start по ID контейнера. Также удалён лишний запуск нового контейнера из checks, где host bind paths не совпадают. Публичные tests не менялись.
## Ограничения
Стенд локальный и учебный: одиночные PostgreSQL, MinIO, брокер, Redis сессий и LB остаются точками отказа. Проверяется restart с сохранением volumes, а не потеря диска или хоста. Нет резервного копирования и TLS для удалённого доступа. При недоступном брокере API может продолжать приём через outbox, пока работают PostgreSQL и staging; недоступный worker не вызывает синхронный resize в API. Readiness API проверяет PostgreSQL, Redis сессий и S3; брокер и worker не критичны для устойчивого приёма. Нет backpressure по длине очереди; оценка предельной нагрузки не проводилась.
## Источники и помощь
Задание, контракт и публичные проверки: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5
Собственная реализация предыдущей работы: ../mtusi2026lab4. Redis Streams: https://redis.io/docs/latest/develop/data-types/streams/ . Pillow: https://pillow.readthedocs.io/en/stable/ . MinIO: https://github.com/minio/minio .
Использованы FastAPI, Psycopg, redis-py, boto3, Pillow и HAProxy. Codex помог написать реализацию, сценарии проверок и пояснения. Разбор для защиты — docs/REVIEW.docx.
+1
View File
@@ -0,0 +1 @@
"""Laboratory 1: a single-process, in-memory messenger."""
+17
View File
@@ -0,0 +1,17 @@
import logging
import os
import uvicorn
def main():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logging.getLogger("psycopg.pool").setLevel(logging.CRITICAL)
logging.info("Starting lab3 environment=%s with shared PostgreSQL and Redis", os.getenv("APP_ENV", "lab"))
uvicorn.run("app.api:app", host=os.getenv("APP_HOST", "0.0.0.0"),
port=int(os.getenv("APP_PORT", "8080")), workers=1,
access_log=False, timeout_graceful_shutdown=5)
if __name__ == "__main__":
main()
+194
View File
@@ -0,0 +1,194 @@
import asyncio
from contextlib import asynccontextmanager
import json
import logging
import os
from pathlib import Path
import re
from time import perf_counter
from uuid import uuid4
from fastapi import Depends, FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from fastapi.responses import Response
import psycopg
from psycopg_pool import PoolTimeout, TooManyRequests
from redis.exceptions import RedisError
from pydantic import ValidationError
from starlette.exceptions import HTTPException
from .config import Settings
from .errors import APIError, invalid, unavailable
from .models import CreateChat, CreateMessage, CreateUser
from .security import basic_credentials, identifier
from .store import Store
from .attachments import Attachments, metadata
from botocore.exceptions import BotoCoreError, ClientError
from starlette.requests import ClientDisconnect
logger = logging.getLogger("messenger")
def error_response(request, status, code, message, headers=None):
return JSONResponse(status_code=status, headers=headers, content={"error": {
"code": code, "message": message, "request_id": request.state.request_id}})
async def body(request, model):
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_type != "application/json":
raise APIError(415, "unsupported_media_type", "Use application/json")
try:
raw = (await request.body()).decode("utf-8")
def reject_constant(value):
raise ValueError("Nonstandard JSON constant")
data = json.loads(raw, parse_constant=reject_constant)
return model.model_validate(data)
except (ValueError, UnicodeError, ValidationError):
raise invalid("Body must be valid UTF-8 JSON with the required fields and types") from None
def pagination(request):
raw = request.query_params.get("limit", "50")
if re.fullmatch(r"[0-9]+", raw) is None or len(raw) > 3 or not 1 <= int(raw) <= 100:
raise invalid("Limit must be an integer from 1 to 100")
return int(raw), request.query_params.get("cursor")
def create_app(settings=None):
@asynccontextmanager
async def lifespan(app):
store = Store(settings or Settings.load())
app.state.store = store
app.state.attachments = Attachments(store)
try:
await store.open()
yield
finally:
await store.close()
instance_id = os.getenv("HOSTNAME", "api") + "-" + uuid4().hex[:12]
app = FastAPI(title="Lab 5 messenger", docs_url=None, redoc_url=None,
openapi_url=None, redirect_slashes=False, lifespan=lifespan)
@app.middleware("http")
async def request_context(request, call_next):
request.state.request_id = str(uuid4())
start = perf_counter()
try:
async with asyncio.timeout(35):
response = await call_next(request)
except (psycopg.Error, PoolTimeout, TooManyRequests, RedisError, BotoCoreError, ClientError, TimeoutError, ClientDisconnect):
error = unavailable()
logger.warning("Storage unavailable request_id=%s", request.state.request_id)
response = error_response(request, error.status, error.code, error.message)
except Exception:
logger.error("Unhandled error request_id=%s", request.state.request_id)
response = error_response(request, 500, "internal_error", "Internal server error")
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = instance_id
response.headers["X-App-Version"] = os.getenv("APP_VERSION", "lab3-v1")
logger.info("request_id=%s method=%s path=%s status=%s duration_ms=%.2f",
request.state.request_id, request.method, request.url.path,
response.status_code, (perf_counter() - start) * 1000)
return response
@app.exception_handler(APIError)
async def api_error(request, error):
return error_response(request, error.status, error.code, error.message, error.headers)
@app.exception_handler(RequestValidationError)
async def validation_error(request, error):
return error_response(request, 400, "invalid_request", "Invalid request parameters")
@app.exception_handler(HTTPException)
async def routing_error(request, error):
code = "not_found" if error.status_code == 404 else "invalid_request"
return error_response(request, error.status_code, code, "Unknown path or unsupported method")
async def current_user(request: Request):
user, token = await app.state.store.authenticate(request.headers.get("authorization"))
request.state.session_token = token
return user
@app.get("/health/live")
async def health():
return {"status": "ok"}
@app.get("/health/ready")
async def readiness():
await app.state.store.ready()
await asyncio.to_thread(app.state.attachments.s3.head_bucket, Bucket="attachments")
return {"status": "ok"}
@app.post("/api/v1/auth/sessions", status_code=201)
async def login(request: Request):
credentials = basic_credentials(request.headers.get("authorization"))
if await request.body():
raise invalid("Session creation does not accept a body")
return await app.state.store.login(*credentials)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
async def logout(request: Request, user=Depends(current_user)):
await app.state.store.logout(request.state.session_token)
return Response(status_code=204)
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
return await app.state.store.create_user(await body(request, CreateUser))
@app.get("/api/v1/users/me")
async def me(user=Depends(current_user)):
return user
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request, user=Depends(current_user)):
return await app.state.store.create_chat(user["id"], await body(request, CreateChat))
@app.get("/api/v1/chats")
async def list_chats(request: Request, user=Depends(current_user)):
return await app.state.store.list_chats(user["id"], *pagination(request))
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(chat_id: str, user=Depends(current_user)):
return await app.state.store.get_chat(user["id"], identifier(chat_id))
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(chat_id: str, request: Request, user=Depends(current_user)):
chat_id = identifier(chat_id)
await app.state.store.get_chat(user["id"], chat_id)
key = request.headers.get("idempotency-key", "")
if re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key, flags=re.ASCII) is None:
raise invalid("A valid Idempotency-Key is required")
return await app.state.store.create_message(user["id"], chat_id,
await body(request, CreateMessage), key)
@app.get("/api/v1/chats/{chat_id}/messages")
async def list_messages(chat_id: str, request: Request, user=Depends(current_user)):
chat_id = identifier(chat_id)
await app.state.store.get_chat(user["id"], chat_id)
return await app.state.store.list_messages(user["id"], chat_id, *pagination(request))
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload(chat_id: str, request: Request, user=Depends(current_user)):
return await app.state.attachments.upload(user["id"], identifier(chat_id), request)
@app.get("/api/v1/attachments/{attachment_id}")
async def attachment(attachment_id: str, user=Depends(current_user)):
return metadata(await app.state.attachments.get(user["id"], identifier(attachment_id)))
@app.get("/api/v1/attachments/{attachment_id}/content")
async def content(attachment_id: str, request: Request, user=Depends(current_user)):
row = await app.state.attachments.get(user["id"], identifier(attachment_id))
return await app.state.attachments.download(row, request.query_params.get("variant", "original"))
@app.get("/openapi.json", include_in_schema=False)
async def contract():
path = Path(__file__).resolve().parents[1] / "contracts" / "openapi.json"
return json.loads(path.read_text(encoding="utf-8"))
return app
app = create_app()
+195
View File
@@ -0,0 +1,195 @@
"""Private S3 originals and durable upload intents."""
import asyncio
from contextlib import asynccontextmanager
import hashlib
import os
from pathlib import Path
import tempfile
import warnings
from uuid import uuid4, UUID
import boto3
from botocore.config import Config
from PIL import Image, UnidentifiedImageError
from starlette.responses import StreamingResponse
from .errors import APIError, invalid
from .store import public
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
BUCKET = 'attachments'
def secret(name):
return Path('/run/secrets/' + name).read_text().strip()
def client():
return boto3.client('s3', endpoint_url='http://minio:9000', region_name='us-east-1',
aws_access_key_id='messenger-api', aws_secret_access_key=secret('s3_secret_key'),
config=Config(connect_timeout=2, read_timeout=3, retries={'max_attempts': 0},
s3={'addressing_style': 'path'}, max_pool_connections=4))
def validate_image(file, content_type):
try:
with warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
file.seek(0)
with Image.open(file) as image:
if image.width * image.height > MAX_PIXELS:
raise APIError(413, 'payload_too_large', 'Too many pixels')
if image.format not in ('PNG', 'JPEG') or Image.MIME[image.format] != content_type:
raise APIError(415, 'unsupported_media_type', 'Image MIME does not match')
image.verify()
file.seek(0)
with Image.open(file) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise APIError(413, 'payload_too_large', 'Too many pixels') from None
except (UnidentifiedImageError, OSError, SyntaxError, ValueError):
raise APIError(415, 'unsupported_media_type', 'Invalid image') from None
finally:
file.seek(0)
def metadata(row):
row = dict(row)
row['owner_id'] = row['uploader_id']
row.setdefault('updated_at', row['created_at'])
return public({k: row[k] for k in ('id','chat_id','owner_id','filename','content_type',
'size_bytes','sha256','status','created_at','updated_at')}) | {'variants': row['variants'], 'error_code': row['error_code']}
async def blocking(function, *args, **kwargs):
# Cancellation must not close the spool or release the SQL lock while a thread uses it.
task = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs))
try:
return await asyncio.shield(task)
except asyncio.CancelledError:
await task
raise
class Attachments:
def __init__(self, store):
self.store = store
self.s3 = client()
# One decode per replica bounds decoded RGBA memory to about 160 MB.
self.slots = asyncio.Semaphore(1)
@asynccontextmanager
async def slot(self):
try:
await asyncio.wait_for(self.slots.acquire(), timeout=10)
except TimeoutError:
raise APIError(503, 'unavailable', 'Upload capacity is busy') from None
try:
yield
finally:
self.slots.release()
async def get(self, actor, attachment_id):
async with self.store.pool.connection() as conn:
cur = await conn.execute('''SELECT a.* FROM attachments a JOIN chat_members m
ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s''',
(attachment_id, UUID(actor)))
row = await cur.fetchone()
if row is None:
raise APIError(404, 'not_found', 'Attachment not found')
return row
async def upload(self, actor, chat_id, request):
await self.store.get_chat(actor, chat_id)
name = request.query_params.get('filename', '')
if not 1 <= len(name) <= 255 or any(c in '/\\' or ord(c) < 32 or ord(c) == 127 for c in name):
raise invalid('Invalid filename')
mime = request.headers.get('content-type', '').strip().lower()
if mime not in ('image/png','image/jpeg'):
raise APIError(415, 'unsupported_media_type', 'Use PNG or JPEG')
length = request.headers.get('content-length')
if length is not None:
if not length.isascii() or not length.isdecimal():
raise invalid('Invalid Content-Length')
if int(length) > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB')
async with self.slot():
# Temporary spool is only staging, never the source of confirmed originals.
with tempfile.TemporaryFile() as file:
size = 0
digest = hashlib.sha256()
async with asyncio.timeout(20):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB')
file.write(chunk)
digest.update(chunk)
await blocking(validate_image, file, mime)
attachment_id = uuid4()
key = 'original/' + attachment_id.hex
# Commit the intent BEFORE PUT: even a killed process leaves a tracked key.
async with self.store.pool.connection() as conn:
await conn.execute('''INSERT INTO attachments
(id,chat_id,uploader_id,object_key,filename,content_type,size_bytes,sha256,status)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')''',
(attachment_id,chat_id,UUID(actor),key,name,mime,size,digest.hexdigest()))
# The pending row tracks the staging key before S3 PUT.
async with self.store.pool.connection() as conn:
await conn.execute('SELECT id FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,))
await blocking(self.s3.put_object, Bucket=BUCKET, Key='staging/'+attachment_id.hex,
Body=file, ContentLength=size, ContentType=mime)
cur = await conn.execute("UPDATE attachments SET status='queued', updated_at=clock_timestamp() WHERE id=%s RETURNING *",
(attachment_id,))
result = metadata(await cur.fetchone())
await conn.execute('INSERT INTO jobs(id,attachment_id) VALUES (%s,%s)', (uuid4(),attachment_id))
# COMMIT of queued + job is the durable handoff; publisher is independent.
return result
async def download(self, row, variant='original'):
if row['status'] != 'ready':
raise APIError(409, 'conflict', 'Attachment is not ready')
row = dict(row)
if variant != 'original':
info = next((v for v in row['variants'] if v['name']==variant), None)
if info is None:
raise APIError(404, 'not_found', 'Variant not found')
row.update(info)
row['object_key'] = 'variants/'+row['id'].hex+'/v1/'+variant
obj = await asyncio.to_thread(self.s3.get_object, Bucket=BUCKET, Key=row['object_key'])
stream = obj['Body']
async def chunks():
try:
while chunk := await asyncio.to_thread(stream.read, 64 * 1024):
yield chunk
finally:
stream.close()
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff'})
async def cleanup(self):
removed = 0
async with self.store.pool.connection() as conn:
cur = await conn.execute("""SELECT id,object_key FROM attachments WHERE status='pending'
AND created_at < clock_timestamp()-interval '1 hour' ORDER BY created_at
LIMIT 100 FOR UPDATE SKIP LOCKED""")
for row in await cur.fetchall():
await asyncio.to_thread(self.s3.delete_object, Bucket=BUCKET, Key='staging/'+row['id'].hex)
await conn.execute("DELETE FROM attachments WHERE id=%s AND status='pending'", (row['id'],))
removed += 1
return removed
async def cleaner():
from .config import Settings
from .store import Store
store = Store(Settings.load())
await store.open()
attachments = Attachments(store)
try:
while True:
try:
print({'removed_pending': await attachments.cleanup()}, flush=True)
except Exception:
print('Cleanup deferred; storage unavailable', flush=True)
await asyncio.sleep(60)
finally:
await store.close()
if __name__ == '__main__':
asyncio.run(cleaner())
+28
View File
@@ -0,0 +1,28 @@
"""Runtime settings; credential values are never logged."""
from dataclasses import dataclass
import os
from pathlib import Path
def secret(name):
return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip()
@dataclass(frozen=True)
class Settings:
database_url: str
cache_url: str
cursor_key: bytes
session_ttl: int
@classmethod
def load(cls):
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
if not 1 <= ttl <= 604800:
raise ValueError("SESSION_TTL_SECONDS must be in 1..604800")
if os.getenv("SESSION_STORE", "redis") != "redis":
raise ValueError("This deployment uses SESSION_STORE=redis")
key = secret("CURSOR_KEY").encode()
if len(key) < 32:
raise ValueError("Cursor key must contain at least 32 bytes")
return cls(secret("DATABASE_URL"), secret("CACHE_URL"), key, ttl)
+22
View File
@@ -0,0 +1,22 @@
class APIError(Exception):
def __init__(self, status: int, code: str, message: str, headers=None):
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
super().__init__(message)
def invalid(message="Invalid request"):
return APIError(400, "invalid_request", message)
def unauthorized(basic=False):
challenge = ('Basic realm="messenger", charset="UTF-8"' if basic
else 'Bearer realm="messenger"')
return APIError(401, "unauthorized", "Valid credentials required",
{"WWW-Authenticate": challenge})
def unavailable():
return APIError(503, "unavailable", "Storage temporarily unavailable")
+37
View File
@@ -0,0 +1,37 @@
"""Apply numbered migrations atomically; safe to repeat and serialize."""
import hashlib
from pathlib import Path
import psycopg
from .config import secret
ROOT = Path(__file__).resolve().parents[1]
def migrate():
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=2,
options="-c statement_timeout=10000 -c lock_timeout=5000") as conn:
conn.execute("SELECT pg_advisory_xact_lock(202602)")
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY, sha256 text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT clock_timestamp())""")
for path in sorted((ROOT / "migrations").glob("[0-9]*.sql")):
# Git checkout may change LF to CRLF on Windows; SQL is unchanged.
content = path.read_text(encoding="utf-8").encode("utf-8")
checksum = hashlib.sha256(content).hexdigest()
row = conn.execute("SELECT sha256 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone()
if row:
if row[0] != checksum:
raise RuntimeError("Applied migration checksum differs: " + path.name)
print("Unchanged:", path.name)
continue
conn.execute(content.decode("utf-8"))
conn.execute("INSERT INTO schema_migrations(version, sha256) VALUES (%s, %s)",
(path.name, checksum))
print("Applied:", path.name)
print("PASS: migrations committed")
if __name__ == "__main__":
migrate()
+73
View File
@@ -0,0 +1,73 @@
"""Strict request models; strings retain their original Unicode content."""
import re
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
class Input(BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")
@field_validator("*", check_fields=False)
@classmethod
def valid_text(cls, value):
if isinstance(value, str):
if not value.strip():
raise ValueError("A string must not be blank")
value.encode("utf-8")
if "\x00" in value:
raise ValueError("NUL is not supported")
return value
class CreateUser(Input):
username: str
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_format(cls, value):
if re.fullmatch(r"[a-z0-9_]{3,32}", value) is None:
raise ValueError("Invalid username")
return value
class CreateChat(Input):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator("member_ids")
@classmethod
def members(cls, values):
normalized = [str(UUID(value)) for value in values]
if len(set(normalized)) != len(normalized):
raise ValueError("Duplicate members")
return normalized
class CreateMessage(Input):
text: str = Field(max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@field_validator("text")
@classmethod
def valid_text(cls, value):
value.encode("utf-8")
if "\x00" in value:
raise ValueError("Invalid text")
return value
@field_validator("attachment_ids")
@classmethod
def attachments(cls, values):
normalized = [str(UUID(value)) for value in values]
if len(set(normalized)) != len(normalized):
raise ValueError("Duplicate attachments")
return normalized
@model_validator(mode="after")
def not_empty(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Message is empty")
return self
+181
View File
@@ -0,0 +1,181 @@
"""PostgreSQL outbox, Redis Streams at-least-once transport and fenced effects."""
import asyncio
import hashlib
import json
import os
from pathlib import Path
import random
import sys
import tempfile
from uuid import uuid4, UUID
from psycopg.types.json import Jsonb
from redis.asyncio import Redis
from redis.exceptions import ResponseError
from .attachments import Attachments, BUCKET
from .config import Settings
from .store import Store
STREAM='images.v1'
GROUP='workers'
MAX_ATTEMPTS=int(os.getenv('MAX_ATTEMPTS','5'))
def broker():
return Redis.from_url(Path('/run/secrets/cache_url').read_text().strip().replace('@redis:', '@broker:'), decode_responses=True)
async def publish(store, queue, attachments):
async with store.pool.connection() as conn:
# Row locks fence stale workers; recovery waits until their connection is gone.
expired=await conn.execute("""SELECT id,attachment_id,attempts FROM jobs WHERE state='processing'
AND lease_until<clock_timestamp() FOR UPDATE SKIP LOCKED""")
for row in await expired.fetchall():
if row['attempts']>=MAX_ATTEMPTS:
await conn.execute("UPDATE jobs SET state='failed',lease_until=NULL WHERE id=%s",(row['id'],))
await conn.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['attachment_id'],))
continue
await conn.execute("UPDATE jobs SET state='queued', available_at=clock_timestamp(),last_published=NULL WHERE id=%s",(row['id'],))
await conn.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=%s",(row['attachment_id'],))
cur=await conn.execute("""SELECT * FROM jobs WHERE state='queued' AND available_at<=clock_timestamp()
AND (last_published IS NULL OR last_published<clock_timestamp()-interval '5 seconds')
ORDER BY available_at LIMIT 50 FOR UPDATE SKIP LOCKED""")
for row in await cur.fetchall():
payload=dict(schema_version=1,job_id=str(row['id']),type='attachment.process.v1',
attachment_id=str(row['attachment_id']),pipeline_version=1,attempt=row['attempts']+1,
created_at=row['created_at'].isoformat())
await queue.xadd(STREAM,{'payload':json.dumps(payload)})
# Redis appendfsync=always confirms its AOF before replying. A lost reply
# or DB rollback causes a duplicate. DB intent stays until terminal state.
await conn.execute('UPDATE jobs SET last_published=clock_timestamp() WHERE id=%s',(row['id'],))
# Delete staging only for safely committed ready; repeated deletion is safe.
async with store.pool.connection() as conn:
cur=await conn.execute("SELECT id FROM attachments WHERE status='ready' AND staging_cleaned=false LIMIT 100")
for row in await cur.fetchall():
await asyncio.to_thread(attachments.s3.delete_object,Bucket=BUCKET,Key='staging/'+row['id'].hex)
await conn.execute('UPDATE attachments SET staging_cleaned=true WHERE id=%s',(row['id'],))
# Failed inputs retained for replay for 7 days, then removed; row remains auditable.
cur=await conn.execute("""SELECT a.id FROM attachments a JOIN jobs j ON j.attachment_id=a.id
WHERE a.status='failed' AND a.updated_at<clock_timestamp()-interval '7 days'
AND NOT a.staging_cleaned LIMIT 100 FOR UPDATE OF j SKIP LOCKED""")
for row in await cur.fetchall():
await asyncio.to_thread(attachments.s3.delete_object,Bucket=BUCKET,Key='staging/'+row['id'].hex)
await conn.execute('UPDATE attachments SET staging_cleaned=true WHERE id=%s',(row['id'],))
async def process(store, attachments, payload):
job_id=UUID(payload['job_id'])
async with store.pool.connection() as conn:
cur=await conn.execute('SELECT * FROM jobs WHERE id=%s FOR UPDATE',(job_id,))
job=await cur.fetchone()
if not job or job['state'] in ('ready','failed'):
return
if str(job['attachment_id'])!=payload['attachment_id'] or payload['pipeline_version']!=1:
return
# DB is authoritative for attempts, availability and lease, never the duplicate payload.
cur=await conn.execute("""UPDATE jobs SET state='processing',attempts=attempts+1,
lease_until=clock_timestamp()+interval '90 seconds' WHERE id=%s AND state='queued'
AND available_at<=clock_timestamp() RETURNING *""",(job_id,))
job=await cur.fetchone()
if not job:
return
await conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s",(job['attachment_id'],))
# Claim is committed and visible. Effect transaction locks the job for fencing.
async with store.pool.connection() as conn:
await conn.execute('SELECT id FROM jobs WHERE id=%s FOR UPDATE',(job_id,))
cur=await conn.execute('SELECT * FROM attachments WHERE id=%s',(job['attachment_id'],))
row=await cur.fetchone()
code=None
try:
if job['attempts']>MAX_ATTEMPTS:
code='retry_exhausted'
else:
with tempfile.TemporaryDirectory() as directory:
source=Path(directory)/'source'
await asyncio.to_thread(attachments.s3.download_file,BUCKET,'staging/'+row['id'].hex,str(source))
if source.stat().st_size!=row['size_bytes'] or hashlib.sha256(source.read_bytes()).hexdigest()!=row['sha256']:
code='invalid_image'
else:
child=await asyncio.create_subprocess_exec(sys.executable,'-m','app.transform',str(source),row['content_type'],directory,
stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.DEVNULL)
try:
stdout,_=await asyncio.wait_for(child.communicate(),30)
except BaseException:
child.kill()
await child.wait()
raise
if child.returncode:
code='invalid_image'
else:
variants=json.loads(stdout)
await asyncio.to_thread(attachments.s3.copy_object,Bucket=BUCKET,Key=row['object_key'],
CopySource={'Bucket':BUCKET,'Key':'staging/'+row['id'].hex})
# Test-only one-shot crash marker on the disposable container filesystem.
if Path('/tmp/crash_after_object').exists():
Path('/tmp/crash_after_object').unlink()
os._exit(86)
for variant in variants:
body=(Path(directory)/variant['name']).read_bytes()
variant.update(size_bytes=len(body),sha256=hashlib.sha256(body).hexdigest())
await asyncio.to_thread(attachments.s3.put_object,Bucket=BUCKET,
Key='variants/'+row['id'].hex+'/v1/'+variant['name'],Body=body,ContentType=variant['content_type'])
await conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(Jsonb(variants),row['id']))
await conn.execute("UPDATE jobs SET state='ready',lease_until=NULL WHERE id=%s",(job_id,))
return
except TimeoutError:
code='processing_timeout'
except Exception:
code='storage_unavailable'
terminal=code=='invalid_image' or job['attempts']>=MAX_ATTEMPTS
if terminal and code!='invalid_image':
code='retry_exhausted'
delay=min(60,2**min(job['attempts'],6))+random.uniform(0,1)
state='failed' if terminal else 'queued'
await conn.execute("UPDATE jobs SET state=%s,available_at=clock_timestamp()+(%s*interval '1 second'),lease_until=NULL,last_published=NULL WHERE id=%s",(state,delay,job_id))
await conn.execute('UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s',(state,code if terminal else None,row['id']))
async def replay(store, job_id, reason):
if not reason.strip():
raise ValueError('Replay reason required')
async with store.pool.connection() as conn:
cur=await conn.execute('SELECT * FROM jobs WHERE id=%s FOR UPDATE',(UUID(job_id),))
job=await cur.fetchone()
if not job or job['state']!='failed':
raise ValueError('Only failed jobs can be replayed')
cur=await conn.execute('SELECT staging_cleaned FROM attachments WHERE id=%s',(job['attachment_id'],))
if (await cur.fetchone())['staging_cleaned']:
raise ValueError('Replay retention expired')
await conn.execute('INSERT INTO replay_audit(job_id,reason) VALUES (%s,%s)',(job['id'],reason))
await conn.execute("UPDATE jobs SET state='queued',attempts=0,available_at=clock_timestamp(),last_published=NULL WHERE id=%s",(job['id'],))
await conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(job['attachment_id'],))
async def main(role):
store=Store(Settings.load()); await store.open()
queue=broker(); attachments=Attachments(store)
try:
if role=='replay':
await replay(store,*sys.argv[2:]); return
if role=='worker':
try: await queue.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
except ResponseError as error:
if 'BUSYGROUP' not in str(error): raise
consumer=uuid4().hex
while True:
try:
Path('/tmp/pipeline_alive').touch()
if role=='publisher':
await publish(store,queue,attachments); await asyncio.sleep(.5)
else:
claimed=await queue.xautoclaim(STREAM,GROUP,consumer,5000,'0-0',count=1)
entries=claimed[1]
if not entries:
batches=await queue.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000)
entries=batches[0][1] if batches else []
for message_id, fields in entries:
await process(store,attachments,json.loads(fields['payload']))
await queue.xack(STREAM,GROUP,message_id)
await queue.xdel(STREAM,message_id)
except Exception:
print(role+': operation deferred',flush=True)
await asyncio.sleep(1)
finally:
await queue.aclose(); await store.close()
if __name__=='__main__':
asyncio.run(main(sys.argv[1]))
+93
View File
@@ -0,0 +1,93 @@
"""scrypt passwords, strict Basic credentials and persistent signed cursors."""
import base64
from datetime import datetime, timezone
import hashlib
import hmac
import json
import re
import secrets
from uuid import UUID
from .errors import invalid, unauthorized
def password_hash(password, salt=None):
salt = salt or secrets.token_bytes(16)
digest = hashlib.scrypt(password.encode("utf-8"), salt=salt, n=16384,
r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024)
return "scrypt$16384$8$1$" + salt.hex() + "$" + digest.hex()
def password_matches(password, encoded):
try:
algorithm, n, r, p, salt, digest = encoded.split("$")
if (algorithm, n, r, p) != ("scrypt", "16384", "8", "1"):
return False
actual = password_hash(password, bytes.fromhex(salt)).rsplit("$", 1)[1]
return hmac.compare_digest(actual, digest)
except (ValueError, UnicodeError):
return False
def basic_credentials(header):
try:
scheme, encoded = header.split(" ", 1)
if scheme.lower() != "basic" or len(encoded) > 2048:
raise ValueError
value = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = value.split(":", 1)
if re.fullmatch(r"[a-z0-9_]{3,32}", username) is None:
raise ValueError
if not 12 <= len(password) <= 128:
raise ValueError
return username, password
except (AttributeError, ValueError, UnicodeError):
raise unauthorized(basic=True) from None
def bearer(header):
try:
scheme, token = header.split(" ", 1)
if scheme.lower() != "bearer" or re.fullmatch(r"[A-Za-z0-9_-]{43}", token) is None:
raise ValueError
return token
except (AttributeError, ValueError):
raise unauthorized() from None
def identifier(value):
try:
return UUID(value)
except (ValueError, AttributeError, TypeError):
raise invalid("Invalid UUID") from None
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
class Cursors:
def __init__(self, key):
self.key = key
def encode(self, scope, item):
payload = json.dumps([scope, item["created_at"], item["id"]], separators=(",", ":")).encode()
signature = hmac.digest(self.key, payload, hashlib.sha256)
return base64.urlsafe_b64encode(signature + payload).decode().rstrip("=")
def decode(self, scope, token):
try:
if not token or len(token) > 1024:
raise ValueError
raw = base64.b64decode(token + "=" * (-len(token) % 4), altchars=b"-_", validate=True)
signature, payload = raw[:32], raw[32:]
if not hmac.compare_digest(signature, hmac.digest(self.key, payload, hashlib.sha256)):
raise ValueError
parts = json.loads(payload)
if not isinstance(parts, list) or len(parts) != 3 or parts[0] != scope:
raise ValueError
if not all(isinstance(x, str) for x in parts) or not parts[1].endswith("Z"):
raise ValueError
return datetime.fromisoformat(parts[1].replace("Z", "+00:00")), UUID(parts[2])
except (ValueError, TypeError, UnicodeError):
raise invalid("Invalid cursor for this resource") from None
+238
View File
@@ -0,0 +1,238 @@
"""Persistent business data and durable Redis sessions; no local state cache."""
import asyncio
from datetime import datetime, timezone
import hashlib
import json
import secrets
from uuid import UUID, uuid4
from psycopg.errors import ForeignKeyViolation, UniqueViolation
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool
import redis.asyncio as redis
from redis.asyncio.retry import Retry
from redis.backoff import NoBackoff
from .errors import APIError, invalid, unauthorized
from .security import Cursors, bearer, password_hash, password_matches, stamp
USER_COLUMNS = "id, username, display_name, created_at"
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at, attachment_ids"
def public(row):
if "attachment_ids" in row:
row["attachment_ids"] = [str(x) for x in row["attachment_ids"]]
return {key: (str(value) if isinstance(value, UUID) else
stamp(value) if isinstance(value, datetime) else value)
for key, value in row.items()}
class Store:
def __init__(self, settings):
self.settings = settings
self.cursors = Cursors(settings.cursor_key)
self.kdf_slots = asyncio.Semaphore(4)
self.dummy_hash = password_hash("dummy-password-for-missing-user", b"0" * 16)
self.pool = AsyncConnectionPool(
settings.database_url, min_size=1, max_size=10, max_waiting=20,
timeout=1, open=False, reconnect_timeout=5,
check=AsyncConnectionPool.check_connection,
kwargs={"row_factory": dict_row, "connect_timeout": 2,
"keepalives": 1, "keepalives_idle": 2, "keepalives_interval": 1,
"keepalives_count": 2, "tcp_user_timeout": 2000,
"options": "-c timezone=UTC -c statement_timeout=2000 -c lock_timeout=1000"})
self.sessions = redis.Redis.from_url(
settings.cache_url, decode_responses=True, max_connections=10,
socket_connect_timeout=1, socket_timeout=1,
retry=Retry(NoBackoff(), 0), retry_on_timeout=False)
async def open(self):
await self.pool.open()
await self.pool.wait(timeout=10)
await self.sessions.ping()
async def close(self):
await self.sessions.aclose()
await self.pool.close()
async def ready(self):
async with self.pool.connection() as conn:
await conn.execute("SELECT 1")
await self.sessions.ping()
async def create_user(self, data):
async with self.kdf_slots:
encoded = await asyncio.to_thread(password_hash, data.password)
try:
async with self.pool.connection() as conn:
cur = await conn.execute(
"INSERT INTO users(id, username, display_name, password_hash) "
"VALUES (%s, %s, %s, %s) RETURNING " + USER_COLUMNS,
(uuid4(), data.username, data.display_name, encoded))
result = public(await cur.fetchone())
return result
except UniqueViolation:
raise APIError(409, "conflict", "Username is already taken") from None
async def login(self, username, password):
async with self.pool.connection() as conn:
cur = await conn.execute("SELECT " + USER_COLUMNS + ", password_hash FROM users WHERE username=%s",
(username,))
row = await cur.fetchone()
async with self.kdf_slots:
valid = await asyncio.to_thread(password_matches, password,
row["password_hash"] if row else self.dummy_hash)
if not valid or row is None:
raise unauthorized(basic=True)
row.pop("password_hash")
token = secrets.token_urlsafe(32)
expires_ms = int(datetime.now(timezone.utc).timestamp() * 1000) + self.settings.session_ttl * 1000
await self.sessions.set(self.session_key(token), json.dumps(
{"user_id": str(row["id"]), "expires_ms": expires_ms}), pxat=expires_ms)
return {"token": token, "token_type": "Bearer",
"expires_at": stamp(datetime.fromtimestamp(expires_ms / 1000, timezone.utc)),
"user": public(row)}
@staticmethod
def session_key(token):
return "session:" + hashlib.sha256(token.encode("ascii")).hexdigest()
async def authenticate(self, header):
token = bearer(header)
value = await self.sessions.get(self.session_key(token))
if value is None:
raise unauthorized()
session = json.loads(value)
if session["expires_ms"] <= int(datetime.now(timezone.utc).timestamp() * 1000):
raise unauthorized()
async with self.pool.connection() as conn:
cur = await conn.execute("SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
(UUID(session["user_id"]),))
row = await cur.fetchone()
if row is None:
raise unauthorized()
return public(row), token
async def logout(self, token):
await self.sessions.delete(self.session_key(token))
async def create_chat(self, actor, data):
if actor not in data.member_ids:
raise invalid("Members must include the current user")
try:
async with self.pool.connection() as conn:
cur = await conn.execute("INSERT INTO chats(id, title) VALUES (%s, %s) "
"RETURNING id, title, created_at", (uuid4(), data.title))
row = await cur.fetchone()
async with conn.cursor() as cur:
await cur.executemany("INSERT INTO chat_members(chat_id, user_id) VALUES (%s, %s)",
[(row["id"], UUID(member)) for member in data.member_ids])
result = public(row)
result["member_ids"] = sorted(data.member_ids)
return result
except ForeignKeyViolation:
raise invalid("All members must exist") from None
async def accessible_chat(self, conn, actor, chat_id):
cur = await conn.execute("""SELECT c.id, c.title, c.created_at FROM chats c
JOIN chat_members m ON m.chat_id=c.id
WHERE c.id=%s AND m.user_id=%s""", (chat_id, UUID(actor)))
row = await cur.fetchone()
if row is None:
raise APIError(404, "not_found", "Chat not found")
return row
async def get_chat(self, actor, chat_id):
async with self.pool.connection() as conn:
row = await self.accessible_chat(conn, actor, chat_id)
cur = await conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (chat_id,))
result = public(row)
result["member_ids"] = [str(x["user_id"]) for x in await cur.fetchall()]
return result
async def create_message(self, actor, chat_id, data, key):
fingerprint = hashlib.sha256(json.dumps({"text": data.text, "attachment_ids": data.attachment_ids},
ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
async with self.pool.connection() as conn:
await self.accessible_chat(conn, actor, chat_id)
# Evaluate clock_timestamp AFTER obtaining the per-chat lock.
await conn.execute("SELECT id FROM chats WHERE id=%s FOR UPDATE", (chat_id,))
# All replicas share this lock. Replay and side effect commit together.
await conn.execute("DELETE FROM message_requests WHERE user_id=%s AND chat_id=%s "
"AND request_key=%s AND expires_at <= clock_timestamp()", (UUID(actor), chat_id, key))
replay = await conn.execute("SELECT request_hash, message_id FROM message_requests "
"WHERE user_id=%s AND chat_id=%s AND request_key=%s", (UUID(actor), chat_id, key))
previous = await replay.fetchone()
if previous:
if previous["request_hash"] != fingerprint:
raise APIError(409, "conflict", "Idempotency key was used with a different message")
saved = await conn.execute("SELECT " + MESSAGE_COLUMNS + " FROM messages WHERE id=%s",
(previous["message_id"],))
return public(await saved.fetchone())
for attachment_id in data.attachment_ids:
found = await conn.execute("SELECT status FROM attachments WHERE id=%s AND chat_id=%s",
(UUID(attachment_id), chat_id))
attachment = await found.fetchone()
if attachment is None:
raise APIError(404, "not_found", "Attachment not found")
if attachment["status"] not in ("ready", "queued", "processing"):
raise APIError(409, "conflict", "Attachment is not ready")
cur = await conn.execute("""UPDATE chats SET last_message_at=
greatest(clock_timestamp(), last_message_at + interval '1 microsecond')
WHERE id=%s RETURNING last_message_at""", (chat_id,))
created_at = (await cur.fetchone())["last_message_at"]
cur = await conn.execute("INSERT INTO messages(id, chat_id, sender_id, text, created_at, attachment_ids) "
"VALUES (%s, %s, %s, %s, %s, %s) RETURNING " + MESSAGE_COLUMNS,
(uuid4(), chat_id, UUID(actor), data.text, created_at, [UUID(x) for x in data.attachment_ids]))
result = public(await cur.fetchone())
await conn.execute("INSERT INTO message_requests "
"(user_id, chat_id, request_key, request_hash, message_id) VALUES (%s,%s,%s,%s,%s)",
(UUID(actor), chat_id, key, fingerprint, UUID(result["id"])))
return result
def page(self, rows, scope, limit):
items = rows[:limit]
return {"items": items, "next_cursor": self.cursors.encode(scope, items[-1])
if len(rows) > limit else None}
async def list_messages(self, actor, chat_id, limit, cursor):
scope = "messages:" + str(chat_id)
boundary = self.cursors.decode(scope, cursor) if cursor is not None else None
async with self.pool.connection() as conn:
await self.accessible_chat(conn, actor, chat_id)
query = "SELECT " + MESSAGE_COLUMNS + " FROM messages WHERE chat_id=%s"
params = [chat_id]
if boundary:
query += " AND (created_at, id) > (%s, %s)"
params.extend(boundary)
query += " ORDER BY created_at, id LIMIT %s"
params.append(limit + 1)
cur = await conn.execute(query, params)
rows = [public(row) for row in await cur.fetchall()]
return self.page(rows, scope, limit)
async def list_chats(self, actor, limit, cursor):
scope = "chats:" + actor
boundary = self.cursors.decode(scope, cursor) if cursor is not None else None
query = """SELECT c.id, c.title, c.created_at FROM chats c
JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s"""
params = [UUID(actor)]
if boundary:
query += " AND (c.created_at, c.id) > (%s, %s)"
params.extend(boundary)
query += " ORDER BY c.created_at, c.id LIMIT %s"
params.append(limit + 1)
async with self.pool.connection() as conn:
cur = await conn.execute(query, params)
rows = [public(row) for row in await cur.fetchall()]
if rows:
ids = [UUID(row["id"]) for row in rows]
cur = await conn.execute("SELECT chat_id, user_id FROM chat_members WHERE chat_id=ANY(%s) ORDER BY user_id",
(ids,))
members = {row["id"]: [] for row in rows}
for member in await cur.fetchall():
members[str(member["chat_id"])].append(str(member["user_id"]))
for row in rows:
row["member_ids"] = members[row["id"]]
return self.page(rows, scope, limit)
+35
View File
@@ -0,0 +1,35 @@
"""Isolated decoder: parent enforces a wall-clock timeout, container bounds RAM."""
import json
from pathlib import Path
import sys
from PIL import Image, ImageOps
from .attachments import validate_image
def transform(source, mime, destination):
with open(source, 'rb') as file:
validate_image(file, mime)
with Image.open(file) as raw:
oriented = ImageOps.exif_transpose(raw).convert('RGBA')
image = Image.new('RGB', oriented.size, 'white')
image.paste(oriented, mask=oriented.getchannel('A'))
variants = []
for name, bound, fmt in [('thumbnail',256,'JPEG'),('crop',128,'JPEG'),('optimized',1280,'WEBP')]:
if name == 'crop':
out = ImageOps.fit(image, (128,128), method=Image.Resampling.LANCZOS, centering=(.5,.5))
else:
w,h = image.size
ratio = min(1, bound/w, bound/h)
out = image.resize((max(1,int(w*ratio)),max(1,int(h*ratio))), Image.Resampling.LANCZOS)
# New RGB image removes EXIF, ICC and text metadata.
clean = Image.new('RGB', out.size)
clean.paste(out)
clean.save(Path(destination)/name, format=fmt, quality=85 if fmt=='JPEG' else 80)
variants.append({'name':name,'width':out.width,'height':out.height,
'content_type':'image/jpeg' if fmt=='JPEG' else 'image/webp'})
return variants
if __name__ == '__main__':
try:
print(json.dumps(transform(*sys.argv[1:])))
except Exception:
sys.exit(2)
+213
View File
@@ -0,0 +1,213 @@
name: mtusi-lab5
x-app: &app
image: mtusi-messenger:lab5
build: {context: ., target: runtime}
environment: &environment
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: ${APP_ENV:-lab}
APP_VERSION: ${APP_VERSION:-lab5-v1}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
MAX_ATTEMPTS: ${MAX_ATTEMPTS:-5}
SESSION_STORE: redis
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_KEY_FILE: /run/secrets/cursor_key
secrets: [database_url, cache_url, cursor_key, s3_secret_key]
networks: [data]
cpus: 1.0
mem_limit: 512m
stop_grace_period: 8s
services:
postgres:
image: postgres:17.6-bookworm
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
networks: [data]
healthcheck:
test: [CMD-SHELL, "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 30
cpus: 1.0
mem_limit: 512m
stop_grace_period: 10s
redis:
image: redis:7.4.5-bookworm
command: [sh, -ec, 'exec redis-server /usr/local/etc/redis/redis.conf --requirepass "$$(cat /run/secrets/redis_password)"']
secrets: [redis_password]
volumes:
- redis_data:/data
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
networks: [data]
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/redis_password)" redis-cli ping | grep -q PONG']
interval: 2s
timeout: 2s
retries: 30
cpus: 0.5
mem_limit: 192m
stop_grace_period: 10s
migrate:
<<: *app
command: [python, -m, app.migrate]
depends_on:
postgres: {condition: service_healthy}
restart: "no"
api:
<<: *app
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
networks: [edge, data]
minio:
image: mtusi-minio:2025-09-07
build: ./infra/s3
command: [server, /data, --console-address, "127.0.0.1:9001"]
environment:
MINIO_ROOT_USER: lab-admin
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
secrets: [s3_root_password]
volumes: [s3_data:/data]
networks: [data]
cpus: 1.0
mem_limit: 512m
healthcheck:
test: [CMD-SHELL, "mc alias set health http://127.0.0.1:9000 lab-admin $$(cat /run/secrets/s3_root_password) >/dev/null && mc ready health"]
interval: 2s
timeout: 2s
retries: 30
bucket-init:
image: mtusi-minio:2025-09-07
build: ./infra/s3
entrypoint: [/bin/sh, /bootstrap/bootstrap.sh]
secrets: [s3_root_password, s3_secret_key]
volumes: [./infra/s3:/bootstrap:ro]
networks: [data]
depends_on:
minio: {condition: service_healthy}
mem_limit: 128m
restart: "no"
cleanup:
<<: *app
command: [python, -m, app.attachments]
healthcheck:
test: [CMD, python, -c, "import os; os.kill(1, 0)"]
interval: 5s
timeout: 2s
retries: 3
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
restart: unless-stopped
broker:
image: redis:7.4.5-bookworm
command: [sh, -ec, 'exec redis-server /usr/local/etc/redis/redis.conf --requirepass "$$(cat /run/secrets/redis_password)"']
secrets: [redis_password]
volumes: [broker_data:/data, ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro]
networks: [data]
mem_limit: 192m
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/redis_password)" redis-cli ping | grep -q PONG']
interval: 2s
timeout: 2s
retries: 30
publisher:
<<: *app
command: [python, -m, app.pipeline, publisher]
healthcheck:
test: [CMD, python, -c, "import time,pathlib; assert time.time()-pathlib.Path('/tmp/pipeline_alive').stat().st_mtime<120"]
interval: 5s
timeout: 2s
retries: 5
restart: unless-stopped
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
worker:
<<: *app
command: [python, -m, app.pipeline, worker]
healthcheck:
test: [CMD, python, -c, "import time,pathlib; assert time.time()-pathlib.Path('/tmp/pipeline_alive').stat().st_mtime<120"]
interval: 5s
timeout: 2s
retries: 5
restart: unless-stopped
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
lb:
image: haproxy:3.0.12-alpine
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:${API_PORT:-8085}:8080"
networks: [edge]
depends_on:
api: {condition: service_healthy}
cpus: 0.5
mem_limit: 128m
stop_grace_period: 8s
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, "http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
retries: 30
checks:
profiles: [tools]
image: mtusi-messenger:lab5-checks
build: {context: ., target: checks}
environment:
<<: *environment
BASE_URL: http://lb:8080
COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-mtusi-lab5}
secrets: [database_url, cache_url, cursor_key, s3_secret_key]
volumes:
- .:/checks
- /var/run/docker.sock:/var/run/docker.sock
networks: [edge, data]
cpus: 1.0
mem_limit: 512m
networks:
edge:
data:
internal: true
volumes:
postgres_data:
redis_data:
s3_data:
broker_data:
secrets:
postgres_password: {file: ./secrets/postgres_password}
redis_password: {file: ./secrets/redis_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
cursor_key: {file: ./secrets/cursor_key}
s3_root_password: {file: ./secrets/s3_root_password}
s3_secret_key: {file: ./secrets/s3_secret_key}
BIN
View File
Binary file not shown.
+13
View File
@@ -0,0 +1,13 @@
# Публикация лабораторной 5
Решение опубликовано в [gwalut/mtusi2026lab5](https://gitea.privcloud.tech/gwalut/mtusi2026lab5), ветка `solution/bpi-2304-salamatin`. Открыт [PR №112](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/112) в `mtusi2026/mtusi2026lab5:main`.
Заголовок: **ЛР 5 — БПИ-2304 — Саламатин — на 4**. Автор — Саламатин, БПИ-2304. Целевая оценка — 4, включая минимум на 3. Треки на 5 не выполнялись.
API принимает исходник в устойчивый staging MinIO и фиксирует задание в PostgreSQL до ответа 202. Отдельные publisher, Redis Streams и worker обеспечивают доставку и обработку. Создаются thumbnail, crop и optimized WebP. Реализованы outbox, ограниченные повторы, lease, реестр failed и replay с аудитом.
Публичный smoke, два теста изображений, сценарии сбоев pipeline, persistence и failover прошли 21 ноября 2026 года. Команды, результаты и ограничения находятся в REPORT.md и evidence/. Понятное объяснение и сценарий защиты — docs/REVIEW.docx.
Проверенный локальный коммит — `8aab4ad`. Публикация выполнена через применение Git binary patch в веб-интерфейсе: коммит реализации на сервере — `790911092460f332d1b6b7de546dc96a452613d6`. Перед созданием PR подтверждено полное совпадение деревьев файлов. Последующие изменения этой памятки относятся только к документации.
Секреты, .env и Docker volumes в публикацию не включены. README задания, контракт и публичные tests сохранены. Право сопровождающих редактировать ветку отключено. PR открыт для проверки, слияние не выполнялось.
+134
View File
@@ -0,0 +1,134 @@
[
{
"service": "api",
"id": "81b9ace8e144c8e38d3ffbc95ecd6428ab9bc5e91d8be9ca733a90c913d035bb",
"name": "mtusi-lab5-api-2",
"started_at": "2026-11-21T08:44:28.046206398Z",
"status": "running",
"image": "sha256:05c8a8b03100fe722dcff4b8dda6175dc3ef57c7d6c4a08a0927ff714b9ac628",
"user": "messenger",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": []
},
{
"service": "api",
"id": "2b83f91f71b2d50a4aa388520b3f3ee1f8b25f1777326a097a8adff30af0f682",
"name": "mtusi-lab5-api-1",
"started_at": "2026-11-21T08:44:27.947752549Z",
"status": "running",
"image": "sha256:05c8a8b03100fe722dcff4b8dda6175dc3ef57c7d6c4a08a0927ff714b9ac628",
"user": "messenger",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": []
},
{
"service": "postgres",
"id": "f7473488534846f3cd65557f7565e2d0863ff55e8dd92cdb50aa157882b3ee81",
"name": "mtusi-lab5-postgres-1",
"started_at": "2026-11-21T08:44:27.517627855Z",
"status": "running",
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
"user": "",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_postgres_data"
]
},
{
"service": "redis",
"id": "f477f71a61df1e8e7b7a435c3bdb29520514ffd3fa59e938090181a4c5205b6c",
"name": "mtusi-lab5-redis-1",
"started_at": "2026-11-21T08:44:27.41536Z",
"status": "running",
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
"user": "",
"memory_bytes": 201326592,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_redis_data"
]
},
{
"service": "broker",
"id": "d27e1d46eed78bff88fde978cda8aab25ae6faef2aacfb56e0ddd0465dbe8250",
"name": "mtusi-lab5-broker-1",
"started_at": "2026-11-21T08:44:27.116236287Z",
"status": "running",
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
"user": "",
"memory_bytes": 201326592,
"nano_cpus": 0,
"published_ports": {},
"volumes": [
"mtusi-lab5_broker_data"
]
},
{
"service": "worker",
"id": "bf9991ff3292dea15f7e4047d05fb61b7c368dd8b831dd92d32a8afa091d8107",
"name": "mtusi-lab5-worker-1",
"started_at": "2026-11-21T08:44:33.034808691Z",
"status": "running",
"image": "sha256:05c8a8b03100fe722dcff4b8dda6175dc3ef57c7d6c4a08a0927ff714b9ac628",
"user": "messenger",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": []
},
{
"service": "publisher",
"id": "848ac5ef740dd11f670ee829ad4c13f56d4bf08c32d2bad0b2573d2e1362bcfe",
"name": "mtusi-lab5-publisher-1",
"started_at": "2026-11-21T08:44:32.974938883Z",
"status": "running",
"image": "sha256:05c8a8b03100fe722dcff4b8dda6175dc3ef57c7d6c4a08a0927ff714b9ac628",
"user": "messenger",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": []
},
{
"service": "minio",
"id": "e9bae9c756da03423d57b3b400b5059c6759a197e24324c51f8187f5b9730519",
"name": "mtusi-lab5-minio-1",
"started_at": "2026-11-21T08:44:27.245193147Z",
"status": "running",
"image": "sha256:22163eb16611d12c5dd11f7f60d8297fda4840075df379a38b03d4c0c15f8350",
"user": "",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_s3_data"
]
},
{
"service": "lb",
"id": "7137b6f4ea583cbf9fb39d9cfdd39990ae5768d1a84a540cfe447b364e42eb3f",
"name": "mtusi-lab5-lb-1",
"started_at": "2026-11-21T08:44:27.6597139Z",
"status": "running",
"image": "sha256:e3229153dd6429fc71f8a9a1f1ccbf8dab6962422d7043c81e84dd6a075e90ee",
"user": "haproxy",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8085"
}
]
},
"volumes": []
}
]
+134
View File
@@ -0,0 +1,134 @@
[
{
"service": "api",
"id": "81b9ace8e144c8e38d3ffbc95ecd6428ab9bc5e91d8be9ca733a90c913d035bb",
"name": "mtusi-lab5-api-2",
"started_at": "2026-11-21T08:41:20.314678435Z",
"status": "running",
"image": "sha256:05c8a8b03100fe722dcff4b8dda6175dc3ef57c7d6c4a08a0927ff714b9ac628",
"user": "messenger",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": []
},
{
"service": "api",
"id": "2b83f91f71b2d50a4aa388520b3f3ee1f8b25f1777326a097a8adff30af0f682",
"name": "mtusi-lab5-api-1",
"started_at": "2026-11-21T08:41:20.715646634Z",
"status": "running",
"image": "sha256:05c8a8b03100fe722dcff4b8dda6175dc3ef57c7d6c4a08a0927ff714b9ac628",
"user": "messenger",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": []
},
{
"service": "postgres",
"id": "f7473488534846f3cd65557f7565e2d0863ff55e8dd92cdb50aa157882b3ee81",
"name": "mtusi-lab5-postgres-1",
"started_at": "2026-11-21T08:34:56.571813343Z",
"status": "running",
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
"user": "",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_postgres_data"
]
},
{
"service": "redis",
"id": "f477f71a61df1e8e7b7a435c3bdb29520514ffd3fa59e938090181a4c5205b6c",
"name": "mtusi-lab5-redis-1",
"started_at": "2026-11-21T08:34:56.684590509Z",
"status": "running",
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
"user": "",
"memory_bytes": 201326592,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_redis_data"
]
},
{
"service": "broker",
"id": "d27e1d46eed78bff88fde978cda8aab25ae6faef2aacfb56e0ddd0465dbe8250",
"name": "mtusi-lab5-broker-1",
"started_at": "2026-11-21T08:41:01.362289693Z",
"status": "running",
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
"user": "",
"memory_bytes": 201326592,
"nano_cpus": 0,
"published_ports": {},
"volumes": [
"mtusi-lab5_broker_data"
]
},
{
"service": "worker",
"id": "bf9991ff3292dea15f7e4047d05fb61b7c368dd8b831dd92d32a8afa091d8107",
"name": "mtusi-lab5-worker-1",
"started_at": "2026-11-21T08:43:38.821404243Z",
"status": "running",
"image": "sha256:05c8a8b03100fe722dcff4b8dda6175dc3ef57c7d6c4a08a0927ff714b9ac628",
"user": "messenger",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": []
},
{
"service": "publisher",
"id": "848ac5ef740dd11f670ee829ad4c13f56d4bf08c32d2bad0b2573d2e1362bcfe",
"name": "mtusi-lab5-publisher-1",
"started_at": "2026-11-21T08:41:23.916156889Z",
"status": "running",
"image": "sha256:05c8a8b03100fe722dcff4b8dda6175dc3ef57c7d6c4a08a0927ff714b9ac628",
"user": "messenger",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": []
},
{
"service": "minio",
"id": "e9bae9c756da03423d57b3b400b5059c6759a197e24324c51f8187f5b9730519",
"name": "mtusi-lab5-minio-1",
"started_at": "2026-11-21T08:44:14.030717549Z",
"status": "running",
"image": "sha256:22163eb16611d12c5dd11f7f60d8297fda4840075df379a38b03d4c0c15f8350",
"user": "",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_s3_data"
]
},
{
"service": "lb",
"id": "7137b6f4ea583cbf9fb39d9cfdd39990ae5768d1a84a540cfe447b364e42eb3f",
"name": "mtusi-lab5-lb-1",
"started_at": "2026-11-21T08:34:56.97949279Z",
"status": "running",
"image": "sha256:e3229153dd6429fc71f8a9a1f1ccbf8dab6962422d7043c81e84dd6a075e90ee",
"user": "haproxy",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8085"
}
]
},
"volumes": []
}
]
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 119, "failed_attempts": 3, "first_recovery_seconds": 2.17}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+7
View File
@@ -0,0 +1,7 @@
test_exif_orientation_and_metadata (test_images.ImageRules.test_exif_orientation_and_metadata) ... ok
test_small_transparent_source (test_images.ImageRules.test_small_transparent_source) ... ok
----------------------------------------------------------------------
Ran 2 tests in 0.060s
OK
+7
View File
@@ -0,0 +1,7 @@
{
"target_instance": "81b9ace8e144-c09077d927cd",
"container": "81b9ace8e144c8e38d3ffbc95ecd6428ab9bc5e91d8be9ca733a90c913d035bb",
"exit_code": 0,
"stop_seconds": 0.868,
"remains_stopped": true
}
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+35
View File
@@ -0,0 +1,35 @@
[
{
"test": "stopped worker and API broker restart",
"result": "PASS",
"attachment_id": "0111d52d-1e37-47c4-9cdb-f75005f8bd11",
"queued_messages": 1
},
{
"test": "crash between COMMIT and publish",
"result": "PASS",
"attachment_id": "fe26b7ea-f68d-4530-b329-e705ee6445a3"
},
{
"test": "crash after S3 write and duplicate delivery",
"result": "PASS",
"attachment_id": "4eb502b5-398d-4894-abef-20ab1313ae33",
"attempts": 2,
"variant_objects": 3
},
{
"test": "permanent failure registry and audited replay",
"result": "PASS",
"attachment_id": "6861c994-551a-4adb-a9ac-9f8d08a43d35"
},
{
"test": "temporary failure backoff and bounded retry",
"result": "PASS",
"attachment_id": "bb745a64-79de-45a1-8dae-a0c716d2fc0e",
"attempts": 5
},
{
"test": "upload limits MIME filename",
"result": "PASS"
}
]
+138
View File
@@ -0,0 +1,138 @@
Container mtusi-lab5-worker-1 Stopping
Container mtusi-lab5-worker-1 Stopped
Container mtusi-lab5-api-1 Restarting
Container mtusi-lab5-api-2 Restarting
Container mtusi-lab5-broker-1 Restarting
Container mtusi-lab5-broker-1 Started
Container mtusi-lab5-api-1 Started
Container mtusi-lab5-api-2 Started
Container mtusi-lab5-minio-1 Waiting
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-minio-1 Healthy
Container mtusi-lab5-bucket-init-1 Starting
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-bucket-init-1 Started
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-worker-1 Starting
Container mtusi-lab5-worker-1 Started
PASS: stopped worker and API broker restart
Container mtusi-lab5-publisher-1 Stopping
Container mtusi-lab5-publisher-1 Stopped
Container mtusi-lab5-api-1 Killing
Container mtusi-lab5-api-2 Killing
Container mtusi-lab5-api-1 Killed
Container mtusi-lab5-api-2 Killed
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-minio-1 Waiting
Container mtusi-lab5-minio-1 Healthy
Container mtusi-lab5-bucket-init-1 Starting
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-bucket-init-1 Started
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-redis-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-redis-1 Healthy
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-api-2 Starting
Container mtusi-lab5-api-2 Started
Container mtusi-lab5-api-1 Starting
Container mtusi-lab5-api-1 Started
Container mtusi-lab5-minio-1 Waiting
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-minio-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-bucket-init-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-bucket-init-1 Started
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-publisher-1 Starting
Container mtusi-lab5-publisher-1 Started
PASS: crash between COMMIT and publish
Container mtusi-lab5-worker-1 Stopping
Container mtusi-lab5-worker-1 Stopped
Container mtusi-lab5-minio-1 Waiting
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-minio-1 Healthy
Container mtusi-lab5-bucket-init-1 Starting
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-bucket-init-1 Started
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-worker-1 Starting
Container mtusi-lab5-worker-1 Started
PASS: crash after S3 write and duplicate delivery
Container mtusi-lab5-worker-1 Stopping
Container mtusi-lab5-worker-1 Stopped
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-minio-1 Waiting
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-minio-1 Healthy
Container mtusi-lab5-bucket-init-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-bucket-init-1 Started
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-worker-1 Starting
Container mtusi-lab5-worker-1 Started
PASS: permanent failure registry and audited replay
Container mtusi-lab5-worker-1 Stopping
Container mtusi-lab5-worker-1 Stopped
Container mtusi-lab5-minio-1 Stopping
Container mtusi-lab5-minio-1 Stopped
Container mtusi-lab5-minio-1 Starting
Container mtusi-lab5-minio-1 Started
PASS: temporary failure backoff and bounded retry
PASS: upload limits MIME filename
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-minio-1 Waiting
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-minio-1 Healthy
Container mtusi-lab5-bucket-init-1 Starting
Container mtusi-lab5-bucket-init-1 Started
Container mtusi-lab5-redis-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-redis-1 Healthy
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-bucket-init-1 Exited
+16
View File
@@ -0,0 +1,16 @@
Container mtusi-lab5-minio-1 Waiting
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-minio-1 Healthy
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-bucket-init-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-bucket-init-1 Started
Container mtusi-lab5-redis-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-redis-1 Healthy
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-api-2 Starting
Container mtusi-lab5-api-2 Started
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://lb:8080"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+4
View File
@@ -0,0 +1,4 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
python3 tests/smoke.py --base-url "http://lb:8080"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+85
View File
@@ -0,0 +1,85 @@
{
"skeleton": "276dbdad8a34da4c5b57fbf1879ca169b581d75b",
"files": [
{
"path": "COURSE.md",
"unchanged": true,
"sha256": "c34d53cd082d194aa4b58f504c4e0348d25b6afec3d6a9c0963dd01c9c81e971"
},
{
"path": "README.md",
"unchanged": true,
"sha256": "0651974eebbd4f34a04cd7c06df167cf61ae686cc67ceb2ac977a115ad70b866"
},
{
"path": "contracts/README.md",
"unchanged": true,
"sha256": "72df27a559f88ffa3fd8c3c79d3c32f6db2c5986a0602677c51994c434a3fc4f"
},
{
"path": "contracts/attachment-states.md",
"unchanged": true,
"sha256": "cf0519e137410e4b34ab632aadb8ad8a381d4b73033e27b354c58eeacbdf86d6"
},
{
"path": "contracts/examples.json",
"unchanged": true,
"sha256": "381cc18c03789e935b391d007fb5843964be9cfd9f7530cb6b1996ed4a1c4c37"
},
{
"path": "contracts/infrastructure.json",
"unchanged": true,
"sha256": "6d4ca861b160118cef8764ae0ca292ac924b541963ccc7be4857eddbef6d77d8"
},
{
"path": "contracts/jobs.schema.json",
"unchanged": true,
"sha256": "9ccf9b11466c0c3a26a941520ac5ff543241fc4935e3a6eebb01ef303779b10d"
},
{
"path": "contracts/openapi.json",
"unchanged": true,
"sha256": "62b6e1d43ca0c6115e9c6ad0ffa9be0835e8fdf9ae664d2fea3c258ef745d56d"
},
{
"path": "tests/README.md",
"unchanged": true,
"sha256": "e0bfa7e196797f6a5bf6e44d8ad2ffeb82651927ff5d8484eb1c2dda1ec7dec5"
},
{
"path": "tests/client.py",
"unchanged": true,
"sha256": "8d50976fc7c7bbb81e88a152f613c61ab25eaf53dba33c6a9dfab19f868d546d"
},
{
"path": "tests/fixtures/README.md",
"unchanged": true,
"sha256": "c001783760b4f87a7a8441a7ba55f5d080ff48e1397d3581d72a88d7691f96c2"
},
{
"path": "tests/fixtures/sample.png",
"unchanged": true,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75"
},
{
"path": "tests/image_probe.py",
"unchanged": true,
"sha256": "a9a4f89181c8e0fda6ce42e1a1af32fb42d8e3ae519e314207127c9b9b448b13"
},
{
"path": "tests/load.py",
"unchanged": true,
"sha256": "6d3511ea719c7a7e841083e73d435c86dcbfd9f4e5c2a07d62eb9614b01d9933"
},
{
"path": "tests/resilience.py",
"unchanged": true,
"sha256": "1f96663a174b3978d24eff66f2741f5f8c9b788d3242261d37a972720c52788b"
},
{
"path": "tests/smoke.py",
"unchanged": true,
"sha256": "52ac38eb343ff7d43e7543588e1da58ca06c9fce74b2b0d0bf0e4982bc23c1db"
}
]
}
+70
View File
@@ -0,0 +1,70 @@
Container mtusi-lab5-broker-1 Running
Container mtusi-lab5-postgres-1 Running
Container mtusi-lab5-minio-1 Running
Container mtusi-lab5-redis-1 Running
Container mtusi-lab5-cleanup-1 Running
Container mtusi-lab5-worker-1 Running
Container mtusi-lab5-api-1 Running
Container mtusi-lab5-api-2 Running
Container mtusi-lab5-publisher-1 Running
Container mtusi-lab5-lb-1 Running
Container mtusi-lab5-minio-1 Waiting
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-minio-1 Healthy
Container mtusi-lab5-bucket-init-1 Starting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-bucket-init-1 Started
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-redis-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-redis-1 Waiting
Container mtusi-lab5-redis-1 Healthy
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-redis-1 Healthy
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-api-2 Waiting
Container mtusi-lab5-api-1 Waiting
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-api-2 Healthy
Container mtusi-lab5-api-1 Healthy
Container mtusi-lab5-minio-1 Waiting
Container mtusi-lab5-lb-1 Waiting
Container mtusi-lab5-api-2 Waiting
Container mtusi-lab5-api-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-bucket-init-1 Waiting
Container mtusi-lab5-redis-1 Waiting
Container mtusi-lab5-publisher-1 Waiting
Container mtusi-lab5-cleanup-1 Waiting
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-worker-1 Waiting
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-cleanup-1 Healthy
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-publisher-1 Healthy
Container mtusi-lab5-redis-1 Healthy
Container mtusi-lab5-minio-1 Healthy
Container mtusi-lab5-bucket-init-1 Exited
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-lb-1 Healthy
Container mtusi-lab5-worker-1 Healthy
Container mtusi-lab5-api-2 Healthy
Container mtusi-lab5-api-1 Healthy
+12
View File
@@ -0,0 +1,12 @@
{
"date": "2026-11-21",
"seconds": {
"public": 5.206,
"images": 3.358,
"pipeline": 211.347,
"persistence": 11.684,
"failover": 20.057,
"final_smoke": 4.441
},
"result": "PASS"
}
+39
View File
@@ -0,0 +1,39 @@
global
log stdout format raw local0
maxconn 512
stats socket ipv4@0.0.0.0:9999 level admin
defaults
mode http
log global
option httplog
option http-server-close
timeout connect 1s
timeout client 40s
timeout server 40s
timeout http-request 5s
timeout check 1s
retries 0
resolvers docker
nameserver embedded 127.0.0.11:53
resolve_retries 2
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
hold timeout 1s
hold refused 1s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 1
server-template api 8 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
+9
View File
@@ -0,0 +1,9 @@
bind 0.0.0.0
protected-mode yes
port 6379
dir /data
appendonly yes
appendfsync always
save ""
maxmemory 128mb
maxmemory-policy noeviction
+9
View File
@@ -0,0 +1,9 @@
FROM golang:1.24.7-bookworm AS build
ENV CGO_ENABLED=0 GOTOOLCHAIN=auto
RUN go install github.com/minio/minio@RELEASE.2025-09-07T16-13-09Z
RUN go install github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
FROM debian:bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
COPY --from=build /go/bin/minio /usr/local/bin/minio
COPY --from=build /go/bin/mc /usr/local/bin/mc
ENTRYPOINT ["minio"]
+1
View File
@@ -0,0 +1 @@
{"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": ["arn:aws:s3:::attachments"]}, {"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], "Resource": ["arn:aws:s3:::attachments/original/*", "arn:aws:s3:::attachments/staging/*", "arn:aws:s3:::attachments/variants/*"]}]}
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
mc alias set local http://minio:9000 lab-admin "$(cat /run/secrets/s3_root_password)" >/dev/null
mc mb --ignore-existing local/attachments
mc anonymous set none local/attachments
mc admin user add local messenger-api "$(cat /run/secrets/s3_secret_key)" >/dev/null
mc admin policy create local messenger-api /bootstrap/api-policy.json
mc admin policy attach local messenger-api --user messenger-api
+32
View File
@@ -0,0 +1,32 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
last_message_at timestamptz NOT NULL DEFAULT '-infinity'
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chat_idx ON chat_members(user_id, chat_id);
CREATE INDEX chats_created_id_idx ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL,
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_chat_created_id_idx ON messages(chat_id, created_at, id);
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE message_requests (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
request_key varchar(128) NOT NULL,
request_hash char(64) NOT NULL,
message_id uuid NOT NULL REFERENCES messages(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL DEFAULT (clock_timestamp() + interval '24 hours'),
PRIMARY KEY (user_id, chat_id, request_key)
);
CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at);
+11
View File
@@ -0,0 +1,11 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
CREATE TABLE attachments (
id uuid PRIMARY KEY, chat_id uuid NOT NULL REFERENCES chats(id),
uploader_id uuid NOT NULL REFERENCES users(id), object_key text UNIQUE NOT NULL,
filename text NOT NULL, content_type text NOT NULL, size_bytes bigint NOT NULL,
sha256 text NOT NULL, status text NOT NULL CHECK(status IN ('pending','ready')),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachments_pending_idx ON attachments(created_at) WHERE status='pending';
+17
View File
@@ -0,0 +1,17 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK(status IN ('pending','queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp();
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
ALTER TABLE attachments ADD COLUMN error_code text;
ALTER TABLE attachments ADD COLUMN staging_cleaned boolean NOT NULL DEFAULT false;
CREATE TABLE jobs (
id uuid PRIMARY KEY, attachment_id uuid UNIQUE NOT NULL REFERENCES attachments(id),
pipeline_version int NOT NULL DEFAULT 1 CHECK(pipeline_version=1),
attempts int NOT NULL DEFAULT 0, state text NOT NULL DEFAULT 'queued',
available_at timestamptz NOT NULL DEFAULT clock_timestamp(), lease_until timestamptz,
last_published timestamptz, created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE replay_audit (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id),
reason text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
+19
View File
@@ -0,0 +1,19 @@
annotated-doc==0.0.5
annotated-types==0.8.0
anyio==4.15.1
click==8.5.0
colorama==0.4.6; sys_platform == 'win32'
fastapi==0.141.1
h11==0.16.0
idna==3.20
pydantic==2.13.5
pydantic-core==2.46.5
starlette==1.6.0
typing-extensions==4.16.0
typing-inspection==0.4.4
uvicorn==0.53.0
psycopg[binary]==3.2.10
psycopg-pool==3.2.6
redis==6.4.0
boto3==1.40.61
Pillow==12.0.0
+35
View File
@@ -0,0 +1,35 @@
"""Create local synthetic credentials once without printing secret values."""
from pathlib import Path
import secrets
ROOT = Path(__file__).resolve().parents[1]
def main():
directory = ROOT / "secrets"
directory.mkdir(exist_ok=True)
for name in ("postgres_password", "redis_password", "cursor_key", "s3_root_password", "s3_secret_key"):
path = directory / name
if not path.exists():
path.write_text(secrets.token_hex(32) + "\n", encoding="utf-8", newline="\n")
else:
# Shell command substitution on Linux must not retain a Windows CR.
value = path.read_text(encoding="utf-8").strip()
path.write_text(value + "\n", encoding="utf-8", newline="\n")
pg_password = (directory / "postgres_password").read_text().strip()
redis_password = (directory / "redis_password").read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{pg_password}@postgres:5432/messenger",
"cache_url": f"redis://:{redis_password}@redis:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n", encoding="utf-8", newline="\n")
env = ROOT / ".env"
if not env.exists():
env.write_bytes((ROOT / ".env.example").read_bytes())
print("PASS: local .env and secret files exist; existing values preserved")
if __name__ == "__main__":
main()
+94
View File
@@ -0,0 +1,94 @@
"""Docker operations scoped to the configured local Compose project."""
import csv
import io
import json
import os
from pathlib import Path
import socket
import subprocess
import time
ROOT = Path(__file__).resolve().parents[1]
def command(args, timeout=120, env=None):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout, env=env)
if result.returncode:
# Commands and stderr can contain secrets. Keep detailed errors local.
raise RuntimeError("Local command failed: " + " ".join(args[:3]) + "\n" + result.stderr[-1500:])
return result.stdout.strip()
def compose(*args, timeout=120, env=None):
return command(["docker", "compose", "--env-file", ".env", *args], timeout, env)
def project():
return os.getenv("COMPOSE_PROJECT_NAME") or json.loads(compose("config", "--format", "json"))["name"]
def containers(service="api", running=True):
args = ["docker", "ps"] + ([] if running else ["-a"])
ids = command(args + ["--filter", "label=com.docker.compose.project=" + project(),
"--filter", "label=com.docker.compose.service=" + service,
"--filter", "label=com.docker.compose.oneoff=False", "--format", "{{.ID}}"])
return json.loads(command(["docker", "inspect", *ids.split()])) if ids else []
def summary():
result = []
for service in ("api", "postgres", "redis", "broker", "worker", "publisher", "minio", "lb"):
for row in containers(service, running=False):
result.append({"service": service, "id": row["Id"], "name": row["Name"].lstrip("/"),
"started_at": row["State"]["StartedAt"], "status": row["State"]["Status"],
"image": row["Image"], "user": row["Config"]["User"],
"memory_bytes": row["HostConfig"]["Memory"], "nano_cpus": row["HostConfig"]["NanoCpus"],
"published_ports": row["HostConfig"]["PortBindings"],
"volumes": [m["Name"] for m in row["Mounts"] if m["Type"] == "volume"]})
return result
def runtime(line):
with socket.create_connection(("lb", 9999), timeout=3) as sock:
sock.sendall((line + "\n").encode())
sock.shutdown(socket.SHUT_WR)
chunks = []
while True:
data = sock.recv(65536)
if not data:
return b"".join(chunks).decode()
chunks.append(data)
def drain(address):
lines = runtime("show servers state messenger").splitlines()
fields = next(line[2:].split() for line in lines if line.startswith("# "))
for line in lines:
values = line.split()
if not values or line.startswith("#") or len(values) != len(fields):
continue
row = dict(zip(fields, values))
if row["srv_addr"] == address:
slot = "messenger/" + row["srv_name"]
runtime("set server " + slot + " state drain")
# Let requests already assigned to this replica finish (HTTP budget 5 s).
deadline = time.monotonic() + 7
while time.monotonic() < deadline:
stats = list(csv.DictReader(io.StringIO(runtime("show stat").lstrip("# "))))
active = [s for s in stats if s["pxname"] == "messenger" and s["svname"] == row["srv_name"]]
if active and int(active[0]["scur"]) == 0:
return slot
time.sleep(.1)
raise RuntimeError("Replica did not drain within 7 seconds")
raise RuntimeError("Backend address was not found in HAProxy")
if __name__ == "__main__":
import sys
if sys.argv[1] == "drain":
print(drain(sys.argv[2]))
elif sys.argv[1] == "ready":
runtime("set server " + sys.argv[2] + " state ready")
elif sys.argv[1] == "summary":
print(json.dumps(summary(), indent=2))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env restart --timeout 6 api postgres redis broker minio lb worker publisher
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python scripts/stop_one.py
+24
View File
@@ -0,0 +1,24 @@
"""Stop exactly the replica selected by the public failover scenario."""
import json
import os
import time
from ops import ROOT, command, containers
def main():
target = os.environ["TARGET_INSTANCE_ID"]
matches = [row for row in containers() if target.startswith(row["Config"]["Hostname"] + "-")]
if len(matches) != 1:
raise RuntimeError("Target does not identify exactly one running API replica")
row = matches[0]
started = time.monotonic()
command(["docker", "stop", "--time", "6", row["Id"]], timeout=9)
state = json.loads(command(["docker", "inspect", row["Id"]]))[0]["State"]
assert not state["Running"] and state["ExitCode"] == 0, "Graceful shutdown failed"
report = {"target_instance": target, "container": row["Id"], "exit_code": state["ExitCode"],
"stop_seconds": round(time.monotonic() - started, 3), "remains_stopped": True}
(ROOT / "evidence" / "last-stop.json").write_text(json.dumps(report, indent=2) + "\n")
if __name__ == "__main__":
main()
+33
View File
@@ -0,0 +1,33 @@
"""Explicit local verification with sanitized logs; never deletes volumes."""
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT=Path(__file__).resolve().parents[1]
EVIDENCE=ROOT/'evidence'
def run(name,args):
start=time.monotonic()
result=subprocess.run(['docker','compose','--env-file','.env',*args],cwd=ROOT,
stdout=subprocess.PIPE,stderr=subprocess.STDOUT,text=True,encoding='utf-8',errors='replace')
(EVIDENCE/name).write_text(result.stdout,encoding='utf-8')
if result.returncode:raise RuntimeError(name+' failed; inspect evidence')
print('PASS',name,flush=True)
return round(time.monotonic()-start,3)
def main():
EVIDENCE.mkdir(exist_ok=True)
results={}
if '--build' in sys.argv:run('build.txt',['build','api','checks'])
run('startup.txt',['up','-d','--no-build','--wait','--wait-timeout','180'])
results['public']=run('smoke.txt',['run','--rm','--no-deps','checks','make','check','test'])
results['images']=run('images.txt',['run','--rm','--no-deps','-e','PYTHONPATH=/app','checks','python','-m','unittest','discover','-s','student-tests','-p','test_images.py','-v'])
results['pipeline']=run('pipeline.txt',['run','--rm','--no-deps','-e','PYTHONPATH=/app','checks','python','student-tests/pipeline_checks.py'])
run('before-restart.json',['run','--rm','--no-deps','checks','python','scripts/ops.py','summary'])
results['persistence']=run('persistence.txt',['run','--rm','--no-deps','checks','sh','-ec','chmod +x scripts/restart.sh; make test-persistence ACTION_SCRIPT=scripts/restart.sh'])
run('after-restart.json',['run','--rm','--no-deps','checks','python','scripts/ops.py','summary'])
results['failover']=run('failover.txt',['run','--rm','--no-deps','checks','sh','-ec','chmod +x scripts/stop-one.sh; make test-failover ACTION_SCRIPT=scripts/stop-one.sh'])
run('restore.txt',['start','api'])
results['final_smoke']=run('smoke-final.txt',['run','--rm','--no-deps','checks','make','test'])
(EVIDENCE/'verification.json').write_text(json.dumps({'date':'2026-11-21','seconds':results,'result':'PASS'},indent=2))
if __name__=='__main__':main()
+13
View File
@@ -0,0 +1,13 @@
$ErrorActionPreference = 'Stop'
Set-Location -LiteralPath $PSScriptRoot
docker info --format '{{.ServerVersion}}'
if ($LASTEXITCODE -ne 0) { throw 'Start Docker Desktop first.' }
docker run --rm --mount "type=bind,source=$PSScriptRoot,target=/work" --workdir /work python:3.13.15-slim-bookworm python scripts/init_local.py
if ($LASTEXITCODE -ne 0) { throw 'Local configuration initialization failed.' }
docker compose --env-file .env build api checks minio
if ($LASTEXITCODE -ne 0) { throw 'Image build failed.' }
docker compose --env-file .env up -d --no-build --wait --wait-timeout 180
if ($LASTEXITCODE -ne 0) { throw 'Stack startup failed; inspect docker compose logs.' }
$taskConfig = docker compose --env-file .env config --format json | ConvertFrom-Json
$taskPort = $taskConfig.services.lb.ports[0].published
Write-Host "Ready: http://127.0.0.1:$taskPort/health/ready"
+136
View File
@@ -0,0 +1,136 @@
"""Explicit fault injection against this lab's disposable Compose project."""
import asyncio
import hashlib
from io import BytesIO
import json
from pathlib import Path
import subprocess
import sys
import time
from uuid import UUID
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
from client import Client
from PIL import Image
import psycopg
from psycopg.rows import dict_row
from app.pipeline import broker
from app.attachments import client as s3client
ROOT=Path(__file__).resolve().parents[1]
results=[]
def dc(*args):
subprocess.run(['docker','compose','--env-file','.env',*args],cwd=ROOT,check=True,stdout=subprocess.DEVNULL)
def sql(query,params=(),one=True):
with psycopg.connect(Path('/run/secrets/database_url').read_text().strip(),row_factory=dict_row) as conn:
cur=conn.execute(query,params)
return cur.fetchone() if cur.description and one else cur.fetchall() if cur.description else None
def record(name,**data):
results.append(dict(test=name,result='PASS',**data))
(ROOT/'evidence/pipeline.json').write_text(json.dumps(results,indent=2))
print('PASS:',name,flush=True)
c=Client(); a,b,e,chat=c.bootstrap(); s3=s3client()
def meta(att):return c.call('GET','/api/v1/attachments/'+att['id'],actor=a,schema='Attachment')['json']
def upload():return c.attachment(a,chat)
def job(att):return sql('SELECT * FROM jobs WHERE attachment_id=%s',(UUID(att['id']),))
def wait_state(att,state,seconds=120):
end=time.monotonic()+seconds
while time.monotonic()<end:
current=meta(att)
if current['status']==state:return current
time.sleep(.3)
raise AssertionError('state timeout '+state)
def variants(att):
ready=c.wait_ready(a,att,120)
for v in ready['variants']:
data=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+v['name'],actor=b)['bytes']
assert hashlib.sha256(data).hexdigest()==v['sha256']
with Image.open(BytesIO(data)) as im:
assert im.size==(v['width'],v['height'])
assert im.format==('WEBP' if v['name']=='optimized' else 'JPEG')
assert not im.getexif()
return ready
try:
dc('stop','worker')
att,original=upload()
c.message(a,chat,'',attachments=[att['id']])
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=a)
c.call('GET','/api/v1/attachments/'+att['id'],404,actor=e)
deadline=time.monotonic()+10
while job(att)['last_published'] is None and time.monotonic()<deadline:time.sleep(.2)
assert job(att)['last_published'] is not None
before=asyncio.run(broker().xlen('images.v1'))
assert before>0
dc('restart','api','broker')
dc('start','worker')
ready=variants(att)
assert c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)['bytes']==original
record('stopped worker and API broker restart',attachment_id=att['id'],queued_messages=before)
dc('stop','publisher')
att,_=upload()
assert job(att)['last_published'] is None
# API is killed after queued+job COMMIT, before any publisher can execute.
dc('kill','api'); dc('start','api'); dc('start','publisher')
variants(att)
record('crash between COMMIT and publish',attachment_id=att['id'])
dc('stop','worker')
# Start worker with one-shot marker inserted while it is running, before upload.
dc('start','worker');dc('exec','-T','worker','touch','/tmp/crash_after_object')
att,_=upload()
ready=variants(att)
assert job(att)['attempts']>=2
original_attempts=job(att)['attempts']
payload=dict(schema_version=1,job_id=str(job(att)['id']),type='attachment.process.v1',attachment_id=att['id'],pipeline_version=1,attempt=1,created_at=att['created_at'])
async def duplicate():
q=broker()
for _ in range(3):await q.xadd('images.v1',{'payload':json.dumps(payload)})
await q.aclose()
asyncio.run(duplicate());time.sleep(3)
assert meta(att)['variants']==ready['variants'] and job(att)['attempts']==original_attempts
listing=s3.list_objects_v2(Bucket='attachments',Prefix='variants/'+UUID(att['id']).hex+'/v1/')
assert listing['KeyCount']==3
record('crash after S3 write and duplicate delivery',attachment_id=att['id'],attempts=original_attempts,variant_objects=3)
dc('stop','worker'); att,_=upload()
# Trusted-storage corruption emulates permanent input failure after acceptance.
s3.put_object(Bucket='attachments',Key='staging/'+UUID(att['id']).hex,Body=b'broken')
dc('start','worker');failed=wait_state(att,'failed')
assert failed['error_code']=='invalid_image'
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=a)
s3.put_object(Bucket='attachments',Key='staging/'+UUID(att['id']).hex,Body=original,ContentType='image/png')
dc('exec','-T','worker','python','-m','app.pipeline','replay',str(job(att)['id']),'restored staging for lab fault test')
variants(att)
assert sql('SELECT count(*) AS n FROM replay_audit WHERE job_id=%s',(job(att)['id'],))['n']==1
record('permanent failure registry and audited replay',attachment_id=att['id'])
dc('stop','worker'); att,_=upload()
worker_id=subprocess.check_output(['docker','compose','ps','-aq','worker'],cwd=ROOT,text=True).strip()
dc('stop','minio')
# docker start intentionally avoids Compose restarting the stopped dependency.
subprocess.run(['docker','start',worker_id],check=True,stdout=subprocess.DEVNULL)
# S3 outage intentionally makes API readiness fail, so HAProxy may return 503.
# Observe the durable job directly until storage and routing are restored.
deadline=time.monotonic()+150
while job(att)['state']!='failed' and time.monotonic()<deadline:time.sleep(.5)
failed=sql('SELECT error_code FROM attachments WHERE id=%s',(UUID(att['id']),))
assert failed['error_code']=='retry_exhausted' and job(att)['attempts']==5
dc('start','minio')
deadline=time.monotonic()+30
while time.monotonic()<deadline:
try:
if meta(att)['status']=='failed':break
except AssertionError:pass
time.sleep(.5)
assert meta(att)['status']=='failed'
record('temporary failure backoff and bounded retry',attachment_id=att['id'],attempts=5)
# Negative upload validation inherited from lab4.
path='/api/v1/chats/'+chat['id']+'/attachments?filename=bad.png'
for raw,mime,status in [(b'bad','image/png',415),(original,'image/jpeg',415),(b'x'*(10*1024*1024+1),'image/png',413)]:
c.call('POST',path,status,actor=a,raw=raw,headers={'Content-Type':mime})
c.call('POST',path.replace('bad.png','../bad.png'),400,actor=a,raw=original,headers={'Content-Type':'image/png'})
record('upload limits MIME filename')
finally:
dc('start','minio','worker','publisher','api')
+27
View File
@@ -0,0 +1,27 @@
import tempfile
from pathlib import Path
import unittest
from PIL import Image
from app.transform import transform
class ImageRules(unittest.TestCase):
def test_small_transparent_source(self):
with tempfile.TemporaryDirectory() as d:
src=Path(d)/'input.png'
Image.new('RGBA',(16,8),(255,0,0,0)).save(src)
rows=transform(src,'image/png',d)
self.assertEqual([(v['width'],v['height']) for v in rows],[(16,8),(128,128),(16,8)])
with Image.open(Path(d)/'thumbnail') as im:
self.assertGreater(min(im.getpixel((0,0))),250)
def test_exif_orientation_and_metadata(self):
with tempfile.TemporaryDirectory() as d:
src=Path(d)/'input.jpg'
exif=Image.Exif();exif[274]=6;exif[315]='remove this author'
Image.new('RGB',(320,200),'blue').save(src,exif=exif)
rows=transform(src,'image/jpeg',d)
self.assertEqual((rows[0]['width'],rows[0]['height']),(160,256))
for name in ('thumbnail','crop','optimized'):
with Image.open(Path(d)/name) as im:self.assertFalse(im.getexif())
if __name__=='__main__':unittest.main()