64 changed files with 10106 additions and 45 deletions
+2
View File
@@ -18,3 +18,5 @@ __pycache__
node_modules
evidence
.DS_Store
build
+11 -22
View File
@@ -1,25 +1,14 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
APP_PORT=18091
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
API_CPUS=1
API_MEMORY=256m
HTTP_WORKERS=16
DB_POOL_MAX=8
DB_ACCESS_MODE=optimized
POSTGRES_CPUS=1
WORKER_CONCURRENCY=1
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
JOB_TIMEOUT_SECONDS=20
JOB_LEASE_SECONDS=60
MAX_PENDING_JOBS=100
+26
View File
@@ -0,0 +1,26 @@
name: Application
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 15
env:
BASE_URL: http://127.0.0.1:18091
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- run: |
python scripts/bootstrap.py
docker compose build api
docker compose up -d --wait
python scripts/wait_ready.py
make check test test-student BASE_URL=$BASE_URL
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=$BASE_URL
docker compose up -d --wait --no-deps api
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$BASE_URL
- if: always()
run: docker compose down
+13
View File
@@ -0,0 +1,13 @@
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2
WORKDIR /app
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt
COPY src/ ./src/
COPY migrations/ ./migrations/
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"]
STOPSIGNAL SIGTERM
CMD ["python", "src/server.py"]
+5
View File
@@ -28,3 +28,8 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: test-student
test-student:
BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' -v
+196 -23
View File
@@ -1,37 +1,210 @@
# Отчёт — лабораторная 5
# Лабораторная 5 — очереди и асинхронные изображения
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
**Майорова Софья, БВТ2301. Целевая оценка: 5. Трек 2 — доказанная идемпотентность versioned pipeline.** Выполнено 27 октября 2026 года. Оценку подтверждает преподаватель.
Продолжено [моё решение lab4, PR 18](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/18). Сохранены две API за LB, сессии PostgreSQL, Redis-кэш, keyset-пагинация, идемпотентность сообщений и приватные S3-вложения. Upload изменён по контракту: 202 queued после устойчивого staging и сохранения задания; resize выполняется отдельным worker. Исходные README/COURSE/contracts/tests не менялись.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Engine/Compose v2 и Python 3.9+ для публичных тестов на хосте. Сервер/worker — Python 3.12.8 внутри Docker. На этом Mac:
## Архитектура и решения
```sh
colima start --profile mtusi-labs
export DOCKER_CONTEXT=colima-mtusi-labs
cd "/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab5"
python3 scripts/bootstrap.py
docker compose build api
docker compose up -d --wait
export BASE_URL=http://127.0.0.1:18091
python3 scripts/wait_ready.py
make check test test-student BASE_URL=$BASE_URL
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
На другой машине клонировать публичный форк lab5 и выбрать `solution/BVT2301-Mayorova`, затем выполнить те же команды из корня clone. `build api` идёт первым: migrate, worker, dispatcher и tools используют тот же построенный образ. Одноразовый migrate под advisory lock применяет 001–005; API DDL не выполняют. Bucket-bootstrap идемпотентно создаёт приватный bucket. После них поднимаются API, dispatcher и worker.
## Доказательства критериев
Bootstrap создаёт случайные локальные пароли и ключи, сохраняет существующие значения и `.env`, пересобирает только генерируемую конфигурацию identities из этих ключей. Каталог secrets — 0700; полный s3_config — **0600**, монтируется только root S3 gateway. Отдельные ключи процесса читаются UID 10001 внутри контейнера; их файлы находятся в закрытом 0700-каталоге хоста. API не получает worker/admin credentials. `.env`, secrets, build и данные исключены из Git и build context.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Остановка с сохранением данных: `docker compose down`. Только для намеренного удаления ненужных **учебных данных данного проекта** — `docker compose down -v`. В restart/failover volumes не удаляются. Изолированный clean-clone стенд использует другое имя проекта, порт и временные volumes, которые удаляются после его проверки.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Архитектура и границы подтверждения
## Эксперименты и отказы
```text
Клиент → LB → API × 2 → PostgreSQL (сессии, чаты, сообщения, attachments + jobs/outbox)
├→ Redis cache профилей (восстанавливаемый)
└→ S3 staging (полные исходные bytes)
↑
PostgreSQL jobs/outbox → dispatcher → Redis Streams broker (AOF always)
↓ XREADGROUP / XAUTOCLAIM
worker → изолированный render subprocess
├→ конечный S3 results/<attachment>/vN/...
└→ атомарный ready + варианты в PostgreSQL → XACK + XDEL
```
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
S3 — SeaweedFS 4.51: отдельный gateway и mini master/volume/filer на сети storage. И байты, и filer metadata устойчиво находятся в `s3_data`; API не имеет доступа к filer, административный UI/WebDAV отключены. PostgreSQL — другой volume. Брокер — **отдельный Redis 7.4.1**, не кэш: AOF, `appendfsync always`, `noeviction`, persistent `broker_data`, без публикации порта на хост. Кэш остаётся disposable Redis без AOF. Единственная публичная точка — LB на **127.0.0.1:18091**.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Образы Python 3.12.8, PostgreSQL 17.2-bookworm, Redis 7.4.1-bookworm, HAProxy 3.0.5-alpine и SeaweedFS 4.51 закреплены digest в Dockerfile/Compose. Например, SeaweedFS: `sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8`. Pillow 13.0.0, boto3 1.40.61, psycopg/redis и транзитивные зависимости закреплены в requirements.txt. Это проверенный набор версий, не обещание отсутствия будущих уязвимостей.
## Ограничения и следующие шаги
| Процесс | CPU | Память | Состояние |
| --- | ---: | ---: | --- |
| API, каждая из двух | 1 | 256 MiB | Без устойчивых локальных originals |
| Worker, concurrency=1 | 1 | 768 MiB | Временный tmpfs; источник/результат в S3 |
| Dispatcher | 1 | 256 MiB | Устойчивый outbox в PostgreSQL |
| PostgreSQL | 1 | 512 MiB | postgres_data |
| Broker | 0,5 | 192 MiB, maxmemory=96mb/noeviction | broker_data/AOF |
| S3 store / gateway | 1 / 0,5 | 512 / 512 MiB | s3_data / без собственных originals |
| LB / cache | 0,5 / 0,25 | 64 / 64 MiB | Нет обязательных устойчивых данных |
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Сумма CPU-лимитов превышает 4 vCPU локальной VM: это overcommit, не выделенные физические ядра. Измерения проводились на macOS arm64, Colima/Linux с 4 vCPU и примерно 4 GiB RAM. Ранее запущенные учебные стенды не останавливались.
### Что сохранено при 202
API проверяет Bearer/доступ к чату, Content-Length ≤10 MiB, допустимый filename/MIME, реальный PNG/JPEG header, размеры ≤40 млн pixels и `verify()`. Исходник читается кусками 64 KiB в ограниченный временный файл, SHA-256 считается потоково. Полного декодирования/resize в API нет. Поэтому JPEG с корректным заголовком, но повреждённым scan data может быть принят и затем завершиться **failed invalid_image** в worker; заведомо чужой формат/несовпадение MIME отклоняется 415, лимиты — 413, опасное имя — 400.
Перед PUT фиксируется internal intent uploading со случайным UUID, server-generated staging key и ожидаемой checksum. Под row lock выполняется полный PUT в приватный `staging/<UUID>`. В одной следующей транзакционной границе с этим lock создаётся queued job/outbox и меняется Attachment на queued с current_job_id. **COMMIT выполнен до 202.** API возвращает сохранённый queued snapshot даже если worker успел закончить раньше HTTP-ответа. Bytes в PostgreSQL и broker не помещаются. Временный файл API не является долговременным staging.
Уже принятая работа не зависит от RAM API или доступности dispatcher/broker в этот момент. Readiness API проверяет PostgreSQL и S3; остановленный worker/broker не запускает resize в HTTP. При `MAX_PENDING_JOBS=100` новые uploading/queued/processing ограничены общим счётчиком PostgreSQL под advisory lock; заполнение даёт 503. Это предел количества, не расчёт disk quota. Неподтверждённый upload не следует прозрачно повторять: контракт идемпотентности относится к POST messages.
## Transactional outbox и at-least-once
Таблица `attachment_jobs` совмещает job registry и transactional outbox. Создание job и queued Attachment атомарно. Dispatcher выбирает due rows `FOR UPDATE SKIP LOCKED`, выполняет XADD с envelope по jobs.schema.json, затем отмечает published_at. При сбое до XADD строка остаётся; после XADD до фиксации marker возможен дубль. XADD response с AOF always подтверждает приём/сохранение брокером согласно его политике. Это ещё не подтверждение обработки изображения.
Worker читает consumer group `image-workers`, **count/prefetch=1 на slot**. XAUTOCLAIM возвращает зависшую pending delivery после lease. Dispatcher также восстанавливает processing с истёкшей lease и повторяет долго не захваченную queued publication. DB-инварианты, а не предположение о единственности сообщения брокера, определяют допустимость обработки.
Broker message содержит только schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at; допускается контрактный trace carrier. Пример фактически проверенного envelope — 243 bytes. Ключи хранилища и credentials берутся worker из доверенных настроек/БД, произвольные URL/filename-пути из сообщения не исполняются. Неверный envelope наблюдаем в ограниченном `rejected-jobs` stream: только причина, исходный message ID и hash, без сохранения произвольного payload. Подтверждение ошибочного envelope происходит после записи этой DLQ-записи.
XACK + XDEL выполняются одной Redis Lua-операцией **после** устойчивого ready/failed либо безопасного no-op. При потере ack доставка повторяется. Ошибка БД, не позволившая сохранить результат/ошибку, не подтверждает delivery. Доставка остаётся at-least-once; сокращение числа логических эффектов не превращает её в exactly-once.
## Worker, варианты, retry и lease
Worker — отдельный контейнер/процесс. Default concurrency=1, конфигурация допускает 1–2; при увеличении нужно отдельно увеличить/измерить WORKER_MEMORY, в отчёте проверен только 1. Производственная lease — 60 s; дочерний renderer ограничен wall timeout 20 s, CPU 15 s (hard 16), virtual address space 512 MiB. Worker контейнер ограничен 768 MiB, tmpfs 128 MiB. Исходник ≤10 MiB скачивается кусками с SHA-256/size и deadline 15 s; S3 connect/read timeouts — 1/3 s, SDK total attempts=1. Самостоятельные retry контролирует job registry.
Pillow полностью декодирует исходник в **изолированном subprocess**, проверяет pixels до load, применяет EXIF orientation, alpha композитит на белом. Размеры рассчитываются с округлением вниз, min 1 px. Для thumbnail/optimized маленький исходник не увеличивается. Варианты сохраняются без EXIF/ICC:
- thumbnail: JPEG quality=85, вписан в 256×256; fixture 320×200 → 256×160;
- crop: центрированный квадрат 128×128, Lanczos; маленький исходник допускает увеличение;
- optimized: WebP, вписан в 1280×1280, method=4, quality=82 для pipeline v1, quality=70 для v2. Уменьшение каждого маленького файла не обещается.
Оригинал публикуется побайтно в конечный S3-prefix отдельно от staging. Ready фиксируется только после original + **всех трёх** вариантов с checksums/размерами. GET content для queued/processing/failed даёт 409, для чужого attachment — 404. Сообщение может ссылаться на queued/processing своего чата, failed прикреплять нельзя. Download остаётся авторизованным proxy через API без redirect.
Attempt увеличивается только при успешном claim; job_id при redelivery/retry не меняется. По умолчанию максимум **5 попыток** на job. Временные storage/timeouts/processing failures возвращают queued с backoff `min(8, 0.5*2^(attempt-1)) + U(0,0.5)` секунд; постоянные invalid_image/source_missing/checksum_mismatch/result_conflict сразу дают failed. Исчерпание бюджета — failed retry_exhausted. В публичной ошибке нет exception details. Реестр failed и job_events доступны оператору, не клиенту.
Повторная delivery уже ready/failed — no-op. Busy/early duplicate не отбирает чужой lease. Истёкший lease снимает ownership и создаёт следующую управляемую попытку. Final COMMIT проверяет current_job_id, pipeline_version, state, lease_token **и срок lease**; потерявший аренду worker не может опубликовать старый результат.
## Версионирование и инварианты трека 2
Ключ эффекта: `(attachment_id, pipeline_version, variant)`. Ключи объектов: `results/<attachment>/vN/original|thumbnail|crop|optimized`. PUT результата условный `If-None-Match: *`; готовый объект с совпадающими size/hash повторно не пишется, конфликтующий не затирается. Результаты детерминированы закреплённым renderer и параметрами версии.
Операторский переход на следующую реализованную версию атомарно помечает старую job superseded, создаёт новую queued job с отдельным job_id и обновляет current_job_id/version. Источник новой job — устойчивый исходник прежней версии либо staging, а не локальный файл API. Сейчас реализованы только v1 и v2; публичный upload создаёт v1.
Проверяемые инварианты:
1. Подтверждённая queued задача сохраняется после смерти API и рестарта broker.
2. Одновременный дубль допускает единственный claim; serial duplicate/reorder не увеличивает число успешно опубликованных вариантов.
3. Падение до/после S3 side effect и до/после результата БД не теряет исходник и не создаёт второй набор объектов той же версии.
4. Устаревшая delivery и уже выполняющийся старый worker не возвращают ready в processing и не затирают результат новой версии.
5. Failed наблюдаем, retries конечны; replay допускается только для текущей failed job с доступным source и оставляет audit.
6. Cleanup не удаляет staging текущей queued/processing работы или свежей failed.
`student-tests/pipeline_checks.py` выполняет настоящие операции PostgreSQL/S3/Redis и завершает отдельные worker/publisher-процессы через **os._exit(77)** в явных операторских fault points. Это не публичный HTTP hook. Проверены: COMMIT без publisher; publisher после XADD до outbox marker; worker before_side_effect, after_original, after_objects, after_commit; повтор/перестановка envelope; два concurrent consumer; pause старого subprocess после записи всех объектов → завершение v2 → продолжение v1, получающее fenced no-op; поздняя v1 delivery после ready v2.
Для первого crash действительно ожидается истечение тестовой lease=5 s; в двух следующих случаях deadline той же тестовой строки ускоряется SQL-изменением lease_until, чтобы не тратить по 60 s. Это воздействие теста на **новые собственные job IDs**, не подмена результата. Production lease=60 s не изменяется. После каждого crash выполняется реальный Redis pending reclaim/ack; в соответствующем prefix остаётся ровно четыре объекта. После after_commit новая delivery не увеличивает attempt.
Transient failure воспроизводится настоящим S3 отказом подписи с заведомо неверным тестовым ключом: budget=2 только для этой job, затем queued → failed retry_exhausted, после явного replay с правильными credentials — ready на attempt 3. Повреждённый JPEG доходит до постоянного failed invalid_image. Проверены EXIF-поворот, прозрачность на белом, отсутствие EXIF в thumbnail и запрет увеличения маленького thumbnail.
Команда исследования, **только на синтетическом стенде** (останавливает фоновую обработку, не API):
```sh
docker compose stop worker dispatcher
docker compose run --rm -T --no-deps -v "$PWD:/work:ro" -w /work audit python student-tests/pipeline_checks.py
docker compose up -d --no-deps worker dispatcher
```
`audit` — tools profile, не запускается обычным up. Только он получает дополнительные локальные секреты для операторских/негативных проверок; API их не получает. Исследование автоматически создаёт новые synthetic users/chats/jobs, не изменяет чужие.
## Операторские команды и очистка
```sh
# Наблюдаемый реестр failed:
docker compose exec worker python src/admin.py failed
# Явный replay текущей failed job; UUID взять из предыдущей команды:
docker compose exec worker python src/admin.py replay JOB_UUID
# Новый pipeline для конкретного вложения:
docker compose exec worker python src/admin.py version ATTACHMENT_UUID --version 2
# Dry-run и явное применение cleanup:
docker compose exec worker python src/admin.py cleanup
docker compose exec worker python src/admin.py cleanup --apply
```
Replay не сбрасывает счётчик, не повторяет ready и не создаёт новую копию успешной job: явно добавляет новый бюджет попыток к тому же job_id, сохраняя историю. Необходим доступный source; после истечения retention может понадобиться внешнее восстановление исходника.
Staging удаляется только **после COMMIT ready**. Crash после COMMIT до очистки безопасен: операторский cleanup идемпотентно удаляет оставшийся staging готовой работы. Failed/незавершённый uploading хранится минимум **24 h**; queued/processing никогда не собираются. Cleanup использует row locks/SKIP LOCKED и фильтр managed staging prefix; dry-run без --apply ничего не меняет. Счётчики cleanup отражают кандидаты по метаданным: часть staging objects уже удалена worker, повторная очистка безопасна. Реестр и сообщения не удаляются вместе с исходником. Неактуальные immutable outputs старых версий пока сохраняются — автоматическая уборка old-version results не заявляется.
## Результаты и воспроизводимость
| Критерий | Проверка и наблюдение | Evidence |
| --- | --- | --- |
| Публичный минимум lab5 | Smoke: 202, ready, исходный SHA-256 и настоящий JPEG thumbnail | final-tests.txt |
| Прежний API и собственные изображения | 18 собственных тестов PASS: Unicode/NUL, auth, гонки/idempotency, variants, ACL, MIME/bytes/pixels/filename | student-tests.txt, final-tests.txt |
| Устойчивый staging/очередь | Worker/dispatcher остановлены, upload 202 и message; обе API убиты SIGKILL до publish; после рестарта со старым Bearer всё сохранено | durability.json |
| Persistence broker | Настоящий Redis Streams XADD; до/после restart с volume — 1/1 запись; затем та же job ready | durability.json |
| Полный restart | API/LB/worker/dispatcher остановлены, PostgreSQL/cache/оба S3/broker перезапущены с volumes; старые сессия и bytes сохранены | persistence.txt, restart-before/after.json |
| Отказ API | 2 реплики, 114 подтверждённых writes, 2 переходные ошибки, восстановление 2,534 s | failover.txt |
| Все уровни retry/final/replay | Реальные S3 signature failures, конечный budget=2, audit/replay и готовый результат на attempt 3 | pipeline.json |
| Outbox и worker crash windows | Отдельные процессы exit 77, реальные redelivery и четыре объекта на версию | pipeline.json |
| Reorder, concurrent duplicate, stale version | 13 групп assertions/invariants; old running process fenced, new ready не изменился | pipeline.json |
| Изображения и лимиты worker | EXIF/alpha/small-image, malformed JPEG failed, 40 Мп RGB ready, subprocess/container limits | pipeline.json, resources.json |
| Cleanup | Dry-run/apply/repeat, защита активного source и свежих failed | pipeline.json |
| Приватность | Anonymous known-key GET/list →403; в envelope только 7 служебных полей, проверены неверные типы/форматы | security.json |
| Полная валидация схемы | OpenAPI 3.1 / JSON Schema PASS | openapi.txt |
| Чистый clone | Отдельные новые секреты, проект, порт и пустые volumes | PASS: новые секреты и пустые PG/S3/broker volumes, 18 тестов, SIGKILL API с queued work и полный restart; clean-clone.txt |
Все сценарии сохраняют tokens только в памяти теста. Snapshot содержит разрешённые поля state/started_at/mounts/resources, не environment или secret values. Fault injection и операторские команды воспроизводимы из репозитория. Дополнительные команды:
```sh
export BASE_URL=http://127.0.0.1:18091
python3 student-tests/durability.py
python3 student-tests/resources.py
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=$BASE_URL
docker compose up -d --wait --no-deps api
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$BASE_URL
```
### Измерение ресурсов и задержек
Это дополнительная проверка bounded worker, **не заявка на трек 1**. Concurrency/prefetch=1, один прогон на профиль, по 10 одновременно начинающих клиентов. Генерация PNG до измерения; отправляется реальное тело, без скрытых retry. Worker перезапускается перед профилем, cgroup peak относится к этому запуску. Считается время HTTP-accept и время до наблюдаемого ready; polling добавляет до примерно 0,3 s. Для N=10 nearest-rank p95/p99 равны максимуму. Сэмплер docker exec также вносит overhead, closed-loop не даёт верхней границы production throughput.
| Pixels / bytes | 202: p95 | До ready: p50 / p95=p99 | Worker cgroup peak / resident anon | Успех |
| --- | ---: | ---: | ---: | --- |
| 64 000 / 1 294 | 0,080 s | 1,834 / 2,923 s | 78,81 / 76,82 MiB | 10/10 |
| 2 520 000 / 7 563 763 | 1,102 s | 4,155 / 7,025 s | 123,46 / 112,50 MiB | 10/10 |
| 40 000 000 / 533 859 | 0,092 s | 6,826 / 11,659 s | 253,67 / 251,36 MiB | 10/10 |
После исследования и восстановления очередь полностью опустела: 85 ready, 3 намеренно failed из-за повреждённых JPEG, 0 queued/processing/uploading, 0 stream entries/pending deliveries; у всех ready по три варианта (jobs-summary.json).
Все 30 приняли 202 и достигли ready; OOM kill=0. `memory.peak` — kernel cgroup максимум контейнера, включая renderer, tmpfs/cache/kernel; resident anon взят как максимум `memory.stat anon` среди samples, это не точный максимальный RSS каждого отдельного PID. CPU counters также сохранены в raw samples. Маленький сжатый файл 40 Мп занимает больше RAM/CPU при декодировании, чем более крупный файл с меньшим количеством pixels. Быстрый 202 не означает мгновенный thumbnail: работа проходит ограниченный worker и ждёт очередь.
![Задержка и память отдельного worker](evidence/pipeline.png)
Основной исследовательский результат трека 2: все внедрённые crash/duplicate/reorder/version сценарии соблюли заявленные инварианты; умирающий после COMMIT worker не заставил повторно публиковать эффект, а живой, но устаревший worker получил fencing вместо перезаписи новой версии. Это измеренная функциональная проверка, не статистическое доказательство отсутствия всех гонок.
Для графиков: Python 3.11+, `pip install -r requirements-research.txt`, `python student-tests/plot.py`. PNG/SVG уже приложены. Workflow Application добавляет сборку/стенд/smoke/own tests/failover/persistence; hosted CI отдельно не объявляется прошедшим без результата платформы.
## Ограничения
Один host/PostgreSQL/S3-store/broker/LB остаются точками отказа. Доказаны процессные сбои и restart с volumes, не потеря диска/всего хоста или гарантия power-loss durability. HTTP и неподключённые к хосту backend-порты допускаются только для локального синтетического стенда; удалённой публикации без TLS нет. Docker operator имеет доступ к секретам.
Существующий immutable результат проверяется по HEAD size/hash metadata, не повторным чтением всего объекта; защита от произвольной подмены привилегированным S3-оператором не заявляется.
Нет распределённой ACID-транзакции. Outbox и периодическая republish закрывают разрывы повтором, поэтому сообщения/CPU могут дублироваться; fences и immutable versioned keys защищают опубликованный результат. Потерявшийся worker может ещё некоторое время тратить CPU до timeout/завершения, но не публиковать старый ready. Нормальные bounded render заканчиваются до lease; выбирая меньшую lease, надо отдельно измерять нагрузку.
Состояния jobs/events и неактуальные versions накапливаются. Cleanup запускает оператор, не скрытый scheduler. Retention failed sources ограничивает возможность replay без внешней копии. Принятая задача может закончиться failed из-за постоянной ошибки или исчерпания retry; это наблюдаемое завершение, а не исчезновение работы. Image parser ограничивает декодированные PNG-текстовые метаданные 1 MiB. Антивирус и UI не входят в работу.
## Источники и ИИ
Собственное решение lab4 (Майорова Софья, PR 18); чужие студенческие решения не копировались. Задание и contracts текущего skeleton — основание обязательного поведения.
- [Redis persistence/AOF](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [XAUTOCLAIM](https://redis.io/docs/latest/commands/xautoclaim/), [Streams](https://redis.io/docs/latest/develop/data-types/streams/).
- [Pillow ImageOps/EXIF/fit](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [Image API](https://pillow.readthedocs.io/en/stable/reference/Image.html).
- [SeaweedFS S3 credentials](https://github.com/seaweedfs/seaweedfs/wiki/S3-Credentials), [PostgreSQL explicit locks](https://www.postgresql.org/docs/17/explicit-locking.html).
Codex использован для реализации, тестов, fault-injection, запуска измерений и оформления. Все PASS/числа получены реальными командами, границы их применимости приведены выше.
+234
View File
@@ -0,0 +1,234 @@
name: mtusi-lab5
x-application: &application
image: mtusi-messenger:lab5
networks: [backend]
environment: &environment
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
HTTP_WORKERS: ${HTTP_WORKERS:-16}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
DB_ACCESS_MODE: ${DB_ACCESS_MODE:-optimized}
LOG_LEVEL: WARNING
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_key
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret
secrets: [database_url, cache_url, s3_api_key, s3_api_secret]
tmpfs: ["/tmp:size=128m,mode=1777"]
mem_limit: ${API_MEMORY:-512m}
cpus: ${API_CPUS:-1}
stop_grace_period: 10s
services:
migrate:
<<: *application
command: [python, src/migrate.py]
healthcheck:
disable: true
deploy:
replicas: 1
depends_on:
postgres:
condition: service_healthy
api:
<<: *application
build: .
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
bucket-bootstrap:
condition: service_completed_successfully
bucket-bootstrap:
<<: *application
command: [python, src/bucket_bootstrap.py]
environment:
<<: *environment
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret
secrets: [s3_admin_key, s3_admin_secret]
healthcheck:
disable: true
depends_on:
s3:
condition: service_healthy
audit:
<<: *application
profiles: [tools]
environment:
<<: *environment
BASE_URL: http://lb:8080
BROKER_URL_FILE: /run/secrets/broker_url
JOB_LEASE_SECONDS: 5
secrets: [database_url, cache_url, s3_api_key, s3_api_secret, s3_admin_key, s3_admin_secret, s3_worker_key, s3_worker_secret, broker_url]
healthcheck:
disable: true
deploy:
replicas: 1
dispatcher:
<<: *application
command: [python, src/pipeline.py, dispatch]
environment:
<<: *environment
BROKER_URL_FILE: /run/secrets/broker_url
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60}
secrets: [database_url, broker_url]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
restart: unless-stopped
worker:
<<: *application
command: [python, src/pipeline.py, work]
environment:
<<: *environment
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_key
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
secrets: [database_url, broker_url, s3_worker_key, s3_worker_secret]
mem_limit: ${WORKER_MEMORY:-768m}
tmpfs: ["/tmp:size=128m,mode=1777"]
cpus: 1
deploy:
replicas: 1
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
bucket-bootstrap:
condition: service_completed_successfully
broker:
condition: service_healthy
restart: unless-stopped
broker:
image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47
command: [redis-server, --appendonly, "yes", --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction, --save, ""]
volumes: [broker_data:/data]
networks: [backend]
mem_limit: 192m
cpus: 0.5
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 1s
retries: 30
s3-store:
image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8
command: [mini, -dir=/data, -ip=s3-store, -s3=false, -admin.ui=false, -webdav=false, -master.telemetry=false, -master.volumeSizeLimitMB=128, -volume.max=16, -volume.fileSizeLimitMB=11, -volume.concurrentUploadLimitMB=64, -volume.concurrentDownloadLimitMB=64]
networks: [storage]
volumes: [s3_data:/data]
mem_limit: 512m
cpus: 1
stop_grace_period: 20s
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8888/]
interval: 2s
timeout: 2s
retries: 45
s3:
image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8
command: [s3, -filer=s3-store:8888, -port=8333, -config=/run/secrets/s3_config, -iam=false, -port.iceberg=0, -port.lance=0, -concurrentFileUploadLimit=16, -concurrentUploadLimitMB=32, -readerCacheSizeMB=16]
secrets: [s3_config]
networks: [backend, storage]
mem_limit: 512m
environment:
GOMEMLIMIT: 384MiB
cpus: 0.5
depends_on:
s3-store:
condition: service_healthy
healthcheck:
test: [CMD-SHELL, 'nc -z 127.0.0.1 8333']
interval: 2s
timeout: 1s
retries: 30
lb:
image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949
ports: ["127.0.0.1:${APP_PORT:-18091}:8080"]
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
networks: [edge, backend]
depends_on:
api:
condition: service_healthy
mem_limit: 64m
cpus: 0.5
stop_grace_period: 12s
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 2s
timeout: 2s
retries: 10
postgres:
image: postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
networks: [backend]
healthcheck:
test: [CMD, pg_isready, -h, 127.0.0.1, -U, messenger, -d, messenger]
interval: 2s
timeout: 1s
retries: 30
mem_limit: 512m
cpus: ${POSTGRES_CPUS:-1}
redis:
image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47
command: [redis-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
networks: [backend]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 1s
retries: 30
mem_limit: 64m
cpus: 0.25
volumes:
postgres_data:
broker_data:
s3_data:
networks:
edge:
storage:
internal: true
backend:
internal: true
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_admin_key:
file: ./secrets/s3_admin_key
s3_admin_secret:
file: ./secrets/s3_admin_secret
s3_api_key:
file: ./secrets/s3_api_key
s3_api_secret:
file: ./secrets/s3_api_secret
s3_config:
file: ./secrets/s3_config
broker_url:
file: ./secrets/broker_url
s3_worker_key:
file: ./secrets/s3_worker_key
s3_worker_secret:
file: ./secrets/s3_worker_secret
+55
View File
@@ -0,0 +1,55 @@
Image mtusi-messenger:lab5 Building
#1 [internal] load local bake definitions
#1 reading from stdin 650B done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 739B done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
#3 DONE 0.0s
#4 [internal] load .dockerignore
#4 transferring context: 196B done
#4 DONE 0.0s
#5 [internal] load build context
#5 transferring context: 15.97kB done
#5 DONE 0.0s
#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done
#6 DONE 0.0s
#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
#7 CACHED
#8 [2/7] WORKDIR /app
#8 CACHED
#9 [4/7] COPY requirements.txt ./
#9 CACHED
#10 [5/7] RUN pip install --no-cache-dir -r requirements.txt
#10 CACHED
#11 [6/7] COPY src/ ./src/
#11 DONE 0.0s
#12 [7/7] COPY migrations/ ./migrations/
#12 DONE 0.0s
#13 exporting to image
#13 exporting layers 0.0s done
#13 exporting manifest sha256:7e99f0c6a7ac94b85161817ff88967106e59e98d4e7dd9de3938b1135e0b12cd done
#13 exporting config sha256:0e7c28f3d856890d8caee1ac1cdc0b496010b55a4e1f60fd6b733acc575f7f2e done
#13 exporting attestation manifest sha256:f9c54687679fe921dd39dde4f93898025e671dd4d91f692ffb6ec14f50df16c6 done
#13 exporting manifest list sha256:14afbe72a221379a2e0438f88629275c862f8347151aee0a33ac60708793d1c2 done
#13 naming to docker.io/library/mtusi-messenger:lab5 done
#13 unpacking to docker.io/library/mtusi-messenger:lab5 0.0s done
#13 DONE 0.1s
#14 resolving provenance for metadata file
#14 DONE 0.0s
Image mtusi-messenger:lab5 Built
File diff suppressed because it is too large Load Diff
+27
View File
@@ -0,0 +1,27 @@
Container mtusi-lab5-worker-1 Stopping
Container mtusi-lab5-dispatcher-1 Stopping
Container mtusi-lab5-dispatcher-1 Stopped
Container mtusi-lab5-worker-1 Stopped
Container mtusi-lab5-api-1 Starting
Container mtusi-lab5-api-1 Started
Container mtusi-lab5-api-2 Starting
Container mtusi-lab5-api-2 Started
Container mtusi-lab5-api-1 Waiting
Container mtusi-lab5-api-2 Waiting
Container mtusi-lab5-api-1 Healthy
Container mtusi-lab5-api-2 Healthy
Container mtusi-lab5-dispatcher-1 Starting
Container mtusi-lab5-dispatcher-1 Started
Container mtusi-lab5-dispatcher-1 Stopping
Container mtusi-lab5-dispatcher-1 Stopped
Container mtusi-lab5-broker-1 Restarting
Container mtusi-lab5-broker-1 Started
Container mtusi-lab5-broker-1 Running
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-dispatcher-1 Starting
Container mtusi-lab5-worker-1 Starting
Container mtusi-lab5-worker-1 Started
Container mtusi-lab5-dispatcher-1 Started
Container mtusi-lab5-dispatcher-1 Running
Container mtusi-lab5-worker-1 Running
+766
View File
@@ -0,0 +1,766 @@
{
"killed_api_containers": [
"e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa",
"e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3"
],
"stream_entries_before_after_broker_restart": [
1,
1
],
"attachment_id": "06570cf0-e36c-4403-b10d-c216f92d9696",
"original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"status": "ready",
"variants": [
"thumbnail",
"crop",
"optimized"
],
"old_bearer_survived": true,
"message_attachment_retained": true,
"before": [
{
"container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa",
"name": "/mtusi-lab5-api-1",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:06:02.475559008Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3",
"name": "/mtusi-lab5-api-2",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:06:02.552220082Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
}
]
},
{
"container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97",
"name": "/mtusi-lab5-broker-1",
"service": "broker",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.074661473Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 201326592,
"nano_cpus": 500000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_broker_data"
}
]
},
{
"container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86",
"name": "/mtusi-lab5-bucket-bootstrap-1",
"service": "bucket-bootstrap",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:06:01.893110991Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_admin_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_admin_secret",
"name": null
}
]
},
{
"container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30",
"name": "/mtusi-lab5-dispatcher-1",
"service": "dispatcher",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:06:01.993890605Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
}
]
},
{
"container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d",
"name": "/mtusi-lab5-lb-1",
"service": "lb",
"image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
"user": "haproxy",
"running": true,
"started_at": "2026-10-27T15:06:08.145248456Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 500000000,
"ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "18091"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
"name": null
}
]
},
{
"container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4",
"name": "/mtusi-lab5-migrate-1",
"service": "migrate",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:06:01.411588564Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
}
]
},
{
"container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3",
"name": "/mtusi-lab5-postgres-1",
"service": "postgres",
"image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.049832274Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"5432/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/var/lib/postgresql/data",
"name": "mtusi-lab5_postgres_data"
},
{
"type": "bind",
"destination": "/run/secrets/postgres_password",
"name": null
}
]
},
{
"container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010",
"name": "/mtusi-lab5-redis-1",
"service": "redis",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.007675374Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 250000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011"
}
]
},
{
"container": "99e7c7f261870f281f7b9185e5a12c752e4e77d9890288e73f66628db5967106",
"name": "/mtusi-lab5-s3-1",
"service": "s3",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T15:07:19.59007169Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 500000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21"
},
{
"type": "bind",
"destination": "/run/secrets/s3_config",
"name": null
}
]
},
{
"container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7",
"name": "/mtusi-lab5-s3-store-1",
"service": "s3-store",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:55.491841572Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_s3_data"
}
]
},
{
"container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed",
"name": "/mtusi-lab5-worker-1",
"service": "worker",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:06:02.475268543Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 805306368,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_secret",
"name": null
}
]
}
],
"after": [
{
"container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa",
"name": "/mtusi-lab5-api-1",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:07:57.608120136Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3",
"name": "/mtusi-lab5-api-2",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:07:57.67064591Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
}
]
},
{
"container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97",
"name": "/mtusi-lab5-broker-1",
"service": "broker",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T15:08:06.296755178Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 201326592,
"nano_cpus": 500000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_broker_data"
}
]
},
{
"container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86",
"name": "/mtusi-lab5-bucket-bootstrap-1",
"service": "bucket-bootstrap",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:06:01.893110991Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_admin_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_admin_secret",
"name": null
}
]
},
{
"container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30",
"name": "/mtusi-lab5-dispatcher-1",
"service": "dispatcher",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:08:08.679298033Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
}
]
},
{
"container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d",
"name": "/mtusi-lab5-lb-1",
"service": "lb",
"image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
"user": "haproxy",
"running": true,
"started_at": "2026-10-27T15:06:08.145248456Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 500000000,
"ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "18091"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
"name": null
}
]
},
{
"container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4",
"name": "/mtusi-lab5-migrate-1",
"service": "migrate",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:06:01.411588564Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
}
]
},
{
"container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3",
"name": "/mtusi-lab5-postgres-1",
"service": "postgres",
"image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.049832274Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"5432/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/var/lib/postgresql/data",
"name": "mtusi-lab5_postgres_data"
},
{
"type": "bind",
"destination": "/run/secrets/postgres_password",
"name": null
}
]
},
{
"container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010",
"name": "/mtusi-lab5-redis-1",
"service": "redis",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.007675374Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 250000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011"
}
]
},
{
"container": "99e7c7f261870f281f7b9185e5a12c752e4e77d9890288e73f66628db5967106",
"name": "/mtusi-lab5-s3-1",
"service": "s3",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T15:07:19.59007169Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 500000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21"
},
{
"type": "bind",
"destination": "/run/secrets/s3_config",
"name": null
}
]
},
{
"container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7",
"name": "/mtusi-lab5-s3-store-1",
"service": "s3-store",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:55.491841572Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_s3_data"
}
]
},
{
"container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed",
"name": "/mtusi-lab5-worker-1",
"service": "worker",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:08:08.679577406Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 805306368,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_secret",
"name": null
}
]
}
]
}
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18091"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 2, "first_recovery_seconds": 2.534}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+28
View File
@@ -0,0 +1,28 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
python3 tests/smoke.py --base-url "http://127.0.0.1:18091"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
BASE_URL="http://127.0.0.1:18091" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
test_cursor_scope_and_validation (test_api.APIContract) ... ok
test_foreign_chat (test_api.APIContract) ... ok
test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
test_idempotency_conflicting_race (test_api.APIContract) ... ok
test_idempotency_invalid_keys (test_api.APIContract) ... ok
test_idempotency_scope (test_api.APIContract) ... ok
test_idempotency_semantic_body (test_api.APIContract) ... ok
test_invalid_chat_creation (test_api.APIContract) ... ok
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
test_protocol_error_headers (test_api.APIContract) ... ok
test_registration_race (test_api.APIContract) ... ok
test_unicode_limits (test_api.APIContract) ... ok
test_unicode_nul_round_trip (test_api.APIContract) ... ok
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
test_async_variants_and_replay (test_images.Images) ... ok
test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
test_private_and_cross_chat (test_images.Images) ... ok
----------------------------------------------------------------------
Ran 18 tests in 5.344s
OK
+73
View File
@@ -0,0 +1,73 @@
{
"attachments_by_state": [
{
"state": "failed",
"count": 3
},
{
"state": "ready",
"count": 85
}
],
"ready_without_three_variants": 0,
"stream_length": 0,
"pending_deliveries": 0,
"job_events": [
{
"event": "claimed",
"count": 107
},
{
"event": "duplicate_or_early_noop",
"count": 3
},
{
"event": "failed:invalid_image",
"count": 3
},
{
"event": "failed:retry_exhausted",
"count": 2
},
{
"event": "failed:source_missing",
"count": 3
},
{
"event": "fenced_noop",
"count": 3
},
{
"event": "operator_replay",
"count": 5
},
{
"event": "operator_supersede",
"count": 3
},
{
"event": "published",
"count": 74
},
{
"event": "ready",
"count": 85
},
{
"event": "retry:lease_expired",
"count": 9
},
{
"event": "retry:storage_unavailable",
"count": 2
},
{
"event": "stale_noop",
"count": 3
},
{
"event": "terminal_noop",
"count": 30
}
]
}
+1
View File
@@ -0,0 +1 @@
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18091"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
Container mtusi-lab5-audit-run-b338f8a5a02a Creating
Container mtusi-lab5-audit-run-b338f8a5a02a Created
+119
View File
@@ -0,0 +1,119 @@
{
"invariants": {
"commit_before_publish_recovered": true,
"publish_before_marker_duplicate_safe": true,
"crashes_before_after_side_effect_and_commit": true,
"invalid_envelope_observable_dlq": true,
"duplicate_and_reorder_no_second_effect": true,
"concurrent_duplicate_single_claim": true,
"stale_running_version_fenced": true,
"old_delivery_after_new_ready_no_overwrite": true,
"failed_registry_explicit_replay_and_ready_refusal": true,
"transient_backoff_exhaustion_then_replay": true,
"invalid_decode_permanent_failed": true,
"cleanup_preserves_current_and_fresh_failed": true,
"ready_staging_cleanup_idempotent": true
},
"faults": [
{
"point": "before_side_effect",
"exit_code": 77,
"state_at_crash": "processing",
"recovered": "ready",
"attempts": 2,
"result_objects": 4
},
{
"point": "after_original",
"exit_code": 77,
"state_at_crash": "processing",
"recovered": "ready",
"attempts": 2,
"result_objects": 4
},
{
"point": "after_objects",
"exit_code": 77,
"state_at_crash": "processing",
"recovered": "ready",
"attempts": 2,
"result_objects": 4
},
{
"point": "after_commit",
"exit_code": 77,
"state_at_crash": "ready",
"recovered": "ready",
"attempts": 1,
"result_objects": 4
}
],
"retry": {
"transitions": [
{
"state": "queued",
"attempt": 1,
"error_code": "storage_unavailable"
},
{
"state": "failed",
"attempt": 2,
"error_code": "retry_exhausted"
}
],
"seconds": 1.941,
"replayed_attempt": 3
},
"transforms": [
{
"kind": "alpha",
"size": [
20,
10
],
"exif_removed": true
},
{
"kind": "exif",
"size": [
256,
160
],
"exif_removed": true
},
{
"kind": "small",
"size": [
5,
3
],
"exif_removed": true
}
],
"cleanup": [
{
"apply": false,
"retention_seconds": 86400,
"counts": {
"ready_staging": 46,
"expired_staging": 0
}
},
{
"apply": true,
"retention_seconds": 86400,
"counts": {
"ready_staging": 46,
"expired_staging": 0
}
},
{
"apply": true,
"retention_seconds": 86400,
"counts": {
"ready_staging": 0,
"expired_staging": 0
}
}
]
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

File diff suppressed because it is too large Load Diff

After

Width:  |  Height:  |  Size: 66 KiB

+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://127.0.0.1:18091"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+6
View File
@@ -0,0 +1,6 @@
Container mtusi-lab5-worker-1 Restarting
Container mtusi-lab5-worker-1 Started
Container mtusi-lab5-worker-1 Restarting
Container mtusi-lab5-worker-1 Started
Container mtusi-lab5-worker-1 Restarting
Container mtusi-lab5-worker-1 Started
File diff suppressed because it is too large Load Diff
+373
View File
@@ -0,0 +1,373 @@
[
{
"container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa",
"name": "/mtusi-lab5-api-1",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:06:02.475559008Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
}
]
},
{
"container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3",
"name": "/mtusi-lab5-api-2",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:06:02.552220082Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97",
"name": "/mtusi-lab5-broker-1",
"service": "broker",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.074661473Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 201326592,
"nano_cpus": 500000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_broker_data"
}
]
},
{
"container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86",
"name": "/mtusi-lab5-bucket-bootstrap-1",
"service": "bucket-bootstrap",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:06:01.893110991Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_admin_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_admin_secret",
"name": null
}
]
},
{
"container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30",
"name": "/mtusi-lab5-dispatcher-1",
"service": "dispatcher",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:06:01.993890605Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
}
]
},
{
"container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d",
"name": "/mtusi-lab5-lb-1",
"service": "lb",
"image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
"user": "haproxy",
"running": true,
"started_at": "2026-10-27T15:06:08.145248456Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 500000000,
"ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "18091"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
"name": null
}
]
},
{
"container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4",
"name": "/mtusi-lab5-migrate-1",
"service": "migrate",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:06:01.411588564Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3",
"name": "/mtusi-lab5-postgres-1",
"service": "postgres",
"image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.049832274Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"5432/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/var/lib/postgresql/data",
"name": "mtusi-lab5_postgres_data"
},
{
"type": "bind",
"destination": "/run/secrets/postgres_password",
"name": null
}
]
},
{
"container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010",
"name": "/mtusi-lab5-redis-1",
"service": "redis",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.007675374Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 250000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011"
}
]
},
{
"container": "b8d84d132c21a7d1d44afe270bce5beef0c9f2ecf28e4fa9e88efe83c04660d3",
"name": "/mtusi-lab5-s3-1",
"service": "s3",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.087804668Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 500000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_config",
"name": null
},
{
"type": "volume",
"destination": "/data",
"name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21"
}
]
},
{
"container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7",
"name": "/mtusi-lab5-s3-store-1",
"service": "s3-store",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:55.491841572Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_s3_data"
}
]
},
{
"container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed",
"name": "/mtusi-lab5-worker-1",
"service": "worker",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:06:02.475268543Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 805306368,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_secret",
"name": null
}
]
}
]
+373
View File
@@ -0,0 +1,373 @@
[
{
"container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa",
"name": "/mtusi-lab5-api-1",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:05:46.437791821Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3",
"name": "/mtusi-lab5-api-2",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:03:32.37188016Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
}
]
},
{
"container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97",
"name": "/mtusi-lab5-broker-1",
"service": "broker",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T15:03:41.154830662Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 201326592,
"nano_cpus": 500000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_broker_data"
}
]
},
{
"container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86",
"name": "/mtusi-lab5-bucket-bootstrap-1",
"service": "bucket-bootstrap",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:03:24.364562897Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_admin_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_admin_secret",
"name": null
}
]
},
{
"container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30",
"name": "/mtusi-lab5-dispatcher-1",
"service": "dispatcher",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:05:23.350954361Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
}
]
},
{
"container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d",
"name": "/mtusi-lab5-lb-1",
"service": "lb",
"image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
"user": "haproxy",
"running": true,
"started_at": "2026-10-27T14:56:14.79412711Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 500000000,
"ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "18091"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
"name": null
}
]
},
{
"container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4",
"name": "/mtusi-lab5-migrate-1",
"service": "migrate",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:03:23.874882274Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3",
"name": "/mtusi-lab5-postgres-1",
"service": "postgres",
"image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
"user": "",
"running": true,
"started_at": "2026-10-27T14:56:01.375415684Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"5432/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/var/lib/postgresql/data",
"name": "mtusi-lab5_postgres_data"
},
{
"type": "bind",
"destination": "/run/secrets/postgres_password",
"name": null
}
]
},
{
"container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010",
"name": "/mtusi-lab5-redis-1",
"service": "redis",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T14:56:01.375672543Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 250000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011"
}
]
},
{
"container": "b8d84d132c21a7d1d44afe270bce5beef0c9f2ecf28e4fa9e88efe83c04660d3",
"name": "/mtusi-lab5-s3-1",
"service": "s3",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T14:56:05.95436716Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 500000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_config",
"name": null
},
{
"type": "volume",
"destination": "/data",
"name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21"
}
]
},
{
"container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7",
"name": "/mtusi-lab5-s3-store-1",
"service": "s3-store",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T14:56:01.374203439Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_s3_data"
}
]
},
{
"container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed",
"name": "/mtusi-lab5-worker-1",
"service": "worker",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:05:23.351252982Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 805306368,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_worker_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_key",
"name": null
}
]
}
]
+373
View File
@@ -0,0 +1,373 @@
[
{
"container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa",
"name": "/mtusi-lab5-api-1",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:07:57.608120136Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
}
]
},
{
"container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3",
"name": "/mtusi-lab5-api-2",
"service": "api",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:07:57.67064591Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97",
"name": "/mtusi-lab5-broker-1",
"service": "broker",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T15:08:06.296755178Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 201326592,
"nano_cpus": 500000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_broker_data"
}
]
},
{
"container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86",
"name": "/mtusi-lab5-bucket-bootstrap-1",
"service": "bucket-bootstrap",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:06:01.893110991Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_admin_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_admin_secret",
"name": null
}
]
},
{
"container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30",
"name": "/mtusi-lab5-dispatcher-1",
"service": "dispatcher",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:08:08.679298033Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
}
]
},
{
"container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d",
"name": "/mtusi-lab5-lb-1",
"service": "lb",
"image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
"user": "haproxy",
"running": true,
"started_at": "2026-10-27T15:06:08.145248456Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 500000000,
"ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "18091"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
"name": null
}
]
},
{
"container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4",
"name": "/mtusi-lab5-migrate-1",
"service": "migrate",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": false,
"started_at": "2026-10-27T15:06:01.411588564Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
}
]
},
{
"container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3",
"name": "/mtusi-lab5-postgres-1",
"service": "postgres",
"image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.049832274Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"5432/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/postgres_password",
"name": null
},
{
"type": "volume",
"destination": "/var/lib/postgresql/data",
"name": "mtusi-lab5_postgres_data"
}
]
},
{
"container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010",
"name": "/mtusi-lab5-redis-1",
"service": "redis",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:54.007675374Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 250000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011"
}
]
},
{
"container": "99e7c7f261870f281f7b9185e5a12c752e4e77d9890288e73f66628db5967106",
"name": "/mtusi-lab5-s3-1",
"service": "s3",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T15:07:19.59007169Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 500000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21"
},
{
"type": "bind",
"destination": "/run/secrets/s3_config",
"name": null
}
]
},
{
"container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7",
"name": "/mtusi-lab5-s3-store-1",
"service": "s3-store",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-10-27T15:05:55.491841572Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab5_s3_data"
}
]
},
{
"container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed",
"name": "/mtusi-lab5-worker-1",
"service": "worker",
"image": "mtusi-messenger:lab5",
"user": "10001:10001",
"running": true,
"started_at": "2026-10-27T15:08:08.679577406Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 805306368,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_secret",
"name": null
}
]
}
]
+2
View File
@@ -0,0 +1,2 @@
Container mtusi-lab5-audit-run-790369e3d6e4 Creating
Container mtusi-lab5-audit-run-790369e3d6e4 Created
+15
View File
@@ -0,0 +1,15 @@
{
"anonymous_object_and_list_403": true,
"job_fields": [
"attachment_id",
"attempt",
"created_at",
"job_id",
"pipeline_version",
"schema_version",
"type"
],
"job_bytes": 243,
"no_source_bytes_or_credentials_in_envelope": true,
"strict_invalid_envelope_checks": true
}
+1
View File
@@ -0,0 +1 @@
{"trickle_seconds": 14.998, "http_status": 400, "absolute_deadline_enforced": true}
+80
View File
@@ -0,0 +1,80 @@
Container mtusi-lab5-broker-1 Running
Container mtusi-lab5-s3-store-1 Running
Container mtusi-lab5-postgres-1 Running
Container mtusi-lab5-redis-1 Running
Container mtusi-lab5-s3-1 Running
Container mtusi-lab5-bucket-bootstrap-1 Recreate
Container mtusi-lab5-migrate-1 Recreate
Container mtusi-lab5-bucket-bootstrap-1 Recreated
Container mtusi-lab5-migrate-1 Recreated
Container mtusi-lab5-worker-1 Recreate
Container mtusi-lab5-api-1 Recreate
Container mtusi-lab5-dispatcher-1 Recreate
Container mtusi-lab5-api-2 Recreate
Container mtusi-lab5-dispatcher-1 Recreated
Container mtusi-lab5-api-1 Recreated
Container mtusi-lab5-api-2 Recreated
Container mtusi-lab5-lb-1 Running
Container mtusi-lab5-worker-1 Recreated
Container mtusi-lab5-s3-store-1 Waiting
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-s3-store-1 Healthy
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-migrate-1 Starting
Container mtusi-lab5-s3-1 Waiting
Container mtusi-lab5-migrate-1 Started
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-s3-1 Healthy
Container mtusi-lab5-bucket-bootstrap-1 Starting
Container mtusi-lab5-bucket-bootstrap-1 Started
Container mtusi-lab5-bucket-bootstrap-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-redis-1 Waiting
Container mtusi-lab5-bucket-bootstrap-1 Waiting
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-dispatcher-1 Starting
Container mtusi-lab5-dispatcher-1 Started
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-redis-1 Healthy
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-bucket-bootstrap-1 Exited
Container mtusi-lab5-bucket-bootstrap-1 Exited
Container mtusi-lab5-worker-1 Starting
Container mtusi-lab5-api-1 Starting
Container mtusi-lab5-worker-1 Started
Container mtusi-lab5-api-1 Started
Container mtusi-lab5-api-2 Starting
Container mtusi-lab5-api-2 Started
Container mtusi-lab5-api-1 Waiting
Container mtusi-lab5-api-2 Waiting
Container mtusi-lab5-api-1 Healthy
Container mtusi-lab5-api-2 Healthy
Container mtusi-lab5-broker-1 Waiting
Container mtusi-lab5-s3-1 Waiting
Container mtusi-lab5-redis-1 Waiting
Container mtusi-lab5-dispatcher-1 Waiting
Container mtusi-lab5-api-1 Waiting
Container mtusi-lab5-api-2 Waiting
Container mtusi-lab5-migrate-1 Waiting
Container mtusi-lab5-lb-1 Waiting
Container mtusi-lab5-bucket-bootstrap-1 Waiting
Container mtusi-lab5-s3-store-1 Waiting
Container mtusi-lab5-postgres-1 Waiting
Container mtusi-lab5-worker-1 Waiting
Container mtusi-lab5-redis-1 Healthy
Container mtusi-lab5-migrate-1 Exited
Container mtusi-lab5-postgres-1 Healthy
Container mtusi-lab5-s3-store-1 Healthy
Container mtusi-lab5-lb-1 Healthy
Container mtusi-lab5-broker-1 Healthy
Container mtusi-lab5-dispatcher-1 Healthy
Container mtusi-lab5-s3-1 Healthy
Container mtusi-lab5-worker-1 Healthy
Container mtusi-lab5-bucket-bootstrap-1 Exited
Container mtusi-lab5-api-1 Healthy
Container mtusi-lab5-api-2 Healthy
+1
View File
@@ -0,0 +1 @@
{"instance": "e4901b6a601a4dd094235d4dfcb0591c", "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", "running": false, "exit_code": 0, "seconds": 0.445}
+26
View File
@@ -0,0 +1,26 @@
python3 tests/smoke.py --base-url "http://127.0.0.1:18091"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
BASE_URL="http://127.0.0.1:18091" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
test_cursor_scope_and_validation (test_api.APIContract) ... ok
test_foreign_chat (test_api.APIContract) ... ok
test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
test_idempotency_conflicting_race (test_api.APIContract) ... ok
test_idempotency_invalid_keys (test_api.APIContract) ... ok
test_idempotency_scope (test_api.APIContract) ... ok
test_idempotency_semantic_body (test_api.APIContract) ... ok
test_invalid_chat_creation (test_api.APIContract) ... ok
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
test_protocol_error_headers (test_api.APIContract) ... ok
test_registration_race (test_api.APIContract) ... ok
test_unicode_limits (test_api.APIContract) ... ok
test_unicode_nul_round_trip (test_api.APIContract) ... ok
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
test_async_variants_and_replay (test_images.Images) ... ok
test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
test_private_and_cross_chat (test_images.Images) ... ok
----------------------------------------------------------------------
Ran 18 tests in 5.184s
OK
+46
View File
@@ -0,0 +1,46 @@
global
maxconn 1024
nbthread 1
hard-stop-after 10s
defaults
mode http
timeout connect 500ms
timeout check 500ms
timeout queue 1s
timeout client 30s
timeout server 30s
timeout http-request 3s
retries 2
retry-on conn-failure
option http-server-close
resolvers docker
nameserver docker_dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 2
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold nx 1s
hold refused 1s
hold timeout 1s
hold other 1s
hold obsolete 1s
frontend public
bind :8080
default_backend replicas
backend replicas
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fall 2 rise 2
server-template api 1-4 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
listen metrics
bind :8404
stats enable
stats uri /stats
+42
View File
@@ -0,0 +1,42 @@
CREATE TABLE users (
id UUID PRIMARY KEY
, username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$')
, display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100)
, password_hash TEXT NOT NULL
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
)
;
CREATE TABLE chats (
id UUID PRIMARY KEY
, title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100)
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, last_message_at TIMESTAMPTZ
)
;
CREATE TABLE chat_members (
chat_id UUID NOT NULL REFERENCES chats(id)
, user_id UUID NOT NULL REFERENCES users(id)
, PRIMARY KEY (chat_id, user_id)
)
;
CREATE INDEX chat_members_user ON chat_members (user_id, chat_id)
;
CREATE TABLE messages (
id UUID PRIMARY KEY
, chat_id UUID NOT NULL REFERENCES chats(id)
, sender_id UUID NOT NULL REFERENCES users(id)
, text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000)
, created_at TIMESTAMPTZ NOT NULL
, FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
)
;
CREATE INDEX messages_history ON messages (chat_id, created_at, id)
;
CREATE TABLE sessions (
token_hash TEXT PRIMARY KEY
, user_id UUID NOT NULL REFERENCES users(id)
, expires_at TIMESTAMPTZ NOT NULL
)
;
CREATE INDEX sessions_expiry ON sessions (expires_at)
;
+17
View File
@@ -0,0 +1,17 @@
-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA.
-- API validation keeps the contract's code-point limits; these bounds cap bytes.
ALTER TABLE users
DROP CONSTRAINT users_display_name_check
, ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8')
, ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400)
;
ALTER TABLE chats
DROP CONSTRAINT chats_title_check
, ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8')
, ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400)
;
ALTER TABLE messages
DROP CONSTRAINT messages_text_check
, ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8')
, ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000)
;
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE idempotency_keys (
user_id UUID NOT NULL REFERENCES users(id)
, chat_id UUID NOT NULL REFERENCES chats(id)
, key VARCHAR(128) NOT NULL
, request_hash CHAR(64) NOT NULL
, message_id UUID REFERENCES messages(id)
, expires_at TIMESTAMPTZ NOT NULL
, PRIMARY KEY (user_id, chat_id, key)
)
;
CREATE INDEX idempotency_expiry ON idempotency_keys (expires_at)
;
+30
View File
@@ -0,0 +1,30 @@
ALTER TABLE messages
DROP CONSTRAINT messages_text_bytes
, ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000)
, ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}'
;
CREATE TABLE attachments (
id UUID PRIMARY KEY
, chat_id UUID NOT NULL REFERENCES chats(id)
, owner_id UUID NOT NULL REFERENCES users(id)
, filename TEXT NOT NULL
, object_key TEXT NOT NULL UNIQUE
, content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg'))
, size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760)
, sha256 CHAR(64) NOT NULL
, state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'storage_missing'))
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id)
)
;
CREATE INDEX attachments_chat ON attachments (chat_id, id)
;
CREATE TABLE message_attachments (
message_id UUID NOT NULL REFERENCES messages(id)
, attachment_id UUID NOT NULL REFERENCES attachments(id)
, PRIMARY KEY (message_id, attachment_id)
)
;
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id)
;
+40
View File
@@ -0,0 +1,40 @@
ALTER TABLE attachments
DROP CONSTRAINT attachments_state_check
, ADD CONSTRAINT attachments_state_check CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed', 'storage_missing'))
, ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1
, ADD COLUMN staging_key TEXT
, ADD COLUMN variants JSONB NOT NULL DEFAULT '[]'
, ADD COLUMN error_code TEXT
, ADD COLUMN current_job_id UUID
;
CREATE TABLE attachment_jobs (
id UUID PRIMARY KEY
, attachment_id UUID NOT NULL REFERENCES attachments(id)
, pipeline_version INTEGER NOT NULL
, source_key TEXT NOT NULL
, state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded'))
, attempt INTEGER NOT NULL DEFAULT 0
, max_attempts INTEGER NOT NULL DEFAULT 5
, lease_token UUID
, lease_until TIMESTAMPTZ
, next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, published_at TIMESTAMPTZ
, error_code TEXT
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, UNIQUE (attachment_id, pipeline_version)
)
;
ALTER TABLE attachments
ADD CONSTRAINT attachments_current_job FOREIGN KEY (current_job_id) REFERENCES attachment_jobs(id)
;
CREATE INDEX jobs_due ON attachment_jobs (state, next_attempt_at)
;
CREATE TABLE job_events (
id BIGSERIAL PRIMARY KEY
, job_id UUID NOT NULL REFERENCES attachment_jobs(id)
, event TEXT NOT NULL
, attempt INTEGER NOT NULL
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
)
;
+2
View File
@@ -0,0 +1,2 @@
# Optional, only for plotting the saved evidence, Python 3.11+.
matplotlib==3.11.2
+12
View File
@@ -0,0 +1,12 @@
psycopg[binary]==3.2.3
psycopg-pool==3.2.4
redis==5.2.1
typing-extensions==4.16.0
boto3==1.40.61
Pillow==13.0.0
botocore==1.40.76
jmespath==1.1.0
s3transfer==0.14.0
python-dateutil==2.9.0.post0
urllib3==2.8.0
six==1.17.0
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Create local credentials without printing or overwriting existing values."""
import json
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
directory = root / 'secrets'
directory.mkdir(mode=0o700, exist_ok=True)
def save(name, value):
path = directory / name
if not path.exists():
path.write_text(value)
path.chmod(0o644) # UID 10001 in Compose; parent is owner-only 0700.
return path.read_text().strip()
password = save('postgres_password', secrets.token_hex(24))
save('database_url', 'postgresql://messenger:' + password + '@postgres:5432/messenger')
save('cache_url', 'redis://redis:6379/0')
save('broker_url', 'redis://broker:6379/0')
identities = []
for role, actions in {
'admin': ['Admin', 'Read', 'Write', 'List'],
'api': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'],
'worker': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'],
}.items():
key = save('s3_' + role + '_key', secrets.token_hex(12))
secret = save('s3_' + role + '_secret', secrets.token_hex(24))
identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions})
configuration = directory / 's3_config'
configuration.write_text(json.dumps({'identities': identities}))
configuration.chmod(0o600) # Only the root S3 gateway mounts the full identity configuration.
env = root / '.env'
if not env.exists():
env.write_text((root / '.env.example').read_text())
env.chmod(0o600)
print('Local configuration ready; no secrets printed.')
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
python3 scripts/snapshot.py > evidence/restart-before.json
docker compose stop api lb worker dispatcher
docker compose restart postgres redis s3-store s3 broker
docker compose up -d --wait postgres redis s3-store s3 broker
docker compose up -d --wait api lb worker dispatcher
python3 scripts/snapshot.py > evidence/restart-after.json
+11
View File
@@ -0,0 +1,11 @@
"""Sanitized process and resource evidence; never dump container environments."""
import json
import subprocess
ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split()
records = []
for container in ids:
raw = subprocess.check_output(['docker', 'inspect', container], text=True)
data = json.loads(raw)[0]
records.append({'container': data['Id'], 'name': data['Name'], 'service': data['Config']['Labels']['com.docker.compose.service'], 'image': data['Config']['Image'], 'user': data['Config']['User'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'exit_code': data['State']['ExitCode'], 'oom_killed': data['State']['OOMKilled'], 'memory_limit': data['HostConfig']['Memory'], 'nano_cpus': data['HostConfig']['NanoCpus'], 'ports': data['NetworkSettings']['Ports'], 'mounts': [{'type': m['Type'], 'destination': m['Destination'], 'name': m.get('Name')} for m in data['Mounts']]})
print(json.dumps(records, indent=2))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop_one.py
+32
View File
@@ -0,0 +1,32 @@
"""Resolve the requested process alias using each actual container, never a fake header."""
import json
import os
from pathlib import Path
import subprocess
import time
def docker(*args):
return subprocess.check_output(['docker', *args], text=True).strip()
def instance(container):
code = "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=1).headers['X-Instance-Id'])"
return docker('exec', container, 'python', '-c', code)
if __name__ == '__main__':
target = os.environ['TARGET_INSTANCE_ID']
started = time.monotonic()
for container in docker('compose', 'ps', '-q', 'api').splitlines():
if instance(container) == target:
docker('stop', '--time', '8', container)
state = json.loads(docker('inspect', '--format', '{{json .State}}', container))
assert not state['Running'], 'target must remain stopped'
record = {'instance': target, 'container': container, 'running': state['Running'], 'exit_code': state['ExitCode'], 'seconds': round(time.monotonic() - started, 3)}
with Path('evidence/stopped-instances.jsonl').open('a') as out:
out.write(json.dumps(record) + '\n')
print(json.dumps(record))
break
else:
raise SystemExit('Requested live API instance not found')
+16
View File
@@ -0,0 +1,16 @@
import os
import time
import urllib.request
base = os.environ.get('BASE_URL', 'http://127.0.0.1:18087')
for attempt in range(120):
try:
with urllib.request.urlopen(base + '/health/ready', timeout=2) as response:
if response.status == 200:
print('Ready through load balancer.')
break
except OSError:
pass
time.sleep(0.5)
else:
raise SystemExit('Readiness timeout')
+127
View File
@@ -0,0 +1,127 @@
"""Explicit local operator tools; no administrative HTTP endpoint."""
import argparse
import json
import os
import uuid
from pipeline import connect, audit
from objects import Objects
from postgres import setting
def replay(job_id):
with connect() as conn, conn.transaction():
job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job_id,)).fetchone()
if job is None or job['state'] != 'failed':
raise ValueError('Replay requires a failed job')
attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND current_job_id = %s FOR UPDATE', (job_id,)).fetchone()
if attachment is None or attachment['state'] != 'failed' or Objects(setting).head(job['source_key']) is None:
raise ValueError('Current failed source is unavailable')
conn.execute('''
UPDATE attachment_jobs
SET state = 'queued'
, max_attempts = attempt + %s
, published_at = NULL
, next_attempt_at = clock_timestamp()
, error_code = NULL
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
''', (int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), job_id))
conn.execute("UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job_id,))
audit(conn, job, 'operator_replay')
return {'job_id': str(job_id), 'replayed': True}
def version(attachment_id, number):
if number not in (1, 2):
raise ValueError('Only implemented pipeline versions 1 and 2 are supported')
with connect() as conn, conn.transaction():
row = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone()
if row is None or number != row['pipeline_version'] + 1 or row['state'] in ('uploading', 'failed', 'storage_missing'):
raise ValueError('Version must advance the current process exactly once')
old = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (row['current_job_id'],)).fetchone()
source = row['object_key'] if row['state'] == 'ready' else old['source_key']
if Objects(setting).head(source) is None:
raise ValueError('Source unavailable')
conn.execute("UPDATE attachment_jobs SET state = 'superseded', lease_token = NULL, lease_until = NULL WHERE 1=1 AND id = %s", (old['id'],))
audit(conn, old, 'operator_supersede')
job_id = uuid.uuid4()
conn.execute('''
INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts)
VALUES (%s, %s, %s, %s, %s)
''', (job_id, attachment_id, number, source, int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))))
conn.execute('''
UPDATE attachments
SET pipeline_version = %s
, current_job_id = %s
, state = 'queued'
, variants = '[]'
, error_code = NULL
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
''', (number, job_id, attachment_id))
return {'job_id': str(job_id), 'version': number}
def cleanup(apply=False, retention=86400):
if retention < 86400:
raise ValueError('Retention must be >=24 hours')
objects = Objects(setting)
counts = {'ready_staging': 0, 'expired_staging': 0}
with connect() as conn:
# Intent rows cover every valid staging PUT. Never collect active jobs.
last = '00000000-0000-0000-0000-000000000000'
while True:
rows = conn.execute('SELECT id FROM attachments WHERE 1=1 AND id > %s ORDER BY id LIMIT 100', (last,)).fetchall()
if not rows:
break
for candidate in rows:
last = candidate['id']
with conn.transaction():
row = conn.execute('''
SELECT *
FROM attachments
WHERE 1=1
AND id = %s
AND (state = 'ready' OR (state IN ('uploading', 'failed') AND updated_at < clock_timestamp() - %s * INTERVAL '1 second'))
FOR UPDATE SKIP LOCKED
''', (last, retention)).fetchone()
if row is None or not row['staging_key'] or not row['staging_key'].startswith('staging/'):
continue
counts['ready_staging' if row['state'] == 'ready' else 'expired_staging'] += 1
if apply:
objects.delete(row['staging_key'])
if row['state'] == 'uploading' and row['current_job_id'] is None:
conn.execute('DELETE FROM attachments WHERE 1=1 AND id = %s', (last,))
else:
conn.execute('UPDATE attachments SET staging_key = NULL WHERE 1=1 AND id = %s', (last,))
return dict(apply=apply, retention_seconds=retention, counts=counts)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('action', choices=['failed', 'replay', 'version', 'cleanup'])
p.add_argument('identifier', nargs='?')
p.add_argument('--version', type=int)
p.add_argument('--apply', action='store_true')
args = p.parse_args()
try:
if args.action == 'failed':
with connect() as conn:
result = conn.execute("SELECT id, attachment_id, pipeline_version, attempt, max_attempts, error_code FROM attachment_jobs WHERE 1=1 AND state = 'failed' ORDER BY created_at LIMIT 100").fetchall()
elif args.action == 'replay':
result = replay(uuid.UUID(args.identifier))
elif args.action == 'version':
result = version(uuid.UUID(args.identifier), args.version)
else:
result = cleanup(args.apply)
print(json.dumps(result, default=str, indent=2))
except Exception:
print(json.dumps({'error': 'Operator action refused or dependency unavailable; no secrets logged.'}))
raise SystemExit(1)
if __name__ == '__main__':
main()
+13
View File
@@ -0,0 +1,13 @@
"""Idempotent private bucket bootstrap, executed only with administrative S3 keys."""
from botocore.exceptions import ClientError
from objects import Objects
from postgres import setting
objects = Objects(setting)
try:
objects.client.create_bucket(Bucket=objects.bucket)
except ClientError as exc:
if exc.response['Error']['Code'] not in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
raise RuntimeError('Bucket bootstrap failed') from None
objects.ready()
print('Private bucket ready; static authenticated identities loaded from local secret config.')
+98
View File
@@ -0,0 +1,98 @@
"""Shared HTTP contract; storage is supplied by each laboratory."""
import base64
import datetime as dt
import json
import re
import uuid
class APIError(Exception):
def __init__(self, status, code, message):
self.status, self.code, self.message = status, code, message
def invalid():
raise APIError(400, 'invalid_request', 'Неверные параметры запроса')
def unauthorized():
raise APIError(401, 'unauthorized', 'Требуется действительная идентификация')
def missing():
raise APIError(404, 'not_found', 'Объект не найден')
def identifier(value):
try:
if not isinstance(value, str):
invalid()
return str(uuid.UUID(value))
except ValueError:
invalid()
def text(value, maximum, minimum=1):
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
invalid()
try:
value.encode('utf-8')
except UnicodeError:
invalid()
return value
def fields(value, required):
if not isinstance(value, dict) or set(value) != set(required):
invalid()
def username(value):
if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value):
invalid()
return value
def now():
return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def page_args(query, scope):
if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
invalid()
limit = query.get('limit', ['50'])[0]
if not re.fullmatch(r'[0-9]+', limit):
invalid()
try:
limit = int(limit)
except ValueError:
invalid()
if not 1 <= limit <= 100:
invalid()
cursor = None
if 'cursor' in query:
raw = query['cursor'][0]
try:
if not raw or len(raw) > 1024:
invalid()
decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True))
if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope:
invalid()
stamp = decoded[1]
if not isinstance(stamp, str) or not stamp.endswith('Z'):
invalid()
stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z')
cursor = (stamp, identifier(decoded[2]))
except (ValueError, TypeError, UnicodeError):
invalid()
return limit, cursor
def page(items, limit, scope):
more = len(items) > limit
items = items[:limit]
cursor = None
if more:
last = items[-1]
cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=')
return {'items': items, 'next_cursor': cursor}
+13
View File
@@ -0,0 +1,13 @@
"""One-shot migration job; replicas never execute DDL."""
from pathlib import Path
import psycopg
from postgres import setting
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn:
conn.execute('SELECT pg_advisory_xact_lock(2301003)')
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)')
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone():
conn.execute(path.read_text())
conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,))
print('Applied', path.name)
+156
View File
@@ -0,0 +1,156 @@
"""Bounded S3 operations. No root credentials or filename-derived object keys."""
import contextlib
import hashlib
import os
import tempfile
import threading
import time
import unicodedata
import warnings
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, PngImagePlugin, UnidentifiedImageError
from common import APIError, invalid
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024
PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
UPLOAD_SLOTS = threading.BoundedSemaphore(8)
DECODE_SLOTS = threading.BoundedSemaphore(1)
def unavailable():
raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно')
def filename(value):
if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip():
invalid()
if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value):
invalid()
try:
value.encode('utf-8')
except UnicodeError:
invalid()
return value
@contextlib.contextmanager
def bounded_upload():
if not UPLOAD_SLOTS.acquire(blocking=False):
raise APIError(503, 'unavailable', 'Все слоты загрузки заняты')
try:
yield
finally:
UPLOAD_SLOTS.release()
@contextlib.contextmanager
def receive(handler):
lengths = handler.headers.get_all('Content-Length', [])
if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'):
invalid()
try:
length = int(lengths[0])
except ValueError:
invalid()
if length > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Максимум 10 MiB')
if length <= 0:
invalid()
mime = handler.headers.get_content_type()
if mime not in ('image/png', 'image/jpeg'):
raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG')
deadline = time.monotonic() + 15
digest = hashlib.sha256()
# Ephemeral bounded spool is never a durable source of a confirmed attachment.
with tempfile.TemporaryFile(mode='w+b') as source:
remaining = length
while remaining:
handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic())))
if time.monotonic() >= deadline:
invalid()
try:
chunk = handler.rfile.read1(min(64 * 1024, remaining))
except (OSError, TimeoutError):
invalid()
if not chunk:
invalid()
digest.update(chunk)
source.write(chunk)
remaining -= len(chunk)
if not DECODE_SLOTS.acquire(timeout=10):
raise APIError(503, 'unavailable', 'Проверка изображений перегружена')
try:
source.flush()
for decode in (False,):
source.seek(0)
# Pillow.close releases pixel buffers and may close its input file.
# A duplicate descriptor preserves the original for the S3 PUT.
with os.fdopen(os.dup(source.fileno()), 'rb') as image_file:
image = Image.open(image_file, formats=['PNG', 'JPEG'])
try:
if image.width * image.height > MAX_PIXELS:
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей')
if Image.MIME[image.format] != mime:
raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению')
if decode:
image.load()
else:
image.verify()
finally:
image.close() # Free pixels BEFORE releasing the decoder semaphore.
except (Image.DecompressionBombWarning, Image.DecompressionBombError):
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None
finally:
DECODE_SLOTS.release()
source.seek(0)
yield source, length, mime, digest.hexdigest()
class Objects:
def __init__(self, setting):
self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments')
self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required'))
def head(self, key):
try:
return self.client.head_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'):
return None
unavailable()
except BotoCoreError:
unavailable()
def ready(self):
try:
self.client.head_bucket(Bucket=self.bucket)
except (ClientError, BotoCoreError):
unavailable()
def put(self, key, source, length, mime, digest):
try:
self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest})
except (ClientError, BotoCoreError):
unavailable()
def get(self, key):
try:
return self.client.get_object(Bucket=self.bucket, Key=key)
except (ClientError, BotoCoreError):
unavailable()
def delete(self, key):
try:
self.client.delete_object(Bucket=self.bucket, Key=key)
except (ClientError, BotoCoreError):
unavailable()
+332
View File
@@ -0,0 +1,332 @@
"""Durable outbox, Redis Streams delivery, bounded worker and fenced result commits."""
import argparse
import concurrent.futures
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import random
import re
import signal
import subprocess
import sys
import tempfile
import threading
import time
import uuid
import psycopg
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
import redis
from botocore.exceptions import ClientError, BotoCoreError
from common import APIError
from objects import Objects, MAX_BYTES
from postgres import setting
STREAM, GROUP = 'attachment-jobs', 'image-workers'
LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '60'))
STOP = threading.Event()
def connect():
return psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, autocommit=True, connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1500')
def broker():
client = redis.Redis.from_url(setting('BROKER_URL'), decode_responses=True, socket_connect_timeout=1, socket_timeout=3)
try:
client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
except redis.ResponseError as error:
if 'BUSYGROUP' not in str(error):
raise
return client
def event(job):
return {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], 'attempt': job['attempt'] + 1, 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z')}
def validate(message):
required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
if not isinstance(message, dict) or set(message) - required - {'traceparent', 'tracestate'} or not required <= set(message):
return False
try:
uuid.UUID(message['job_id']); uuid.UUID(message['attachment_id'])
stamp = dt.datetime.fromisoformat(message['created_at'].replace('Z', '+00:00'))
if stamp.tzinfo is None or not message['created_at'].endswith('Z'):
return False
if 'traceparent' in message and (not isinstance(message['traceparent'], str) or not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', message['traceparent'])):
return False
if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512):
return False
return type(message['schema_version']) is int and message['schema_version'] == 1 and message['type'] == 'attachment.process.v1' and all(type(message[k]) is int and message[k] >= 1 for k in ['pipeline_version', 'attempt'])
except (ValueError, TypeError, AttributeError):
return False
def audit(conn, job, name):
conn.execute('INSERT INTO job_events (job_id, event, attempt) VALUES (%s, %s, %s)', (job['id'], name, job['attempt']))
def retry(conn, job, code, permanent=False):
failed = permanent or job['attempt'] >= job['max_attempts']
state = 'failed' if failed else 'queued'
code = code if permanent or not failed else 'retry_exhausted'
delay = min(8, .5 * 2 ** min(job['attempt'] - 1, 4)) + random.uniform(0, .5)
conn.execute('''
UPDATE attachment_jobs
SET state = %s
, error_code = %s
, lease_token = NULL
, lease_until = NULL
, published_at = NULL
, next_attempt_at = clock_timestamp() + %s * INTERVAL '1 second'
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
''', (state, code, delay, job['id']))
conn.execute('''
UPDATE attachments
SET state = %s
, error_code = %s
, updated_at = clock_timestamp()
WHERE 1=1
AND current_job_id = %s
''', (state, code if failed else None, job['id']))
audit(conn, job, 'failed:' + code if failed else 'retry:' + code)
def dispatch_once(client, fault=None):
with connect() as conn:
with conn.transaction():
expired = conn.execute("SELECT * FROM attachment_jobs WHERE 1=1 AND state = 'processing' AND lease_until < clock_timestamp() FOR UPDATE SKIP LOCKED LIMIT 20").fetchall()
for job in expired:
retry(conn, job, 'lease_expired')
with conn.transaction():
jobs = conn.execute('''
SELECT j.*
FROM attachment_jobs j
JOIN attachments a ON a.current_job_id = j.id
WHERE 1=1
AND j.state = 'queued'
AND j.next_attempt_at <= clock_timestamp()
AND (j.published_at IS NULL OR j.published_at < clock_timestamp() - %s * INTERVAL '1 second')
ORDER BY j.created_at
LIMIT 10
FOR UPDATE OF j SKIP LOCKED
''', (LEASE,)).fetchall()
for job in jobs:
client.xadd(STREAM, {'event': json.dumps(event(job), separators=(',', ':'))})
if fault:
fault('after_publish', job)
conn.execute('UPDATE attachment_jobs SET published_at = clock_timestamp() WHERE 1=1 AND id = %s', (job['id'],))
audit(conn, job, 'published')
return len(jobs)
def claim(message):
with connect() as conn, conn.transaction():
job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (message['job_id'],)).fetchone()
if job is None:
return None
attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (job['attachment_id'],)).fetchone()
if str(job['attachment_id']) != message['attachment_id'] or job['pipeline_version'] != message['pipeline_version']:
return None
if attachment['current_job_id'] != job['id'] or attachment['pipeline_version'] != job['pipeline_version']:
audit(conn, job, 'stale_noop')
return None
if job['state'] in ('ready', 'failed', 'superseded'):
audit(conn, job, 'terminal_noop')
return None
if job['state'] != 'queued' or job['next_attempt_at'] > dt.datetime.now(dt.timezone.utc) or message['attempt'] != job['attempt'] + 1:
audit(conn, job, 'duplicate_or_early_noop')
return None
token = uuid.uuid4()
job = conn.execute('''
UPDATE attachment_jobs
SET state = 'processing'
, attempt = attempt + 1
, lease_token = %s
, lease_until = clock_timestamp() + %s * INTERVAL '1 second'
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
RETURNING *
''', (token, LEASE, job['id'])).fetchone()
conn.execute("UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job['id'],))
audit(conn, job, 'claimed')
return job, attachment
class Permanent(Exception):
pass
def immutable(objects, key, path, mime, sha):
head = objects.head(key)
if head is not None:
if head['ContentLength'] != path.stat().st_size or head.get('Metadata', {}).get('sha256') != sha:
raise Permanent('result_conflict')
return
try:
with path.open('rb') as source:
objects.client.put_object(Bucket=objects.bucket, Key=key, Body=source, ContentLength=path.stat().st_size, ContentType=mime, Metadata={'sha256': sha}, IfNoneMatch='*')
except ClientError as error:
if error.response['ResponseMetadata']['HTTPStatusCode'] in (409, 412):
head = objects.head(key)
if head and head['ContentLength'] == path.stat().st_size and head.get('Metadata', {}).get('sha256') == sha:
return
raise
def process(message, fault=None):
claimed = claim(message)
if claimed is None:
return 'noop'
job, attachment = claimed
objects = Objects(setting)
try:
if fault:
fault('before_side_effect', job)
with tempfile.TemporaryDirectory(prefix='job-') as temporary:
directory = Path(temporary)
source = directory / 'original'
head = objects.head(job['source_key'])
if head is None:
raise Permanent('source_missing')
response = objects.get(job['source_key'])
digest, size = hashlib.sha256(), 0
deadline = time.monotonic() + 15
with response['Body'] as stream, source.open('wb') as out:
for chunk in iter(lambda: stream.read(64 * 1024), b''):
size += len(chunk)
if size > MAX_BYTES or time.monotonic() > deadline:
raise Permanent('invalid_image')
digest.update(chunk); out.write(chunk)
if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']:
raise Permanent('checksum_mismatch')
result = subprocess.run([sys.executable, str(Path(__file__).with_name('render.py')), str(source), str(directory), str(job['pipeline_version'])], capture_output=True, timeout=int(os.environ.get('JOB_TIMEOUT_SECONDS', '20')))
if result.returncode == 2:
raise Permanent('invalid_image')
if result.returncode:
raise RuntimeError('render_failed')
variants = json.loads(result.stdout)
prefix = 'results/' + str(attachment['id']) + '/v' + str(job['pipeline_version']) + '/'
original_key = prefix + 'original'
immutable(objects, original_key, source, attachment['content_type'], attachment['sha256'])
if fault:
fault('after_original', job)
for variant in variants:
variant['object_key'] = prefix + variant['name']
immutable(objects, variant['object_key'], directory / variant['name'], variant['content_type'], variant['sha256'])
if fault:
fault('after_objects', job)
with connect() as conn, conn.transaction():
current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
changed = conn.execute('''
UPDATE attachments
SET state = 'ready'
, object_key = %s
, variants = %s
, error_code = NULL
, updated_at = clock_timestamp()
WHERE 1=1
AND current_job_id = %s
AND pipeline_version = %s
AND %s
RETURNING id
''', (original_key, Jsonb(variants), job['id'], job['pipeline_version'], current['state'] == 'processing' and current['lease_token'] == job['lease_token'] and current['lease_until'] > dt.datetime.now(dt.timezone.utc))).fetchone()
if changed:
conn.execute("UPDATE attachment_jobs SET state = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (job['id'],))
audit(conn, job, 'ready')
else:
audit(conn, job, 'fenced_noop')
if fault:
fault('after_commit', job)
if changed and job['source_key'].startswith('staging/'):
objects.delete(job['source_key'])
return 'ready' if changed else 'fenced'
except Exception as error:
code = str(error) if isinstance(error, Permanent) else 'processing_timeout' if isinstance(error, subprocess.TimeoutExpired) else 'storage_unavailable' if isinstance(error, (APIError, ClientError, BotoCoreError)) else 'processing_failed'
with connect() as conn, conn.transaction():
current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
if current['state'] == 'processing' and current['lease_token'] == job['lease_token']:
retry(conn, current, code, isinstance(error, Permanent))
return 'failed_or_retry'
finally:
objects.client.close()
def ack(client, identifier):
# Atomic ack+delete; PostgreSQL is the durable completion/replay registry.
client.eval("redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])", 1, STREAM, GROUP, identifier)
def consume(once=False, fault=None):
client = broker()
consumer = uuid.uuid4().hex
while not STOP.is_set():
messages = []
try:
pending = client.xautoclaim(STREAM, GROUP, consumer, min_idle_time=LEASE * 1000, start_id='0-0', count=1)[1]
if pending:
messages = pending
else:
batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=500)
messages = batches[0][1] if batches else []
for identifier, fields in messages:
raw = fields.get('event', '')
try:
message = json.loads(raw) if len(raw) <= 4096 else None
except ValueError:
message = None
if validate(message):
process(message, fault)
else:
client.xadd('rejected-jobs', {'source_message_id': identifier, 'sha256': hashlib.sha256(raw.encode()).hexdigest(), 'reason': 'invalid_envelope'}, maxlen=1000, approximate=False)
ack(client, identifier)
if once:
return
except (redis.RedisError, psycopg.Error):
STOP.wait(.5)
if once and not messages:
return
client.close()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('mode', choices=['dispatch', 'work'])
parser.add_argument('--once', action='store_true')
parser.add_argument('--fault-point', choices=['before_side_effect', 'after_original', 'after_objects', 'after_commit', 'after_publish'])
args = parser.parse_args()
# Operator-only opt-in injection; no request header or public endpoint controls it.
def fault(point, job):
if point == args.fault_point:
os._exit(77)
signal.signal(signal.SIGTERM, lambda *_: STOP.set())
signal.signal(signal.SIGINT, lambda *_: STOP.set())
if args.mode == 'dispatch':
client = broker()
while not STOP.is_set():
try:
dispatch_once(client, fault)
except (redis.RedisError, psycopg.Error):
pass
if args.once:
break
STOP.wait(.2)
else:
count = int(os.environ.get('WORKER_CONCURRENCY', '1'))
if not 1 <= count <= 2:
raise ValueError('WORKER_CONCURRENCY must be 1..2')
with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool:
futures = [pool.submit(consume, args.once, fault) for _ in range(count)]
for result in futures:
result.result()
if __name__ == '__main__':
main()
+409
View File
@@ -0,0 +1,409 @@
import base64
import contextlib
import datetime as dt
import hashlib
import hmac
import json
import os
import re
from pathlib import Path
import secrets
import threading
import uuid
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
import redis
from objects import Objects, unavailable
from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username
KDF_SLOTS = threading.BoundedSemaphore(2)
def setting(name):
path = os.environ.get(name + '_FILE')
return Path(path).read_text().strip() if path else os.environ[name]
def public(row):
return {k: (v.decode('utf-8') if isinstance(v, bytes) else [str(x) for x in v] if isinstance(v, list) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'}
def password_hash(password, salt=None):
salt = salt or secrets.token_bytes(16)
if not KDF_SLOTS.acquire(timeout=1.5):
raise APIError(503, 'unavailable', 'Аутентификация временно перегружена')
try:
digest = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024)
finally:
KDF_SLOTS.release()
return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex()
class Store:
persistent = True
def __init__(self):
self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600'))
if not 1 <= self.ttl <= 86400:
raise ValueError('SESSION_TTL_SECONDS must be 1..86400')
self.access_mode = os.environ.get('DB_ACCESS_MODE', 'optimized')
if self.access_mode not in ('baseline', 'optimized'):
raise ValueError('DB_ACCESS_MODE must be baseline or optimized')
pool_size = int(os.environ.get('DB_POOL_MAX', '8'))
if not 1 <= pool_size <= 32:
raise ValueError('DB_POOL_MAX must be 1..32')
self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=pool_size, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True)
self.pool.wait(timeout=10)
self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True)
self.objects = Objects(setting)
self.dummy_hash = password_hash('dummy-password-for-unknown-users')
with self.connection() as conn:
if not conn.execute("SELECT version FROM schema_migrations WHERE 1=1 AND version = '005_pipeline.sql'").fetchone():
raise RuntimeError('Run the separate migration service first')
@contextlib.contextmanager
def connection(self):
try:
with self.pool.connection() as conn:
yield conn
except psycopg.errors.UniqueViolation:
raise APIError(409, 'conflict', 'Имя пользователя занято') from None
except (psycopg.Error, PoolTimeout):
raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None
def close(self):
self.pool.close()
self.cache.close()
self.objects.client.close()
def ready(self):
self.objects.ready()
with self.connection() as conn:
conn.execute('SELECT 1')
def register(self, body):
fields(body, ('username', 'display_name', 'password'))
name, display = username(body['username']), text(body['display_name'], 100)
# Passwords may contain spaces and colons; only length/UTF-8 are restricted.
password = body['password']
if not isinstance(password, str) or not 12 <= len(password) <= 128:
invalid()
try:
password.encode('utf-8')
except UnicodeError:
invalid()
hashed = password_hash(password)
with self.connection() as conn:
row = conn.execute('''
INSERT INTO users (id, username, display_name, password_hash)
VALUES (%s, %s, %s, %s)
RETURNING id, username, display_name, created_at
''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone()
return public(row)
def login(self, headers):
authorization = headers.get('Authorization', '')
try:
scheme, encoded = authorization.split(' ', 1)
if scheme.lower() != 'basic':
unauthorized()
name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
except (ValueError, UnicodeError):
unauthorized()
if not re_username(name) or not 12 <= len(password) <= 128:
unauthorized()
with self.connection() as conn:
row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone()
hashed = row['password_hash'] if row else self.dummy_hash
verified = hmac.compare_digest(password_hash(password, bytes.fromhex(hashed.split('$')[4])), hashed)
if row is None or not verified:
unauthorized()
token = secrets.token_urlsafe(32)
with self.connection() as conn:
expiry = conn.execute('''
INSERT INTO sessions (token_hash, user_id, expires_at)
VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second')
RETURNING expires_at
''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at']
return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)}
def token_hash(self, headers):
parts = headers.get('Authorization', '').split(' ')
if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii():
unauthorized()
return hashlib.sha256(parts[1].encode()).hexdigest()
def authenticate(self, headers):
token_hash = self.token_hash(headers)
with self.connection() as conn:
session = conn.execute('''
SELECT user_id
FROM sessions
WHERE 1=1
AND token_hash = %s
AND expires_at > clock_timestamp()
''', (token_hash,)).fetchone()
if session is None:
unauthorized()
key = 'user:' + str(session['user_id'])
try:
cached = self.cache.get(key)
if cached:
return json.loads(cached)
except (redis.RedisError, ValueError):
pass
row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone()
if row is None:
unauthorized()
user = public(row)
try:
self.cache.setex(key, 60, json.dumps(user))
except redis.RedisError:
pass
return user
def logout(self, headers):
with self.connection() as conn:
conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),))
def chat_in(self, conn, actor, chat_id):
row = conn.execute('''
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE 1=1
AND c.id = %s
AND cm.user_id = %s
''', (chat_id, actor['id'])).fetchone()
if row is None:
missing()
members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall()
return dict(public(row), member_ids=[str(v['user_id']) for v in members])
def chat(self, actor, chat_id):
with self.connection() as conn:
return self.chat_in(conn, actor, chat_id)
def create_chat(self, actor, body):
fields(body, ('title', 'member_ids'))
title = text(body['title'], 100)
members = body['member_ids']
if not isinstance(members, list) or not 1 <= len(members) <= 100:
invalid()
members = [identifier(v) for v in members]
if len(set(members)) != len(members) or actor['id'] not in members:
invalid()
with self.connection() as conn:
chat_id = uuid.uuid4()
conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8')))
# Checking after INSERT exercises rollback of the whole composite operation.
found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall()
if len(found) != len(members):
invalid()
with conn.cursor() as cur:
cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members])
return self.chat_in(conn, actor, chat_id)
def list_chats(self, actor, limit, cursor, scope):
with self.connection() as conn:
where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else ''
params = [actor['id']] + list(cursor or ()) + [limit + 1]
rows = conn.execute('''
SELECT c.id
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE 1=1
AND cm.user_id = %s
''' + where + '''
ORDER BY c.created_at, c.id
LIMIT %s
''', params).fetchall()
return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope)
def require_member(self, conn, actor, chat_id):
if self.access_mode == 'baseline':
return self.chat_in(conn, actor, chat_id)
if not conn.execute('''
SELECT 1
FROM chat_members
WHERE 1=1
AND chat_id = %s
AND user_id = %s
''', (chat_id, actor['id'])).fetchone():
missing()
def send(self, actor, chat_id, body, key):
if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
invalid()
if not isinstance(body, dict) or set(body) - {'text', 'attachment_ids'} or 'text' not in body:
invalid()
attachments = body.get('attachment_ids', [])
if not isinstance(attachments, list) or len(attachments) > 10:
invalid()
attachments = [identifier(v) for v in attachments]
if len(set(attachments)) != len(attachments):
invalid()
content = body['text']
if not isinstance(content, str) or len(content) > 4000 or (not attachments and not content.strip()):
invalid()
try:
content.encode('utf-8')
except UnicodeError:
invalid()
canonical = json.dumps({'text': content, 'attachment_ids': attachments}, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
with self.connection() as conn:
self.require_member(conn, actor, chat_id)
# The unique key arbitrates concurrent processes; everything commits together.
claimed = conn.execute('''
INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, expires_at)
VALUES (%s, %s, %s, %s, clock_timestamp() + INTERVAL '24 hours')
ON CONFLICT (user_id, chat_id, key) DO UPDATE
SET request_hash = EXCLUDED.request_hash
, message_id = NULL
, expires_at = EXCLUDED.expires_at
WHERE 1=1
AND idempotency_keys.expires_at <= clock_timestamp()
RETURNING request_hash
''', (actor['id'], chat_id, key, digest)).fetchone()
if claimed is None:
previous = conn.execute('''
SELECT ik.request_hash
, m.id
, m.chat_id
, m.sender_id
, m.text
, m.created_at
, m.attachment_ids
FROM idempotency_keys ik
JOIN messages m ON m.id = ik.message_id
WHERE 1=1
AND ik.user_id = %s
AND ik.chat_id = %s
AND ik.key = %s
''', (actor['id'], chat_id, key)).fetchone()
if previous['request_hash'] != digest:
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
previous.pop('request_hash')
return public(previous)
if attachments:
rows = conn.execute('''
SELECT id, state
FROM attachments
WHERE 1=1
AND id = ANY(%s::uuid[])
AND chat_id = %s
ORDER BY id
FOR SHARE
''', (attachments, chat_id)).fetchall()
if len(rows) != len(attachments):
missing()
if any(v['state'] not in ('ready', 'queued', 'processing') for v in rows):
raise APIError(409, 'conflict', 'Вложение ещё не готово')
# Per-chat row lock preserves committed history order across replicas/clocks.
stamp = conn.execute('''
UPDATE chats
SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond')
WHERE 1=1
AND id = %s
RETURNING last_message_at
''', (chat_id,)).fetchone()['last_message_at']
row = conn.execute('''
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
VALUES (%s, %s, %s, %s, %s, %s::uuid[])
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids
''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp, attachments)).fetchone()
if attachments:
with conn.cursor() as cur:
cur.executemany('INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)', [(row['id'], v) for v in attachments])
conn.execute('''
UPDATE idempotency_keys
SET message_id = %s
WHERE 1=1
AND user_id = %s
AND chat_id = %s
AND key = %s
''', (row['id'], actor['id'], chat_id, key))
return public(row)
def history(self, actor, chat_id, limit, cursor, scope):
with self.connection() as conn:
self.require_member(conn, actor, chat_id)
where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else ''
params = [chat_id] + list(cursor or ()) + [limit + 1]
rows = conn.execute('''
SELECT id, chat_id, sender_id, text, created_at, attachment_ids
FROM messages
WHERE 1=1
AND chat_id = %s
''' + where + '''
ORDER BY created_at, id
LIMIT %s
''', params).fetchall()
return page([public(v) for v in rows], limit, scope)
def upload(self, actor, chat_id, name, source, length, mime, digest):
attachment_id, job_id = uuid.uuid4(), uuid.uuid4()
key = 'staging/' + str(attachment_id)
with self.connection() as conn:
self.require_member(conn, actor, chat_id)
conn.execute('SELECT pg_advisory_xact_lock(2301005)')
pending = conn.execute("SELECT count(*) AS n FROM attachments WHERE 1=1 AND state IN ('uploading', 'queued', 'processing')").fetchone()['n']
if pending >= int(os.environ.get('MAX_PENDING_JOBS', '100')):
raise APIError(503, 'unavailable', 'Очередь принятых работ заполнена')
conn.execute('''
INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256, state)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')
''', (attachment_id, chat_id, actor['id'], name, 'results/' + str(attachment_id) + '/v1/original', key, mime, length, digest))
with self.connection() as conn:
if conn.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() is None:
unavailable()
self.objects.put(key, source, length, mime, digest)
conn.execute('''
INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts)
VALUES (%s, %s, 1, %s, %s)
''', (job_id, attachment_id, key, int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))))
row = conn.execute('''
UPDATE attachments
SET state = 'queued'
, current_job_id = %s
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
RETURNING *
''', (job_id, attachment_id)).fetchone()
# Capture the queued snapshot, even if the worker finishes before HTTP reply.
return self.attachment_public(row)
def attachment_public(self, row):
keys = ('id', 'chat_id', 'owner_id', 'filename', 'content_type', 'size_bytes', 'sha256', 'created_at', 'updated_at')
result = public({k: row[k] for k in keys})
variants = [{k: v for k, v in item.items() if k != 'object_key'} for item in row['variants']]
return dict(result, status=row['state'], variants=variants, error_code=row['error_code'])
def attachment(self, actor, attachment_id):
with self.connection() as conn:
row = conn.execute('''
SELECT a.*
FROM attachments a
JOIN chat_members cm ON cm.chat_id = a.chat_id
WHERE 1=1
AND a.id = %s
AND cm.user_id = %s
''', (attachment_id, actor['id'])).fetchone()
if row is None:
missing()
if row['state'] == 'storage_missing':
unavailable()
if row['state'] == 'uploading':
raise APIError(409, 'conflict', 'Приём не завершён')
return row
def re_username(value):
import re
return re.fullmatch(r'[a-z0-9_]{3,32}', value)
+64
View File
@@ -0,0 +1,64 @@
"""Isolated image decoder/resizer; no network, database or credentials."""
import hashlib
import json
import math
from pathlib import Path
import resource
import sys
import warnings
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
resource.setrlimit(resource.RLIMIT_CPU, (15, 16))
from PIL import Image, ImageOps, PngImagePlugin
Image.MAX_IMAGE_PIXELS = 40_000_000
PngImagePlugin.MAX_TEXT_CHUNK = PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
def render(source, directory, version):
output = []
with Image.open(source, formats=['PNG', 'JPEG']) as image:
if image.width * image.height > 40_000_000:
raise ValueError('pixels')
image.load()
ImageOps.exif_transpose(image, in_place=True)
rgba = None
if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info):
rgba = image if image.mode == 'RGBA' else image.convert('RGBA')
rgb = Image.new('RGB', rgba.size, 'white')
alpha = rgba.getchannel('A')
rgb.paste(rgba, mask=alpha)
alpha.close()
else:
rgb = image if image.mode == 'RGB' else image.convert('RGB')
try:
for name, bound, fmt, quality in [('thumbnail', 256, 'JPEG', 85), ('crop', 128, 'JPEG', 85), ('optimized', 1280, 'WEBP', 82 if version == 1 else 70)]:
if name == 'crop':
result = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(.5, .5))
else:
scale = min(1, bound / max(rgb.size))
size = tuple(max(1, math.floor(v * scale)) for v in rgb.size)
result = rgb.resize(size, Image.Resampling.LANCZOS)
try:
path = directory / name
options = {'quality': quality, 'exif': b'', 'icc_profile': None}
if fmt == 'WEBP':
options['method'] = 4
result.save(path, format=fmt, **options)
raw = path.read_bytes()
output.append({'name': name, 'width': result.width, 'height': result.height, 'content_type': 'image/jpeg' if fmt == 'JPEG' else 'image/webp', 'size_bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()})
finally:
result.close()
finally:
if rgb is not image:
rgb.close()
if rgba is not None and rgba is not image:
rgba.close()
return output
if __name__ == '__main__':
try:
print(json.dumps(render(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3]))))
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning):
raise SystemExit(2)
+296
View File
@@ -0,0 +1,296 @@
"""Bounded threaded API; persistent correctness state is shared across replicas."""
import concurrent.futures
import http.server
import json
import logging
import os
import time
import signal
import socket
import threading
import uuid
from urllib.parse import parse_qs, urlsplit
from objects import bounded_upload, receive, filename, unavailable
from common import APIError, identifier, invalid, missing, page_args
LOG = logging.getLogger('messenger')
def unique_object(pairs):
result = {}
for key, value in pairs:
if key in result:
invalid()
result[key] = value
return result
class Handler(http.server.BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
def setup(self):
super().setup()
self.connection.settimeout(3)
def send_error(self, code, message=None, explain=None):
# BaseHTTPRequestHandler otherwise emits HTML without contract headers.
self.request_id = uuid.uuid4().hex
self.route = ''
self.close_connection = True
status = 413 if code == 413 else 400
self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}})
def log_message(self, *args):
pass
def reply(self, status, body):
if status == 200 and isinstance(body, tuple):
return self.download(*body)
raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
if status >= 400:
self.close_connection = True
self.send_response(status)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(raw)))
if self.close_connection:
self.send_header('Connection', 'close')
self.send_header('X-Request-Id', self.request_id)
self.send_header('X-Instance-Id', self.server.instance_id)
if status == 401 and self.server.store.persistent:
self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
self.end_headers()
if self.command != 'HEAD':
self.wfile.write(raw)
def download(self, row, response, chunk):
stream = response['Body']
try:
self.send_response(200)
self.send_header('Content-Type', row['content_type'])
self.send_header('Content-Length', str(row['size_bytes']))
self.send_header('X-Request-Id', self.request_id)
self.send_header('X-Instance-Id', self.server.instance_id)
self.send_header('X-Content-Type-Options', 'nosniff')
self.send_header('Cache-Control', 'private, no-store')
self.end_headers()
deadline = time.monotonic() + 15
while chunk:
self.wfile.write(chunk)
if time.monotonic() > deadline:
raise TimeoutError()
chunk = stream.read(64 * 1024)
finally:
stream.close()
def body(self):
lengths = self.headers.get_all('Content-Length', [])
if len(lengths) != 1 or self.headers.get('Transfer-Encoding'):
invalid()
try:
length = int(lengths[0])
except ValueError:
invalid()
if length > 32768:
raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт')
if length <= 0:
invalid()
if self.headers.get_content_type() != 'application/json':
raise APIError(415, 'unsupported_media_type', 'Требуется application/json')
try:
raw = self.rfile.read(length)
if len(raw) != length:
invalid()
return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid())
except (ValueError, UnicodeError, RecursionError):
invalid()
def dispatch(self):
self.request_id = uuid.uuid4().hex
self.route = urlsplit(self.path).path
status = 500
try:
status, result = self.route_request()
except APIError as exc:
status = exc.status
result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}}
except Exception:
# No bodies, credentials, tokens, SQL or exception details in logs/responses.
LOG.error('request_failed request_id=%s', self.request_id)
result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}}
try:
self.reply(status, result)
except Exception:
self.close_connection = True
LOG.info('method=%s status=%s request_id=%s', self.command, status, self.request_id)
def route_request(self):
method, path, store = self.command, self.route, self.server.store
if method == 'GET' and path in ('/health/live', '/health/ready'):
if path.endswith('ready'):
store.ready()
return 200, {'status': 'ok'}
if method == 'POST' and path == '/api/v1/users':
return 201, store.register(self.body())
if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions':
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
invalid()
return 201, store.login(self.headers)
actor = store.authenticate(self.headers)
if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current':
store.logout(self.headers)
return 204, None
if method == 'GET' and path == '/api/v1/users/me':
return 200, actor
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
if path == '/api/v1/chats':
if method == 'POST':
return 201, store.create_chat(actor, self.body())
if method == 'GET':
scope = 'chats:' + actor['id']
limit, cursor = page_args(query, scope)
return 200, store.list_chats(actor, limit, cursor, scope)
parts = path.split('/')
if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET':
row = store.attachment(actor, identifier(parts[4]))
if len(parts) == 5:
if query:
invalid()
return 200, store.attachment_public(row)
if parts[5] == 'content':
if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()):
invalid()
if row['state'] != 'ready':
raise APIError(409, 'conflict', 'Вложение ещё не готово')
variant = query.get('variant', ['original'])[0]
if variant != 'original':
item = next((v for v in row['variants'] if v['name'] == variant), None)
if item is None:
missing()
row = dict(row, object_key=item['object_key'], content_type=item['content_type'], size_bytes=item['size_bytes'])
response = store.objects.get(row['object_key'])
try:
chunk = response['Body'].read(64 * 1024)
if response['ContentLength'] != row['size_bytes'] or not chunk:
unavailable()
except Exception:
response['Body'].close()
unavailable()
return 200, (row, response, chunk)
if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']:
chat_id = identifier(parts[4])
if store.access_mode == 'baseline':
store.chat(actor, chat_id)
if len(parts) == 5 and method == 'GET':
return 200, store.chat(actor, chat_id)
if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST':
store.chat(actor, chat_id)
if set(query) != {'filename'} or len(query['filename']) != 1:
invalid()
name = filename(query['filename'][0])
with bounded_upload():
with receive(self) as (source, length, mime, digest):
return 202, store.upload(actor, chat_id, name, source, length, mime, digest)
if len(parts) == 6 and parts[5] == 'messages':
if method == 'POST':
keys = self.headers.get_all('Idempotency-Key', [])
return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None)
if method == 'GET':
scope = 'messages:' + chat_id
limit, cursor = page_args(query, scope)
return 200, store.history(actor, chat_id, limit, cursor, scope)
missing()
do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch
class Server(http.server.HTTPServer):
request_queue_size = 64
allow_reuse_address = True
def __init__(self, address, store):
self.store = store
self.instance_id = uuid.uuid4().hex
workers = int(os.environ.get('HTTP_WORKERS', '16'))
if not 1 <= workers <= 128:
raise ValueError('HTTP_WORKERS must be 1..128')
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers)
self.slots = threading.BoundedSemaphore(workers * 2)
self.connections = set()
self.connections_lock = threading.Lock()
super().__init__(address, Handler)
def process_request(self, request, address):
if not self.slots.acquire(blocking=False):
rid = uuid.uuid4().hex
raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode()
try:
request.settimeout(0.2)
request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw)
except OSError:
pass
self.shutdown_request(request)
return
with self.connections_lock:
self.connections.add(request)
self.executor.submit(self.worker, request, address)
def worker(self, request, address):
try:
self.finish_request(request, address)
except OSError:
pass
finally:
self.shutdown_request(request)
with self.connections_lock:
self.connections.discard(request)
self.slots.release()
def close_connections(self):
with self.connections_lock:
connections = list(self.connections)
for connection in connections:
try:
connection.shutdown(socket.SHUT_RDWR)
except OSError:
pass
def server_close(self):
super().server_close()
# Bound draining even when a slow client keeps an incomplete request alive.
deadline = threading.Timer(5, self.close_connections)
deadline.start()
try:
self.executor.shutdown(wait=True)
finally:
deadline.cancel()
def main():
logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s')
from postgres import Store
store = Store()
server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store)
stopping = threading.Event()
def stop(signum, frame):
if not stopping.is_set():
stopping.set()
LOG.info('shutdown_started signal=%s', signum)
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats())))
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
LOG.info('started port=%s', server.server_address[1])
try:
server.serve_forever(poll_interval=0.1)
finally:
server.server_close()
store.close()
LOG.info('shutdown_complete')
if __name__ == '__main__':
main()
+52
View File
@@ -0,0 +1,52 @@
"""Stop worker, accept queued, restart API/broker, then complete the same attachment."""
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
from client import Client,Failure
def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip()
def main():
c=Client();alice,bob,eve,chat=c.bootstrap();record={}
before=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True))
try:
docker('compose','stop','worker','dispatcher')
queued,raw=c.attachment(alice,chat)
assert queued['status']=='queued'
message=c.message(alice,chat,'accepted with worker stopped',attachments=[queued['id']])['json']
c.call('GET','/api/v1/attachments/'+queued['id']+'/content',409,actor=bob)
killed=docker('compose','ps','-q','api').splitlines()
for container in killed:docker('kill','--signal=KILL',container)
record['killed_api_containers']=killed
docker('compose','up','-d','--wait','--no-deps','api')
time.sleep(2)
assert c.call('GET','/api/v1/attachments/'+queued['id'],actor=bob,schema='Attachment')['json']['status']=='queued'
docker('compose','up','-d','--no-deps','dispatcher')
deadline=time.monotonic()+10
while int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))<1:
assert time.monotonic()<deadline;time.sleep(.1)
docker('compose','stop','dispatcher')
length_before=int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))
docker('compose','restart','broker')
docker('compose','up','-d','--wait','--no-deps','broker')
length_after=int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))
assert length_after>=length_before>=1
record['stream_entries_before_after_broker_restart']=[length_before,length_after]
docker('compose','up','-d','--no-deps','worker','dispatcher')
ready=c.wait_ready(bob,queued)
assert c.call('GET','/api/v1/attachments/'+queued['id']+'/content',actor=bob)['bytes']==raw
record.update(attachment_id=queued['id'],original_sha256=hashlib.sha256(raw).hexdigest(),status=ready['status'],variants=[v['name'] for v in ready['variants']],old_bearer_survived=True,message_attachment_retained=message['attachment_ids']==[queued['id']])
finally:docker('compose','up','-d','--no-deps','worker','dispatcher')
after=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True))
record['before']=before;record['after']=after
print(json.dumps(record,indent=2))
if __name__=='__main__':main()
+24
View File
@@ -0,0 +1,24 @@
"""Small deterministic PNG helpers; generation happens outside measurement."""
import random
import struct
import zlib
def chunk(name, data):
return struct.pack('!I', len(data)) + name + data + struct.pack('!I', zlib.crc32(name + data) & 0xffffffff)
def png(width, height, noise=False, rgb=True):
channels = 3 if rgb else 1
rand = random.Random(2301)
encoded = zlib.compressobj(1)
parts = []
for _ in range(height):
line = bytes(rand.getrandbits(8) for _ in range(width * channels)) if noise else b'\x7f' * (width * channels)
parts.append(encoded.compress(b'\0' + line))
parts.append(encoded.flush())
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2 if rgb else 0, 0, 0, 0)) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'')
def bomb():
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'x')) + chunk(b'IEND', b'')
+188
View File
@@ -0,0 +1,188 @@
"""Real DB/S3/Redis fault injection. Stop production worker/dispatcher first."""
import concurrent.futures
import hashlib
import io
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
sys.path[:0]=['/app/src',str(Path.cwd()/'tests')]
from PIL import Image
from client import Client
from pipeline import STREAM,GROUP,broker,connect,event,dispatch_once,process,ack
from objects import Objects
from postgres import setting
from admin import replay,version,cleanup
def main():
c=Client();alice,bob,eve,chat=c.bootstrap()
bus=broker(); objects=Objects(setting)
report={'invariants':{},'faults':[]}
def fetch(aid):
with connect() as conn:
a=conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s',(aid,)).fetchone()
j=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(a['current_job_id'],)).fetchone()
return a,j
def upload(raw=None,mime='image/png'):
if raw is None: a,_=c.attachment(alice,chat)
else:a=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=research.png',202,actor=alice,raw=raw,headers={'Content-Type':mime},schema='Attachment')['json']
return a,fetch(a['id'])[1]
def run_once(fault=None):
args=[sys.executable,'/app/src/pipeline.py','work','--once']
if fault:args+=['--fault-point',fault]
return subprocess.run(args,capture_output=True,timeout=30).returncode
def drain():
for _ in range(200):
batch=bus.xreadgroup(GROUP,'research-drain',{STREAM:'>'},count=1)
if not batch:break
for mid,data in batch[0][1]:process(json.loads(data['event']));ack(bus,mid)
def expire(job):
with connect() as conn:
conn.execute("UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE 1=1 AND id = %s",(job['id'],))
def recover(aid):
until=time.monotonic()+20
while time.monotonic()<until:
dispatch_once(bus);drain()
a,j=fetch(aid)
if a['state']=='ready':return a,j
time.sleep(.1)
raise AssertionError('not recovered')
for mid,data in bus.xautoclaim(STREAM,GROUP,'initial-recovery',min_idle_time=0,start_id='0-0',count=100)[1]:
process(json.loads(data['event']));ack(bus,mid)
# COMMIT -> no publisher yet: one queued row, no Redis message, then automatic outbox pass.
a,j=upload()
assert fetch(a['id'])[0]['state']=='queued'
assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',409,actor=bob)
c.message(bob,chat,'queued is linkable',attachments=[a['id']])
dispatch_once(bus);drain();assert fetch(a['id'])[0]['state']=='ready'
report['invariants']['commit_before_publish_recovered']=True
# Broker accepted XADD but publisher died before marking outbox: true exit(77).
a,j=upload()
code=subprocess.run([sys.executable,'/app/src/pipeline.py','dispatch','--once','--fault-point','after_publish'],capture_output=True,timeout=10).returncode
assert code==77
dispatch_once(bus);drain()
assert fetch(a['id'])[0]['state']=='ready'
report['invariants']['publish_before_marker_duplicate_safe']=True
for point in ['before_side_effect','after_original','after_objects','after_commit']:
a,j=upload();dispatch_once(bus)
code=run_once(point);assert code==77,(point,code)
before,job=fetch(a['id'])
if point not in ('after_commit','before_side_effect'):expire(job)
result,job=recover(a['id'])
assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',actor=bob)['bytes']==(Path.cwd()/'tests/fixtures/sample.png').read_bytes()
reclaimed=bus.xautoclaim(STREAM,GROUP,'post-crash-redelivery',min_idle_time=0,start_id='0-0',count=100)[1]
for mid,data in reclaimed:
process(json.loads(data['event']));ack(bus,mid)
assert bus.xpending(STREAM,GROUP)['pending']==0
keys=objects.client.list_objects_v2(Bucket=objects.bucket,Prefix='results/'+a['id']+'/v1/').get('Contents',[])
assert len(keys)==4,(point,len(keys))
report['faults'].append({'point':point,'exit_code':code,'state_at_crash':before['state'],'recovered':result['state'],'attempts':job['attempt'],'result_objects':len(keys)})
report['invariants']['crashes_before_after_side_effect_and_commit']=True
bus.xadd(STREAM,{'event':json.dumps({'not':'a job'})})
assert run_once()==0
assert bus.xlen('rejected-jobs')>=1
report['invariants']['invalid_envelope_observable_dlq']=True
# Duplicate/reorder through broker on a completed job: no extra attempt/effect.
a,j=upload();dispatch_once(bus);drain();before,done=fetch(a['id'])
old=event(j)
for attempt in [2,1,3,1,1]:bus.xadd(STREAM,{'event':json.dumps(dict(old,attempt=attempt))})
drain();after,done2=fetch(a['id'])
assert before['variants']==after['variants'] and done2['attempt']==done['attempt']
report['invariants']['duplicate_and_reorder_no_second_effect']=True
# Race two consumers with the SAME logical job, not just one serial duplicate.
a,j=upload();e=event(j)
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: results=list(pool.map(lambda _:process(e),range(2)))
assert fetch(a['id'])[0]['state']=='ready' and sorted(results)==['noop','ready']
report['invariants']['concurrent_duplicate_single_claim']=True
# Pause actual old process after all immutable side effects, advance version and finish v2.
a,j=upload();e=event(j)
directory=Path('/tmp')/('fence-'+uuid.uuid4().hex);directory.mkdir();marker=directory/'paused';release=directory/'release'
code="import sys,json,time;from pathlib import Path;sys.path.insert(0,'/app/src');from pipeline import process\ndef hook(point,job):\n if point=='after_objects':\n Path(sys.argv[2]).write_text('paused')\n while not Path(sys.argv[3]).exists():time.sleep(.05)\nprint(process(json.loads(sys.argv[1]),hook))"
child=subprocess.Popen([sys.executable,'-c',code,json.dumps(e),str(marker),str(release)],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
try:
deadline=time.monotonic()+20
while not marker.exists():
assert child.poll() is None
assert time.monotonic()<deadline;time.sleep(.05)
new=version(uuid.UUID(a['id']),2);newrow,newjob=fetch(a['id'])
assert process(event(newjob))=='ready'
current,finished=fetch(a['id']);signature=current['variants']
release.write_text('go');stdout,stderr=child.communicate(timeout=10)
assert child.returncode==0 and stdout.strip()=='fenced',(stdout,stderr)
bus.xadd(STREAM,{'event':json.dumps(e)});drain()
final,finished=fetch(a['id'])
assert final['pipeline_version']==2 and final['variants']==signature and final['state']=='ready'
assert all('/v2/' in v['object_key'] for v in final['variants'])
report['invariants']['stale_running_version_fenced']=True
report['invariants']['old_delivery_after_new_ready_no_overwrite']=True
finally:
release.write_text('go')
if child.poll() is None:child.kill();child.wait()
# Transient failures consume a finite budget; failed registry and explicit replay.
a,j=upload()
with connect() as conn:conn.execute('UPDATE attachment_jobs SET max_attempts = 2 WHERE 1=1 AND id = %s',(j['id'],))
# Exact staging copy saved externally; missing source is permanent and observable.
raw=(Path.cwd()/'tests/fixtures/sample.png').read_bytes();objects.delete(j['source_key'])
process(event(j));failed,fj=fetch(a['id']);assert failed['state']=='failed' and failed['error_code']=='source_missing'
objects.put(j['source_key'],io.BytesIO(raw),len(raw),'image/png',hashlib.sha256(raw).hexdigest())
replay(j['id']);fresh,fj=fetch(a['id']);assert process(event(fj))=='ready'
try:replay(j['id']);raise AssertionError('replayed ready')
except ValueError:pass
report['invariants']['failed_registry_explicit_replay_and_ready_refusal']=True
# Real S3 signature failures are transient, with two attempts and measured backoff.
a,j=upload()
with connect() as conn:conn.execute('UPDATE attachment_jobs SET max_attempts = 2 WHERE 1=1 AND id = %s',(j['id'],))
wrong=Path('/tmp/wrong-secret');wrong.write_text('synthetic-invalid-key-not-a-real-secret')
env=dict(os.environ,S3_SECRET_ACCESS_KEY_FILE=str(wrong))
transitions=[];started=time.monotonic()
for _ in range(2):
row,j=fetch(a['id'])
delay=max(0,(j['next_attempt_at']-__import__('datetime').datetime.now(__import__('datetime').timezone.utc)).total_seconds())
time.sleep(delay+.01)
code="import sys,json;sys.path.insert(0,'/app/src');from pipeline import process;print(process(json.loads(sys.argv[1])))"
run=subprocess.run([sys.executable,'-c',code,json.dumps(event(j))],env=env,capture_output=True,timeout=10)
assert run.returncode==0
row,j=fetch(a['id']);transitions.append({'state':j['state'],'attempt':j['attempt'],'error_code':j['error_code']})
assert row['state']=='failed' and row['error_code']=='retry_exhausted' and j['attempt']==2
replay(j['id']);row,j=fetch(a['id']);assert process(event(j))=='ready'
report['retry']={'transitions':transitions,'seconds':round(time.monotonic()-started,3),'replayed_attempt':fetch(a['id'])[1]['attempt']}
report['invariants']['transient_backoff_exhaustion_then_replay']=True
# Real corrupted JPEG whose header is valid: heavy decoder fails safely in worker.
jpg=io.BytesIO();Image.new('RGB',(320,200),'blue').save(jpg,format='JPEG');truncated=jpg.getvalue()[:-100]
a,j=upload(truncated,'image/jpeg');process(event(j));bad,_=fetch(a['id'])
assert bad['state']=='failed' and bad['error_code']=='invalid_image'
report['invariants']['invalid_decode_permanent_failed']=True
# EXIF, alpha, small image and all on-disk variants checked by the real decoder.
transforms=[]
for kind in ['alpha','exif','small']:
out=io.BytesIO()
if kind=='alpha':
Image.new('RGBA',(20,10),(255,0,0,0)).save(out,format='PNG');mime='image/png'
elif kind=='exif':
image=Image.new('RGB',(200,320),'green');exif=Image.Exif();exif[274]=6;image.save(out,format='JPEG',exif=exif);mime='image/jpeg'
else:Image.new('RGB',(5,3),'yellow').save(out,format='PNG');mime='image/png'
a,j=upload(out.getvalue(),mime);assert process(event(j))=='ready'
r=c.call('GET','/api/v1/attachments/'+a['id']+'/content?variant=thumbnail',actor=bob)
with Image.open(io.BytesIO(r['bytes'])) as thumb:
expected=(20,10) if kind=='alpha' else (256,160) if kind=='exif' else (5,3)
assert thumb.size==expected
assert not thumb.getexif()
if kind=='alpha':assert min(thumb.getpixel((0,0)))>=250
transforms.append({'kind':kind,'size':thumb.size,'exif_removed':not bool(thumb.getexif())})
report['transforms']=transforms
pending,pending_job=upload()
dry=cleanup(False);fixed=cleanup(True);repeat=cleanup(True)
assert fetch(pending['id'])[0]['state']=='queued' and objects.head(pending_job['source_key']) is not None
report['invariants']['cleanup_preserves_current_and_fresh_failed']=True
assert repeat['counts']['ready_staging']==0
report['invariants']['ready_staging_cleanup_idempotent']=True
report['cleanup']=[dry,fixed,repeat]
print(json.dumps(report,indent=2))
if __name__=='__main__': main()
+28
View File
@@ -0,0 +1,28 @@
"""Figures from real accepted/ready and cgroup samples, no synthetic results."""
import json
from pathlib import Path
import statistics
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
r=Path(__file__).resolve().parents[1]/'evidence'
x=json.loads((r/'resources.json').read_text())['experiments']
fig,axes=plt.subplots(1,2,figsize=(11,4.3),layout='constrained')
labels=['320×200\n1,3 KiB','1800×1400\n7,21 MiB','10000×4000\n0,51 MiB']
a=[max(v['accepted_seconds'] for v in e['replies']) for e in x]
b=[max(v['ready_seconds'] for v in e['replies']) for e in x]
axes[0].bar([i-.18 for i in range(3)],a,.36,label='p95 до 202')
axes[0].bar([i+.18 for i in range(3)],b,.36,label='p95 до ready')
axes[0].set(xticks=range(3),xticklabels=labels,ylabel='Секунды',title='10 одновременных загрузок · worker=1')
axes[0].legend()
peak=[int(e['after']['memory.peak'])/1048576 for e in x]
anon=[max(int(dict(line.split() for line in s['worker']['memory.stat'].splitlines())['anon']) for s in e['samples'])/1048576 for e in x]
axes[1].bar([i-.18 for i in range(3)],peak,.36,label='cgroup memory.peak')
axes[1].bar([i+.18 for i in range(3)],anon,.36,label='Пик resident anon')
axes[1].axhline(768,color='#d62728',linestyle='--',label='Лимит worker')
axes[1].set(xticks=range(3),xticklabels=labels,ylabel='MiB',title='Worker + дочерний декодер')
axes[1].legend(fontsize=8)
for ax in axes:ax.grid(axis='y',alpha=.2);ax.set_axisbelow(True)
fig.suptitle('ЛР 5 · реальные измерения · concurrency=1, prefetch=1')
fig.savefig(r/'pipeline.png',dpi=150)
fig.savefig(r/'pipeline.svg')
+52
View File
@@ -0,0 +1,52 @@
"""Finite simultaneous jobs including 40 MP; record actual worker cgroup peak/RSS."""
import concurrent.futures
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
from client import Client
from images import png
def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip()
def memory(container):
code="import json;from pathlib import Path;p=Path('/sys/fs/cgroup');print(json.dumps({k:(p/k).read_text().strip() for k in ['memory.current','memory.peak','memory.events','memory.stat','cpu.stat']}))"
return json.loads(docker('exec',container,'python','-c',code))
def main():
c=Client(timeout=30);a,b,e,chat=c.bootstrap()
profiles=[(320,200,False),(1800,1400,True),(10000,4000,False)]
result=[]
for w,h,noise in profiles:
content=png(w,h,noise=noise)
docker('compose','restart','worker');container=docker('compose','ps','-q','worker')
before=memory(container);samples=[];done=threading.Event();start=time.monotonic()
def sample():
while not done.is_set():
samples.append({'seconds':round(time.monotonic()-start,4),'worker':memory(container)})
done.wait(.05)
sampler=threading.Thread(target=sample);sampler.start()
barrier=threading.Barrier(10)
def upload(i):
barrier.wait();t=time.monotonic()
r=Client(timeout=30).call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=load.png',202,actor=a,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
accepted=time.monotonic()-t
ready=Client(timeout=30).wait_ready(b,r['json'])
assert ready['sha256']==hashlib.sha256(content).hexdigest()
return {'index':i,'accepted_seconds':accepted,'ready_seconds':time.monotonic()-t,'status':ready['status']}
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: replies=list(pool.map(upload,range(10)))
finally:done.set();sampler.join()
after=memory(container);assert 'oom_kill 0' in after['memory.events']
result.append({'width':w,'height':h,'bytes':len(content),'clients':10,'before':before,'after':after,'samples':samples,'replies':replies,'seconds':time.monotonic()-start})
print(json.dumps({'method':'10 simultaneous uploads, worker concurrency=1; cgroup v2 samples include child renderer; anon is resident anonymous memory, memory.peak includes tmpfs/kernel/cache','experiments':result},indent=2))
if __name__=='__main__':main()
+27
View File
@@ -0,0 +1,27 @@
"""Private S3 and job envelopes do not expose source bytes or credentials."""
import json
from pathlib import Path
import sys
import urllib.error
import urllib.request
sys.path[:0]=['/app/src',str(Path.cwd()/'tests')]
from client import Client
from pipeline import connect,validate,event
from objects import Objects
from postgres import setting
c=Client();a,b,e,chat=c.bootstrap();att,raw=c.attachment(a,chat);ready=c.wait_ready(b,att)
with connect() as conn:
row=conn.execute('SELECT object_key, current_job_id FROM attachments WHERE 1=1 AND id = %s',(att['id'],)).fetchone()
job=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(row['current_job_id'],)).fetchone()
message=event(job);message['attempt']=max(1,job['attempt'])
assert validate(message)
assert set(message)=={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'}
assert len(json.dumps(message))<1024
for path in ('/'+Objects(setting).bucket+'/'+row['object_key'],'/'+Objects(setting).bucket+'?list-type=2'):
try:urllib.request.urlopen('http://s3:8333'+path,timeout=2);raise AssertionError('anonymous access')
except urllib.error.HTTPError as error:assert error.code==403
for key in ['schema_version','pipeline_version','attempt']:
bad=dict(message);bad[key]=True;assert not validate(bad)
for body in [dict(message,extra='x'),dict(message,created_at='bad'),dict(message,traceparent='bad')]:assert not validate(body)
print(json.dumps({'anonymous_object_and_list_403':True,'job_fields':sorted(message),'job_bytes':len(json.dumps(message)),'no_source_bytes_or_credentials_in_envelope':True,'strict_invalid_envelope_checks':True},indent=2))
+32
View File
@@ -0,0 +1,32 @@
"""A continuously trickling client still reaches the absolute upload deadline."""
import json
from pathlib import Path
import select
import socket
import sys
import time
import uuid
from urllib.parse import urlsplit
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client
c = Client()
alice, bob, eve, chat = c.bootstrap()
target = urlsplit(c.base)
connection = socket.create_connection((target.hostname, target.port), timeout=3)
header = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=slow_' + uuid.uuid4().hex + '.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n'
started = time.monotonic()
connection.sendall(header.encode())
response = b''
while time.monotonic() - started < 20:
if select.select([connection], [], [], .15)[0]:
response += connection.recv(4096)
if b'\r\n\r\n' in response:
break
connection.sendall(b'x')
connection.close()
elapsed = time.monotonic() - started
assert response.startswith(b'HTTP/1.1 400') or response.startswith(b'HTTP/1.0 400'), response[:100]
assert 14 <= elapsed < 19
assert b'x-instance-id:' in response.lower() and b'x-request-id:' in response.lower()
print(json.dumps({'trickle_seconds': round(elapsed, 3), 'http_status': 400, 'absolute_deadline_enforced': True}))
+177
View File
@@ -0,0 +1,177 @@
"""Extra black-box tests, without changing the course's public tests."""
import concurrent.futures
from pathlib import Path
import sys
import unittest
from urllib.parse import quote
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client as PublicClient, LAB
class Client(PublicClient):
def call(self, method, path, *args, headers=None, **kwargs):
headers = dict(headers or {})
if method == 'POST' and path.endswith('/messages'):
headers.setdefault('Idempotency-Key', uuid.uuid4().hex)
return super().call(method, path, *args, headers=headers, **kwargs)
class APIContract(unittest.TestCase):
def setUp(self):
self.client = Client()
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
self.path = '/api/v1/chats/' + self.chat['id']
def test_malformed_json_and_extra_fields(self):
for raw in (b'{', b'[]', b'{"text":"x","text":"y"}', b'{"text":NaN}', b'{"text":"\\ud800"}'):
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'})
for body in ({'text': 'ok', 'sender_id': self.eve['id']}, {'text': True}, {'text': ' '}, {'text': ''}, {'text': None}):
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body=body)
deep = b'{"text":' + b'[' * 1200 + b'0' + b']' * 1200 + b'}'
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=deep, headers={'Content-Type': 'application/json'})
def test_unicode_limits(self):
result = self.client.message(self.alice, self.chat, '👩' * 4000)['json']
self.assertEqual(result['text'], '👩' * 4000)
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '👩' * 4001})
def test_unicode_nul_round_trip(self):
import base64
value = 'A\x00B\\u0000👩'
registration = dict(self.alice['registration'], username='nul_' + uuid.uuid4().hex[:20], display_name=value)
user = self.client.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json']
self.assertEqual(user['display_name'], value)
credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
self.assertEqual(session['user'], user)
actor = {'id': user['id'], 'token': session['token']}
# Exercise the database path and then the Redis profile cache.
for _ in range(2):
self.assertEqual(self.client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'], user)
chat = self.client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': value, 'member_ids': [actor['id']]}, schema='Chat')['json']
self.assertEqual(chat['title'], value)
self.assertEqual(self.client.call('GET', '/api/v1/chats/' + chat['id'], actor=actor, schema='Chat')['json'], chat)
contents = [value, '\x00' * 4000, '👩' * 3999 + '\x00']
for content in contents:
self.assertEqual(self.client.message(actor, chat, content)['json']['text'], content)
history = self.client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor, schema='MessagePage')['json']
self.assertEqual([message['text'] for message in history['items']], contents)
def test_foreign_chat(self):
for suffix in ('', '/messages'):
self.client.call('GET', self.path + suffix, 404, actor=self.eve)
self.client.call('POST', self.path + '/messages', 404, actor=self.eve, body={'text': 'hidden'})
self.client.call('GET', '/api/v1/chats/' + str(uuid.uuid4()), 404, actor=self.alice)
def test_registration_race(self):
body = dict(self.alice['registration'], username='race_' + uuid.uuid4().hex[:20])
def register(_):
return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status']
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
statuses = list(executor.map(register, range(10)))
self.assertEqual(statuses.count(201), 1)
self.assertEqual(statuses.count(409), 9)
def test_invalid_chat_creation(self):
members = ([self.alice['id'], self.alice['id']], [self.bob['id']], [self.alice['id'], str(uuid.uuid4())], [], [True])
for member_ids in members:
self.client.call('POST', '/api/v1/chats', 400, actor=self.alice, body={'title': 'rollback', 'member_ids': member_ids})
def test_cursor_scope_and_validation(self):
for i in range(3):
self.client.message(self.alice, self.chat, str(i))
cursor = self.client.call('GET', self.path + '/messages?limit=1', actor=self.alice, schema='MessagePage')['json']['next_cursor']
other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json']
self.client.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(cursor), 400, actor=self.alice)
for query in ('limit=0', 'limit=101', 'limit=' + '9' * 5000, 'limit=true', 'limit=1&limit=2', 'cursor=', 'cursor=bad', 'cursor=e30=', 'offset=1'):
self.client.call('GET', self.path + '/messages?' + query, 400, actor=self.alice)
def test_concurrent_messages_and_pagination(self):
def send(i):
return Client().message(self.alice if i % 2 else self.bob, self.chat, 'parallel-' + str(i))['json']
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
sent = list(executor.map(send, range(100)))
found = []
cursor = None
while True:
url = self.path + '/messages?limit=7' + ('&cursor=' + quote(cursor) if cursor else '')
page = self.client.call('GET', url, actor=self.alice, schema='MessagePage')['json']
found.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
break
self.assertEqual(len(found), 100)
self.assertEqual(len({v['id'] for v in found}), 100)
self.assertEqual({v['id'] for v in found}, {v['id'] for v in sent})
self.assertEqual(found, sorted(found, key=lambda v: (v['created_at'], v['id'])))
def test_idempotency_concurrent_replicas(self):
key = uuid.uuid4().hex
def send(_):
return Client().message(self.alice, self.chat, 'Один раз\x00👩', key=key)
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
replies = list(executor.map(send, range(32)))
self.assertGreaterEqual(len({v['headers']['X-Instance-Id'] for v in replies}), 2)
self.assertTrue(all(v['json'] == replies[0]['json'] for v in replies))
page = self.client.call('GET', self.path + '/messages', actor=self.bob, schema='MessagePage')['json']
self.assertEqual(page['items'], [replies[0]['json']])
self.client.call('POST', self.path + '/messages', 409, actor=self.alice, body={'text': 'different'}, headers={'Idempotency-Key': key})
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=self.alice)
self.client.call('POST', self.path + '/messages', 401, actor=self.alice, body={'text': 'Один раз\x00👩'}, headers={'Idempotency-Key': key})
def test_protocol_error_headers(self):
self.client.call('TRACE', '/api/v1/users/me', 400, actor=self.alice)
def test_idempotency_scope(self):
key = uuid.uuid4().hex
one = self.client.message(self.alice, self.chat, 'same', key=key)['json']
two = self.client.message(self.bob, self.chat, 'same', key=key)['json']
other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'scope', 'member_ids': [self.alice['id']]}, schema='Chat')['json']
three = self.client.message(self.alice, other, 'same', key=key)['json']
self.assertEqual(len({one['id'], two['id'], three['id']}), 3)
def test_idempotency_semantic_body(self):
key = uuid.uuid4().hex
one = self.client.message(self.alice, self.chat, 'semantic', key=key)['json']
two = self.client.call('POST', self.path + '/messages', 201, actor=self.alice, body={'attachment_ids': [], 'text': 'semantic'}, headers={'Idempotency-Key': key}, schema='Message')['json']
self.assertEqual(one, two)
def test_idempotency_invalid_keys(self):
for key in (None, '', 'x' * 129, 'space key', 'bad,key', 'slash/key'):
headers = {} if key is None else {'Idempotency-Key': key}
PublicClient().call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': 'valid'}, headers=headers)
for raw in (b'{"text":true}', b'{"text":"ok","extra":1}', b'{"text":"ok","attachment_ids":false}'):
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'})
def test_idempotency_conflicting_race(self):
key = uuid.uuid4().hex
def send(i):
return Client().call('POST', self.path + '/messages', [201, 409], actor=self.alice, body={'text': str(i % 2)}, headers={'Idempotency-Key': key}, schema=None)
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
replies = list(executor.map(send, range(32)))
self.assertEqual(sum(v['status'] == 201 for v in replies), 16)
self.assertEqual(sum(v['status'] == 409 for v in replies), 16)
page = self.client.call('GET', self.path + '/messages', actor=self.alice, schema='MessagePage')['json']
self.assertEqual(len(page['items']), 1)
@unittest.skipUnless(LAB >= 2, 'sessions start in lab2')
def test_unicode_password_basic_and_logout(self):
import base64
registration = {'username': 'unicode_' + uuid.uuid4().hex[:16], 'display_name': 'Софья', 'password': '🔐' * 127 + ':'}
self.client.call('POST', '/api/v1/users', 201, body=registration)
encoded = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
headers = {'Authorization': 'Basic ' + encoded}
session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers=headers, schema='Session')['json']
actor = {'id': session['user']['id'], 'token': session['token']}
self.client.call('GET', '/api/v1/users/me', actor=actor)
self.client.call('GET', '/api/v1/users/me', 401, headers=headers)
for authorization in ('Basic !!!', 'Bearer wrong', '', 'Basic Zm9v'):
self.client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': authorization})
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
self.client.call('GET', '/api/v1/users/me', 401, actor=actor)
if __name__ == '__main__':
unittest.main(verbosity=2)
+52
View File
@@ -0,0 +1,52 @@
"""Independent tests for async image metadata, transformations and access."""
import io
from pathlib import Path
import sys
import unittest
import uuid
from urllib.parse import quote
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'tests')]
from client import Client
from images import bomb
class Images(unittest.TestCase):
def setUp(self):
self.c=Client(); self.alice,self.bob,self.eve,self.chat=self.c.bootstrap()
self.path='/api/v1/chats/'+self.chat['id']
self.raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes()
def test_async_variants_and_replay(self):
queued,_=self.c.attachment(self.alice,self.chat)
self.assertEqual(queued['status'],'queued')
key=uuid.uuid4().hex
one=self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json']
self.assertEqual(self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'],one)
ready=self.c.wait_ready(self.alice,queued)
self.assertEqual({v['name'] for v in ready['variants']},{'thumbnail','crop','optimized'})
dimensions={v['name']:(v['width'],v['height']) for v in ready['variants']}
self.assertEqual(dimensions,{'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)})
for v in ready['variants']:
import hashlib
r=self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant='+v['name'],actor=self.bob)
self.assertEqual(hashlib.sha256(r['bytes']).hexdigest(),v['sha256'])
self.assertEqual(len(r['bytes']),v['size_bytes'])
self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant=absent',404,actor=self.bob)
def test_mime_bytes_pixels_and_filename(self):
path=self.path+'/attachments?filename=test.png'
for raw,mime,status in [(b'not image','image/png',415),(self.raw[:20],'image/png',415),(self.raw,'image/jpeg',415),(bomb(),'image/png',413),(b'x'*(10485760+1),'image/png',413)]:
self.c.call('POST',path,status,actor=self.alice,raw=raw,headers={'Content-Type':mime})
for name in ('../x.png','a\\x.png','a\0.png','a\n.png','a'*256,''):
self.c.call('POST',self.path+'/attachments?filename='+quote(name,safe=''),400,actor=self.alice,raw=self.raw,headers={'Content-Type':'image/png'})
def test_private_and_cross_chat(self):
att,_=self.c.attachment(self.alice,self.chat)
for suffix in ('','/content'):
self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=self.eve)
self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,401)
other=self.c.call('POST','/api/v1/chats',201,actor=self.alice,body={'title':'other','member_ids':[self.alice['id']]},schema='Chat')['json']
self.c.call('POST','/api/v1/chats/'+other['id']+'/messages',404,actor=self.alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
if __name__=='__main__': unittest.main(verbosity=2)