Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d506539f5e | ||
|
|
2773115e1c | ||
|
|
5dbb905364 | ||
|
|
518c98fd3e |
@@ -18,3 +18,9 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
research
|
||||
student-tests
|
||||
scripts
|
||||
docs
|
||||
requirements-research.txt
|
||||
*.md
|
||||
|
||||
@@ -1,25 +1,36 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Секреты (пароль БД, DATABASE_URL) создаёт scripts/init-secrets.sh в каталоге secrets/ — не в .env.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Срок жизни сессии; для проверки expiry запускайте с SESSION_TTL_SECONDS=2 (см. student-tests/test_expiry.sh)
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
# TTL записи в кэше Valkey (не больше остатка жизни сессии)
|
||||
SESSION_CACHE_TTL_SECONDS=60
|
||||
# Источник сессий: PostgreSQL (истина) + Valkey (кэш). Адрес кэша задан в compose.yaml (VALKEY_URL).
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
DB_POOL_MAX=10
|
||||
# Файлы секретов внутри контейнера (создаются scripts/init-secrets.sh)
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# --- lab3 ---
|
||||
# Число API-реплик (compose deploy.replicas); также можно: docker compose up -d --scale api=4
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
API_CPUS=1.0
|
||||
API_MEM=256m
|
||||
POSTGRES_CPUS=1.0
|
||||
LB_CPUS=1.0
|
||||
# Пауза между SIGTERM и закрытием listener: LB успевает исключить реплику (rolling update без ошибок)
|
||||
DRAIN_SECONDS=4
|
||||
# --- lab4 ---
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
S3_CPUS=1.0
|
||||
UPLOAD_CONCURRENCY=4
|
||||
# --- lab5 ---
|
||||
RABBITMQ_CPUS=1.0
|
||||
WORKER_CPUS=1.0
|
||||
WORKER_MEM=768m
|
||||
WORKER_REPLICAS=1
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
MAX_ATTEMPTS=5
|
||||
LEASE_SECONDS=60
|
||||
MAX_QUEUED_JOBS=200
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# Keep shell hooks runnable after a Windows clone with core.autocrlf=true.
|
||||
*.sh text eol=lf
|
||||
@@ -0,0 +1,25 @@
|
||||
name: Application smoke
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Prepare isolated lab credentials
|
||||
run: |
|
||||
sh scripts/init-secrets.sh
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
- name: Build and wait for application
|
||||
run: docker compose --env-file .env up -d --build --wait --wait-timeout 120
|
||||
- name: Check contract and public scenario
|
||||
run: make check test PYTHON=python BASE_URL=http://127.0.0.1:8080
|
||||
- name: Stop disposable CI stack
|
||||
if: always()
|
||||
run: docker compose --env-file .env down
|
||||
@@ -0,0 +1,32 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# Базовый образ закреплён тегом и digest (python 3.12.12, Debian bookworm).
|
||||
ARG PYTHON_IMAGE=python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
|
||||
|
||||
# --- сборка зависимостей в отдельном слое (в runtime не попадают pip и кэш) ---
|
||||
FROM ${PYTHON_IMAGE} AS builder
|
||||
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_CACHE_DIR=1
|
||||
WORKDIR /build
|
||||
COPY requirements.txt .
|
||||
RUN python -m venv /opt/venv && /opt/venv/bin/pip install -r requirements.txt
|
||||
|
||||
# --- runtime ---
|
||||
FROM ${PYTHON_IMAGE}
|
||||
ENV PATH=/opt/venv/bin:$PATH \
|
||||
PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/srv/src \
|
||||
APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab LIMIT_CONCURRENCY=1000 MIGRATIONS_DIR=/srv/migrations
|
||||
RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home app
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
WORKDIR /srv
|
||||
COPY src/ /srv/src/
|
||||
COPY migrations/ /srv/migrations/
|
||||
# ревизия образа (меняется при rolling update; видна в журнале запуска)
|
||||
ARG REV=dev
|
||||
LABEL org.opencontainers.image.revision=$REV
|
||||
ENV APP_REVISION=$REV
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request,os,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT','8080'), timeout=2).status == 200 else 1)"]
|
||||
# exec-форма: сервер = PID 1. Состояние хранится в PostgreSQL/Valkey, процесс stateless;
|
||||
# масштабирование — репликами контейнера. SIGTERM: drain (readiness 503) -> graceful shutdown (см. app/serve.py).
|
||||
CMD ["python", "-m", "app.serve"]
|
||||
@@ -28,3 +28,19 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# --- собственные цели студента ---
|
||||
.PHONY: secrets up down student-test
|
||||
secrets:
|
||||
sh scripts/init-secrets.sh
|
||||
|
||||
up: secrets
|
||||
docker compose --env-file .env up -d --build
|
||||
|
||||
down:
|
||||
docker compose --env-file .env down
|
||||
|
||||
student-test:
|
||||
$(PYTHON) -m unittest discover -s student-tests -p "test_api.py" -v
|
||||
$(PYTHON) -m unittest discover -s student-tests -p "test_attachments.py" -v
|
||||
$(PYTHON) -m unittest discover -s student-tests -p "test_pipeline.py" -v
|
||||
|
||||
@@ -1,37 +1,190 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Кушнаренко Николай Вячеславович, БПИ2302
|
||||
- Целевая оценка; выбранный трек 5, если нужен: **5**, трек 1 «Очередь под перегрузкой»
|
||||
- Commit/PR: [PR №52](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/52), ветка `solution/bpi2302-kushnarenko` из [форка kolya](https://gitea.privcloud.tech/kolya/mtusi2026lab5) в `mtusi2026/mtusi2026lab5:main`; проверенная базовая реализация — `5dbb905`.
|
||||
- Использованные источники, библиотеки и помощь ИИ: Python 3.12, uvicorn, asyncpg, aiobotocore, aio-pika (RabbitMQ), Pillow (обработка изображений); RabbitMQ 4.1.8, SeaweedFS 4.48, PostgreSQL 17, Valkey 8, HAProxy 3.0. Код lab4 перенесён. ИИ-ассистенты использовались для помощи с кодом, тестами и отчётом; Codex проверил подготовку к сдаче, исправил перенос reconciler, усилил методику исследования и выполнил реальные запуски 12.11.2026
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker Compose v2, bash (Git Bash на Windows), Python ≥ 3.9; на Windows `PYTHONUTF8=1`.
|
||||
Версии: `python:3.12.12-slim-bookworm@sha256:593bd06e…`, `postgres:17.11-alpine@sha256:b0f9560a…`, `valkey/valkey:8.1.10-alpine@sha256:081c2f5c…`, `haproxy:3.0-alpine@sha256:56b887da…`, `chrislusf/seaweedfs:4.48@sha256:4e61d15f…`, `rabbitmq:4.1-alpine@sha256:6a0f7984…` (4.1.8), зависимости — `requirements.txt` (закреплены).
|
||||
|
||||
```sh
|
||||
sh scripts/init-secrets.sh # secrets/*: БД, S3 (admin + ключи API), RabbitMQ (rabbitmq.conf, broker_url)
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 120 # postgres, valkey, seaweedfs, rabbitmq -> migrate + s3-init -> api x2, worker -> lb
|
||||
curl http://127.0.0.1:8080/health/ready
|
||||
python3 tests/smoke.py --base-url http://127.0.0.1:8080 # upload -> 202 -> ready -> thumbnail 256x160
|
||||
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
|
||||
python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
|
||||
sh scripts/restore-replicas.sh
|
||||
# собственные тесты
|
||||
python3 -m unittest discover -s student-tests -p "test_api.py" -v # + test_idempotency.py, test_attachments.py
|
||||
python3 student-tests/test_pipeline.py # Pillow желателен (пиксельные проверки)
|
||||
python3 student-tests/test_resilience5.py # семь окон отказа
|
||||
python3 student-tests/test_reconcile5.py # варианты и staging не удаляются
|
||||
python3 student-tests/test_expiry.py
|
||||
python3 student-tests/test_dependencies.py
|
||||
# операторские команды
|
||||
docker compose --env-file .env run --rm -T worker python -m app.replay list
|
||||
docker compose --env-file .env run --rm -T worker python -m app.replay replay <attachment_id>
|
||||
```
|
||||
|
||||
Windows: `scripts/restart.cmd`, `scripts/stop-one.cmd`. Остановка с сохранением данных — `docker compose --env-file .env stop`; **намеренное удаление учебных данных** — `down -v` (volumes `pgdata`, `valkeydata`, `seaweeddata`, `rabbitmqdata`). Учётные данные RabbitMQ задаются конфигурационным файлом-секретом `secrets/rabbitmq.conf`; hostname контейнера фиксирован (`rabbitmq`), иначе очередь не переживает пересоздание контейнера.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
Клиент ─▶ LB ─▶ API ─┬─(1) staging в S3 (приватный prefix staging/)
|
||||
└─(2) ОДНА транзакция PostgreSQL: attachments(status=queued) + outbox ──▶ 202 queued
|
||||
outbox ──relay (в API и worker, SKIP LOCKED)──▶ RabbitMQ (quorum, persistent, publisher confirm) ──▶ worker
|
||||
worker: claim(queued→processing, lease) → staging → thumbnail/crop/WebP → S3 att/<id>/… → ОДНА транзакция: варианты + ready → ack → очистка staging
|
||||
```
|
||||
|
||||
Все сервисы находятся в сети Compose `backend`; наружу опубликован только LB. Полные digest перечислены в compose.yaml и Dockerfile.
|
||||
|
||||
| Роль → service name | Образ / ресурсы | Health/readiness | Volume | Порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| API → `api` × 2 | собственный образ, Python 3.12.12; по 1 CPU / 256 MiB | `/health/ready`, отдельно `/health/live` | нет локального состояния | 8080, только backend | файловые secrets database_url, S3, broker_url |
|
||||
| БД → `postgres` | PostgreSQL 17.11; 1 CPU / 512 MiB | `pg_isready` | `pgdata` | 5432, backend | postgres_password; клиентам database_url |
|
||||
| Кэш сессий → `valkey` | Valkey 8.1.10; 0.5 CPU / 128 MiB | `valkey-cli ping` | `valkeydata`, AOF | 6379, backend | отдельного пароля нет; не опубликован на host |
|
||||
| S3 → `seaweedfs` | SeaweedFS 4.48; 1 CPU / 768 MiB | master cluster status + S3 socket; restart hook дополнительно читает объекты | `seaweeddata` | 8333 и внутренние служебные порты, backend | seaweed_s3_config; admin только для bootstrap, API-ключ ограничен bucket |
|
||||
| Брокер → `rabbitmq` | RabbitMQ 4.1.8; 1 CPU / 512 MiB | `rabbitmq-diagnostics ping` | `rabbitmqdata` | 5672, backend | rabbitmq_conf; клиентам broker_url |
|
||||
| Worker → `worker` | собственный образ, Python 3.12.12; 1 CPU / 768 MiB | heartbeat-файл моложе 30 с | нет локального устойчивого состояния | нет слушающего порта | файловые secrets БД/S3/брокер |
|
||||
| Балансировщик → `lb` | HAProxy 3.0; 1 CPU / 128 MiB | внутренний `:8081/healthz`; backend checks API | конфигурация read-only | **127.0.0.1:8080** | credentials не нужны |
|
||||
|
||||
Bootstrap: init-secrets.sh создаёт недостающие локальные файлы без перезаписи; migrate применяет версионные SQL-миграции; s3-init создаёт bucket при отсутствии и проверяет права; API/worker идемпотентно объявляют durable RabbitMQ topology. Повтор s3-init проверен дважды. Секреты и volumes не переносятся из предыдущей лабораторной. `.gitattributes` сохраняет LF в shell hooks при Windows clone.
|
||||
|
||||
- **Границы приёма (до 202)**: байты проверены (формат, целостность, размеры) и записаны в **устойчивый staging** (приватный S3-prefix `staging/<id>`, не локальный диск); метаданные и **намерение обработки** записаны одной транзакцией PostgreSQL (таблицы `attachments` и `outbox`). Только после COMMIT клиент получает `202` со снимком `status=queued`. Если брокер недоступен — это не мешает приёму: задание остаётся в outbox и публикуется позже. HTTP-передача байт клиентом по-прежнему синхронна (честная граница: staging уже в S3).
|
||||
- **Состояния и переходы** (`contracts/attachment-states.md`): `queued → processing → ready | failed`; `processing → queued` при временной ошибке (backoff) и при истечении lease (sweeper); `failed → queued` только явным операторским replay. Повторная доставка готовой задачи — no-op с ack; завершённое состояние не откатывается дубликатом. Условия UPDATE (`status='queued'`, `lease_owner=…`) исключают конкурирующую запись.
|
||||
- **Задание в брокере** (`jobs.schema.json`): только `schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at` — без байт, ключей S3 и credentials; worker берёт доверенные метаданные (ключи, тип) из БД. `job_id` не меняется при redelivery; `attempt` — управляемая попытка.
|
||||
- **Доставка и подтверждения**: durable quorum-очередь `attachment.process`, persistent-сообщения, **publisher confirm** (подтверждение приёма брокером) → только затем `published_at` в outbox; **ack** consumer — только после COMMIT результата (устойчивый эффект). Это разные границы. Доставка at-least-once; эффект идемпотентен: ключ `(attachment_id, pipeline_version, name)` (PK `attachment_variants`, `ON CONFLICT DO NOTHING`), детерминированные ключи объектов S3.
|
||||
- **Обработка** (`src/app/pipeline.py`): **thumbnail** — JPEG, вписан в 256×256 без увеличения, EXIF-ориентация применяется, alpha на белый, размеры округляются вниз (320×200 → 256×160), quality 85, метаданные не сохраняются; **crop** — JPEG, центрированный квадрат 128×128 (мелкие допускается увеличивать); **optimized** — WebP, вписан в 1280×1280 без увеличения, `quality=80, method=4`, EXIF/ICC не сохраняются, прозрачность сохраняется. `size_bytes/sha256/content_type` вложения всегда описывают оригинал; у вариантов — свои поля. Готовность (`ready`) ставится после сохранения оригинала и всех вариантов.
|
||||
- **Лимиты обработки**: API отклоняет файл >10 MiB (413) и >40 млн пикселей (413) до распаковки; worker — таймаут задания 45 с (`JOB_TIMEOUT`), число пикселей, `WORKER_CONCURRENCY` (= prefetch = число потоков), лимит памяти контейнера 768 MiB; JPEG >2560×2560 декодируется сразу в масштабе 1/2.
|
||||
- **Retry**: ограниченное число попыток `MAX_ATTEMPTS=5`; временные ошибки (S3/БД/таймаут) → `queued` + новая запись outbox с отложенной публикацией; задержка — **экспоненциальный backoff с jitter** (случайное значение в [50 %, 100 %] от `min(60 с, 2 с × 2^(n−1))`); постоянные ошибки (нечитаемое изображение) → сразу `failed` с `error_code` (`invalid_image`, `image_too_large`, `processing_failed`); исчерпание попыток → `failed/retry_exhausted`.
|
||||
- **Реестр failed и DLQ**: реестр — таблица `attachments` со `status=failed` + `job_events` (журнал событий, без секретов); `python -m app.replay list`. Нечитаемые сообщения брокера (`schema_version`, типы) отправляются в DLQ `attachment.process.dead` через DLX. **Replay** (`python -m app.replay replay <id>`) допускается только для `failed` (иначе `skipped`): новая попытка, новый `job_id`, запись `operator_replay` в `job_events`; повторное исполнение уже успешной работы невозможно.
|
||||
- **Lease и зависшие задачи**: при захвате задание получает lease (`LEASE_SECONDS=60`); sweeper worker возвращает просроченные `processing` в `queued` (или `failed/retry_exhausted`); публиковать результат может только владелец lease (`FOR UPDATE` + проверка владельца).
|
||||
- **Outbox** (уровень 4): relay в API и worker читает `FOR UPDATE SKIP LOCKED`; после краха между COMMIT и publish задание публикуется автоматически (проверено). Повторная публикация после краха между confirm и `published_at` даёт дубликат, обрабатываемый как no-op.
|
||||
- **Очистка staging**: удаляется после COMMIT результата; для `failed` хранится `STAGING_FAILED_RETENTION_SECONDS` (7 суток) для разбора/replay; фоновая очистка не затрагивает текущую работу.
|
||||
- **Backpressure**: перед приёмом тела API проверяет число активных заданий; при `queued+processing ≥ MAX_QUEUED_JOBS` — `503 unavailable` + `Retry-After` (контракт допускает 503). Readiness API не зависит от брокера и worker (их отказ не должен останавливать приём/чтение).
|
||||
|
||||
## Исправления при подготовке к сдаче — 12.11.2026
|
||||
|
||||
Перенесено исправление HAProxy из ЛР4: когда общая БД недоступна и все API теряют readiness, резервный backend по liveness позволяет приложению вернуть контрактный JSON, Retry-After и X-Request-Id; /health/live остаётся доступен. Автоматический retry HAProxy ограничен ошибкой установления соединения, чтобы повтор уже отправленного upload POST не создавал лишнюю загрузку. Восстановлены собственные проверки TTL, PostgreSQL и Valkey.
|
||||
|
||||
**Исправлен опасный перенос reconciler из ЛР4.** Старый dry-run считал три готовых варианта осиротевшими, поскольку учитывал только attachments.s3_key; вывод до исправления сохранён в [reconcile_before_fix.json](evidence/recheck_2026-11-12/reconcile_before_fix.json). Теперь учитываются метаданные вариантов, staging и владелец серверного префикса att/<id>/… . Объект внутри префикса существующего вложения защищён даже до COMMIT метаданных worker. Перед удалением перепроверяются references и владелец. Такой подход консервативен: неизвестный объект внутри существующего вложения остаётся до отдельного операторского разбора.
|
||||
|
||||
Для staging можно явно выполнить `docker compose run --rm -T api python -m app.reconcile --prefix staging/` (dry-run). Рабочий grace по умолчанию 900 с; `--apply` включает исправление. В тестах grace=0 применяется только к созданным тестом префиксам после окончания записи. Staging принятой задачи не удаляется; корректные оригинал и варианты не становятся failed. Очистку staging после ready и по retention для failed по-прежнему выполняет worker.
|
||||
|
||||
Повторный dry-run того же вложения после исправления: [reconcile_after_fix.json](evidence/recheck_2026-11-12/reconcile_after_fix.json) — четыре объекта, orphan/missing/size mismatch = 0, изменений нет.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Текущий skeleton — `276dbda`. README, COURSE, CONTRIBUTING, публичные tests/ и contracts/ сохранены без изменений. В Windows запущены Python-команды, соответствующие make check, make validate и make test. Полный реальный вывод — [checks_2026-11-12.txt](evidence/checks_2026-11-12.txt); команды всех собственных тестов — [student-tests/README.md](student-tests/README.md).
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Проверка | Результат |
|
||||
| --- | --- |
|
||||
| check_contract, полная OpenAPI-валидация, публичный smoke | PASS; upload → 202 queued → ready; original и JPEG thumbnail проверены |
|
||||
| API и аутентификация | 12 тестов OK |
|
||||
| Идемпотентность и несколько API-реплик | 9 тестов OK |
|
||||
| Изображения, лимиты, MIME, filename, ACL, private bucket, прерывание upload | 11 тестов OK |
|
||||
| Топология, порты, лимиты, непривилегированный пользователь | 6 тестов OK |
|
||||
| Варианты, EXIF, alpha, размеры, оригинал, redelivery, cleanup | 9 тестов OK, включая пиксельные проверки Pillow |
|
||||
| Безопасная сверка вариантов и staging | 2 теста OK; сохраняются все варианты и незавершённый S3-эффект, orphan защищён grace и удаляется после него |
|
||||
| Семь сценариев отказа pipeline | 7 тестов OK: worker/API/broker, outbox, crash до COMMIT/ack, lease, retry exhaustion, registry/replay |
|
||||
| TTL сессий и отказ PostgreSQL/Valkey | 2 + 2 теста OK |
|
||||
| Persistence и failover | PASS; 121 успешных записей, 0 неуспешных попыток; восстановление 2.344 с |
|
||||
| Bootstrap S3 дважды | OK: bucket сохранён, анонимный доступ 403, создание bucket ключом API запрещено |
|
||||
|
||||
## Эксперименты и отказы
|
||||
Итого **60 собственных тестов**. После экспериментов первый дополнительный persistence выявил гонку старта SeaweedFS: S3-порт уже открыт, но тома ещё не зарегистрированы, и чтение объекта получает 503. Этот неуспешный запуск сохранён в commands.txt. Исправлены операторские scripts/restart.sh и .cmd: после healthy модуль app.s3_recovery ждёт LIST и ranged GET существующих объектов, читая по одному байту с параллелизмом 8 и общим таймаутом 45 с. Это отдельная проверка восстановления, а не изменение публичного теста или обещание полной готовности S3 по /health/ready API. Для большого хранилища время такого обхода может превысить лимит; hook завершится ошибкой.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
После исправления **три последовательных persistence и финальный smoke прошли**: [restart_recheck.txt](evidence/recheck_2026-11-12/restart_recheck.txt). [lifecycle.json](evidence/recheck_2026-11-12/lifecycle.json) подтверждает в каждом повторе изменение StartedAt всех семи ролей (восемь контейнеров, включая две API-реплики), сохранение имён volumes и healthy после restart. SHA-256 изменённых после матрицы скриптов восстановления — [restart_source_metadata.json](evidence/recheck_2026-11-12/restart_source_metadata.json); обработка API/worker и измерительные скрипты матрицы не менялись.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
## Эксперименты и отказы (трек 1)
|
||||
|
||||
**Гипотеза.** Увеличение concurrency/prefetch ускоряет работу до насыщения CPU, но повышает память и может ухудшить latency крупных файлов. Ограничение приёма по глубине активной работы даёт предсказуемую перегрузку: отклонённые запросы получают Retry-After, а принятые задачи завершаются; после разгрузки приём восстанавливается.
|
||||
|
||||
**Методика.** Один worker, фактические лимиты проверяются через docker inspect: **1 CPU / 768 MiB**, concurrency=prefetch ∈ {1, 2, 4}. Девять точек: small 320×200 PNG (109 KiB, 80 загрузок), medium 2400×1600 PNG (1961 KiB, 24 загрузки), large 6000×4000 JPEG (3100 KiB, 12 загрузок). Перед каждой точкой worker пересоздаётся, чтобы VmHWM начинался заново. Клиент — один Python-процесс, 8 одновременных загрузчиков; HTTP-соединения не переиспользуются после блокирующих Docker-команд. Неопределённый результат upload при сетевой ошибке автоматически не повторяется.
|
||||
|
||||
Клиент: Python 3.11.9, aiohttp 3.14.3, Pillow 12.3.0, matplotlib 3.11.2. Стенд: Docker Desktop, Linux-контейнеры; версии приложения и инфраструктуры закреплены в Git. На компьютере оставался другой Docker-проект. Время до ready = ready_at − accepted_at из PostgreSQL; p95 — nearest rank. Throughput — число ready / интервал от первого accepted_at до последнего ready_at: это измерение конвейера, включающее темп приёма следующих файлов. Queue lag — возраст старейшего активного задания (queued + processing); это не только ожидание в брокере. Глубина и возраст снимаются примерно каждые 0,4 с плюс время SQL/CLI. Пик RSS — VmHWM процесса worker, не память всего стенда.
|
||||
|
||||
| concurrency | Изображение | ready | Throughput, файлов/с | p95, с | RSS, MiB | Пик queued+processing | Возраст старейшего, с |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| 1 | small | 80 | 23.84 | 2.84 | 72.5 | 67 | 2.7 |
|
||||
| 1 | medium | 24 | 4.09 | 5.21 | 133.2 | 22 | 5.4 |
|
||||
| 1 | large | 12 | 3.38 | 3.35 | 160.4 | 11 | 3.3 |
|
||||
| 2 | small | 80 | 44.13 | 1.48 | 75.4 | 56 | 1.1 |
|
||||
| 2 | medium | 24 | 4.6 | 4.78 | 194.9 | 22 | 4.3 |
|
||||
| 2 | large | 12 | 3.84 | 2.98 | 250.9 | 12 | 2.7 |
|
||||
| 4 | small | 80 | 52.3 | 1.25 | 79.9 | 50 | 1.1 |
|
||||
| 4 | medium | 24 | 4.56 | 4.82 | 318.9 | 24 | 4.8 |
|
||||
| 4 | large | 12 | 3.39 | 3.36 | 430.0 | 12 | 2.8 |
|
||||
|
||||
Все **348 из 348** загрузок матрицы достигли ready; failed=0, OOMKilled=false. Максимальный измеренный RSS — **430.0 MiB** из 768. Наибольший throughput крупных изображений в этом прогоне получен при concurrency=2 (3.84 файлов/с, p95=2.98 с). Это наблюдение одного прогона на точку, а не статистически доказанный оптимум; близкие результаты следует считать сопоставимыми. На мелких файлах throughput изменился с 23.84 до 52.3 файлов/с при concurrency 1 → 4. Память крупных изображений: 160.4 → 430.0 MiB.
|
||||
|
||||
### Backpressure и восстановление
|
||||
|
||||
MAX_QUEUED_JOBS=30, один worker concurrency=1, burst из 200 мелких файлов с 32 клиентами. Принято **54**, получено **146 ответов 503**; Retry-After реально прочитан и проверен (5 с). Пик активной очереди — **48**, время разгрузки от начала burst — 2.7 с; все 54 принятых задач стали ready, failed=0, p95=1.88 с. После разгрузки три новые загрузки приняты и также достигли ready.
|
||||
|
||||
Порог является **мягким**, а не строгим резервированием мест: несколько запросов могут одновременно проверить старую глубину очереди перед приёмом тела. Поэтому пик может превышать 30. Число находящихся в приёме запросов дополнительно ограничено API concurrency/очередью и MAX_INFLIGHT. Для строгого лимита нужна атомарная резервация слотов. Этот эксперимент подтверждает ограничение приёма и восстановление, но не доказывает универсальную границу памяти при любом размере/параллелизме.
|
||||
|
||||
Из исследовательского скрипта удалён заранее выставленный `retry_after_header_seen=True`: результат определяется HTTP-заголовками. Проверяются коды/число ответов, отсутствие транспортных ошибок, готовность всех подтверждённых задач, восстановление, фактические ресурсы и OOM. Ошибка Docker или измерения не маскируется; промежуточные результаты сохраняются после каждой точки. Настройки API/worker восстанавливаются в finally.
|
||||
|
||||
Воспроизведение: `python -m pip install -r research/requirements.txt`, затем `python research/queue_overload.py --out evidence/recheck_2026-11-12`. Данные: [queue_overload.json](evidence/recheck_2026-11-12/queue_overload.json), [вывод команд](evidence/recheck_2026-11-12/commands.txt), [SHA-256 исходников](evidence/recheck_2026-11-12/source_metadata.json).
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
**Ограничения.** Один компьютер, по одному прогону на точку, фиксированный worker CPU=1; фоновые процессы и темп клиента влияют на абсолютные числа. При нескольких worker или другом CPU результаты требуют повторной проверки. Октябрьские данные в evidence/queue_overload.json и старые PNG сохранены как исторические; текущие выводы основаны на повторе 12.11.2026.
|
||||
|
||||
### Окна отказа (`contracts/attachment-states.md`)
|
||||
|
||||
| Окно | Воздействие | Результат |
|
||||
| --- | --- | --- |
|
||||
| после staging до записи задания | механизм компенсации в коде; отдельная инъекция именно в это окно не выполнялась | исключение вызывает удаление staging; остатки доступны исправленному reconcile с --prefix staging/ и grace; безопасность reconcile проверена отдельно |
|
||||
| после COMMIT до publish | остановка брокера + kill всех API | outbox довёл задание |
|
||||
| после publish до confirm | механизм повторной публикации в коде; отдельная инъекция именно в это окно не выполнялась | outbox сохраняет неопубликованное намерение; повторная доставка проверена отдельно |
|
||||
| после записи S3 до COMMIT результата | `crash_once:after_s3_before_commit` | lease → повтор тем же результатом |
|
||||
| после COMMIT результата до ack | `crash_once:after_commit_before_ack` | redelivery = no-op + ack |
|
||||
|
||||
## Дополнительная сверка PR №52 с файлами задания
|
||||
|
||||
Проверены README.md (уровни 3/4 и трек 1), COURSE.md (переходы и измерения), CONTRIBUTING.md (ветка/fork/PR/CI), .github/pull_request_template.md, contracts/README.md, contracts/attachment-states.md, contracts/infrastructure.json, tests/README.md и infra/README.md. Обязательные материалы не изменены относительно skeleton 276dbda. PR открыт из правильной ветки, заголовок соответствует шаблону; сведения об ИИ, переносе, ограничениях и доказательствах заполнены.
|
||||
|
||||
**Аппаратура.** Intel Core i9-12900KS, 16 физических / 24 логических ядра, 102830256128 байт RAM (около 95.8 GiB). Docker Desktop Engine 29.5.3, Linux x86_64: 24 CPU и 50392047616 байт RAM (около 46.9 GiB), Compose 5.1.4. Лимиты каждой роли приведены выше; матрица использует одну worker-реплику с 1 CPU / 768 MiB. Фоновые процессы другого проекта оставались включены.
|
||||
|
||||
**Прогрев.** Перед точкой worker пересоздаётся, ожидается healthy, затем выдерживаются 3 с; при пересоздании API — 6 с после healthy. Изображения для прогрева отдельно не обрабатываются: это короткие burst после холодного старта worker, а не установившаяся предельная производительность. Для каждого размера повторяются одинаковые байты внутри серии. Фиксированного RPS нет: матрица — 8 клиентов, перегрузка — 32.
|
||||
|
||||
**Второй независимый прогон** сохранён в [pr52_audit_2026-11-12](evidence/pr52_audit_2026-11-12/), отдельно от первого. Все 348/348 снова ready, failed/OOM=0. В матрице HTTP non-2xx=0 %. p50/p95/p99 — nearest-rank по ready_at − accepted_at; при N≤80 p99 равен максимуму выборки. CPU — разность utime+stime из /proc/1/stat (включая потоки worker), делённая на монотонный интервал между снимками; 100 % = одно ядро. Интервал включает приём и ожидание завершения с шагом polling, поэтому он немного длиннее span самого конвейера; CPU других сервисов не включён. В JSON сохранены CPU-seconds, интервалы, HTTP-коды и синтетические attachment IDs без токенов.
|
||||
|
||||
| concurrency/размер | Окно, с | ready/с / 202 HTTP RPS за окно | p50/p95/p99, с | CPU worker, % ядра | RSS, MiB |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 1/small | 3.488 | 26.26 / 22.94 | 1.4/2.57/2.71 | 45.59 | 72.7 |
|
||||
| 1/medium | 6.264 | 4.09 / 3.83 | 2.78/5.23/5.44 | 82.21 | 133.7 |
|
||||
| 1/large | 4.058 | 3.33 / 2.96 | 1.73/3.34/3.34 | 75.16 | 160.9 |
|
||||
| 2/small | 2.687 | 43.4 / 29.77 | 0.83/1.57/1.65 | 56.93 | 77.0 |
|
||||
| 2/medium | 5.726 | 4.6 / 4.19 | 2.48/4.64/4.65 | 91.34 | 196.4 |
|
||||
| 2/large | 4.012 | 3.7 / 2.99 | 1.68/3.04/3.04 | 80.0 | 251.0 |
|
||||
| 4/small | 1.991 | 50.21 / 40.17 | 0.84/1.33/1.37 | 73.82 | 78.3 |
|
||||
| 4/medium | 6.405 | 4.37 / 3.75 | 2.69/4.89/4.93 | 85.4 | 319.1 |
|
||||
| 4/large | 4.061 | 3.48 / 2.96 | 2.3/3.3/3.3 | 82.75 | 434.5 |
|
||||
|
||||
Повтор backpressure: 61 ответов 202 и 139 ответов 503 из 200; HTTP non-2xx = 69.5 % (ожидаемое отклонение перегрузки), Retry-After проверен, 61 принятых готовы, новые три после разгрузки готовы. p50/p95/p99 = 1.46/2.31/2.4 с; CPU worker = 35.49 % за окно 3.72 с. Числа первой серии выше и на исходных графиках относятся к первому прогону; данные не подменялись. Два коротких прогона не доказывают статистический оптимум.
|
||||
|
||||
**Обе API-реплики.** Публичный failover повторён с восстановлением между отказами; фактически остановлены mtusi-lab5-api-1, mtusi-lab5-api-2. Результаты — [both_replicas.json](evidence/pr52_audit_2026-11-12/both_replicas.json) и [commands.txt](evidence/pr52_audit_2026-11-12/commands.txt). Подтверждённые сообщения и вложение сохранены, дубликаты проверены публичным тестом.
|
||||
|
||||
**Чистый clone.** Отдельный clone коммита 2773115 с core.autocrlf=true, новые секреты и изначально отсутствующие volumes: bootstrap, config, build, readiness, check_contract, smoke и persistence прошли. LF в shell-скриптах закреплён в .gitattributes; это устраняет CRLF в Windows/Git Bash. Исходный стенд остановлен на время проверки и затем восстановлен без удаления его volumes. Вывод — [clean_clone.txt](evidence/pr52_audit_2026-11-12/clean_clone.txt).
|
||||
|
||||
**CI.** Добавлен отдельный application job в [.github/workflows/application.yml](.github/workflows/application.yml): bootstrap, сборка, ожидание readiness, make check test, остановка временного стенда. События push/pull_request, permissions contents:read; секреты репозитория и pull_request_target не используются. Это GitHub workflow из предусмотренного заданием каталога; автоматический запуск CI на Gitea не подтверждён. Реальное локальное выполнение эквивалентных команд и проверка чистого clone приложены; зелёный статус серверного CI не заявляется.
|
||||
|
||||
Повтор измерений без перезаписи доказательств: `python research/queue_overload.py --out evidence/my-queue-run`. Python для research/requirements.txt — 3.11+; публичным smoke достаточно Python 3.9+. Аппаратные данные и SHA-256 скриптов второго прогона: [metadata.json](evidence/pr52_audit_2026-11-12/metadata.json).
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один узел RabbitMQ, S3, PostgreSQL (SPOF). Отложенные повторы реализованы через outbox (брокерных delayed-exchange нет).
|
||||
- Один worker-процесс по умолчанию (`WORKER_REPLICAS` можно увеличить); lease и `SKIP LOCKED`/условные UPDATE допускают несколько worker.
|
||||
- Конкурентные записи старой и новой `pipeline_version` защищены ключом эффекта и условием `pipeline_version <= job.pipeline_version`; версионирование нескольких пайплайнов (трек 2) не реализовано.
|
||||
- Подтверждено экспериментами: различия измеренных throughput/p95/RSS, backpressure и восстановление; гипотеза без доказательства: поведение при >1 worker и при другом лимите CPU.
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
# Стенд lab5: HAProxy -> API x N -> PostgreSQL (метаданные) + SeaweedFS (приватный S3, байты) + Valkey (кэш сессий).
|
||||
# Секреты: scripts/init-secrets.sh создаёт secrets/* (в Git не попадают). Запуск и проверки — см. REPORT.md.
|
||||
name: mtusi-lab5
|
||||
|
||||
x-common: &common
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options: {max-size: "10m", max-file: "3"}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
<<: *common
|
||||
image: postgres:17.11-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
command: ["postgres", "-c", "max_connections=200", "-c", "shared_buffers=128MB", "-c", "log_min_duration_statement=500"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
mem_limit: 512m
|
||||
cpus: ${POSTGRES_CPUS:-1.0}
|
||||
networks: [backend]
|
||||
|
||||
valkey:
|
||||
<<: *common
|
||||
image: valkey/valkey:8.1.10-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
|
||||
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec",
|
||||
"--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl", "--save", ""]
|
||||
volumes:
|
||||
- valkeydata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
networks: [backend]
|
||||
|
||||
# Приватное S3-совместимое хранилище SeaweedFS (all-in-one: master + volume + filer + S3-шлюз).
|
||||
# Порты наружу не публикуются: доступ только из сети backend и только по ключам из s3.json (identities).
|
||||
# Анонимный доступ запрещён (при заданных identities запрос без подписи получает 403).
|
||||
seaweedfs:
|
||||
<<: *common
|
||||
image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command: ["exec weed server -dir=/data -ip=seaweedfs -master.volumeSizeLimitMB=512 -volume.max=30 -filer -s3 -s3.port=8333 -s3.config=/run/secrets/seaweed_s3_config"]
|
||||
secrets: [seaweed_s3_config]
|
||||
volumes:
|
||||
- seaweeddata:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status && nc -z 127.0.0.1 8333"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 10s
|
||||
mem_limit: 768m
|
||||
cpus: ${S3_CPUS:-1.0}
|
||||
networks: [backend]
|
||||
|
||||
# Идемпотентный bootstrap: bucket создаётся, если его нет; проверяется, что анонимный доступ запрещён и что ключ API работает.
|
||||
s3-init:
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["python", "-m", "app.s3_bootstrap"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://seaweedfs:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key
|
||||
S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
secrets: [s3_admin_access_key, s3_admin_secret_key, s3_access_key, s3_secret_key]
|
||||
depends_on:
|
||||
seaweedfs: {condition: service_healthy}
|
||||
restart: "no"
|
||||
networks: [backend]
|
||||
|
||||
# Брокер: durable quorum-очередь, persistent-сообщения, publisher confirms; данные на volume rabbitmqdata.
|
||||
# hostname фиксирован: имя узла RabbitMQ определяет каталог данных, иначе очередь не переживёт пересоздание контейнера.
|
||||
rabbitmq:
|
||||
<<: *common
|
||||
image: rabbitmq:4.1-alpine@sha256:6a0f7984185cd2cdbf78a184b4c17abe88ecbbda6006a7acf17dbc7e4307f4e2
|
||||
hostname: rabbitmq
|
||||
# RabbitMQ 4.x: RABBITMQ_DEFAULT_USER/PASS объявлены устаревшими -> учётные данные в конфигурационном файле (secret)
|
||||
secrets:
|
||||
- source: rabbitmq_conf
|
||||
target: /etc/rabbitmq/conf.d/20-credentials.conf
|
||||
mode: 0444
|
||||
volumes:
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 10s
|
||||
timeout: 8s
|
||||
retries: 12
|
||||
mem_limit: 512m
|
||||
cpus: ${RABBITMQ_CPUS:-1.0}
|
||||
networks: [backend]
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
restart: "no"
|
||||
networks: [backend]
|
||||
|
||||
api:
|
||||
<<: *common
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
environment:
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
VALKEY_URL: redis://valkey:6379/0
|
||||
S3_ENDPOINT: http://seaweedfs:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-4}
|
||||
MAX_INFLIGHT: ${MAX_INFLIGHT:-64}
|
||||
ACCESS_LOG: ${ACCESS_LOG:-1}
|
||||
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4}
|
||||
FAULT_INJECTION: ${API_FAULT_INJECTION:-}
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
MAX_QUEUED_JOBS: ${MAX_QUEUED_JOBS:-200}
|
||||
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
valkey: {condition: service_healthy}
|
||||
stop_grace_period: 15s
|
||||
mem_limit: ${API_MEM:-256m}
|
||||
cpus: ${API_CPUS:-1.0}
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
networks: [backend]
|
||||
|
||||
# Worker: отдельный процесс (тот же образ). Берёт задания из RabbitMQ, обрабатывает изображения, пишет результат в S3 и БД.
|
||||
worker:
|
||||
<<: *common
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["python", "-m", "app.worker"]
|
||||
environment:
|
||||
SERVICE_NAME: messenger-worker
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://seaweedfs:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
MAX_ATTEMPTS: ${MAX_ATTEMPTS:-5}
|
||||
LEASE_SECONDS: ${LEASE_SECONDS:-60}
|
||||
RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2}
|
||||
FAULT_INJECTION: ${WORKER_FAULT_INJECTION:-}
|
||||
ACCESS_LOG: "0"
|
||||
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
rabbitmq: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os,time,sys; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker.alive') < 30 else 1)"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
stop_grace_period: 30s
|
||||
mem_limit: ${WORKER_MEM:-768m}
|
||||
cpus: ${WORKER_CPUS:-1.0}
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
networks: [backend]
|
||||
|
||||
lb:
|
||||
<<: *common
|
||||
image: haproxy:3.0-alpine@sha256:56b887da77428b7a6621e59e480cdbd330cc805c22d3cedb66ceea76ffdea2c6
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
- api
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/healthz"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 6
|
||||
mem_limit: 128m
|
||||
cpus: ${LB_CPUS:-1.0}
|
||||
stop_grace_period: 10s
|
||||
networks: [backend]
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
seaweed_s3_config:
|
||||
file: ./secrets/seaweed_s3.json
|
||||
s3_admin_access_key:
|
||||
file: ./secrets/s3_admin_access_key
|
||||
s3_admin_secret_key:
|
||||
file: ./secrets/s3_admin_secret_key
|
||||
s3_access_key:
|
||||
file: ./secrets/s3_access_key
|
||||
s3_secret_key:
|
||||
file: ./secrets/s3_secret_key
|
||||
rabbitmq_conf:
|
||||
file: ./secrets/rabbitmq.conf
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
valkeydata:
|
||||
seaweeddata:
|
||||
rabbitmqdata:
|
||||
|
||||
networks:
|
||||
backend:
|
||||
@@ -0,0 +1,52 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: Кушнаренко Николай Вячеславович, БПИ2302
|
||||
Целевая оценка (3/4/5): 5
|
||||
Трек 5, если заявлен: Трек 1 — очередь под перегрузкой
|
||||
|
||||
Что реализовано и что перенесено:
|
||||
Из ЛР4 перенесены PostgreSQL, сессии/Valkey, две API-реплики за HAProxy, идемпотентность сообщений, приватный S3 и вложения. Upload возвращает 202 queued только после сохранения staging в S3 и одной транзакции attachments + outbox. Relay публикует durable задания в RabbitMQ с publisher confirm; отдельный worker создаёт JPEG thumbnail, crop 128×128 и optimized WebP, сохраняет результат и подтверждает сообщение после COMMIT. Есть ограниченные retry с backoff/jitter, lease, failed registry, DLQ, безопасный replay и очистка staging. Исправлен перенесённый reconciler: варианты и незавершённые эффекты worker больше не считаются orphan. Сохранено исправление HAProxy для контрактных ошибок при отказе общей БД.
|
||||
|
||||
Команды запуска из чистого clone (bash / Git Bash):
|
||||
|
||||
```sh
|
||||
sh scripts/init-secrets.sh
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 120
|
||||
python3 scripts/check_contract.py
|
||||
python3 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
|
||||
python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
|
||||
sh scripts/restore-replicas.sh
|
||||
```
|
||||
|
||||
Windows: PYTHONUTF8=1, hooks scripts/restart.cmd и scripts/stop-one.cmd. Все собственные проверки и исследование — [student-tests/README.md](https://gitea.privcloud.tech/kolya/mtusi2026lab5/src/branch/solution/bpi2302-kushnarenko/student-tests/README.md).
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md (повтор 12.11.2026):
|
||||
|
||||
- check_contract, полная OpenAPI-валидация, публичный smoke — PASS;
|
||||
- **60 собственных тестов — OK**, включая EXIF/alpha/размеры вариантов, оригинал/ACL, идемпотентность, безопасную сверку S3, TTL и отказы зависимостей;
|
||||
- семь сценариев pipeline: worker down, API/broker restart, broker down/outbox, crash COMMIT→publish, crash worker до COMMIT/ack, lease recovery, retry exhaustion и replay — OK;
|
||||
- persistence и failover — PASS; 121 успешных записей, 0 неуспешных попыток, восстановление 2.344 с;
|
||||
- дополнительный рестарт после нагрузки выявил раннюю готовность S3; hook исправлен и три последовательных persistence прошли. Проверено чтение 3670 объектов после восстановления; исходный сбой и повторы сохранены в evidence;
|
||||
- bootstrap дважды: приватность и ограниченные права API подтверждены;
|
||||
- трек 1: матрица 3×3 при worker 1 CPU / 768 MiB; 348/348 загрузок ready, failed=0, OOM нет, пик RSS 430.0 MiB; измерены throughput, p95, глубина и возраст очереди;
|
||||
- burst 200: принято 54, ответов 503 — 146 с реально проверенным Retry-After; все принятые ready; после разгрузки три новые загрузки тоже ready. Порог 30 мягкий, измеренный пик 48.
|
||||
|
||||
Отчёт: [REPORT.md](https://gitea.privcloud.tech/kolya/mtusi2026lab5/src/branch/solution/bpi2302-kushnarenko/REPORT.md), [Word](https://gitea.privcloud.tech/kolya/mtusi2026lab5/src/branch/solution/bpi2302-kushnarenko/docs/report_lab5.docx). Доказательства: [checks_2026-11-12.txt](https://gitea.privcloud.tech/kolya/mtusi2026lab5/src/branch/solution/bpi2302-kushnarenko/evidence/checks_2026-11-12.txt), [JSON и графики](https://gitea.privcloud.tech/kolya/mtusi2026lab5/src/branch/solution/bpi2302-kushnarenko/evidence/recheck_2026-11-12).
|
||||
|
||||
Ограничения / известные ошибки:
|
||||
Один узел PostgreSQL, RabbitMQ и S3; at-least-once доставка с идемпотентным эффектом. Версионирование нескольких pipeline как исследование трека 2 не заявлено. Ограничение очереди мягкое: конкурентный приём может превысить порог; строгая атомарная резервация слотов не реализована. Reconciler сохраняет неизвестные объекты внутри префикса существующего вложения; их удаление требует отдельного разбора. Замеры — по одному прогону на точку на одном компьютере с фоновой нагрузкой; устойчивый оптимум concurrency не доказан.
|
||||
|
||||
Источники и использование ИИ:
|
||||
Python, uvicorn, asyncpg, aiobotocore, aio-pika, Pillow, RabbitMQ, SeaweedFS, PostgreSQL, Valkey, HAProxy; первичные материалы перечислены в README задания. ИИ-ассистенты помогали с кодом, тестами и отчётом; Codex исправил перенос reconciler и методику исследования, проверил результаты реальными запусками.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены через эквивалентные Python-команды; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
Дополнительная сверка после открытия PR №52: в REPORT.md добавлены аппаратная конфигурация, прогрев, второй прогон с CPU/p50/p95/p99, проверка отказа обеих API-реплик и чистого clone. Добавлены отдельный application CI-job и LF для shell scripts. Серверный CI на Gitea не заявляется; реальные локальные результаты — evidence/pr52_audit_2026-11-12/.
|
||||
@@ -0,0 +1,49 @@
|
||||
$ python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
$ python3 tests/smoke.py (upload -> 202 queued -> ready, thumbnail 256x160)
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
# контейнеры до перезапуска:
|
||||
/mtusi-lab5-api-1 StartedAt=2026-10-02T23:58:35.341793495Z
|
||||
/mtusi-lab5-api-2 StartedAt=2026-10-02T23:58:35.523444484Z
|
||||
/mtusi-lab5-worker-1 StartedAt=2026-10-02T23:58:48.76269165Z
|
||||
/mtusi-lab5-rabbitmq-1 StartedAt=2026-10-02T23:53:22.014370477Z
|
||||
/mtusi-lab5-seaweedfs-1 StartedAt=2026-10-02T23:46:39.837097835Z
|
||||
/mtusi-lab5-postgres-1 StartedAt=2026-10-02T23:46:39.850517882Z
|
||||
/mtusi-lab5-valkey-1 StartedAt=2026-10-02T23:46:39.827928529Z
|
||||
$ python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh (Windows: restart.cmd)
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
# контейнеры после перезапуска:
|
||||
/mtusi-lab5-api-1 StartedAt=2026-10-02T23:59:33.742707516Z
|
||||
/mtusi-lab5-api-2 StartedAt=2026-10-02T23:59:32.383092893Z
|
||||
/mtusi-lab5-worker-1 StartedAt=2026-10-02T23:59:28.718812327Z
|
||||
/mtusi-lab5-rabbitmq-1 StartedAt=2026-10-02T23:59:29.238592291Z
|
||||
/mtusi-lab5-seaweedfs-1 StartedAt=2026-10-02T23:59:29.413843719Z
|
||||
/mtusi-lab5-postgres-1 StartedAt=2026-10-02T23:59:28.458564542Z
|
||||
/mtusi-lab5-valkey-1 StartedAt=2026-10-02T23:59:29.099168068Z
|
||||
$ python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh (1-я реплика)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 0, "first_recovery_seconds": 2.328}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
(2-я реплика)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 108, "failed_attempts": 0, "first_recovery_seconds": 2.328}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ python -m unittest test_api.py
|
||||
Ran 12 tests in 2.390s
|
||||
OK
|
||||
$ python -m unittest test_idempotency.py
|
||||
Ran 9 tests in 1.971s
|
||||
OK
|
||||
$ python -m unittest test_attachments.py
|
||||
Ran 11 tests in 5.686s
|
||||
OK
|
||||
$ python student-tests/test_pipeline.py (интерпретатор с Pillow: пиксельные проверки alpha -> белый фон)
|
||||
Ran 9 tests in 10.648s
|
||||
OK
|
||||
$ python student-tests/test_topology.py
|
||||
Ran 6 tests in 1.514s
|
||||
OK
|
||||
$ python student-tests/test_resilience5.py (окна отказа: worker/брокер/outbox/lease/retry/replay)
|
||||
Ran 7 tests in 139.354s
|
||||
OK
|
||||
@@ -0,0 +1,209 @@
|
||||
$ python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
exit=0
|
||||
|
||||
$ python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_api.py -v
|
||||
test_bearer_rules (test_api.AuthTest.test_bearer_rules) ... ok
|
||||
test_concurrent_registration_of_same_username (test_api.AuthTest.test_concurrent_registration_of_same_username) ... ok
|
||||
test_logout_revokes_in_database_and_cache (test_api.AuthTest.test_logout_revokes_in_database_and_cache) ... ok
|
||||
test_password_is_not_exposed_or_stored_plain (test_api.AuthTest.test_password_is_not_exposed_or_stored_plain) ... ok
|
||||
test_register_validation (test_api.AuthTest.test_register_validation) ... ok
|
||||
test_session_shape_entropy_and_logout (test_api.AuthTest.test_session_shape_entropy_and_logout) ... ok
|
||||
test_token_only_stored_as_hash (test_api.AuthTest.test_token_only_stored_as_hash) ... ok
|
||||
test_wrong_password_and_unknown_user_are_indistinguishable (test_api.AuthTest.test_wrong_password_and_unknown_user_are_indistinguishable) ... ok
|
||||
test_chat_creation_is_atomic (test_api.DataTest.test_chat_creation_is_atomic) ... ok
|
||||
test_foreign_chat_empty_message_bad_json (test_api.DataTest.test_foreign_chat_empty_message_bad_json) ... ok
|
||||
test_multi_page_history_and_cursor_binding (test_api.DataTest.test_multi_page_history_and_cursor_binding) ... ok
|
||||
test_parallel_writes_from_many_clients (test_api.DataTest.test_parallel_writes_from_many_clients) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 12 tests in 2.559s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_idempotency.py -v
|
||||
test_concurrent_same_key_creates_exactly_one_message_across_replicas (test_idempotency.IdempotencyTest.test_concurrent_same_key_creates_exactly_one_message_across_replicas) ... ok
|
||||
test_different_body_same_key_is_409 (test_idempotency.IdempotencyTest.test_different_body_same_key_is_409) ... ok
|
||||
test_expired_key_is_reusable_after_ttl (test_idempotency.IdempotencyTest.test_expired_key_is_reusable_after_ttl) ... ok
|
||||
test_key_validation (test_idempotency.IdempotencyTest.test_key_validation) ... ok
|
||||
test_replay_rechecks_session_and_membership (test_idempotency.IdempotencyTest.test_replay_rechecks_session_and_membership) ... ok
|
||||
test_replay_returns_same_message_and_creates_no_duplicate (test_idempotency.IdempotencyTest.test_replay_returns_same_message_and_creates_no_duplicate) ... ok
|
||||
test_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_scope_is_user_and_chat) ... ok
|
||||
test_semantically_equal_body_is_replayed (test_idempotency.IdempotencyTest.test_semantically_equal_body_is_replayed) ... ok
|
||||
test_session_is_shared_between_replicas_without_relogin (test_idempotency.IdempotencyTest.test_session_is_shared_between_replicas_without_relogin) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 9 tests in 1.806s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_attachments.py -v
|
||||
test_aborted_upload_leaves_nothing (test_attachments.AttachmentTest.test_aborted_upload_leaves_nothing) ... ok
|
||||
test_bucket_is_private_outside_the_api (test_attachments.AttachmentTest.test_bucket_is_private_outside_the_api) ... ok
|
||||
test_corrupted_and_spoofed_files_are_415_not_500 (test_attachments.AttachmentTest.test_corrupted_and_spoofed_files_are_415_not_500) ... ok
|
||||
test_dangerous_filenames_are_400 (test_attachments.AttachmentTest.test_dangerous_filenames_are_400) ... ok
|
||||
test_decompression_bomb_header_is_413 (test_attachments.AttachmentTest.test_decompression_bomb_header_is_413) ... ok
|
||||
test_empty_body_is_400 (test_attachments.AttachmentTest.test_empty_body_is_400) ... ok
|
||||
test_foreign_user_cannot_upload_read_or_download (test_attachments.AttachmentTest.test_foreign_user_cannot_upload_read_or_download) ... ok
|
||||
test_idempotent_replay_with_attachments (test_attachments.AttachmentTest.test_idempotent_replay_with_attachments) ... ok
|
||||
test_message_with_attachments_and_chat_binding (test_attachments.AttachmentTest.test_message_with_attachments_and_chat_binding) ... ok
|
||||
test_png_and_jpeg_roundtrip_with_checksum (test_attachments.AttachmentTest.test_png_and_jpeg_roundtrip_with_checksum) ... ok
|
||||
test_too_large_is_413_and_nothing_is_stored (test_attachments.AttachmentTest.test_too_large_is_413_and_nothing_is_stored) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 11 tests in 5.441s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_topology.py -v
|
||||
test_api_runs_as_non_root (test_topology.TopologyTest.test_api_runs_as_non_root) ... ok
|
||||
test_at_least_two_api_replicas_without_published_ports (test_topology.TopologyTest.test_at_least_two_api_replicas_without_published_ports) ... ok
|
||||
test_no_fixed_container_names (test_topology.TopologyTest.test_no_fixed_container_names) ... ok
|
||||
test_only_lb_is_published_and_only_on_localhost (test_topology.TopologyTest.test_only_lb_is_published_and_only_on_localhost) ... ok
|
||||
test_resource_limits_are_set (test_topology.TopologyTest.test_resource_limits_are_set) ... ok
|
||||
test_stop_grace_period_covers_drain (test_topology.TopologyTest.test_stop_grace_period_covers_drain) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 6 tests in 1.582s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_pipeline.py -v
|
||||
test_alpha_is_composited_on_white (test_pipeline.PipelineTest.test_alpha_is_composited_on_white) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab5\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab5\\student-tests\\fixtures\\alpha.png'>
|
||||
return open(os.path.join(FIX, name), "rb").read()
|
||||
ResourceWarning: Enable tracemalloc to get the object allocation traceback
|
||||
ok
|
||||
test_content_is_409_while_not_ready_and_original_is_untouched (test_pipeline.PipelineTest.test_content_is_409_while_not_ready_and_original_is_untouched) ... ok
|
||||
test_exif_orientation_is_applied (test_pipeline.PipelineTest.test_exif_orientation_is_applied) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab5\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab5\\student-tests\\fixtures\\exif_rot6.jpg'>
|
||||
return open(os.path.join(FIX, name), "rb").read()
|
||||
ResourceWarning: Enable tracemalloc to get the object allocation traceback
|
||||
ok
|
||||
test_message_can_reference_attachment_before_ready_and_failed_cannot (test_pipeline.PipelineTest.test_message_can_reference_attachment_before_ready_and_failed_cannot) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab5\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab5\\student-tests\\fixtures\\poison.png'>
|
||||
return open(os.path.join(FIX, name), "rb").read()
|
||||
ResourceWarning: Enable tracemalloc to get the object allocation traceback
|
||||
ok
|
||||
test_other_variants_visible_only_to_chat_members (test_pipeline.PipelineTest.test_other_variants_visible_only_to_chat_members) ... ok
|
||||
test_ready_job_redelivery_does_not_duplicate_variants (test_pipeline.PipelineTest.test_ready_job_redelivery_does_not_duplicate_variants) ... ok
|
||||
test_sample_png_produces_all_variants_per_contract (test_pipeline.PipelineTest.test_sample_png_produces_all_variants_per_contract) ... ok
|
||||
test_small_image_is_not_upscaled_but_crop_is_128 (test_pipeline.PipelineTest.test_small_image_is_not_upscaled_but_crop_is_128) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab5\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab5\\student-tests\\fixtures\\small.png'>
|
||||
return open(os.path.join(FIX, name), "rb").read()
|
||||
ResourceWarning: Enable tracemalloc to get the object allocation traceback
|
||||
ok
|
||||
test_staging_is_cleaned_after_ready_and_kept_for_failed (test_pipeline.PipelineTest.test_staging_is_cleaned_after_ready_and_kept_for_failed) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 9 tests in 10.719s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_reconcile5.py -v
|
||||
test_ready_variants_and_uncommitted_side_effect_are_preserved (test_reconcile5.ReconcilePipelineTest.test_ready_variants_and_uncommitted_side_effect_are_preserved) ... ok
|
||||
test_staging_is_protected_and_unowned_orphan_obeys_grace (test_reconcile5.ReconcilePipelineTest.test_staging_is_protected_and_unowned_orphan_obeys_grace) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 2 tests in 28.990s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_resilience5.py -v
|
||||
test_1_worker_down_upload_api_restart_worker_up (test_resilience5.ResilienceTest.test_1_worker_down_upload_api_restart_worker_up) ... ok
|
||||
test_2_broker_restart_keeps_accepted_jobs (test_resilience5.ResilienceTest.test_2_broker_restart_keeps_accepted_jobs) ... ok
|
||||
test_3_broker_down_during_acceptance_outbox_delivers_later (test_resilience5.ResilienceTest.test_3_broker_down_during_acceptance_outbox_delivers_later) ... ok
|
||||
test_4_api_crash_between_commit_and_publish (test_resilience5.ResilienceTest.test_4_api_crash_between_commit_and_publish) ... ok
|
||||
test_5_worker_crash_after_s3_before_commit_lease_recovery (test_resilience5.ResilienceTest.test_5_worker_crash_after_s3_before_commit_lease_recovery) ... ok
|
||||
test_6_worker_crash_after_commit_before_ack_redelivery_is_noop (test_resilience5.ResilienceTest.test_6_worker_crash_after_commit_before_ack_redelivery_is_noop) ... ok
|
||||
test_7_retry_exhaustion_registry_and_operator_replay (test_resilience5.ResilienceTest.test_7_retry_exhaustion_registry_and_operator_replay) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 7 tests in 141.784s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 2.344}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ docker compose --env-file .env start api
|
||||
Container mtusi-lab5-seaweedfs-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-seaweedfs-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Starting
|
||||
Container mtusi-lab5-s3-init-1 Starting
|
||||
Container mtusi-lab5-migrate-1 Started
|
||||
Container mtusi-lab5-s3-init-1 Started
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-s3-init-1 Waiting
|
||||
Container mtusi-lab5-valkey-1 Waiting
|
||||
Container mtusi-lab5-valkey-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-s3-init-1 Exited
|
||||
Container mtusi-lab5-api-1 Starting
|
||||
Container mtusi-lab5-api-1 Started
|
||||
exit=0
|
||||
|
||||
$ python student-tests/test_expiry.py
|
||||
..
|
||||
----------------------------------------------------------------------
|
||||
Ran 2 tests in 19.786s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python student-tests/test_dependencies.py
|
||||
..
|
||||
----------------------------------------------------------------------
|
||||
Ran 2 tests in 20.171s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ docker compose --env-file .env run --rm -T s3-init
|
||||
Container mtusi-lab5-seaweedfs-1 Running
|
||||
Container mtusi-lab5-seaweedfs-1 Waiting
|
||||
Container mtusi-lab5-seaweedfs-1 Healthy
|
||||
Container mtusi-lab5-s3-init-run-54461bd2fe34 Creating
|
||||
Container mtusi-lab5-s3-init-run-54461bd2fe34 Created
|
||||
bucket messenger-attachments уже существует
|
||||
анонимный доступ запрещён (HTTP 403)
|
||||
ключ API ограничен одним bucket (create_bucket отклонён: AccessDenied)
|
||||
bootstrap завершён
|
||||
exit=0
|
||||
|
||||
$ docker compose --env-file .env run --rm -T s3-init
|
||||
Container mtusi-lab5-seaweedfs-1 Running
|
||||
Container mtusi-lab5-seaweedfs-1 Waiting
|
||||
Container mtusi-lab5-seaweedfs-1 Healthy
|
||||
Container mtusi-lab5-s3-init-run-84c641a11c75 Creating
|
||||
Container mtusi-lab5-s3-init-run-84c641a11c75 Created
|
||||
bucket messenger-attachments уже существует
|
||||
анонимный доступ запрещён (HTTP 403)
|
||||
ключ API ограничен одним bucket (create_bucket отклонён: AccessDenied)
|
||||
bootstrap завершён
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
@@ -0,0 +1 @@
|
||||
PASS: normative files unchanged; PR title/base/head checked; matrix 348/348, CPU/p50/p95/p99, backpressure and recovery; both API replicas exercised; clean clone with new volumes; safe application CI YAML; source hashes matched
|
||||
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"MAX_QUEUED_JOBS": 30,
|
||||
"burst": 200,
|
||||
"attempt_statuses": {
|
||||
"202": 61,
|
||||
"503": 139
|
||||
},
|
||||
"accepted": 61,
|
||||
"peak_queue_depth": 57,
|
||||
"all_accepted_finished": true,
|
||||
"accepted_in_s": 0.6,
|
||||
"drained_after_s": 3.7,
|
||||
"retry_after_header_seen": true,
|
||||
"retry_after_values": [
|
||||
"5"
|
||||
],
|
||||
"transport_errors": [],
|
||||
"recovered_new_uploads_accepted": true,
|
||||
"recovered_new_uploads_ready": true,
|
||||
"no_accepted_job_lost": true,
|
||||
"ready": 61,
|
||||
"failed": 0,
|
||||
"p50_s": 1.46,
|
||||
"p95_s": 2.31,
|
||||
"p99_s": 2.4,
|
||||
"max_s": 2.4,
|
||||
"throughput_per_s": 22.15,
|
||||
"span_s": 2.8,
|
||||
"measurement_window_s": 3.72,
|
||||
"worker_cpu_seconds": 1.32,
|
||||
"worker_cpu_percent_one_core": 35.49,
|
||||
"http_non_2xx_fraction": 0.695,
|
||||
"accepted_http_rps_over_window": 16.4,
|
||||
"attachment_ids": [
|
||||
"66ea19e2-e3e8-4d74-acfc-b3a9429f997c",
|
||||
"72f006e1-db6c-49f0-bcea-a5062bfe4e94",
|
||||
"a8754c2b-bb4e-41de-8569-a1509672c382",
|
||||
"273fd71e-1d8b-478a-9fb7-6e4f17f85323",
|
||||
"c7963cf2-4c2f-4dc1-890c-f400d78f97e6",
|
||||
"076e695d-ea2d-4a93-a3a9-638f0f5edd64",
|
||||
"b2fa531f-0cff-4869-b8fd-63c543cdef03",
|
||||
"0dc2753b-470f-4d8c-9f6f-46e72d18017b",
|
||||
"d691e18c-fd06-420d-9b3a-4676300f933b",
|
||||
"22349666-78df-48d7-805b-233357cfcd28",
|
||||
"ab17e0d7-6b8b-4dbc-b1a5-00c60f82dcd4",
|
||||
"e76b055b-de11-4ea1-a47b-d687672791ce",
|
||||
"768f98f9-3717-4ad7-81c7-3e5505e55f62",
|
||||
"5b5d86da-d290-4ec4-bbbf-9bc0b1f3a501",
|
||||
"8dad57b3-1a3a-409f-a4eb-60de9f0aaa09",
|
||||
"e8736a1c-b13f-47f7-a85f-cd6ca415209b",
|
||||
"c0a44b33-e8bf-4c5d-9e59-ccda96cdd1da",
|
||||
"504cab76-9082-4801-a6d0-87e4f4914352",
|
||||
"1d02876b-89ae-4380-a460-89fb29e55a93",
|
||||
"c6adc9c6-def5-49cd-900b-d63ecadaaad8",
|
||||
"989bd05b-eccc-4b7d-b806-0c468a331e9a",
|
||||
"1da0e1c5-ec99-4884-a3ad-0fdaae4dbc6c",
|
||||
"d6d37e06-6c8d-4d44-af3b-45fb22e88a59",
|
||||
"d9faea0c-2f02-4b76-8b02-44623b36b808",
|
||||
"a89cbba3-c4a8-4c9a-afec-c099ae7e46d3",
|
||||
"d9f0f0b5-70d1-4d09-a298-06a947b2a86d",
|
||||
"49b1c6e4-ba7f-49ac-9616-2d762185fd22",
|
||||
"452cd743-2644-46f8-9535-23c9122efc84",
|
||||
"94e6c1b6-1bdf-4b1d-b7bf-6812b67b7d05",
|
||||
"6d5279e2-eb5b-4757-befe-bf0d954dabb9",
|
||||
"447023af-7290-487c-8f52-8645a127d540",
|
||||
"08f41743-156e-44f0-9ad3-ab99cedd23ca",
|
||||
"c02cc601-04f5-4a81-bdcb-5a557395a506",
|
||||
"71c12d21-aaea-473f-8565-30014caa52e7",
|
||||
"68e91241-0f9a-4c4e-9080-5862bcbc472e",
|
||||
"3afb2d2e-31c9-4f70-aa3c-34393af6dc92",
|
||||
"a3014daa-2802-44a5-85fb-a86d74d2198d",
|
||||
"c205fb0f-183b-40e7-9ffa-6cb1fefff297",
|
||||
"af5996bf-3182-4778-920b-eeee2009cbce",
|
||||
"6fa05261-09b2-41ea-8ec3-8cf11d5e8a25",
|
||||
"976193b5-f91e-47cc-87f1-e0d8dc5347d1",
|
||||
"9ff6485e-6b41-4384-af91-1014fb114cc5",
|
||||
"0edaa06c-9403-48bc-bc1e-311e32762134",
|
||||
"79a51b1c-082f-48d5-acbf-f912f8792b54",
|
||||
"9cd5ff6b-ca74-4679-959e-7630f6ff3dc0",
|
||||
"10480ccc-186b-4586-bb2d-b3bdab50a2ea",
|
||||
"db1d6478-cbf2-47a9-b6ad-ca739b3d43aa",
|
||||
"f5d2e3ce-6e8d-4dad-8ee6-6905d9053867",
|
||||
"c36fab10-726f-4556-8fae-595ef41ad828",
|
||||
"76f0ac29-0fe9-495d-9873-620dd43019c1",
|
||||
"84a99df5-ca4d-492f-bfee-d8efa996b896",
|
||||
"9cbb0bc9-2700-4813-95c1-6123cec043b1",
|
||||
"e3a3eda1-1cbe-4e1a-883d-0420199073c6",
|
||||
"5dc6f8e5-235a-4072-a7c9-36623d163ff3",
|
||||
"5acb6911-ba65-4545-943d-68ed3c7b5c29",
|
||||
"710959d2-f69b-4bc6-a033-cb412c701f92",
|
||||
"99e4d956-77be-4501-b036-78978c13479d",
|
||||
"472b87fb-87ac-4d75-805b-21232db08a8c",
|
||||
"b8f2ec4e-4734-4267-83e3-3acf47c86cb9",
|
||||
"f127c53e-d159-41b8-8cc3-5f0dcf6ef1fd",
|
||||
"793bc8ef-82ff-45ea-adb2-c799164ce621"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"rounds": [
|
||||
{
|
||||
"round": 1,
|
||||
"stopped": "mtusi-lab5-api-2",
|
||||
"public_test": "PASS"
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"stopped": "mtusi-lab5-api-1",
|
||||
"public_test": "PASS"
|
||||
}
|
||||
],
|
||||
"distinct_stopped": [
|
||||
"mtusi-lab5-api-1",
|
||||
"mtusi-lab5-api-2"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
Clean clone commit: 2773115e1cc7ce4e17a61eddaaf35300021c11a8
|
||||
Existing volumes: 0
|
||||
Windows autocrlf=true; shell scripts LF
|
||||
$ docker compose stop
|
||||
Container mtusi-lab5-worker-1 Stopping
|
||||
Container mtusi-lab5-lb-1 Stopping
|
||||
Container mtusi-lab5-lb-1 Stopped
|
||||
Container mtusi-lab5-api-1 Stopping
|
||||
Container mtusi-lab5-api-2 Stopping
|
||||
Container mtusi-lab5-worker-1 Stopped
|
||||
Container mtusi-lab5-rabbitmq-1 Stopping
|
||||
Container mtusi-lab5-rabbitmq-1 Stopped
|
||||
Container mtusi-lab5-api-2 Stopped
|
||||
Container mtusi-lab5-api-1 Stopped
|
||||
Container mtusi-lab5-migrate-1 Stopping
|
||||
Container mtusi-lab5-s3-init-1 Stopping
|
||||
Container mtusi-lab5-valkey-1 Stopping
|
||||
Container mtusi-lab5-s3-init-1 Stopped
|
||||
Container mtusi-lab5-migrate-1 Stopped
|
||||
Container mtusi-lab5-seaweedfs-1 Stopping
|
||||
Container mtusi-lab5-postgres-1 Stopping
|
||||
Container mtusi-lab5-valkey-1 Stopped
|
||||
Container mtusi-lab5-postgres-1 Stopped
|
||||
Container mtusi-lab5-seaweedfs-1 Stopped
|
||||
exit=0
|
||||
|
||||
$ C:/Program Files/Git/bin/bash.exe scripts/init-secrets.sh
|
||||
exit=0
|
||||
|
||||
$ docker compose --env-file .env config --quiet
|
||||
exit=0
|
||||
|
||||
$ docker compose --env-file .env up -d --build --wait --wait-timeout 120
|
||||
Image mtusi-messenger:lab5 Building
|
||||
Image mtusi-messenger:lab5 Building
|
||||
Image mtusi-messenger:lab5 Building
|
||||
Image mtusi-messenger:lab5 Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 2.18kB 0.0s done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [migrate internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 1.91kB done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [s3-init] resolve image config for docker-image://docker.io/docker/dockerfile:1
|
||||
#3 DONE 4.5s
|
||||
|
||||
#4 [s3-init] docker-image://docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e
|
||||
#4 resolve docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e 0.0s done
|
||||
#4 CACHED
|
||||
|
||||
#5 [api internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
|
||||
#5 DONE 1.8s
|
||||
|
||||
#6 [s3-init internal] load .dockerignore
|
||||
#6 transferring context: 289B done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [worker builder 1/4] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
|
||||
#7 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c 0.0s done
|
||||
#7 DONE 0.0s
|
||||
|
||||
#8 [api internal] load build context
|
||||
#8 transferring context: 136.33kB 0.0s done
|
||||
#8 DONE 0.1s
|
||||
|
||||
#8 [worker internal] load build context
|
||||
#8 DONE 0.1s
|
||||
|
||||
#9 [worker builder 4/4] RUN python -m venv /opt/venv && /opt/venv/bin/pip install -r requirements.txt
|
||||
#9 CACHED
|
||||
|
||||
#10 [worker builder 2/4] WORKDIR /build
|
||||
#10 CACHED
|
||||
|
||||
#11 [worker stage-1 3/6] COPY --from=builder /opt/venv /opt/venv
|
||||
#11 CACHED
|
||||
|
||||
#12 [worker stage-1 2/6] RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home app
|
||||
#12 CACHED
|
||||
|
||||
#13 [worker builder 3/4] COPY requirements.txt .
|
||||
#13 CACHED
|
||||
|
||||
#14 [worker stage-1 4/6] WORKDIR /srv
|
||||
#14 CACHED
|
||||
|
||||
#15 [s3-init stage-1 5/6] COPY src/ /srv/src/
|
||||
#15 DONE 0.0s
|
||||
|
||||
#16 [migrate stage-1 6/6] COPY migrations/ /srv/migrations/
|
||||
#16 DONE 0.1s
|
||||
|
||||
#16 [api stage-1 6/6] COPY migrations/ /srv/migrations/
|
||||
#16 DONE 0.1s
|
||||
|
||||
#17 [api] exporting to image
|
||||
#17 exporting layers
|
||||
#17 exporting layers 0.2s done
|
||||
#17 exporting manifest sha256:7c8a2e7f6d3531b54acce69267a6f15bfde9011a86134cb8f9b3b62ab7fc6b1d 0.1s done
|
||||
#17 exporting config sha256:550a2137f427b5e241f100b4201ff0ab516965bcc9807094320eb0a2a39381d8 0.1s done
|
||||
#17 exporting attestation manifest sha256:673785b83b53b75bbfecf57f523e82e0ecb01284bc04921a0ddfdc003169837f
|
||||
#17 exporting attestation manifest sha256:673785b83b53b75bbfecf57f523e82e0ecb01284bc04921a0ddfdc003169837f 0.1s done
|
||||
#17 exporting manifest list sha256:93039b4a256e412514d09b22e342dadece55558c809948be2d0246b08e584fe7
|
||||
#17 exporting manifest list sha256:93039b4a256e412514d09b22e342dadece55558c809948be2d0246b08e584fe7 0.1s done
|
||||
#17 naming to docker.io/library/mtusi-messenger:lab5 0.0s done
|
||||
#17 unpacking to docker.io/library/mtusi-messenger:lab5
|
||||
#17 unpacking to docker.io/library/mtusi-messenger:lab5 0.2s done
|
||||
#17 DONE 0.8s
|
||||
|
||||
#18 [worker] exporting to image
|
||||
#18 exporting layers 0.2s done
|
||||
#18 exporting manifest sha256:aeab46f0e41076c208e8475321742224b4eef5fc9aac488ebb43e6a0dc78aa09 0.1s done
|
||||
#18 exporting config sha256:60adae1d7c2de7901745fe23b102ff8e113752f4f0c1eadab00be936eadf0d18 0.1s done
|
||||
#18 exporting attestation manifest sha256:c12e12514bd2bc3c2475af2bc996779405ea8d18a631d5dd728977d6b4e363cb 0.1s done
|
||||
#18 exporting manifest list sha256:b06cee700bb6b70f15aefd37f7a76c5e2d6b83a04c470a66c17d79c3e074adde 0.1s done
|
||||
#18 naming to docker.io/library/mtusi-messenger:lab5 0.0s done
|
||||
#18 unpacking to docker.io/library/mtusi-messenger:lab5 0.2s done
|
||||
#18 DONE 0.9s
|
||||
|
||||
#19 [s3-init] exporting to image
|
||||
#19 exporting layers 0.2s done
|
||||
#19 exporting manifest sha256:e79158153c8204a73a2036a5d97f01d2f5661f6ef7c0dfa28e59b2d195c97e2a 0.1s done
|
||||
#19 exporting config sha256:522ce18dbcf909998851355da58b1e8603c36270c648d08350dee73c291b7411 0.1s done
|
||||
#19 exporting attestation manifest sha256:f800544448d358811c6df265f36d6fd267861aa992cb3fd1b1efe2ed28f5cdaa 0.1s done
|
||||
#19 exporting manifest list sha256:dc8d60b8c8800df1f411bf267146709e95328a7d69a02e63e4b435d17bb99fb7 0.1s done
|
||||
#19 naming to docker.io/library/mtusi-messenger:lab5 0.0s done
|
||||
#19 unpacking to docker.io/library/mtusi-messenger:lab5 0.2s done
|
||||
#19 DONE 0.9s
|
||||
|
||||
#20 [migrate] exporting to image
|
||||
#20 exporting layers 0.2s done
|
||||
#20 exporting manifest sha256:b19127753bc5ace630dbed8ac8a8d06bbb2602c70cfa4297eb13279a056f6cf1 0.1s done
|
||||
#20 exporting config sha256:d247ea63b01ed555f888bd4a1c06d7ff1abf6fd53306bf89696ea803ff769a00 0.1s done
|
||||
#20 exporting attestation manifest sha256:aff4ad1cadd4ce9f4a68f09775d5293dd008682956ac13026ee1b621732d9281 0.1s done
|
||||
#20 exporting manifest list sha256:79dfbef57710eafe62b233b48989fb3913e740274c40f663f0dd382d3d396b42 0.1s done
|
||||
#20 naming to docker.io/library/mtusi-messenger:lab5 0.0s done
|
||||
#20 unpacking to docker.io/library/mtusi-messenger:lab5 0.1s done
|
||||
#20 DONE 0.9s
|
||||
|
||||
#21 [api] resolving provenance for metadata file
|
||||
#21 DONE 0.2s
|
||||
|
||||
#22 [worker] resolving provenance for metadata file
|
||||
#22 DONE 0.2s
|
||||
|
||||
#23 [s3-init] resolving provenance for metadata file
|
||||
#23 DONE 0.1s
|
||||
|
||||
#24 [migrate] resolving provenance for metadata file
|
||||
#24 DONE 0.0s
|
||||
Image mtusi-messenger:lab5 Built
|
||||
Image mtusi-messenger:lab5 Built
|
||||
Image mtusi-messenger:lab5 Built
|
||||
Image mtusi-messenger:lab5 Built
|
||||
Network mtusi-lab5-clean-pr52_backend Creating
|
||||
Network mtusi-lab5-clean-pr52_backend Created
|
||||
Volume mtusi-lab5-clean-pr52_valkeydata Creating
|
||||
Volume mtusi-lab5-clean-pr52_valkeydata Created
|
||||
Volume mtusi-lab5-clean-pr52_rabbitmqdata Creating
|
||||
Volume mtusi-lab5-clean-pr52_rabbitmqdata Created
|
||||
Volume mtusi-lab5-clean-pr52_pgdata Creating
|
||||
Volume mtusi-lab5-clean-pr52_pgdata Created
|
||||
Volume mtusi-lab5-clean-pr52_seaweeddata Creating
|
||||
Volume mtusi-lab5-clean-pr52_seaweeddata Created
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Creating
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Creating
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Creating
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Creating
|
||||
time="2026-11-12T04:10:46+03:00" level=warning msg="secrets `uid`, `gid` and `mode` are not supported, they will be ignored"
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Created
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Created
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Created
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Creating
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Creating
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Created
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Created
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Created
|
||||
Container mtusi-lab5-clean-pr52-worker-1 Creating
|
||||
Container mtusi-lab5-clean-pr52-api-2 Creating
|
||||
Container mtusi-lab5-clean-pr52-api-1 Creating
|
||||
Container mtusi-lab5-clean-pr52-api-2 Created
|
||||
Container mtusi-lab5-clean-pr52-worker-1 Created
|
||||
Container mtusi-lab5-clean-pr52-api-1 Created
|
||||
Container mtusi-lab5-clean-pr52-lb-1 Creating
|
||||
Container mtusi-lab5-clean-pr52-lb-1 Created
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Starting
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Starting
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Starting
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Starting
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Started
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Started
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Started
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Started
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Starting
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Starting
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Started
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Started
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Exited
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Exited
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Exited
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Exited
|
||||
Container mtusi-lab5-clean-pr52-api-1 Starting
|
||||
Container mtusi-lab5-clean-pr52-api-1 Started
|
||||
Container mtusi-lab5-clean-pr52-api-2 Starting
|
||||
Container mtusi-lab5-clean-pr52-api-2 Started
|
||||
Container mtusi-lab5-clean-pr52-lb-1 Starting
|
||||
Container mtusi-lab5-clean-pr52-lb-1 Started
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-worker-1 Starting
|
||||
Container mtusi-lab5-clean-pr52-worker-1 Started
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-worker-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-lb-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-api-1 Waiting
|
||||
Container mtusi-lab5-clean-pr52-api-2 Waiting
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Exited
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Exited
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-lb-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-api-1 Healthy
|
||||
Container mtusi-lab5-clean-pr52-api-2 Healthy
|
||||
Container mtusi-lab5-clean-pr52-worker-1 Healthy
|
||||
exit=0
|
||||
|
||||
$ python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ C:/Program Files/Git/bin/bash.exe -n scripts/restart.sh
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
|
||||
PASS: fresh clone, new secrets/volumes, build, readiness, contract, smoke, persistence
|
||||
$ docker compose stop
|
||||
Container mtusi-lab5-clean-pr52-worker-1 Stopping
|
||||
Container mtusi-lab5-clean-pr52-lb-1 Stopping
|
||||
Container mtusi-lab5-clean-pr52-lb-1 Stopped
|
||||
Container mtusi-lab5-clean-pr52-api-2 Stopping
|
||||
Container mtusi-lab5-clean-pr52-api-1 Stopping
|
||||
Container mtusi-lab5-clean-pr52-worker-1 Stopped
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Stopping
|
||||
Container mtusi-lab5-clean-pr52-rabbitmq-1 Stopped
|
||||
Container mtusi-lab5-clean-pr52-api-2 Stopped
|
||||
Container mtusi-lab5-clean-pr52-api-1 Stopped
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Stopping
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Stopping
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Stopping
|
||||
Container mtusi-lab5-clean-pr52-migrate-1 Stopped
|
||||
Container mtusi-lab5-clean-pr52-s3-init-1 Stopped
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Stopping
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Stopping
|
||||
Container mtusi-lab5-clean-pr52-postgres-1 Stopped
|
||||
Container mtusi-lab5-clean-pr52-valkey-1 Stopped
|
||||
Container mtusi-lab5-clean-pr52-seaweedfs-1 Stopped
|
||||
exit=0
|
||||
|
||||
$ docker compose up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb
|
||||
Container mtusi-lab5-seaweedfs-1 Starting
|
||||
Container mtusi-lab5-postgres-1 Starting
|
||||
Container mtusi-lab5-valkey-1 Starting
|
||||
Container mtusi-lab5-rabbitmq-1 Starting
|
||||
Container mtusi-lab5-valkey-1 Started
|
||||
Container mtusi-lab5-valkey-1 Waiting
|
||||
Container mtusi-lab5-seaweedfs-1 Started
|
||||
Container mtusi-lab5-postgres-1 Started
|
||||
Container mtusi-lab5-rabbitmq-1 Started
|
||||
Container mtusi-lab5-rabbitmq-1 Waiting
|
||||
Container mtusi-lab5-valkey-1 Healthy
|
||||
Container mtusi-lab5-api-2 Starting
|
||||
Container mtusi-lab5-api-2 Started
|
||||
Container mtusi-lab5-api-1 Starting
|
||||
Container mtusi-lab5-api-1 Started
|
||||
Container mtusi-lab5-lb-1 Starting
|
||||
Container mtusi-lab5-lb-1 Started
|
||||
Container mtusi-lab5-rabbitmq-1 Healthy
|
||||
Container mtusi-lab5-worker-1 Starting
|
||||
Container mtusi-lab5-worker-1 Started
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-lb-1 Waiting
|
||||
Container mtusi-lab5-valkey-1 Waiting
|
||||
Container mtusi-lab5-rabbitmq-1 Waiting
|
||||
Container mtusi-lab5-seaweedfs-1 Waiting
|
||||
Container mtusi-lab5-worker-1 Waiting
|
||||
Container mtusi-lab5-api-2 Waiting
|
||||
Container mtusi-lab5-api-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-rabbitmq-1 Healthy
|
||||
Container mtusi-lab5-lb-1 Healthy
|
||||
Container mtusi-lab5-valkey-1 Healthy
|
||||
Container mtusi-lab5-api-2 Healthy
|
||||
Container mtusi-lab5-api-1 Healthy
|
||||
Container mtusi-lab5-seaweedfs-1 Healthy
|
||||
Container mtusi-lab5-worker-1 Healthy
|
||||
exit=0
|
||||
|
||||
$ docker compose exec -T api python -m app.s3_recovery
|
||||
PASS: S3 recovery, 5334 existing objects readable
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
@@ -0,0 +1,49 @@
|
||||
$ python -u research/queue_overload.py --out evidence\pr52_audit_2026-11-12
|
||||
{'small': '109 KiB', 'medium': '1961 KiB', 'large': '3100 KiB'}
|
||||
{"concurrency": 1, "image": "small", "image_kib": 109, "uploads": 80, "accepted": 80, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 80}, "transport_errors": [], "accepted_in_s": 0.4, "all_finished": true, "peak_queue_depth": 66, "peak_oldest_age_s": 2.7, "worker_peak_rss_mib": 72.7, "worker_oom_killed": "false", "ready": 80, "failed": 0, "p50_s": 1.4, "p95_s": 2.57, "p99_s": 2.71, "max_s": 2.71, "throughput_per_s": 26.26, "span_s": 3.0, "measurement_window_s": 3.488, "worker_cpu_seconds": 1.59, "worker_cpu_percent_one_core": 45.59, "http_non_2xx_fraction": 0.0, "accepted_http_rps_over_window": 22.94, "attachment_ids": ["f3755317-a6f1-4c38-958e-572d2e72521c", "e1866acf-772a-4e44-9f6d-96d6d6c855ae", "4c88f7ec-94f3-42ef-a80d-2cd5506f288f", "9d76e306-5c0f-4dc0-b417-0e328d8c7ff5", "5d7435aa-9667-4b0d-b772-2a5cd07c6d18", "91734a85-8d57-4ae7-9f80-023bb9cde252", "321ea613-76a4-4ce7-a2e8-da57f8d6fa0e", "d5903675-86be-41fb-8d23-79835bc7d3e7", "dfdc7b08-fedf-45b7-9ebd-a15d00670784", "2e2ec815-68f6-444d-b83f-094c1af5713b", "80d48257-49e5-41df-a73c-1d31715ad117", "816c620e-82d5-4feb-9bdf-def2dd4da03f", "c9b770c8-de70-4476-81eb-1e872ddee1dc", "90239c75-8017-4e4e-b098-3f2901c46f2d", "a29229ce-376b-43a2-bb2e-bc4457e47a5f", "5b45de93-1b7b-4167-a739-07c494cb6d9d", "a35306c2-6cd2-4745-9aab-c5acfaf902ca", "85996ad6-fe4e-41b1-b42d-bb524176f64e", "2f3c4a2c-462b-4d53-b1d2-cb463b63b265", "7315c4f5-397d-41bd-8ada-1199b5ff8d20", "5ca87add-c93e-4bee-b3a6-e28b51b42d28", "93dcb192-6666-45cf-a55d-df0959d6bddd", "d2b44888-c498-4900-9814-340f27e79c12", "bb7b4f52-ffb4-4b75-b271-cfa705c14e50", "5df28e25-a9b3-47f2-92f5-71bd048959fb", "cc44923d-6e23-43bd-8ea0-3a158de52a58", "b352814b-b29c-4043-b862-670fcfe50002", "21ab21a9-1959-4432-8283-eed80a3ae6cc", "d33a072f-937f-4873-8dbc-3eaba5e9a06b", "e1630cdc-c746-4f95-8712-0f0ec4501224", "e6680ab1-e568-4cf3-b372-e165553c8277", "bdd7584a-d8f7-4420-b2f9-127aac3f67e4", "1024b092-18f7-42ad-8269-15137dcb07b5", "c2623c1a-eec4-43c0-a870-4202f1b97542", "6a16b4e3-e6ef-45ef-8a2a-898e0dc590d0", "594a7d05-50f3-4d5c-9114-1a5383869296", "d6c712a8-e930-4e45-862b-2ee20ea4c380", "9147c9ce-32dd-458e-bbda-762bbe1c60de", "902a3269-9937-4c44-91ce-058e67005c87", "df198815-5898-4293-bb85-087679a4efbf", "8fd86d7e-efbe-4c84-8f17-cd03e510f508", "67489317-2bcf-4be0-a97b-52afce0e7c44", "28185ccf-516a-45d8-b851-ae202869b7cc", "bff9004a-3bc8-44df-ab54-2295c3e7e27c", "c0ea5554-7a66-49f2-8ec3-c4fde067abdf", "4a126067-ce1a-4c57-8817-cf2854e40cde", "5a76425e-88ce-4e1d-b335-b0ef76cd88aa", "8ea27a23-6e62-47d4-8523-a6ca1f5ad458", "6e03b1c2-d4a7-4772-89cf-f41ffde6db55", "469e7a1d-3ac9-4224-8661-0e57e3c37d03", "81b4d8b6-7e29-4fe1-b062-15c87101cbfd", "64399d7a-7df1-45fa-b7a7-653e5da60bd0", "b21be428-e7d0-4417-9023-ffd232ba830a", "56029d32-3104-4255-8006-cd775fb22a1f", "84308a9f-59ba-4797-8e42-a990ebf2fefc", "77f0efa1-b5d6-481d-b1a6-a8317af5201b", "5d3d0106-8851-4cc0-9fa8-178e7a4e1852", "190e83e7-1c5a-4cee-8447-5315a54ab817", "ab5363c6-da59-4565-a074-1b3acb1433d0", "19e569fa-412b-4fef-9e89-2c4218293613", "90fb1725-33a3-4011-b430-f28f188d500c", "1bf62c59-0ad5-4225-ac03-9bd5a4965900", "dac8a042-a731-4bdf-92f7-cbf6377a3496", "6370b8dd-8ce2-4e9e-8b20-cdc07d8a4218", "3e3a0ca8-756b-4c9c-a42b-55d0ca984f31", "63f82f8f-419a-4fe3-8aff-180249c946ec", "c2a64395-4403-418f-a4ea-5e32689e4dc1", "6cecaaa4-3159-400d-9e28-e7c0dd26bcf6", "5e5567e8-9d1a-401b-bf15-d7fb2ac4c6a3", "1c4371d1-39c1-475d-9055-63c3823f9859", "c6ab5b7e-2c37-4e42-8abc-3bfe922370d8", "8a52c2ca-e625-4549-9661-f89d3c973c7e", "e6993de3-6dd1-40ed-8aa0-7f811d311087", "3a3bd631-10b6-478f-b25b-575053f0088d", "a906674b-bd9c-4b34-9c6e-d25e23d453fe", "d4aee6be-963c-4803-956b-0dcce87c6dd2", "4510f909-6250-4a54-9488-8c0edd064353", "19a3b5ac-5784-4e91-95cf-a95b66de7d23", "c25cd565-459d-4ec1-83bf-d21859dd3634", "d28d6581-7aef-4762-9d4b-3ae016f33832"]}
|
||||
{"concurrency": 1, "image": "medium", "image_kib": 1961, "uploads": 24, "accepted": 24, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 24}, "transport_errors": [], "accepted_in_s": 0.5, "all_finished": true, "peak_queue_depth": 23, "peak_oldest_age_s": 5.4, "worker_peak_rss_mib": 133.7, "worker_oom_killed": "false", "ready": 24, "failed": 0, "p50_s": 2.78, "p95_s": 5.23, "p99_s": 5.44, "max_s": 5.44, "throughput_per_s": 4.09, "span_s": 5.9, "measurement_window_s": 6.264, "worker_cpu_seconds": 5.15, "worker_cpu_percent_one_core": 82.21, "http_non_2xx_fraction": 0.0, "accepted_http_rps_over_window": 3.83, "attachment_ids": ["d336e33f-283a-49e9-8ca9-9c4d0f17c904", "dfa1d79f-ffe8-4937-8c46-c456143a9394", "a8bb7c8b-931c-492e-bee2-5b9920e6355e", "6de05c1c-ca5a-4b86-8dd3-af63b1aa321f", "dd696951-5e88-44b4-80a2-c868326a2102", "4253bdfd-86c2-4116-ad68-72f685481005", "7afb0faf-2c94-4014-8be5-07731cdfb483", "624e290a-0463-417d-8e53-448a75bfcd31", "0b6f4492-3378-42bd-8fd4-47983dd7a54d", "c4fa5d79-c3f1-48d1-98bb-bf4e73c55efe", "8c25109b-d8ba-4608-a1d7-1d7abc1f47b8", "0361352b-61aa-41c0-ba62-0e767f954a62", "bcf3989b-f1fe-4619-962c-34d22a984261", "80dd3ed7-6e10-4dfd-aca5-3a3e4382326a", "9ca80d24-2c9f-44ac-8811-deeabcb8ff0f", "f6c7cc70-25ba-4a5a-aa8d-75f22a8ce836", "0e8d2721-f293-4ccc-9d10-5873c8fd90b9", "95ce7a82-0133-4a9d-9899-dbed974ee9ec", "5fd7ea2c-685f-4cfc-9f0b-dabaeba4129d", "773dfbf2-fc82-4888-8208-032b25035a4c", "d3ba0d4a-952b-4095-821c-398fce6165f0", "9362cf03-cdd4-482c-8786-42379ce6bbf0", "d3258e0e-78b6-40ea-86b8-9403d2d8671f", "ceaaa543-c950-4398-af14-924025c652bb"]}
|
||||
{"concurrency": 1, "image": "large", "image_kib": 3100, "uploads": 12, "accepted": 12, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 12}, "transport_errors": [], "accepted_in_s": 0.4, "all_finished": true, "peak_queue_depth": 11, "peak_oldest_age_s": 3.3, "worker_peak_rss_mib": 160.9, "worker_oom_killed": "false", "ready": 12, "failed": 0, "p50_s": 1.73, "p95_s": 3.34, "p99_s": 3.34, "max_s": 3.34, "throughput_per_s": 3.33, "span_s": 3.6, "measurement_window_s": 4.058, "worker_cpu_seconds": 3.05, "worker_cpu_percent_one_core": 75.16, "http_non_2xx_fraction": 0.0, "accepted_http_rps_over_window": 2.96, "attachment_ids": ["cd228696-224b-4b17-87c7-f0af12762e19", "0c3b34a6-c495-4450-ba46-bfe72d46b941", "00a409ba-8b6a-4021-aad5-3f717b9133ee", "f239a54e-13c6-4252-bb47-0e513b310008", "e0e8e733-b7cd-48e7-884e-ea54cac614de", "51498e5d-f689-47f9-930d-9c21c7c7cc28", "12591ec9-056b-483a-b9a4-b81a3ad6f9ae", "b2b0d9b2-616a-4874-81ef-18b5b631f7cc", "262d2c63-d351-41d4-a641-4fcb86a28df5", "62c53389-0ec2-4b52-89d9-cc4734867d36", "a0631868-6ed3-4ce9-8adf-8518ec884e13", "95f1942f-300a-43b4-a4cd-7915f2291552"]}
|
||||
{"concurrency": 2, "image": "small", "image_kib": 109, "uploads": 80, "accepted": 80, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 80}, "transport_errors": [], "accepted_in_s": 0.3, "all_finished": true, "peak_queue_depth": 52, "peak_oldest_age_s": 1.2, "worker_peak_rss_mib": 77.0, "worker_oom_killed": "false", "ready": 80, "failed": 0, "p50_s": 0.83, "p95_s": 1.57, "p99_s": 1.65, "max_s": 1.65, "throughput_per_s": 43.4, "span_s": 1.8, "measurement_window_s": 2.687, "worker_cpu_seconds": 1.53, "worker_cpu_percent_one_core": 56.93, "http_non_2xx_fraction": 0.0, "accepted_http_rps_over_window": 29.77, "attachment_ids": ["986c0488-655c-45ee-80b0-ffa2ec88e1b0", "5356f279-0ff8-46d0-9597-6bf80decdba8", "59374d10-af30-4ee2-a9e0-748f64b9c843", "04de697f-cdbb-400b-9c8b-b0b5999f77cb", "e6154e04-3099-49d4-b3f6-4b3cfaefefb2", "b6686d31-c28f-4eda-8cdb-3beaeca14ab2", "7f439c58-449b-4d3f-b23a-eabadea55d48", "e3fd12a4-ba9e-461a-b2e6-62db4ff2934c", "ac5fb8e0-e691-4959-96eb-78e78e51d66d", "b58af054-ffc2-4e6c-8db8-5fed9e7e6030", "99b957f2-c307-4aef-b8ea-cec34efdb314", "5eb6d8c5-a04d-4200-99ed-6cac496df2af", "3878ed43-cf40-4438-b7ef-c21a047be400", "1d2d3647-ad62-4d99-b467-cb69403005e5", "57df0b4a-c241-45dd-83a3-55c52d1b5e78", "2836c401-af66-4514-be08-434267941a12", "1ee32d6e-5152-4186-8916-52cebaae5307", "34be6229-ee00-4996-ae1a-af9c4cb99602", "3a99829b-3693-49c5-9edd-39b3009457fb", "c025c320-f8f4-4ee6-a55a-571dfb5ff359", "4b5dd6d7-c1a7-4741-95a2-6a29f817ab2f", "1b2b74d0-176b-4725-90c1-e58bd788477f", "238dd238-c3e4-4672-bf96-ce07b9dd6604", "ee14148a-40b0-4b6d-b3f8-a338becbebce", "b6396801-9578-4845-b4b1-c79588b3848a", "d935deab-1e92-4152-9869-daf58666a98a", "e92f1c43-80b9-4344-bd1d-f29031d5d1ca", "80f24a3f-d238-4b38-a3fe-c3f3d223ab67", "defdf421-0f51-4c7a-ad0d-1222ac5c5629", "f67bea9b-5bca-4a94-ae97-e976be775a7b", "22028e60-bab4-414f-b096-8725076b391b", "88c729e9-5bee-4a39-832a-a0a43e8b0e5f", "c33daf06-9b41-4e0a-a3fb-586769c480a9", "67ad759d-27b5-46d2-829d-d131c5b62e6d", "ab39f2c2-f6b2-4fb2-b7a8-6e71ee6be134", "55495a88-864d-4dee-bb9a-54bd4125ca57", "9bea0324-df5e-4d01-812a-7df52022c588", "9be5cf96-5b29-4af2-a59c-34fe1505720c", "4bbbd180-25cd-4959-a776-77591b79d991", "51ae7211-ce4b-4a52-8df2-5feec4c545f9", "e28a30a0-bb78-4186-95df-b2a865e0da3d", "c20f5ea6-2691-459d-a2a8-974e152fd596", "cda9bf31-a528-4835-8b5b-82039322bd14", "dced05fc-8996-4907-b975-52806064b50e", "efedf949-15ae-490a-8831-dc515b633d9c", "bd50c7e9-a7e5-4cde-bd9e-2dd8f7455543", "10734543-ea50-44f6-86ea-0d33e1623a63", "241f409f-f390-42d7-a897-7f116cea8b7c", "0b9a97e6-534f-4caa-9f6b-5a186240946c", "33106c0c-1238-4ee9-ae2c-48b03822644e", "137d32d5-28cc-4ac8-91e1-58f1ebe2f343", "e1c72e28-f2b1-4106-bc19-0d7aa5fab077", "5c21cd6d-e19a-4633-b2a9-cf91fd1dfedf", "5957208f-c047-48b7-97d2-1965e62c685e", "58004bef-87ee-4e9d-86d8-ef918592840f", "ef6a9958-1749-4fb4-8eaa-075226783465", "146cca2d-5e1d-4ccf-b6e8-674340122d81", "470c1ca5-142e-4a92-b3b7-b7009d98e22a", "5387645a-eec3-46e6-8cf3-ee8d210421ec", "99d8ea69-1068-4412-a723-abcad61c4489", "e2da2436-4c30-4ab2-904c-2865a74c7a7e", "a372fbca-08fa-48b5-8052-2bac654b56e8", "b9e469d9-6def-4073-adbb-2ea7db538380", "4745a33e-df4d-482d-95a2-137d6970932b", "101e6317-5750-402c-a48e-52dea4c8f25a", "a4d721a4-99ef-4081-8551-a8503fc6f06c", "986a0077-ecfb-4cfc-be8e-2936ee5671df", "d1beeab5-bc82-4227-a995-46bb5e495a6f", "2ada1a7f-fa40-48ee-ae9a-44693d424d4b", "c9d3c291-0c97-4f7d-bebd-1a1b9976b863", "e9e9fac5-8c9d-4149-935f-70ef215a9287", "6ca3dabe-a52e-4a61-a0de-61326a890509", "d8934eaf-2e62-4ff0-beb2-dcf1ef9e4051", "fe98a8e9-5d90-45c9-98e6-92ba24d319b0", "a6fcee5c-34b3-44e4-ae6c-d72c8541f53a", "a01be823-169d-4f37-bfa8-5c54cfd8fb40", "e3825a5c-8de2-41d4-839f-79b7bd15dc68", "9dd41734-aeb3-4127-9358-8c52bab31136", "84621a70-ce86-4f9d-b5ff-2d9b1ce8a7fc", "6f0e2b52-719b-4ee4-bf20-c73154838ca5"]}
|
||||
{"concurrency": 2, "image": "medium", "image_kib": 1961, "uploads": 24, "accepted": 24, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 24}, "transport_errors": [], "accepted_in_s": 0.7, "all_finished": true, "peak_queue_depth": 22, "peak_oldest_age_s": 4.2, "worker_peak_rss_mib": 196.4, "worker_oom_killed": "false", "ready": 24, "failed": 0, "p50_s": 2.48, "p95_s": 4.64, "p99_s": 4.65, "max_s": 4.65, "throughput_per_s": 4.6, "span_s": 5.2, "measurement_window_s": 5.726, "worker_cpu_seconds": 5.23, "worker_cpu_percent_one_core": 91.34, "http_non_2xx_fraction": 0.0, "accepted_http_rps_over_window": 4.19, "attachment_ids": ["0df1caad-9cfe-4efa-96a7-dcc3606702fc", "a0815e57-91e5-44aa-98d2-131c2930b9d4", "625b6ac4-8ded-4f17-b08f-665f77ad599a", "c3497e58-dbd6-4a1a-9510-bfd6588ea56e", "81ca832a-5bc7-4ca0-9e89-b185cd37d694", "d1d0e4c3-cdfd-410c-95e5-9e6bbd73edd1", "5c671809-eaa0-4e95-a94e-5cc9dadf57f3", "43097921-3eca-4122-ae45-0c191de23771", "a5a03c4d-9d01-4c72-b7cd-7183bb32b044", "4269c35d-33b7-4fbf-a783-7209a959693d", "6f3ee881-1fa6-4789-b916-f21860f83c4f", "e20f0e91-9fb5-490f-957d-2a827f347087", "b8ab1e8b-ec0e-4c21-9e82-ccd4f589812d", "93c0fcd0-6c99-4b17-8120-f452bc1b51ce", "bec68cde-cdce-40a5-9973-a1c09fc1c06e", "7e02a10a-c6fa-485d-b4ab-0095260a2cc6", "c87a37cf-590e-4562-a098-06f1ae001690", "e9ffca81-951d-4546-a1e1-4d9d0b6d6ed9", "bb21974f-3888-458f-b475-6b8670ec79da", "cec87457-c37d-42db-9e7f-b12f33fa7949", "0450c693-16d6-41d5-a55a-9d7093fe7c27", "4c97331a-1879-46d9-8959-133697728a2f", "1f52d3ed-f6d1-498d-a6a0-455140ef3230", "13239c5e-6d5c-4bd7-8945-bd6a0790a4a9"]}
|
||||
{"concurrency": 2, "image": "large", "image_kib": 3100, "uploads": 12, "accepted": 12, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 12}, "transport_errors": [], "accepted_in_s": 0.3, "all_finished": true, "peak_queue_depth": 12, "peak_oldest_age_s": 2.7, "worker_peak_rss_mib": 251.0, "worker_oom_killed": "false", "ready": 12, "failed": 0, "p50_s": 1.68, "p95_s": 3.04, "p99_s": 3.04, "max_s": 3.04, "throughput_per_s": 3.7, "span_s": 3.2, "measurement_window_s": 4.012, "worker_cpu_seconds": 3.21, "worker_cpu_percent_one_core": 80.0, "http_non_2xx_fraction": 0.0, "accepted_http_rps_over_window": 2.99, "attachment_ids": ["d75b173f-519d-4ff9-b8fd-05eee6b618d1", "9c3feb38-f6e3-474d-8530-d0855825d56c", "cb3cd3c9-5343-4d29-a655-fc314f718628", "a1015575-62d6-4467-a4bd-faade05754b6", "ab578be5-5c71-41aa-83b0-1e2893649e14", "a4b3667f-d3b4-4903-bc65-e02bd71df865", "a2591bac-d552-4ef3-8637-b96335d73cf4", "e11f7030-71d3-4783-a7bb-599a0d326a5b", "ff5cec4b-4c12-4bb5-b73e-829ea1649504", "02893780-69c4-4026-b296-929a5b14fbc8", "faa4cdff-572f-4a4e-9fad-1798a142ca4f", "de2d34f7-d8e8-4871-934a-4fb3180066f7"]}
|
||||
{"concurrency": 4, "image": "small", "image_kib": 109, "uploads": 80, "accepted": 80, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 80}, "transport_errors": [], "accepted_in_s": 0.3, "all_finished": true, "peak_queue_depth": 54, "peak_oldest_age_s": 1.2, "worker_peak_rss_mib": 78.3, "worker_oom_killed": "false", "ready": 80, "failed": 0, "p50_s": 0.84, "p95_s": 1.33, "p99_s": 1.37, "max_s": 1.37, "throughput_per_s": 50.21, "span_s": 1.6, "measurement_window_s": 1.991, "worker_cpu_seconds": 1.47, "worker_cpu_percent_one_core": 73.82, "http_non_2xx_fraction": 0.0, "accepted_http_rps_over_window": 40.17, "attachment_ids": ["369a2d06-c8de-42a2-aa65-8556e5011d95", "6a0c9c3d-3699-4e6d-8469-89106a11f635", "1a373bdb-41bf-498b-a54a-3cc562ae78cd", "f8377da3-8ddc-4fbb-8c4e-e608082fb039", "3698f728-059d-4480-af59-a44b20ea9cdf", "a42a8bcf-09ce-48d7-8b12-45dc08ca1554", "e0721646-ce31-4e3c-ab42-df59f72be3b6", "59c00e91-97d7-4be4-8d06-6906185d424e", "3fe8235a-40ac-475a-808a-24b768125601", "055b554b-2cc9-434f-b7da-44cbdccaa703", "98e66361-570f-4296-9a08-ea14385c0457", "afdeb335-f2eb-4a46-98a8-42875242ad6b", "e6eb3a8e-8621-44ae-9665-61d71a32566b", "6e3380de-9588-4f29-8729-99ea18ba2225", "6fc5d716-966f-4354-8124-7717fcfd42e0", "0501f9db-df18-4626-a739-d5a46018d961", "c3eff4b1-b119-433c-85a9-1b2e44b966c6", "e3209e5c-75c1-4bdd-9e76-8866c527f81c", "fd483860-6dfe-43a0-a067-17ec289d0ba4", "99c38cc4-0b82-432e-9491-4192dab60c54", "8b5c51ff-de77-4fdc-9acf-bb78fe0d6cdb", "cfb32500-3940-47d7-bb6a-9ec075738012", "2fb57612-22d0-42f2-9ebe-0a0de48dfb4c", "4958f233-5968-4084-bfd8-4028601d7242", "6d3d6524-b973-4759-a353-da475ed69a36", "5df49d78-c5e4-4e24-a8ac-6036dab6a4e8", "cb39ff33-391d-49d3-902c-6de656603e9b", "119305a1-c164-4f42-b551-4cbcfae387a4", "51d3b68d-4a62-403e-8b7f-085d83cdd0a6", "781bd5c5-8aa5-4f69-8c5b-f11c24e60253", "9f289f53-f379-4a8b-abca-9512b39233bd", "162d17fe-be83-4d0a-968d-144e466a6bc6", "d674ca52-c3cf-4f59-a2d6-18ddd8770fc4", "30bf8a4c-6988-475e-b093-3d00ab1dc1a8", "8ba37238-1072-4301-9630-084e5e653777", "b28dcff7-1552-4286-89f3-1a5f83fead59", "5ead051c-3026-45bf-bf55-ee529e6f4ff4", "2e51963f-ae62-4f92-8d3f-8a8a454fd7ac", "2d536590-5932-49c1-9d55-05064097ba2a", "cf124a44-f16e-4f14-9b01-34ad758c5bee", "2a0bd5ff-449f-45ee-b3eb-b95c2271be3d", "767b88a3-f5d7-4bf2-b818-3f7e19d78cb0", "efa00e11-08e0-4416-bb7b-3736e1766339", "8fc529e7-0607-4e4f-8e2f-bbf8a94aed17", "4152271d-f5e7-4fd1-bc91-c26ea1d46f1b", "553a6c7e-0630-49cf-b2e5-3e792655d902", "3fe348ef-85f2-434d-96b9-224bf3759d06", "f27b350b-50d5-47e8-a620-a32088068daa", "2e415149-552d-477e-9f5b-fa738a51eb74", "54e255a8-1abd-4283-89d1-714349db1f0b", "7b626082-147b-4107-acad-23431f02843b", "c9de03cc-c4eb-47f1-af2c-aab97b910c2b", "87957fb0-63e3-4c91-8f6c-a0d88a81291a", "b1d1f957-af29-4ccb-99cb-0fd47f108b8e", "5a9c9b2c-5867-4627-9d22-890abffa11e2", "f7a24e55-969a-49e9-b485-ab3d76e887f9", "264295e7-c3d0-4073-ab78-0447a436b135", "5a209d9d-0413-4363-aeea-12e881a014c0", "ebbf2d91-4961-47f6-8b4d-9e5f7ec83bc2", "8821988c-d93b-4d60-ad32-0781f77682b3", "65ac051b-a319-4c85-8cdc-6dfe3bca6da2", "b6ccfaa1-be04-4b8c-b74f-07ca0a394097", "63151877-74a8-4986-9911-2e47a1fc5061", "f4b8aead-e8b9-425c-9dc5-bc5c88044f34", "1932d7d7-575f-402c-b56c-272ab7587a64", "2079ec0c-5e8d-46cd-95da-4a77f2c64b17", "559ff00e-2679-4674-974f-4fd3a1362c84", "36b39e61-fb6c-49d2-97f4-5687f069754b", "f844cda3-3ddd-45fe-97a2-b418d5e2ca95", "8253f768-0b35-4150-80f7-cb0c1a1e600e", "84699d62-6aae-4c61-a3a9-aefc85761a51", "f94a444d-8bc0-4a24-9f46-db29c824b288", "60be5424-9940-4cf2-acdb-dc3a68d6e453", "7d2d85cc-39a1-4dda-be06-142ec2d6c542", "cf4f344d-c52e-4c5b-8d25-9495a1034dd4", "a7ca7b03-92e2-4e0f-9d9f-3dc4588062e1", "0fd0d132-76b5-46e5-add3-eaf1f5c8577e", "8d269940-4187-461a-9624-c0ed1c1e0823", "a57d54d6-996d-45ae-af6f-f2d37f3a2ceb", "c45136fe-130d-4257-a6ca-1d58296b089e"]}
|
||||
{"concurrency": 4, "image": "medium", "image_kib": 1961, "uploads": 24, "accepted": 24, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 24}, "transport_errors": [], "accepted_in_s": 0.7, "all_finished": true, "peak_queue_depth": 24, "peak_oldest_age_s": 4.8, "worker_peak_rss_mib": 319.1, "worker_oom_killed": "false", "ready": 24, "failed": 0, "p50_s": 2.69, "p95_s": 4.89, "p99_s": 4.93, "max_s": 4.93, "throughput_per_s": 4.37, "span_s": 5.5, "measurement_window_s": 6.405, "worker_cpu_seconds": 5.47, "worker_cpu_percent_one_core": 85.4, "http_non_2xx_fraction": 0.0, "accepted_http_rps_over_window": 3.75, "attachment_ids": ["16ed740f-59f2-4b65-8aa7-3026600e5fdd", "9e9509a1-f58f-4d57-8e5a-ac5b689091cb", "6a3614a2-6461-48a4-808f-d500ddd7157f", "9b15f285-5da4-4b04-85ec-09603af7370c", "71a77ae7-8dba-45d1-986c-19fddc94af3d", "46fe0fd9-c9ae-47f3-a47c-9393c8d3c2a6", "e2410aae-675e-4259-824d-cde0b479b667", "dcef7518-6931-46d6-ac51-f53af10a7018", "8d38362f-34af-4cd4-8988-8525edfcb171", "6f8ba7d2-f77f-42c8-89f4-13b644294d2d", "e23348ac-6fee-49c8-b492-5fc78a853548", "c4fa1ca7-1a46-4e3f-843b-531ea3664359", "66014e42-5f76-4fe8-ab67-9cab84fe5a16", "83a2cbee-f650-4452-9270-4a6921f51ff6", "95b35d8e-d2e1-4c6c-8c8d-058182bd12e8", "0d7d6d37-f538-46b7-b314-281923bc583b", "b389353c-5e56-43b3-a8b2-1bb6e7d865db", "840e21f4-2a44-487d-bedd-b9cf1cd94548", "8b865dac-9945-4314-acd2-2abc09053a23", "d60650ab-0ee4-4e95-9207-688f4af14b9c", "f8e8de12-30b8-4c73-b81f-53482b79fc44", "88b2cfc3-f26b-4515-aaf2-f23fbb78dc28", "d56513e0-c8f4-4a48-9b83-98ef17760dad", "eed9e89b-f6ff-44e4-b2cc-16f871185a19"]}
|
||||
{"concurrency": 4, "image": "large", "image_kib": 3100, "uploads": 12, "accepted": 12, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 12}, "transport_errors": [], "accepted_in_s": 0.3, "all_finished": true, "peak_queue_depth": 12, "peak_oldest_age_s": 2.9, "worker_peak_rss_mib": 434.5, "worker_oom_killed": "false", "ready": 12, "failed": 0, "p50_s": 2.3, "p95_s": 3.3, "p99_s": 3.3, "max_s": 3.3, "throughput_per_s": 3.48, "span_s": 3.4, "measurement_window_s": 4.061, "worker_cpu_seconds": 3.36, "worker_cpu_percent_one_core": 82.75, "http_non_2xx_fraction": 0.0, "accepted_http_rps_over_window": 2.96, "attachment_ids": ["7f5b994f-f3de-4b60-a5e9-f87ffd0835b6", "95d47438-8f87-45fe-9129-cb9563531a6b", "891e1344-f240-4efc-8b89-f3787d67002e", "719431c3-cd94-4cbc-92e5-5c301ae8005d", "85bffd8c-0007-4f7e-88e0-c8f84c9a575a", "32b180e0-5fda-4d53-ad12-7e2fe8715b51", "726dd68e-83f9-4f05-897f-d79f403b8064", "1950ca75-72e0-45c5-9e4e-5d54630fb3b4", "6742055a-66eb-49e1-a26a-b333995e1142", "d770a827-5abb-4df3-8a03-ba24ae2dbe38", "54bcd9c6-6d07-47c6-9aae-cc06fcbb9ca1", "d38c2e12-76ed-412a-a120-bae6d7be1142"]}
|
||||
{"MAX_QUEUED_JOBS": 30, "burst": 200, "attempt_statuses": {"202": 61, "503": 139}, "accepted": 61, "peak_queue_depth": 57, "all_accepted_finished": true, "accepted_in_s": 0.6, "drained_after_s": 3.7, "retry_after_header_seen": true, "retry_after_values": ["5"], "transport_errors": [], "recovered_new_uploads_accepted": true, "recovered_new_uploads_ready": true, "no_accepted_job_lost": true, "ready": 61, "failed": 0, "p50_s": 1.46, "p95_s": 2.31, "p99_s": 2.4, "max_s": 2.4, "throughput_per_s": 22.15, "span_s": 2.8, "measurement_window_s": 3.72, "worker_cpu_seconds": 1.32, "worker_cpu_percent_one_core": 35.49, "http_non_2xx_fraction": 0.695, "accepted_http_rps_over_window": 16.4, "attachment_ids": ["66ea19e2-e3e8-4d74-acfc-b3a9429f997c", "72f006e1-db6c-49f0-bcea-a5062bfe4e94", "a8754c2b-bb4e-41de-8569-a1509672c382", "273fd71e-1d8b-478a-9fb7-6e4f17f85323", "c7963cf2-4c2f-4dc1-890c-f400d78f97e6", "076e695d-ea2d-4a93-a3a9-638f0f5edd64", "b2fa531f-0cff-4869-b8fd-63c543cdef03", "0dc2753b-470f-4d8c-9f6f-46e72d18017b", "d691e18c-fd06-420d-9b3a-4676300f933b", "22349666-78df-48d7-805b-233357cfcd28", "ab17e0d7-6b8b-4dbc-b1a5-00c60f82dcd4", "e76b055b-de11-4ea1-a47b-d687672791ce", "768f98f9-3717-4ad7-81c7-3e5505e55f62", "5b5d86da-d290-4ec4-bbbf-9bc0b1f3a501", "8dad57b3-1a3a-409f-a4eb-60de9f0aaa09", "e8736a1c-b13f-47f7-a85f-cd6ca415209b", "c0a44b33-e8bf-4c5d-9e59-ccda96cdd1da", "504cab76-9082-4801-a6d0-87e4f4914352", "1d02876b-89ae-4380-a460-89fb29e55a93", "c6adc9c6-def5-49cd-900b-d63ecadaaad8", "989bd05b-eccc-4b7d-b806-0c468a331e9a", "1da0e1c5-ec99-4884-a3ad-0fdaae4dbc6c", "d6d37e06-6c8d-4d44-af3b-45fb22e88a59", "d9faea0c-2f02-4b76-8b02-44623b36b808", "a89cbba3-c4a8-4c9a-afec-c099ae7e46d3", "d9f0f0b5-70d1-4d09-a298-06a947b2a86d", "49b1c6e4-ba7f-49ac-9616-2d762185fd22", "452cd743-2644-46f8-9535-23c9122efc84", "94e6c1b6-1bdf-4b1d-b7bf-6812b67b7d05", "6d5279e2-eb5b-4757-befe-bf0d954dabb9", "447023af-7290-487c-8f52-8645a127d540", "08f41743-156e-44f0-9ad3-ab99cedd23ca", "c02cc601-04f5-4a81-bdcb-5a557395a506", "71c12d21-aaea-473f-8565-30014caa52e7", "68e91241-0f9a-4c4e-9080-5862bcbc472e", "3afb2d2e-31c9-4f70-aa3c-34393af6dc92", "a3014daa-2802-44a5-85fb-a86d74d2198d", "c205fb0f-183b-40e7-9ffa-6cb1fefff297", "af5996bf-3182-4778-920b-eeee2009cbce", "6fa05261-09b2-41ea-8ec3-8cf11d5e8a25", "976193b5-f91e-47cc-87f1-e0d8dc5347d1", "9ff6485e-6b41-4384-af91-1014fb114cc5", "0edaa06c-9403-48bc-bc1e-311e32762134", "79a51b1c-082f-48d5-acbf-f912f8792b54", "9cd5ff6b-ca74-4679-959e-7630f6ff3dc0", "10480ccc-186b-4586-bb2d-b3bdab50a2ea", "db1d6478-cbf2-47a9-b6ad-ca739b3d43aa", "f5d2e3ce-6e8d-4dad-8ee6-6905d9053867", "c36fab10-726f-4556-8fae-595ef41ad828", "76f0ac29-0fe9-495d-9873-620dd43019c1", "84a99df5-ca4d-492f-bfee-d8efa996b896", "9cbb0bc9-2700-4813-95c1-6123cec043b1", "e3a3eda1-1cbe-4e1a-883d-0420199073c6", "5dc6f8e5-235a-4072-a7c9-36623d163ff3", "5acb6911-ba65-4545-943d-68ed3c7b5c29", "710959d2-f69b-4bc6-a033-cb412c701f92", "99e4d956-77be-4501-b036-78978c13479d", "472b87fb-87ac-4d75-805b-21232db08a8c", "b8f2ec4e-4734-4267-83e3-3acf47c86cb9", "f127c53e-d159-41b8-8cc3-5f0dcf6ef1fd", "793bc8ef-82ff-45ea-adb2-c799164ce621"]}
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 2.328}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ docker compose up -d --wait --no-recreate --no-deps api
|
||||
Container mtusi-lab5-api-1 Running
|
||||
Container mtusi-lab5-api-2 Starting
|
||||
Container mtusi-lab5-api-2 Started
|
||||
Container mtusi-lab5-api-2 Waiting
|
||||
Container mtusi-lab5-api-1 Waiting
|
||||
Container mtusi-lab5-api-2 Healthy
|
||||
Container mtusi-lab5-api-1 Healthy
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 0, "first_recovery_seconds": 2.328}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ docker compose up -d --wait --no-recreate --no-deps api
|
||||
Container mtusi-lab5-api-2 Running
|
||||
Container mtusi-lab5-api-1 Starting
|
||||
Container mtusi-lab5-api-1 Started
|
||||
Container mtusi-lab5-api-2 Waiting
|
||||
Container mtusi-lab5-api-1 Waiting
|
||||
Container mtusi-lab5-api-2 Healthy
|
||||
Container mtusi-lab5-api-1 Healthy
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
@@ -0,0 +1,674 @@
|
||||
[
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 66,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 72.7,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 1.4,
|
||||
"p95_s": 2.57,
|
||||
"p99_s": 2.71,
|
||||
"max_s": 2.71,
|
||||
"throughput_per_s": 26.26,
|
||||
"span_s": 3.0,
|
||||
"measurement_window_s": 3.488,
|
||||
"worker_cpu_seconds": 1.59,
|
||||
"worker_cpu_percent_one_core": 45.59,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 22.94,
|
||||
"attachment_ids": [
|
||||
"f3755317-a6f1-4c38-958e-572d2e72521c",
|
||||
"e1866acf-772a-4e44-9f6d-96d6d6c855ae",
|
||||
"4c88f7ec-94f3-42ef-a80d-2cd5506f288f",
|
||||
"9d76e306-5c0f-4dc0-b417-0e328d8c7ff5",
|
||||
"5d7435aa-9667-4b0d-b772-2a5cd07c6d18",
|
||||
"91734a85-8d57-4ae7-9f80-023bb9cde252",
|
||||
"321ea613-76a4-4ce7-a2e8-da57f8d6fa0e",
|
||||
"d5903675-86be-41fb-8d23-79835bc7d3e7",
|
||||
"dfdc7b08-fedf-45b7-9ebd-a15d00670784",
|
||||
"2e2ec815-68f6-444d-b83f-094c1af5713b",
|
||||
"80d48257-49e5-41df-a73c-1d31715ad117",
|
||||
"816c620e-82d5-4feb-9bdf-def2dd4da03f",
|
||||
"c9b770c8-de70-4476-81eb-1e872ddee1dc",
|
||||
"90239c75-8017-4e4e-b098-3f2901c46f2d",
|
||||
"a29229ce-376b-43a2-bb2e-bc4457e47a5f",
|
||||
"5b45de93-1b7b-4167-a739-07c494cb6d9d",
|
||||
"a35306c2-6cd2-4745-9aab-c5acfaf902ca",
|
||||
"85996ad6-fe4e-41b1-b42d-bb524176f64e",
|
||||
"2f3c4a2c-462b-4d53-b1d2-cb463b63b265",
|
||||
"7315c4f5-397d-41bd-8ada-1199b5ff8d20",
|
||||
"5ca87add-c93e-4bee-b3a6-e28b51b42d28",
|
||||
"93dcb192-6666-45cf-a55d-df0959d6bddd",
|
||||
"d2b44888-c498-4900-9814-340f27e79c12",
|
||||
"bb7b4f52-ffb4-4b75-b271-cfa705c14e50",
|
||||
"5df28e25-a9b3-47f2-92f5-71bd048959fb",
|
||||
"cc44923d-6e23-43bd-8ea0-3a158de52a58",
|
||||
"b352814b-b29c-4043-b862-670fcfe50002",
|
||||
"21ab21a9-1959-4432-8283-eed80a3ae6cc",
|
||||
"d33a072f-937f-4873-8dbc-3eaba5e9a06b",
|
||||
"e1630cdc-c746-4f95-8712-0f0ec4501224",
|
||||
"e6680ab1-e568-4cf3-b372-e165553c8277",
|
||||
"bdd7584a-d8f7-4420-b2f9-127aac3f67e4",
|
||||
"1024b092-18f7-42ad-8269-15137dcb07b5",
|
||||
"c2623c1a-eec4-43c0-a870-4202f1b97542",
|
||||
"6a16b4e3-e6ef-45ef-8a2a-898e0dc590d0",
|
||||
"594a7d05-50f3-4d5c-9114-1a5383869296",
|
||||
"d6c712a8-e930-4e45-862b-2ee20ea4c380",
|
||||
"9147c9ce-32dd-458e-bbda-762bbe1c60de",
|
||||
"902a3269-9937-4c44-91ce-058e67005c87",
|
||||
"df198815-5898-4293-bb85-087679a4efbf",
|
||||
"8fd86d7e-efbe-4c84-8f17-cd03e510f508",
|
||||
"67489317-2bcf-4be0-a97b-52afce0e7c44",
|
||||
"28185ccf-516a-45d8-b851-ae202869b7cc",
|
||||
"bff9004a-3bc8-44df-ab54-2295c3e7e27c",
|
||||
"c0ea5554-7a66-49f2-8ec3-c4fde067abdf",
|
||||
"4a126067-ce1a-4c57-8817-cf2854e40cde",
|
||||
"5a76425e-88ce-4e1d-b335-b0ef76cd88aa",
|
||||
"8ea27a23-6e62-47d4-8523-a6ca1f5ad458",
|
||||
"6e03b1c2-d4a7-4772-89cf-f41ffde6db55",
|
||||
"469e7a1d-3ac9-4224-8661-0e57e3c37d03",
|
||||
"81b4d8b6-7e29-4fe1-b062-15c87101cbfd",
|
||||
"64399d7a-7df1-45fa-b7a7-653e5da60bd0",
|
||||
"b21be428-e7d0-4417-9023-ffd232ba830a",
|
||||
"56029d32-3104-4255-8006-cd775fb22a1f",
|
||||
"84308a9f-59ba-4797-8e42-a990ebf2fefc",
|
||||
"77f0efa1-b5d6-481d-b1a6-a8317af5201b",
|
||||
"5d3d0106-8851-4cc0-9fa8-178e7a4e1852",
|
||||
"190e83e7-1c5a-4cee-8447-5315a54ab817",
|
||||
"ab5363c6-da59-4565-a074-1b3acb1433d0",
|
||||
"19e569fa-412b-4fef-9e89-2c4218293613",
|
||||
"90fb1725-33a3-4011-b430-f28f188d500c",
|
||||
"1bf62c59-0ad5-4225-ac03-9bd5a4965900",
|
||||
"dac8a042-a731-4bdf-92f7-cbf6377a3496",
|
||||
"6370b8dd-8ce2-4e9e-8b20-cdc07d8a4218",
|
||||
"3e3a0ca8-756b-4c9c-a42b-55d0ca984f31",
|
||||
"63f82f8f-419a-4fe3-8aff-180249c946ec",
|
||||
"c2a64395-4403-418f-a4ea-5e32689e4dc1",
|
||||
"6cecaaa4-3159-400d-9e28-e7c0dd26bcf6",
|
||||
"5e5567e8-9d1a-401b-bf15-d7fb2ac4c6a3",
|
||||
"1c4371d1-39c1-475d-9055-63c3823f9859",
|
||||
"c6ab5b7e-2c37-4e42-8abc-3bfe922370d8",
|
||||
"8a52c2ca-e625-4549-9661-f89d3c973c7e",
|
||||
"e6993de3-6dd1-40ed-8aa0-7f811d311087",
|
||||
"3a3bd631-10b6-478f-b25b-575053f0088d",
|
||||
"a906674b-bd9c-4b34-9c6e-d25e23d453fe",
|
||||
"d4aee6be-963c-4803-956b-0dcce87c6dd2",
|
||||
"4510f909-6250-4a54-9488-8c0edd064353",
|
||||
"19a3b5ac-5784-4e91-95cf-a95b66de7d23",
|
||||
"c25cd565-459d-4ec1-83bf-d21859dd3634",
|
||||
"d28d6581-7aef-4762-9d4b-3ae016f33832"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 23,
|
||||
"peak_oldest_age_s": 5.4,
|
||||
"worker_peak_rss_mib": 133.7,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.78,
|
||||
"p95_s": 5.23,
|
||||
"p99_s": 5.44,
|
||||
"max_s": 5.44,
|
||||
"throughput_per_s": 4.09,
|
||||
"span_s": 5.9,
|
||||
"measurement_window_s": 6.264,
|
||||
"worker_cpu_seconds": 5.15,
|
||||
"worker_cpu_percent_one_core": 82.21,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 3.83,
|
||||
"attachment_ids": [
|
||||
"d336e33f-283a-49e9-8ca9-9c4d0f17c904",
|
||||
"dfa1d79f-ffe8-4937-8c46-c456143a9394",
|
||||
"a8bb7c8b-931c-492e-bee2-5b9920e6355e",
|
||||
"6de05c1c-ca5a-4b86-8dd3-af63b1aa321f",
|
||||
"dd696951-5e88-44b4-80a2-c868326a2102",
|
||||
"4253bdfd-86c2-4116-ad68-72f685481005",
|
||||
"7afb0faf-2c94-4014-8be5-07731cdfb483",
|
||||
"624e290a-0463-417d-8e53-448a75bfcd31",
|
||||
"0b6f4492-3378-42bd-8fd4-47983dd7a54d",
|
||||
"c4fa5d79-c3f1-48d1-98bb-bf4e73c55efe",
|
||||
"8c25109b-d8ba-4608-a1d7-1d7abc1f47b8",
|
||||
"0361352b-61aa-41c0-ba62-0e767f954a62",
|
||||
"bcf3989b-f1fe-4619-962c-34d22a984261",
|
||||
"80dd3ed7-6e10-4dfd-aca5-3a3e4382326a",
|
||||
"9ca80d24-2c9f-44ac-8811-deeabcb8ff0f",
|
||||
"f6c7cc70-25ba-4a5a-aa8d-75f22a8ce836",
|
||||
"0e8d2721-f293-4ccc-9d10-5873c8fd90b9",
|
||||
"95ce7a82-0133-4a9d-9899-dbed974ee9ec",
|
||||
"5fd7ea2c-685f-4cfc-9f0b-dabaeba4129d",
|
||||
"773dfbf2-fc82-4888-8208-032b25035a4c",
|
||||
"d3ba0d4a-952b-4095-821c-398fce6165f0",
|
||||
"9362cf03-cdd4-482c-8786-42379ce6bbf0",
|
||||
"d3258e0e-78b6-40ea-86b8-9403d2d8671f",
|
||||
"ceaaa543-c950-4398-af14-924025c652bb"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 11,
|
||||
"peak_oldest_age_s": 3.3,
|
||||
"worker_peak_rss_mib": 160.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 1.73,
|
||||
"p95_s": 3.34,
|
||||
"p99_s": 3.34,
|
||||
"max_s": 3.34,
|
||||
"throughput_per_s": 3.33,
|
||||
"span_s": 3.6,
|
||||
"measurement_window_s": 4.058,
|
||||
"worker_cpu_seconds": 3.05,
|
||||
"worker_cpu_percent_one_core": 75.16,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 2.96,
|
||||
"attachment_ids": [
|
||||
"cd228696-224b-4b17-87c7-f0af12762e19",
|
||||
"0c3b34a6-c495-4450-ba46-bfe72d46b941",
|
||||
"00a409ba-8b6a-4021-aad5-3f717b9133ee",
|
||||
"f239a54e-13c6-4252-bb47-0e513b310008",
|
||||
"e0e8e733-b7cd-48e7-884e-ea54cac614de",
|
||||
"51498e5d-f689-47f9-930d-9c21c7c7cc28",
|
||||
"12591ec9-056b-483a-b9a4-b81a3ad6f9ae",
|
||||
"b2b0d9b2-616a-4874-81ef-18b5b631f7cc",
|
||||
"262d2c63-d351-41d4-a641-4fcb86a28df5",
|
||||
"62c53389-0ec2-4b52-89d9-cc4734867d36",
|
||||
"a0631868-6ed3-4ce9-8adf-8518ec884e13",
|
||||
"95f1942f-300a-43b4-a4cd-7915f2291552"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 52,
|
||||
"peak_oldest_age_s": 1.2,
|
||||
"worker_peak_rss_mib": 77.0,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.83,
|
||||
"p95_s": 1.57,
|
||||
"p99_s": 1.65,
|
||||
"max_s": 1.65,
|
||||
"throughput_per_s": 43.4,
|
||||
"span_s": 1.8,
|
||||
"measurement_window_s": 2.687,
|
||||
"worker_cpu_seconds": 1.53,
|
||||
"worker_cpu_percent_one_core": 56.93,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 29.77,
|
||||
"attachment_ids": [
|
||||
"986c0488-655c-45ee-80b0-ffa2ec88e1b0",
|
||||
"5356f279-0ff8-46d0-9597-6bf80decdba8",
|
||||
"59374d10-af30-4ee2-a9e0-748f64b9c843",
|
||||
"04de697f-cdbb-400b-9c8b-b0b5999f77cb",
|
||||
"e6154e04-3099-49d4-b3f6-4b3cfaefefb2",
|
||||
"b6686d31-c28f-4eda-8cdb-3beaeca14ab2",
|
||||
"7f439c58-449b-4d3f-b23a-eabadea55d48",
|
||||
"e3fd12a4-ba9e-461a-b2e6-62db4ff2934c",
|
||||
"ac5fb8e0-e691-4959-96eb-78e78e51d66d",
|
||||
"b58af054-ffc2-4e6c-8db8-5fed9e7e6030",
|
||||
"99b957f2-c307-4aef-b8ea-cec34efdb314",
|
||||
"5eb6d8c5-a04d-4200-99ed-6cac496df2af",
|
||||
"3878ed43-cf40-4438-b7ef-c21a047be400",
|
||||
"1d2d3647-ad62-4d99-b467-cb69403005e5",
|
||||
"57df0b4a-c241-45dd-83a3-55c52d1b5e78",
|
||||
"2836c401-af66-4514-be08-434267941a12",
|
||||
"1ee32d6e-5152-4186-8916-52cebaae5307",
|
||||
"34be6229-ee00-4996-ae1a-af9c4cb99602",
|
||||
"3a99829b-3693-49c5-9edd-39b3009457fb",
|
||||
"c025c320-f8f4-4ee6-a55a-571dfb5ff359",
|
||||
"4b5dd6d7-c1a7-4741-95a2-6a29f817ab2f",
|
||||
"1b2b74d0-176b-4725-90c1-e58bd788477f",
|
||||
"238dd238-c3e4-4672-bf96-ce07b9dd6604",
|
||||
"ee14148a-40b0-4b6d-b3f8-a338becbebce",
|
||||
"b6396801-9578-4845-b4b1-c79588b3848a",
|
||||
"d935deab-1e92-4152-9869-daf58666a98a",
|
||||
"e92f1c43-80b9-4344-bd1d-f29031d5d1ca",
|
||||
"80f24a3f-d238-4b38-a3fe-c3f3d223ab67",
|
||||
"defdf421-0f51-4c7a-ad0d-1222ac5c5629",
|
||||
"f67bea9b-5bca-4a94-ae97-e976be775a7b",
|
||||
"22028e60-bab4-414f-b096-8725076b391b",
|
||||
"88c729e9-5bee-4a39-832a-a0a43e8b0e5f",
|
||||
"c33daf06-9b41-4e0a-a3fb-586769c480a9",
|
||||
"67ad759d-27b5-46d2-829d-d131c5b62e6d",
|
||||
"ab39f2c2-f6b2-4fb2-b7a8-6e71ee6be134",
|
||||
"55495a88-864d-4dee-bb9a-54bd4125ca57",
|
||||
"9bea0324-df5e-4d01-812a-7df52022c588",
|
||||
"9be5cf96-5b29-4af2-a59c-34fe1505720c",
|
||||
"4bbbd180-25cd-4959-a776-77591b79d991",
|
||||
"51ae7211-ce4b-4a52-8df2-5feec4c545f9",
|
||||
"e28a30a0-bb78-4186-95df-b2a865e0da3d",
|
||||
"c20f5ea6-2691-459d-a2a8-974e152fd596",
|
||||
"cda9bf31-a528-4835-8b5b-82039322bd14",
|
||||
"dced05fc-8996-4907-b975-52806064b50e",
|
||||
"efedf949-15ae-490a-8831-dc515b633d9c",
|
||||
"bd50c7e9-a7e5-4cde-bd9e-2dd8f7455543",
|
||||
"10734543-ea50-44f6-86ea-0d33e1623a63",
|
||||
"241f409f-f390-42d7-a897-7f116cea8b7c",
|
||||
"0b9a97e6-534f-4caa-9f6b-5a186240946c",
|
||||
"33106c0c-1238-4ee9-ae2c-48b03822644e",
|
||||
"137d32d5-28cc-4ac8-91e1-58f1ebe2f343",
|
||||
"e1c72e28-f2b1-4106-bc19-0d7aa5fab077",
|
||||
"5c21cd6d-e19a-4633-b2a9-cf91fd1dfedf",
|
||||
"5957208f-c047-48b7-97d2-1965e62c685e",
|
||||
"58004bef-87ee-4e9d-86d8-ef918592840f",
|
||||
"ef6a9958-1749-4fb4-8eaa-075226783465",
|
||||
"146cca2d-5e1d-4ccf-b6e8-674340122d81",
|
||||
"470c1ca5-142e-4a92-b3b7-b7009d98e22a",
|
||||
"5387645a-eec3-46e6-8cf3-ee8d210421ec",
|
||||
"99d8ea69-1068-4412-a723-abcad61c4489",
|
||||
"e2da2436-4c30-4ab2-904c-2865a74c7a7e",
|
||||
"a372fbca-08fa-48b5-8052-2bac654b56e8",
|
||||
"b9e469d9-6def-4073-adbb-2ea7db538380",
|
||||
"4745a33e-df4d-482d-95a2-137d6970932b",
|
||||
"101e6317-5750-402c-a48e-52dea4c8f25a",
|
||||
"a4d721a4-99ef-4081-8551-a8503fc6f06c",
|
||||
"986a0077-ecfb-4cfc-be8e-2936ee5671df",
|
||||
"d1beeab5-bc82-4227-a995-46bb5e495a6f",
|
||||
"2ada1a7f-fa40-48ee-ae9a-44693d424d4b",
|
||||
"c9d3c291-0c97-4f7d-bebd-1a1b9976b863",
|
||||
"e9e9fac5-8c9d-4149-935f-70ef215a9287",
|
||||
"6ca3dabe-a52e-4a61-a0de-61326a890509",
|
||||
"d8934eaf-2e62-4ff0-beb2-dcf1ef9e4051",
|
||||
"fe98a8e9-5d90-45c9-98e6-92ba24d319b0",
|
||||
"a6fcee5c-34b3-44e4-ae6c-d72c8541f53a",
|
||||
"a01be823-169d-4f37-bfa8-5c54cfd8fb40",
|
||||
"e3825a5c-8de2-41d4-839f-79b7bd15dc68",
|
||||
"9dd41734-aeb3-4127-9358-8c52bab31136",
|
||||
"84621a70-ce86-4f9d-b5ff-2d9b1ce8a7fc",
|
||||
"6f0e2b52-719b-4ee4-bf20-c73154838ca5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.7,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 22,
|
||||
"peak_oldest_age_s": 4.2,
|
||||
"worker_peak_rss_mib": 196.4,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.48,
|
||||
"p95_s": 4.64,
|
||||
"p99_s": 4.65,
|
||||
"max_s": 4.65,
|
||||
"throughput_per_s": 4.6,
|
||||
"span_s": 5.2,
|
||||
"measurement_window_s": 5.726,
|
||||
"worker_cpu_seconds": 5.23,
|
||||
"worker_cpu_percent_one_core": 91.34,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 4.19,
|
||||
"attachment_ids": [
|
||||
"0df1caad-9cfe-4efa-96a7-dcc3606702fc",
|
||||
"a0815e57-91e5-44aa-98d2-131c2930b9d4",
|
||||
"625b6ac4-8ded-4f17-b08f-665f77ad599a",
|
||||
"c3497e58-dbd6-4a1a-9510-bfd6588ea56e",
|
||||
"81ca832a-5bc7-4ca0-9e89-b185cd37d694",
|
||||
"d1d0e4c3-cdfd-410c-95e5-9e6bbd73edd1",
|
||||
"5c671809-eaa0-4e95-a94e-5cc9dadf57f3",
|
||||
"43097921-3eca-4122-ae45-0c191de23771",
|
||||
"a5a03c4d-9d01-4c72-b7cd-7183bb32b044",
|
||||
"4269c35d-33b7-4fbf-a783-7209a959693d",
|
||||
"6f3ee881-1fa6-4789-b916-f21860f83c4f",
|
||||
"e20f0e91-9fb5-490f-957d-2a827f347087",
|
||||
"b8ab1e8b-ec0e-4c21-9e82-ccd4f589812d",
|
||||
"93c0fcd0-6c99-4b17-8120-f452bc1b51ce",
|
||||
"bec68cde-cdce-40a5-9973-a1c09fc1c06e",
|
||||
"7e02a10a-c6fa-485d-b4ab-0095260a2cc6",
|
||||
"c87a37cf-590e-4562-a098-06f1ae001690",
|
||||
"e9ffca81-951d-4546-a1e1-4d9d0b6d6ed9",
|
||||
"bb21974f-3888-458f-b475-6b8670ec79da",
|
||||
"cec87457-c37d-42db-9e7f-b12f33fa7949",
|
||||
"0450c693-16d6-41d5-a55a-9d7093fe7c27",
|
||||
"4c97331a-1879-46d9-8959-133697728a2f",
|
||||
"1f52d3ed-f6d1-498d-a6a0-455140ef3230",
|
||||
"13239c5e-6d5c-4bd7-8945-bd6a0790a4a9"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 251.0,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 1.68,
|
||||
"p95_s": 3.04,
|
||||
"p99_s": 3.04,
|
||||
"max_s": 3.04,
|
||||
"throughput_per_s": 3.7,
|
||||
"span_s": 3.2,
|
||||
"measurement_window_s": 4.012,
|
||||
"worker_cpu_seconds": 3.21,
|
||||
"worker_cpu_percent_one_core": 80.0,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 2.99,
|
||||
"attachment_ids": [
|
||||
"d75b173f-519d-4ff9-b8fd-05eee6b618d1",
|
||||
"9c3feb38-f6e3-474d-8530-d0855825d56c",
|
||||
"cb3cd3c9-5343-4d29-a655-fc314f718628",
|
||||
"a1015575-62d6-4467-a4bd-faade05754b6",
|
||||
"ab578be5-5c71-41aa-83b0-1e2893649e14",
|
||||
"a4b3667f-d3b4-4903-bc65-e02bd71df865",
|
||||
"a2591bac-d552-4ef3-8637-b96335d73cf4",
|
||||
"e11f7030-71d3-4783-a7bb-599a0d326a5b",
|
||||
"ff5cec4b-4c12-4bb5-b73e-829ea1649504",
|
||||
"02893780-69c4-4026-b296-929a5b14fbc8",
|
||||
"faa4cdff-572f-4a4e-9fad-1798a142ca4f",
|
||||
"de2d34f7-d8e8-4871-934a-4fb3180066f7"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 54,
|
||||
"peak_oldest_age_s": 1.2,
|
||||
"worker_peak_rss_mib": 78.3,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.84,
|
||||
"p95_s": 1.33,
|
||||
"p99_s": 1.37,
|
||||
"max_s": 1.37,
|
||||
"throughput_per_s": 50.21,
|
||||
"span_s": 1.6,
|
||||
"measurement_window_s": 1.991,
|
||||
"worker_cpu_seconds": 1.47,
|
||||
"worker_cpu_percent_one_core": 73.82,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 40.17,
|
||||
"attachment_ids": [
|
||||
"369a2d06-c8de-42a2-aa65-8556e5011d95",
|
||||
"6a0c9c3d-3699-4e6d-8469-89106a11f635",
|
||||
"1a373bdb-41bf-498b-a54a-3cc562ae78cd",
|
||||
"f8377da3-8ddc-4fbb-8c4e-e608082fb039",
|
||||
"3698f728-059d-4480-af59-a44b20ea9cdf",
|
||||
"a42a8bcf-09ce-48d7-8b12-45dc08ca1554",
|
||||
"e0721646-ce31-4e3c-ab42-df59f72be3b6",
|
||||
"59c00e91-97d7-4be4-8d06-6906185d424e",
|
||||
"3fe8235a-40ac-475a-808a-24b768125601",
|
||||
"055b554b-2cc9-434f-b7da-44cbdccaa703",
|
||||
"98e66361-570f-4296-9a08-ea14385c0457",
|
||||
"afdeb335-f2eb-4a46-98a8-42875242ad6b",
|
||||
"e6eb3a8e-8621-44ae-9665-61d71a32566b",
|
||||
"6e3380de-9588-4f29-8729-99ea18ba2225",
|
||||
"6fc5d716-966f-4354-8124-7717fcfd42e0",
|
||||
"0501f9db-df18-4626-a739-d5a46018d961",
|
||||
"c3eff4b1-b119-433c-85a9-1b2e44b966c6",
|
||||
"e3209e5c-75c1-4bdd-9e76-8866c527f81c",
|
||||
"fd483860-6dfe-43a0-a067-17ec289d0ba4",
|
||||
"99c38cc4-0b82-432e-9491-4192dab60c54",
|
||||
"8b5c51ff-de77-4fdc-9acf-bb78fe0d6cdb",
|
||||
"cfb32500-3940-47d7-bb6a-9ec075738012",
|
||||
"2fb57612-22d0-42f2-9ebe-0a0de48dfb4c",
|
||||
"4958f233-5968-4084-bfd8-4028601d7242",
|
||||
"6d3d6524-b973-4759-a353-da475ed69a36",
|
||||
"5df49d78-c5e4-4e24-a8ac-6036dab6a4e8",
|
||||
"cb39ff33-391d-49d3-902c-6de656603e9b",
|
||||
"119305a1-c164-4f42-b551-4cbcfae387a4",
|
||||
"51d3b68d-4a62-403e-8b7f-085d83cdd0a6",
|
||||
"781bd5c5-8aa5-4f69-8c5b-f11c24e60253",
|
||||
"9f289f53-f379-4a8b-abca-9512b39233bd",
|
||||
"162d17fe-be83-4d0a-968d-144e466a6bc6",
|
||||
"d674ca52-c3cf-4f59-a2d6-18ddd8770fc4",
|
||||
"30bf8a4c-6988-475e-b093-3d00ab1dc1a8",
|
||||
"8ba37238-1072-4301-9630-084e5e653777",
|
||||
"b28dcff7-1552-4286-89f3-1a5f83fead59",
|
||||
"5ead051c-3026-45bf-bf55-ee529e6f4ff4",
|
||||
"2e51963f-ae62-4f92-8d3f-8a8a454fd7ac",
|
||||
"2d536590-5932-49c1-9d55-05064097ba2a",
|
||||
"cf124a44-f16e-4f14-9b01-34ad758c5bee",
|
||||
"2a0bd5ff-449f-45ee-b3eb-b95c2271be3d",
|
||||
"767b88a3-f5d7-4bf2-b818-3f7e19d78cb0",
|
||||
"efa00e11-08e0-4416-bb7b-3736e1766339",
|
||||
"8fc529e7-0607-4e4f-8e2f-bbf8a94aed17",
|
||||
"4152271d-f5e7-4fd1-bc91-c26ea1d46f1b",
|
||||
"553a6c7e-0630-49cf-b2e5-3e792655d902",
|
||||
"3fe348ef-85f2-434d-96b9-224bf3759d06",
|
||||
"f27b350b-50d5-47e8-a620-a32088068daa",
|
||||
"2e415149-552d-477e-9f5b-fa738a51eb74",
|
||||
"54e255a8-1abd-4283-89d1-714349db1f0b",
|
||||
"7b626082-147b-4107-acad-23431f02843b",
|
||||
"c9de03cc-c4eb-47f1-af2c-aab97b910c2b",
|
||||
"87957fb0-63e3-4c91-8f6c-a0d88a81291a",
|
||||
"b1d1f957-af29-4ccb-99cb-0fd47f108b8e",
|
||||
"5a9c9b2c-5867-4627-9d22-890abffa11e2",
|
||||
"f7a24e55-969a-49e9-b485-ab3d76e887f9",
|
||||
"264295e7-c3d0-4073-ab78-0447a436b135",
|
||||
"5a209d9d-0413-4363-aeea-12e881a014c0",
|
||||
"ebbf2d91-4961-47f6-8b4d-9e5f7ec83bc2",
|
||||
"8821988c-d93b-4d60-ad32-0781f77682b3",
|
||||
"65ac051b-a319-4c85-8cdc-6dfe3bca6da2",
|
||||
"b6ccfaa1-be04-4b8c-b74f-07ca0a394097",
|
||||
"63151877-74a8-4986-9911-2e47a1fc5061",
|
||||
"f4b8aead-e8b9-425c-9dc5-bc5c88044f34",
|
||||
"1932d7d7-575f-402c-b56c-272ab7587a64",
|
||||
"2079ec0c-5e8d-46cd-95da-4a77f2c64b17",
|
||||
"559ff00e-2679-4674-974f-4fd3a1362c84",
|
||||
"36b39e61-fb6c-49d2-97f4-5687f069754b",
|
||||
"f844cda3-3ddd-45fe-97a2-b418d5e2ca95",
|
||||
"8253f768-0b35-4150-80f7-cb0c1a1e600e",
|
||||
"84699d62-6aae-4c61-a3a9-aefc85761a51",
|
||||
"f94a444d-8bc0-4a24-9f46-db29c824b288",
|
||||
"60be5424-9940-4cf2-acdb-dc3a68d6e453",
|
||||
"7d2d85cc-39a1-4dda-be06-142ec2d6c542",
|
||||
"cf4f344d-c52e-4c5b-8d25-9495a1034dd4",
|
||||
"a7ca7b03-92e2-4e0f-9d9f-3dc4588062e1",
|
||||
"0fd0d132-76b5-46e5-add3-eaf1f5c8577e",
|
||||
"8d269940-4187-461a-9624-c0ed1c1e0823",
|
||||
"a57d54d6-996d-45ae-af6f-f2d37f3a2ceb",
|
||||
"c45136fe-130d-4257-a6ca-1d58296b089e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.7,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 24,
|
||||
"peak_oldest_age_s": 4.8,
|
||||
"worker_peak_rss_mib": 319.1,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.69,
|
||||
"p95_s": 4.89,
|
||||
"p99_s": 4.93,
|
||||
"max_s": 4.93,
|
||||
"throughput_per_s": 4.37,
|
||||
"span_s": 5.5,
|
||||
"measurement_window_s": 6.405,
|
||||
"worker_cpu_seconds": 5.47,
|
||||
"worker_cpu_percent_one_core": 85.4,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 3.75,
|
||||
"attachment_ids": [
|
||||
"16ed740f-59f2-4b65-8aa7-3026600e5fdd",
|
||||
"9e9509a1-f58f-4d57-8e5a-ac5b689091cb",
|
||||
"6a3614a2-6461-48a4-808f-d500ddd7157f",
|
||||
"9b15f285-5da4-4b04-85ec-09603af7370c",
|
||||
"71a77ae7-8dba-45d1-986c-19fddc94af3d",
|
||||
"46fe0fd9-c9ae-47f3-a47c-9393c8d3c2a6",
|
||||
"e2410aae-675e-4259-824d-cde0b479b667",
|
||||
"dcef7518-6931-46d6-ac51-f53af10a7018",
|
||||
"8d38362f-34af-4cd4-8988-8525edfcb171",
|
||||
"6f8ba7d2-f77f-42c8-89f4-13b644294d2d",
|
||||
"e23348ac-6fee-49c8-b492-5fc78a853548",
|
||||
"c4fa1ca7-1a46-4e3f-843b-531ea3664359",
|
||||
"66014e42-5f76-4fe8-ab67-9cab84fe5a16",
|
||||
"83a2cbee-f650-4452-9270-4a6921f51ff6",
|
||||
"95b35d8e-d2e1-4c6c-8c8d-058182bd12e8",
|
||||
"0d7d6d37-f538-46b7-b314-281923bc583b",
|
||||
"b389353c-5e56-43b3-a8b2-1bb6e7d865db",
|
||||
"840e21f4-2a44-487d-bedd-b9cf1cd94548",
|
||||
"8b865dac-9945-4314-acd2-2abc09053a23",
|
||||
"d60650ab-0ee4-4e95-9207-688f4af14b9c",
|
||||
"f8e8de12-30b8-4c73-b81f-53482b79fc44",
|
||||
"88b2cfc3-f26b-4515-aaf2-f23fbb78dc28",
|
||||
"d56513e0-c8f4-4a48-9b83-98ef17760dad",
|
||||
"eed9e89b-f6ff-44e4-b2cc-16f871185a19"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.9,
|
||||
"worker_peak_rss_mib": 434.5,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 2.3,
|
||||
"p95_s": 3.3,
|
||||
"p99_s": 3.3,
|
||||
"max_s": 3.3,
|
||||
"throughput_per_s": 3.48,
|
||||
"span_s": 3.4,
|
||||
"measurement_window_s": 4.061,
|
||||
"worker_cpu_seconds": 3.36,
|
||||
"worker_cpu_percent_one_core": 82.75,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 2.96,
|
||||
"attachment_ids": [
|
||||
"7f5b994f-f3de-4b60-a5e9-f87ffd0835b6",
|
||||
"95d47438-8f87-45fe-9129-cb9563531a6b",
|
||||
"891e1344-f240-4efc-8b89-f3787d67002e",
|
||||
"719431c3-cd94-4cbc-92e5-5c301ae8005d",
|
||||
"85bffd8c-0007-4f7e-88e0-c8f84c9a575a",
|
||||
"32b180e0-5fda-4d53-ad12-7e2fe8715b51",
|
||||
"726dd68e-83f9-4f05-897f-d79f403b8064",
|
||||
"1950ca75-72e0-45c5-9e4e-5d54630fb3b4",
|
||||
"6742055a-66eb-49e1-a26a-b333995e1142",
|
||||
"d770a827-5abb-4df3-8a03-ba24ae2dbe38",
|
||||
"54bcd9c6-6d07-47c6-9aae-cc06fcbb9ca1",
|
||||
"d38c2e12-76ed-412a-a120-bae6d7be1142"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"base_commit": "5dbb905364476e54f105ce901d5b3a5fef7a98db",
|
||||
"source_sha256": {
|
||||
"research/queue_overload.py": "f8ac1dc1c5e2af8fa363e6c018f1abf7bf4ef9e4e820f1d4973a918add10a79e",
|
||||
"scripts/stop-one.cmd": "0e6d46bc7abc2f81403422f985ee9d0d2ad02e0cfd9db2db0fccc873020d8997",
|
||||
".github/workflows/application.yml": "ef1de7d11d22b94f3375c1607f043268171b7c4bb9655f225e005f7e88e05372"
|
||||
},
|
||||
"host": {
|
||||
"cpu": "Intel Core i9-12900KS",
|
||||
"physical_cores": 16,
|
||||
"logical_cpus": 24,
|
||||
"memory_bytes": 102830256128
|
||||
},
|
||||
"docker": {
|
||||
"server_version": "29.5.3",
|
||||
"os": "Docker Desktop",
|
||||
"architecture": "x86_64",
|
||||
"cpus": 24,
|
||||
"memory_bytes": 50392047616
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,834 @@
|
||||
{
|
||||
"date": "2026-11-12 04:06:06",
|
||||
"args": {
|
||||
"out": "evidence\\pr52_audit_2026-11-12",
|
||||
"concurrency": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"n_small": 80,
|
||||
"n_medium": 24,
|
||||
"n_large": 12,
|
||||
"limit": 30,
|
||||
"burst": 200,
|
||||
"b_medium": false
|
||||
},
|
||||
"worker_limits": {
|
||||
"cpus": 1.0,
|
||||
"memory": "768m"
|
||||
},
|
||||
"image_kib": {
|
||||
"small": 109,
|
||||
"medium": 1961,
|
||||
"large": 3100
|
||||
},
|
||||
"matrix": [
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 66,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 72.7,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 1.4,
|
||||
"p95_s": 2.57,
|
||||
"p99_s": 2.71,
|
||||
"max_s": 2.71,
|
||||
"throughput_per_s": 26.26,
|
||||
"span_s": 3.0,
|
||||
"measurement_window_s": 3.488,
|
||||
"worker_cpu_seconds": 1.59,
|
||||
"worker_cpu_percent_one_core": 45.59,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 22.94,
|
||||
"attachment_ids": [
|
||||
"f3755317-a6f1-4c38-958e-572d2e72521c",
|
||||
"e1866acf-772a-4e44-9f6d-96d6d6c855ae",
|
||||
"4c88f7ec-94f3-42ef-a80d-2cd5506f288f",
|
||||
"9d76e306-5c0f-4dc0-b417-0e328d8c7ff5",
|
||||
"5d7435aa-9667-4b0d-b772-2a5cd07c6d18",
|
||||
"91734a85-8d57-4ae7-9f80-023bb9cde252",
|
||||
"321ea613-76a4-4ce7-a2e8-da57f8d6fa0e",
|
||||
"d5903675-86be-41fb-8d23-79835bc7d3e7",
|
||||
"dfdc7b08-fedf-45b7-9ebd-a15d00670784",
|
||||
"2e2ec815-68f6-444d-b83f-094c1af5713b",
|
||||
"80d48257-49e5-41df-a73c-1d31715ad117",
|
||||
"816c620e-82d5-4feb-9bdf-def2dd4da03f",
|
||||
"c9b770c8-de70-4476-81eb-1e872ddee1dc",
|
||||
"90239c75-8017-4e4e-b098-3f2901c46f2d",
|
||||
"a29229ce-376b-43a2-bb2e-bc4457e47a5f",
|
||||
"5b45de93-1b7b-4167-a739-07c494cb6d9d",
|
||||
"a35306c2-6cd2-4745-9aab-c5acfaf902ca",
|
||||
"85996ad6-fe4e-41b1-b42d-bb524176f64e",
|
||||
"2f3c4a2c-462b-4d53-b1d2-cb463b63b265",
|
||||
"7315c4f5-397d-41bd-8ada-1199b5ff8d20",
|
||||
"5ca87add-c93e-4bee-b3a6-e28b51b42d28",
|
||||
"93dcb192-6666-45cf-a55d-df0959d6bddd",
|
||||
"d2b44888-c498-4900-9814-340f27e79c12",
|
||||
"bb7b4f52-ffb4-4b75-b271-cfa705c14e50",
|
||||
"5df28e25-a9b3-47f2-92f5-71bd048959fb",
|
||||
"cc44923d-6e23-43bd-8ea0-3a158de52a58",
|
||||
"b352814b-b29c-4043-b862-670fcfe50002",
|
||||
"21ab21a9-1959-4432-8283-eed80a3ae6cc",
|
||||
"d33a072f-937f-4873-8dbc-3eaba5e9a06b",
|
||||
"e1630cdc-c746-4f95-8712-0f0ec4501224",
|
||||
"e6680ab1-e568-4cf3-b372-e165553c8277",
|
||||
"bdd7584a-d8f7-4420-b2f9-127aac3f67e4",
|
||||
"1024b092-18f7-42ad-8269-15137dcb07b5",
|
||||
"c2623c1a-eec4-43c0-a870-4202f1b97542",
|
||||
"6a16b4e3-e6ef-45ef-8a2a-898e0dc590d0",
|
||||
"594a7d05-50f3-4d5c-9114-1a5383869296",
|
||||
"d6c712a8-e930-4e45-862b-2ee20ea4c380",
|
||||
"9147c9ce-32dd-458e-bbda-762bbe1c60de",
|
||||
"902a3269-9937-4c44-91ce-058e67005c87",
|
||||
"df198815-5898-4293-bb85-087679a4efbf",
|
||||
"8fd86d7e-efbe-4c84-8f17-cd03e510f508",
|
||||
"67489317-2bcf-4be0-a97b-52afce0e7c44",
|
||||
"28185ccf-516a-45d8-b851-ae202869b7cc",
|
||||
"bff9004a-3bc8-44df-ab54-2295c3e7e27c",
|
||||
"c0ea5554-7a66-49f2-8ec3-c4fde067abdf",
|
||||
"4a126067-ce1a-4c57-8817-cf2854e40cde",
|
||||
"5a76425e-88ce-4e1d-b335-b0ef76cd88aa",
|
||||
"8ea27a23-6e62-47d4-8523-a6ca1f5ad458",
|
||||
"6e03b1c2-d4a7-4772-89cf-f41ffde6db55",
|
||||
"469e7a1d-3ac9-4224-8661-0e57e3c37d03",
|
||||
"81b4d8b6-7e29-4fe1-b062-15c87101cbfd",
|
||||
"64399d7a-7df1-45fa-b7a7-653e5da60bd0",
|
||||
"b21be428-e7d0-4417-9023-ffd232ba830a",
|
||||
"56029d32-3104-4255-8006-cd775fb22a1f",
|
||||
"84308a9f-59ba-4797-8e42-a990ebf2fefc",
|
||||
"77f0efa1-b5d6-481d-b1a6-a8317af5201b",
|
||||
"5d3d0106-8851-4cc0-9fa8-178e7a4e1852",
|
||||
"190e83e7-1c5a-4cee-8447-5315a54ab817",
|
||||
"ab5363c6-da59-4565-a074-1b3acb1433d0",
|
||||
"19e569fa-412b-4fef-9e89-2c4218293613",
|
||||
"90fb1725-33a3-4011-b430-f28f188d500c",
|
||||
"1bf62c59-0ad5-4225-ac03-9bd5a4965900",
|
||||
"dac8a042-a731-4bdf-92f7-cbf6377a3496",
|
||||
"6370b8dd-8ce2-4e9e-8b20-cdc07d8a4218",
|
||||
"3e3a0ca8-756b-4c9c-a42b-55d0ca984f31",
|
||||
"63f82f8f-419a-4fe3-8aff-180249c946ec",
|
||||
"c2a64395-4403-418f-a4ea-5e32689e4dc1",
|
||||
"6cecaaa4-3159-400d-9e28-e7c0dd26bcf6",
|
||||
"5e5567e8-9d1a-401b-bf15-d7fb2ac4c6a3",
|
||||
"1c4371d1-39c1-475d-9055-63c3823f9859",
|
||||
"c6ab5b7e-2c37-4e42-8abc-3bfe922370d8",
|
||||
"8a52c2ca-e625-4549-9661-f89d3c973c7e",
|
||||
"e6993de3-6dd1-40ed-8aa0-7f811d311087",
|
||||
"3a3bd631-10b6-478f-b25b-575053f0088d",
|
||||
"a906674b-bd9c-4b34-9c6e-d25e23d453fe",
|
||||
"d4aee6be-963c-4803-956b-0dcce87c6dd2",
|
||||
"4510f909-6250-4a54-9488-8c0edd064353",
|
||||
"19a3b5ac-5784-4e91-95cf-a95b66de7d23",
|
||||
"c25cd565-459d-4ec1-83bf-d21859dd3634",
|
||||
"d28d6581-7aef-4762-9d4b-3ae016f33832"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 23,
|
||||
"peak_oldest_age_s": 5.4,
|
||||
"worker_peak_rss_mib": 133.7,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.78,
|
||||
"p95_s": 5.23,
|
||||
"p99_s": 5.44,
|
||||
"max_s": 5.44,
|
||||
"throughput_per_s": 4.09,
|
||||
"span_s": 5.9,
|
||||
"measurement_window_s": 6.264,
|
||||
"worker_cpu_seconds": 5.15,
|
||||
"worker_cpu_percent_one_core": 82.21,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 3.83,
|
||||
"attachment_ids": [
|
||||
"d336e33f-283a-49e9-8ca9-9c4d0f17c904",
|
||||
"dfa1d79f-ffe8-4937-8c46-c456143a9394",
|
||||
"a8bb7c8b-931c-492e-bee2-5b9920e6355e",
|
||||
"6de05c1c-ca5a-4b86-8dd3-af63b1aa321f",
|
||||
"dd696951-5e88-44b4-80a2-c868326a2102",
|
||||
"4253bdfd-86c2-4116-ad68-72f685481005",
|
||||
"7afb0faf-2c94-4014-8be5-07731cdfb483",
|
||||
"624e290a-0463-417d-8e53-448a75bfcd31",
|
||||
"0b6f4492-3378-42bd-8fd4-47983dd7a54d",
|
||||
"c4fa5d79-c3f1-48d1-98bb-bf4e73c55efe",
|
||||
"8c25109b-d8ba-4608-a1d7-1d7abc1f47b8",
|
||||
"0361352b-61aa-41c0-ba62-0e767f954a62",
|
||||
"bcf3989b-f1fe-4619-962c-34d22a984261",
|
||||
"80dd3ed7-6e10-4dfd-aca5-3a3e4382326a",
|
||||
"9ca80d24-2c9f-44ac-8811-deeabcb8ff0f",
|
||||
"f6c7cc70-25ba-4a5a-aa8d-75f22a8ce836",
|
||||
"0e8d2721-f293-4ccc-9d10-5873c8fd90b9",
|
||||
"95ce7a82-0133-4a9d-9899-dbed974ee9ec",
|
||||
"5fd7ea2c-685f-4cfc-9f0b-dabaeba4129d",
|
||||
"773dfbf2-fc82-4888-8208-032b25035a4c",
|
||||
"d3ba0d4a-952b-4095-821c-398fce6165f0",
|
||||
"9362cf03-cdd4-482c-8786-42379ce6bbf0",
|
||||
"d3258e0e-78b6-40ea-86b8-9403d2d8671f",
|
||||
"ceaaa543-c950-4398-af14-924025c652bb"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 11,
|
||||
"peak_oldest_age_s": 3.3,
|
||||
"worker_peak_rss_mib": 160.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 1.73,
|
||||
"p95_s": 3.34,
|
||||
"p99_s": 3.34,
|
||||
"max_s": 3.34,
|
||||
"throughput_per_s": 3.33,
|
||||
"span_s": 3.6,
|
||||
"measurement_window_s": 4.058,
|
||||
"worker_cpu_seconds": 3.05,
|
||||
"worker_cpu_percent_one_core": 75.16,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 2.96,
|
||||
"attachment_ids": [
|
||||
"cd228696-224b-4b17-87c7-f0af12762e19",
|
||||
"0c3b34a6-c495-4450-ba46-bfe72d46b941",
|
||||
"00a409ba-8b6a-4021-aad5-3f717b9133ee",
|
||||
"f239a54e-13c6-4252-bb47-0e513b310008",
|
||||
"e0e8e733-b7cd-48e7-884e-ea54cac614de",
|
||||
"51498e5d-f689-47f9-930d-9c21c7c7cc28",
|
||||
"12591ec9-056b-483a-b9a4-b81a3ad6f9ae",
|
||||
"b2b0d9b2-616a-4874-81ef-18b5b631f7cc",
|
||||
"262d2c63-d351-41d4-a641-4fcb86a28df5",
|
||||
"62c53389-0ec2-4b52-89d9-cc4734867d36",
|
||||
"a0631868-6ed3-4ce9-8adf-8518ec884e13",
|
||||
"95f1942f-300a-43b4-a4cd-7915f2291552"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 52,
|
||||
"peak_oldest_age_s": 1.2,
|
||||
"worker_peak_rss_mib": 77.0,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.83,
|
||||
"p95_s": 1.57,
|
||||
"p99_s": 1.65,
|
||||
"max_s": 1.65,
|
||||
"throughput_per_s": 43.4,
|
||||
"span_s": 1.8,
|
||||
"measurement_window_s": 2.687,
|
||||
"worker_cpu_seconds": 1.53,
|
||||
"worker_cpu_percent_one_core": 56.93,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 29.77,
|
||||
"attachment_ids": [
|
||||
"986c0488-655c-45ee-80b0-ffa2ec88e1b0",
|
||||
"5356f279-0ff8-46d0-9597-6bf80decdba8",
|
||||
"59374d10-af30-4ee2-a9e0-748f64b9c843",
|
||||
"04de697f-cdbb-400b-9c8b-b0b5999f77cb",
|
||||
"e6154e04-3099-49d4-b3f6-4b3cfaefefb2",
|
||||
"b6686d31-c28f-4eda-8cdb-3beaeca14ab2",
|
||||
"7f439c58-449b-4d3f-b23a-eabadea55d48",
|
||||
"e3fd12a4-ba9e-461a-b2e6-62db4ff2934c",
|
||||
"ac5fb8e0-e691-4959-96eb-78e78e51d66d",
|
||||
"b58af054-ffc2-4e6c-8db8-5fed9e7e6030",
|
||||
"99b957f2-c307-4aef-b8ea-cec34efdb314",
|
||||
"5eb6d8c5-a04d-4200-99ed-6cac496df2af",
|
||||
"3878ed43-cf40-4438-b7ef-c21a047be400",
|
||||
"1d2d3647-ad62-4d99-b467-cb69403005e5",
|
||||
"57df0b4a-c241-45dd-83a3-55c52d1b5e78",
|
||||
"2836c401-af66-4514-be08-434267941a12",
|
||||
"1ee32d6e-5152-4186-8916-52cebaae5307",
|
||||
"34be6229-ee00-4996-ae1a-af9c4cb99602",
|
||||
"3a99829b-3693-49c5-9edd-39b3009457fb",
|
||||
"c025c320-f8f4-4ee6-a55a-571dfb5ff359",
|
||||
"4b5dd6d7-c1a7-4741-95a2-6a29f817ab2f",
|
||||
"1b2b74d0-176b-4725-90c1-e58bd788477f",
|
||||
"238dd238-c3e4-4672-bf96-ce07b9dd6604",
|
||||
"ee14148a-40b0-4b6d-b3f8-a338becbebce",
|
||||
"b6396801-9578-4845-b4b1-c79588b3848a",
|
||||
"d935deab-1e92-4152-9869-daf58666a98a",
|
||||
"e92f1c43-80b9-4344-bd1d-f29031d5d1ca",
|
||||
"80f24a3f-d238-4b38-a3fe-c3f3d223ab67",
|
||||
"defdf421-0f51-4c7a-ad0d-1222ac5c5629",
|
||||
"f67bea9b-5bca-4a94-ae97-e976be775a7b",
|
||||
"22028e60-bab4-414f-b096-8725076b391b",
|
||||
"88c729e9-5bee-4a39-832a-a0a43e8b0e5f",
|
||||
"c33daf06-9b41-4e0a-a3fb-586769c480a9",
|
||||
"67ad759d-27b5-46d2-829d-d131c5b62e6d",
|
||||
"ab39f2c2-f6b2-4fb2-b7a8-6e71ee6be134",
|
||||
"55495a88-864d-4dee-bb9a-54bd4125ca57",
|
||||
"9bea0324-df5e-4d01-812a-7df52022c588",
|
||||
"9be5cf96-5b29-4af2-a59c-34fe1505720c",
|
||||
"4bbbd180-25cd-4959-a776-77591b79d991",
|
||||
"51ae7211-ce4b-4a52-8df2-5feec4c545f9",
|
||||
"e28a30a0-bb78-4186-95df-b2a865e0da3d",
|
||||
"c20f5ea6-2691-459d-a2a8-974e152fd596",
|
||||
"cda9bf31-a528-4835-8b5b-82039322bd14",
|
||||
"dced05fc-8996-4907-b975-52806064b50e",
|
||||
"efedf949-15ae-490a-8831-dc515b633d9c",
|
||||
"bd50c7e9-a7e5-4cde-bd9e-2dd8f7455543",
|
||||
"10734543-ea50-44f6-86ea-0d33e1623a63",
|
||||
"241f409f-f390-42d7-a897-7f116cea8b7c",
|
||||
"0b9a97e6-534f-4caa-9f6b-5a186240946c",
|
||||
"33106c0c-1238-4ee9-ae2c-48b03822644e",
|
||||
"137d32d5-28cc-4ac8-91e1-58f1ebe2f343",
|
||||
"e1c72e28-f2b1-4106-bc19-0d7aa5fab077",
|
||||
"5c21cd6d-e19a-4633-b2a9-cf91fd1dfedf",
|
||||
"5957208f-c047-48b7-97d2-1965e62c685e",
|
||||
"58004bef-87ee-4e9d-86d8-ef918592840f",
|
||||
"ef6a9958-1749-4fb4-8eaa-075226783465",
|
||||
"146cca2d-5e1d-4ccf-b6e8-674340122d81",
|
||||
"470c1ca5-142e-4a92-b3b7-b7009d98e22a",
|
||||
"5387645a-eec3-46e6-8cf3-ee8d210421ec",
|
||||
"99d8ea69-1068-4412-a723-abcad61c4489",
|
||||
"e2da2436-4c30-4ab2-904c-2865a74c7a7e",
|
||||
"a372fbca-08fa-48b5-8052-2bac654b56e8",
|
||||
"b9e469d9-6def-4073-adbb-2ea7db538380",
|
||||
"4745a33e-df4d-482d-95a2-137d6970932b",
|
||||
"101e6317-5750-402c-a48e-52dea4c8f25a",
|
||||
"a4d721a4-99ef-4081-8551-a8503fc6f06c",
|
||||
"986a0077-ecfb-4cfc-be8e-2936ee5671df",
|
||||
"d1beeab5-bc82-4227-a995-46bb5e495a6f",
|
||||
"2ada1a7f-fa40-48ee-ae9a-44693d424d4b",
|
||||
"c9d3c291-0c97-4f7d-bebd-1a1b9976b863",
|
||||
"e9e9fac5-8c9d-4149-935f-70ef215a9287",
|
||||
"6ca3dabe-a52e-4a61-a0de-61326a890509",
|
||||
"d8934eaf-2e62-4ff0-beb2-dcf1ef9e4051",
|
||||
"fe98a8e9-5d90-45c9-98e6-92ba24d319b0",
|
||||
"a6fcee5c-34b3-44e4-ae6c-d72c8541f53a",
|
||||
"a01be823-169d-4f37-bfa8-5c54cfd8fb40",
|
||||
"e3825a5c-8de2-41d4-839f-79b7bd15dc68",
|
||||
"9dd41734-aeb3-4127-9358-8c52bab31136",
|
||||
"84621a70-ce86-4f9d-b5ff-2d9b1ce8a7fc",
|
||||
"6f0e2b52-719b-4ee4-bf20-c73154838ca5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.7,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 22,
|
||||
"peak_oldest_age_s": 4.2,
|
||||
"worker_peak_rss_mib": 196.4,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.48,
|
||||
"p95_s": 4.64,
|
||||
"p99_s": 4.65,
|
||||
"max_s": 4.65,
|
||||
"throughput_per_s": 4.6,
|
||||
"span_s": 5.2,
|
||||
"measurement_window_s": 5.726,
|
||||
"worker_cpu_seconds": 5.23,
|
||||
"worker_cpu_percent_one_core": 91.34,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 4.19,
|
||||
"attachment_ids": [
|
||||
"0df1caad-9cfe-4efa-96a7-dcc3606702fc",
|
||||
"a0815e57-91e5-44aa-98d2-131c2930b9d4",
|
||||
"625b6ac4-8ded-4f17-b08f-665f77ad599a",
|
||||
"c3497e58-dbd6-4a1a-9510-bfd6588ea56e",
|
||||
"81ca832a-5bc7-4ca0-9e89-b185cd37d694",
|
||||
"d1d0e4c3-cdfd-410c-95e5-9e6bbd73edd1",
|
||||
"5c671809-eaa0-4e95-a94e-5cc9dadf57f3",
|
||||
"43097921-3eca-4122-ae45-0c191de23771",
|
||||
"a5a03c4d-9d01-4c72-b7cd-7183bb32b044",
|
||||
"4269c35d-33b7-4fbf-a783-7209a959693d",
|
||||
"6f3ee881-1fa6-4789-b916-f21860f83c4f",
|
||||
"e20f0e91-9fb5-490f-957d-2a827f347087",
|
||||
"b8ab1e8b-ec0e-4c21-9e82-ccd4f589812d",
|
||||
"93c0fcd0-6c99-4b17-8120-f452bc1b51ce",
|
||||
"bec68cde-cdce-40a5-9973-a1c09fc1c06e",
|
||||
"7e02a10a-c6fa-485d-b4ab-0095260a2cc6",
|
||||
"c87a37cf-590e-4562-a098-06f1ae001690",
|
||||
"e9ffca81-951d-4546-a1e1-4d9d0b6d6ed9",
|
||||
"bb21974f-3888-458f-b475-6b8670ec79da",
|
||||
"cec87457-c37d-42db-9e7f-b12f33fa7949",
|
||||
"0450c693-16d6-41d5-a55a-9d7093fe7c27",
|
||||
"4c97331a-1879-46d9-8959-133697728a2f",
|
||||
"1f52d3ed-f6d1-498d-a6a0-455140ef3230",
|
||||
"13239c5e-6d5c-4bd7-8945-bd6a0790a4a9"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 251.0,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 1.68,
|
||||
"p95_s": 3.04,
|
||||
"p99_s": 3.04,
|
||||
"max_s": 3.04,
|
||||
"throughput_per_s": 3.7,
|
||||
"span_s": 3.2,
|
||||
"measurement_window_s": 4.012,
|
||||
"worker_cpu_seconds": 3.21,
|
||||
"worker_cpu_percent_one_core": 80.0,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 2.99,
|
||||
"attachment_ids": [
|
||||
"d75b173f-519d-4ff9-b8fd-05eee6b618d1",
|
||||
"9c3feb38-f6e3-474d-8530-d0855825d56c",
|
||||
"cb3cd3c9-5343-4d29-a655-fc314f718628",
|
||||
"a1015575-62d6-4467-a4bd-faade05754b6",
|
||||
"ab578be5-5c71-41aa-83b0-1e2893649e14",
|
||||
"a4b3667f-d3b4-4903-bc65-e02bd71df865",
|
||||
"a2591bac-d552-4ef3-8637-b96335d73cf4",
|
||||
"e11f7030-71d3-4783-a7bb-599a0d326a5b",
|
||||
"ff5cec4b-4c12-4bb5-b73e-829ea1649504",
|
||||
"02893780-69c4-4026-b296-929a5b14fbc8",
|
||||
"faa4cdff-572f-4a4e-9fad-1798a142ca4f",
|
||||
"de2d34f7-d8e8-4871-934a-4fb3180066f7"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 54,
|
||||
"peak_oldest_age_s": 1.2,
|
||||
"worker_peak_rss_mib": 78.3,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.84,
|
||||
"p95_s": 1.33,
|
||||
"p99_s": 1.37,
|
||||
"max_s": 1.37,
|
||||
"throughput_per_s": 50.21,
|
||||
"span_s": 1.6,
|
||||
"measurement_window_s": 1.991,
|
||||
"worker_cpu_seconds": 1.47,
|
||||
"worker_cpu_percent_one_core": 73.82,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 40.17,
|
||||
"attachment_ids": [
|
||||
"369a2d06-c8de-42a2-aa65-8556e5011d95",
|
||||
"6a0c9c3d-3699-4e6d-8469-89106a11f635",
|
||||
"1a373bdb-41bf-498b-a54a-3cc562ae78cd",
|
||||
"f8377da3-8ddc-4fbb-8c4e-e608082fb039",
|
||||
"3698f728-059d-4480-af59-a44b20ea9cdf",
|
||||
"a42a8bcf-09ce-48d7-8b12-45dc08ca1554",
|
||||
"e0721646-ce31-4e3c-ab42-df59f72be3b6",
|
||||
"59c00e91-97d7-4be4-8d06-6906185d424e",
|
||||
"3fe8235a-40ac-475a-808a-24b768125601",
|
||||
"055b554b-2cc9-434f-b7da-44cbdccaa703",
|
||||
"98e66361-570f-4296-9a08-ea14385c0457",
|
||||
"afdeb335-f2eb-4a46-98a8-42875242ad6b",
|
||||
"e6eb3a8e-8621-44ae-9665-61d71a32566b",
|
||||
"6e3380de-9588-4f29-8729-99ea18ba2225",
|
||||
"6fc5d716-966f-4354-8124-7717fcfd42e0",
|
||||
"0501f9db-df18-4626-a739-d5a46018d961",
|
||||
"c3eff4b1-b119-433c-85a9-1b2e44b966c6",
|
||||
"e3209e5c-75c1-4bdd-9e76-8866c527f81c",
|
||||
"fd483860-6dfe-43a0-a067-17ec289d0ba4",
|
||||
"99c38cc4-0b82-432e-9491-4192dab60c54",
|
||||
"8b5c51ff-de77-4fdc-9acf-bb78fe0d6cdb",
|
||||
"cfb32500-3940-47d7-bb6a-9ec075738012",
|
||||
"2fb57612-22d0-42f2-9ebe-0a0de48dfb4c",
|
||||
"4958f233-5968-4084-bfd8-4028601d7242",
|
||||
"6d3d6524-b973-4759-a353-da475ed69a36",
|
||||
"5df49d78-c5e4-4e24-a8ac-6036dab6a4e8",
|
||||
"cb39ff33-391d-49d3-902c-6de656603e9b",
|
||||
"119305a1-c164-4f42-b551-4cbcfae387a4",
|
||||
"51d3b68d-4a62-403e-8b7f-085d83cdd0a6",
|
||||
"781bd5c5-8aa5-4f69-8c5b-f11c24e60253",
|
||||
"9f289f53-f379-4a8b-abca-9512b39233bd",
|
||||
"162d17fe-be83-4d0a-968d-144e466a6bc6",
|
||||
"d674ca52-c3cf-4f59-a2d6-18ddd8770fc4",
|
||||
"30bf8a4c-6988-475e-b093-3d00ab1dc1a8",
|
||||
"8ba37238-1072-4301-9630-084e5e653777",
|
||||
"b28dcff7-1552-4286-89f3-1a5f83fead59",
|
||||
"5ead051c-3026-45bf-bf55-ee529e6f4ff4",
|
||||
"2e51963f-ae62-4f92-8d3f-8a8a454fd7ac",
|
||||
"2d536590-5932-49c1-9d55-05064097ba2a",
|
||||
"cf124a44-f16e-4f14-9b01-34ad758c5bee",
|
||||
"2a0bd5ff-449f-45ee-b3eb-b95c2271be3d",
|
||||
"767b88a3-f5d7-4bf2-b818-3f7e19d78cb0",
|
||||
"efa00e11-08e0-4416-bb7b-3736e1766339",
|
||||
"8fc529e7-0607-4e4f-8e2f-bbf8a94aed17",
|
||||
"4152271d-f5e7-4fd1-bc91-c26ea1d46f1b",
|
||||
"553a6c7e-0630-49cf-b2e5-3e792655d902",
|
||||
"3fe348ef-85f2-434d-96b9-224bf3759d06",
|
||||
"f27b350b-50d5-47e8-a620-a32088068daa",
|
||||
"2e415149-552d-477e-9f5b-fa738a51eb74",
|
||||
"54e255a8-1abd-4283-89d1-714349db1f0b",
|
||||
"7b626082-147b-4107-acad-23431f02843b",
|
||||
"c9de03cc-c4eb-47f1-af2c-aab97b910c2b",
|
||||
"87957fb0-63e3-4c91-8f6c-a0d88a81291a",
|
||||
"b1d1f957-af29-4ccb-99cb-0fd47f108b8e",
|
||||
"5a9c9b2c-5867-4627-9d22-890abffa11e2",
|
||||
"f7a24e55-969a-49e9-b485-ab3d76e887f9",
|
||||
"264295e7-c3d0-4073-ab78-0447a436b135",
|
||||
"5a209d9d-0413-4363-aeea-12e881a014c0",
|
||||
"ebbf2d91-4961-47f6-8b4d-9e5f7ec83bc2",
|
||||
"8821988c-d93b-4d60-ad32-0781f77682b3",
|
||||
"65ac051b-a319-4c85-8cdc-6dfe3bca6da2",
|
||||
"b6ccfaa1-be04-4b8c-b74f-07ca0a394097",
|
||||
"63151877-74a8-4986-9911-2e47a1fc5061",
|
||||
"f4b8aead-e8b9-425c-9dc5-bc5c88044f34",
|
||||
"1932d7d7-575f-402c-b56c-272ab7587a64",
|
||||
"2079ec0c-5e8d-46cd-95da-4a77f2c64b17",
|
||||
"559ff00e-2679-4674-974f-4fd3a1362c84",
|
||||
"36b39e61-fb6c-49d2-97f4-5687f069754b",
|
||||
"f844cda3-3ddd-45fe-97a2-b418d5e2ca95",
|
||||
"8253f768-0b35-4150-80f7-cb0c1a1e600e",
|
||||
"84699d62-6aae-4c61-a3a9-aefc85761a51",
|
||||
"f94a444d-8bc0-4a24-9f46-db29c824b288",
|
||||
"60be5424-9940-4cf2-acdb-dc3a68d6e453",
|
||||
"7d2d85cc-39a1-4dda-be06-142ec2d6c542",
|
||||
"cf4f344d-c52e-4c5b-8d25-9495a1034dd4",
|
||||
"a7ca7b03-92e2-4e0f-9d9f-3dc4588062e1",
|
||||
"0fd0d132-76b5-46e5-add3-eaf1f5c8577e",
|
||||
"8d269940-4187-461a-9624-c0ed1c1e0823",
|
||||
"a57d54d6-996d-45ae-af6f-f2d37f3a2ceb",
|
||||
"c45136fe-130d-4257-a6ca-1d58296b089e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.7,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 24,
|
||||
"peak_oldest_age_s": 4.8,
|
||||
"worker_peak_rss_mib": 319.1,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.69,
|
||||
"p95_s": 4.89,
|
||||
"p99_s": 4.93,
|
||||
"max_s": 4.93,
|
||||
"throughput_per_s": 4.37,
|
||||
"span_s": 5.5,
|
||||
"measurement_window_s": 6.405,
|
||||
"worker_cpu_seconds": 5.47,
|
||||
"worker_cpu_percent_one_core": 85.4,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 3.75,
|
||||
"attachment_ids": [
|
||||
"16ed740f-59f2-4b65-8aa7-3026600e5fdd",
|
||||
"9e9509a1-f58f-4d57-8e5a-ac5b689091cb",
|
||||
"6a3614a2-6461-48a4-808f-d500ddd7157f",
|
||||
"9b15f285-5da4-4b04-85ec-09603af7370c",
|
||||
"71a77ae7-8dba-45d1-986c-19fddc94af3d",
|
||||
"46fe0fd9-c9ae-47f3-a47c-9393c8d3c2a6",
|
||||
"e2410aae-675e-4259-824d-cde0b479b667",
|
||||
"dcef7518-6931-46d6-ac51-f53af10a7018",
|
||||
"8d38362f-34af-4cd4-8988-8525edfcb171",
|
||||
"6f8ba7d2-f77f-42c8-89f4-13b644294d2d",
|
||||
"e23348ac-6fee-49c8-b492-5fc78a853548",
|
||||
"c4fa1ca7-1a46-4e3f-843b-531ea3664359",
|
||||
"66014e42-5f76-4fe8-ab67-9cab84fe5a16",
|
||||
"83a2cbee-f650-4452-9270-4a6921f51ff6",
|
||||
"95b35d8e-d2e1-4c6c-8c8d-058182bd12e8",
|
||||
"0d7d6d37-f538-46b7-b314-281923bc583b",
|
||||
"b389353c-5e56-43b3-a8b2-1bb6e7d865db",
|
||||
"840e21f4-2a44-487d-bedd-b9cf1cd94548",
|
||||
"8b865dac-9945-4314-acd2-2abc09053a23",
|
||||
"d60650ab-0ee4-4e95-9207-688f4af14b9c",
|
||||
"f8e8de12-30b8-4c73-b81f-53482b79fc44",
|
||||
"88b2cfc3-f26b-4515-aaf2-f23fbb78dc28",
|
||||
"d56513e0-c8f4-4a48-9b83-98ef17760dad",
|
||||
"eed9e89b-f6ff-44e4-b2cc-16f871185a19"
|
||||
]
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.9,
|
||||
"worker_peak_rss_mib": 434.5,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 2.3,
|
||||
"p95_s": 3.3,
|
||||
"p99_s": 3.3,
|
||||
"max_s": 3.3,
|
||||
"throughput_per_s": 3.48,
|
||||
"span_s": 3.4,
|
||||
"measurement_window_s": 4.061,
|
||||
"worker_cpu_seconds": 3.36,
|
||||
"worker_cpu_percent_one_core": 82.75,
|
||||
"http_non_2xx_fraction": 0.0,
|
||||
"accepted_http_rps_over_window": 2.96,
|
||||
"attachment_ids": [
|
||||
"7f5b994f-f3de-4b60-a5e9-f87ffd0835b6",
|
||||
"95d47438-8f87-45fe-9129-cb9563531a6b",
|
||||
"891e1344-f240-4efc-8b89-f3787d67002e",
|
||||
"719431c3-cd94-4cbc-92e5-5c301ae8005d",
|
||||
"85bffd8c-0007-4f7e-88e0-c8f84c9a575a",
|
||||
"32b180e0-5fda-4d53-ad12-7e2fe8715b51",
|
||||
"726dd68e-83f9-4f05-897f-d79f403b8064",
|
||||
"1950ca75-72e0-45c5-9e4e-5d54630fb3b4",
|
||||
"6742055a-66eb-49e1-a26a-b333995e1142",
|
||||
"d770a827-5abb-4df3-8a03-ba24ae2dbe38",
|
||||
"54bcd9c6-6d07-47c6-9aae-cc06fcbb9ca1",
|
||||
"d38c2e12-76ed-412a-a120-bae6d7be1142"
|
||||
]
|
||||
}
|
||||
],
|
||||
"backpressure": {
|
||||
"MAX_QUEUED_JOBS": 30,
|
||||
"burst": 200,
|
||||
"attempt_statuses": {
|
||||
"202": 61,
|
||||
"503": 139
|
||||
},
|
||||
"accepted": 61,
|
||||
"peak_queue_depth": 57,
|
||||
"all_accepted_finished": true,
|
||||
"accepted_in_s": 0.6,
|
||||
"drained_after_s": 3.7,
|
||||
"retry_after_header_seen": true,
|
||||
"retry_after_values": [
|
||||
"5"
|
||||
],
|
||||
"transport_errors": [],
|
||||
"recovered_new_uploads_accepted": true,
|
||||
"recovered_new_uploads_ready": true,
|
||||
"no_accepted_job_lost": true,
|
||||
"ready": 61,
|
||||
"failed": 0,
|
||||
"p50_s": 1.46,
|
||||
"p95_s": 2.31,
|
||||
"p99_s": 2.4,
|
||||
"max_s": 2.4,
|
||||
"throughput_per_s": 22.15,
|
||||
"span_s": 2.8,
|
||||
"measurement_window_s": 3.72,
|
||||
"worker_cpu_seconds": 1.32,
|
||||
"worker_cpu_percent_one_core": 35.49,
|
||||
"http_non_2xx_fraction": 0.695,
|
||||
"accepted_http_rps_over_window": 16.4,
|
||||
"attachment_ids": [
|
||||
"66ea19e2-e3e8-4d74-acfc-b3a9429f997c",
|
||||
"72f006e1-db6c-49f0-bcea-a5062bfe4e94",
|
||||
"a8754c2b-bb4e-41de-8569-a1509672c382",
|
||||
"273fd71e-1d8b-478a-9fb7-6e4f17f85323",
|
||||
"c7963cf2-4c2f-4dc1-890c-f400d78f97e6",
|
||||
"076e695d-ea2d-4a93-a3a9-638f0f5edd64",
|
||||
"b2fa531f-0cff-4869-b8fd-63c543cdef03",
|
||||
"0dc2753b-470f-4d8c-9f6f-46e72d18017b",
|
||||
"d691e18c-fd06-420d-9b3a-4676300f933b",
|
||||
"22349666-78df-48d7-805b-233357cfcd28",
|
||||
"ab17e0d7-6b8b-4dbc-b1a5-00c60f82dcd4",
|
||||
"e76b055b-de11-4ea1-a47b-d687672791ce",
|
||||
"768f98f9-3717-4ad7-81c7-3e5505e55f62",
|
||||
"5b5d86da-d290-4ec4-bbbf-9bc0b1f3a501",
|
||||
"8dad57b3-1a3a-409f-a4eb-60de9f0aaa09",
|
||||
"e8736a1c-b13f-47f7-a85f-cd6ca415209b",
|
||||
"c0a44b33-e8bf-4c5d-9e59-ccda96cdd1da",
|
||||
"504cab76-9082-4801-a6d0-87e4f4914352",
|
||||
"1d02876b-89ae-4380-a460-89fb29e55a93",
|
||||
"c6adc9c6-def5-49cd-900b-d63ecadaaad8",
|
||||
"989bd05b-eccc-4b7d-b806-0c468a331e9a",
|
||||
"1da0e1c5-ec99-4884-a3ad-0fdaae4dbc6c",
|
||||
"d6d37e06-6c8d-4d44-af3b-45fb22e88a59",
|
||||
"d9faea0c-2f02-4b76-8b02-44623b36b808",
|
||||
"a89cbba3-c4a8-4c9a-afec-c099ae7e46d3",
|
||||
"d9f0f0b5-70d1-4d09-a298-06a947b2a86d",
|
||||
"49b1c6e4-ba7f-49ac-9616-2d762185fd22",
|
||||
"452cd743-2644-46f8-9535-23c9122efc84",
|
||||
"94e6c1b6-1bdf-4b1d-b7bf-6812b67b7d05",
|
||||
"6d5279e2-eb5b-4757-befe-bf0d954dabb9",
|
||||
"447023af-7290-487c-8f52-8645a127d540",
|
||||
"08f41743-156e-44f0-9ad3-ab99cedd23ca",
|
||||
"c02cc601-04f5-4a81-bdcb-5a557395a506",
|
||||
"71c12d21-aaea-473f-8565-30014caa52e7",
|
||||
"68e91241-0f9a-4c4e-9080-5862bcbc472e",
|
||||
"3afb2d2e-31c9-4f70-aa3c-34393af6dc92",
|
||||
"a3014daa-2802-44a5-85fb-a86d74d2198d",
|
||||
"c205fb0f-183b-40e7-9ffa-6cb1fefff297",
|
||||
"af5996bf-3182-4778-920b-eeee2009cbce",
|
||||
"6fa05261-09b2-41ea-8ec3-8cf11d5e8a25",
|
||||
"976193b5-f91e-47cc-87f1-e0d8dc5347d1",
|
||||
"9ff6485e-6b41-4384-af91-1014fb114cc5",
|
||||
"0edaa06c-9403-48bc-bc1e-311e32762134",
|
||||
"79a51b1c-082f-48d5-acbf-f912f8792b54",
|
||||
"9cd5ff6b-ca74-4679-959e-7630f6ff3dc0",
|
||||
"10480ccc-186b-4586-bb2d-b3bdab50a2ea",
|
||||
"db1d6478-cbf2-47a9-b6ad-ca739b3d43aa",
|
||||
"f5d2e3ce-6e8d-4dad-8ee6-6905d9053867",
|
||||
"c36fab10-726f-4556-8fae-595ef41ad828",
|
||||
"76f0ac29-0fe9-495d-9873-620dd43019c1",
|
||||
"84a99df5-ca4d-492f-bfee-d8efa996b896",
|
||||
"9cbb0bc9-2700-4813-95c1-6123cec043b1",
|
||||
"e3a3eda1-1cbe-4e1a-883d-0420199073c6",
|
||||
"5dc6f8e5-235a-4072-a7c9-36623d163ff3",
|
||||
"5acb6911-ba65-4545-943d-68ed3c7b5c29",
|
||||
"710959d2-f69b-4bc6-a033-cb412c701f92",
|
||||
"99e4d956-77be-4501-b036-78978c13479d",
|
||||
"472b87fb-87ac-4d75-805b-21232db08a8c",
|
||||
"b8f2ec4e-4734-4267-83e3-3acf47c86cb9",
|
||||
"f127c53e-d159-41b8-8cc3-5f0dcf6ef1fd",
|
||||
"793bc8ef-82ff-45ea-adb2-c799164ce621"
|
||||
]
|
||||
},
|
||||
"backpressure_queue_samples": [
|
||||
[
|
||||
0.2,
|
||||
0,
|
||||
0.0
|
||||
],
|
||||
[
|
||||
0.8,
|
||||
57,
|
||||
0.5
|
||||
],
|
||||
[
|
||||
1.4,
|
||||
45,
|
||||
1.0
|
||||
],
|
||||
[
|
||||
1.9,
|
||||
30,
|
||||
1.5
|
||||
],
|
||||
[
|
||||
2.5,
|
||||
16,
|
||||
1.9
|
||||
],
|
||||
[
|
||||
3.1,
|
||||
0,
|
||||
0.0
|
||||
],
|
||||
[
|
||||
3.7,
|
||||
0,
|
||||
0.0
|
||||
]
|
||||
]
|
||||
}
|
||||
|
After Width: | Height: | Size: 51 KiB |
|
After Width: | Height: | Size: 37 KiB |
@@ -0,0 +1,254 @@
|
||||
{
|
||||
"date": "2026-10-03 02:55:33",
|
||||
"args": {
|
||||
"out": "evidence",
|
||||
"concurrency": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"n_small": 80,
|
||||
"n_medium": 24,
|
||||
"n_large": 12,
|
||||
"limit": 30,
|
||||
"burst": 200,
|
||||
"b_medium": false
|
||||
},
|
||||
"worker_limits": {
|
||||
"cpus": 1.0,
|
||||
"memory": "768m"
|
||||
},
|
||||
"image_kib": {
|
||||
"small": 109,
|
||||
"medium": 1961,
|
||||
"large": 3100
|
||||
},
|
||||
"matrix": [
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 66,
|
||||
"peak_oldest_age_s": 2.2,
|
||||
"worker_peak_rss_mib": 74.7,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 1.33,
|
||||
"p95_s": 2.46,
|
||||
"max_s": 2.55,
|
||||
"throughput_per_s": 28.9,
|
||||
"span_s": 2.8
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 22,
|
||||
"peak_oldest_age_s": 4.8,
|
||||
"worker_peak_rss_mib": 135.6,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.9,
|
||||
"p95_s": 5.05,
|
||||
"max_s": 5.29,
|
||||
"throughput_per_s": 4.21,
|
||||
"span_s": 5.7
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 11,
|
||||
"peak_oldest_age_s": 3.3,
|
||||
"worker_peak_rss_mib": 161.2,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 2.03,
|
||||
"p95_s": 3.37,
|
||||
"max_s": 3.37,
|
||||
"throughput_per_s": 3.34,
|
||||
"span_s": 3.6
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"accepted_in_s": 0.2,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 53,
|
||||
"peak_oldest_age_s": 1.2,
|
||||
"worker_peak_rss_mib": 76.3,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.86,
|
||||
"p95_s": 1.6,
|
||||
"max_s": 1.64,
|
||||
"throughput_per_s": 43.67,
|
||||
"span_s": 1.8
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 22,
|
||||
"peak_oldest_age_s": 4.2,
|
||||
"worker_peak_rss_mib": 194.1,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.8,
|
||||
"p95_s": 4.69,
|
||||
"max_s": 4.7,
|
||||
"throughput_per_s": 4.71,
|
||||
"span_s": 5.1
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 253.8,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 2.03,
|
||||
"p95_s": 2.96,
|
||||
"max_s": 2.96,
|
||||
"throughput_per_s": 3.77,
|
||||
"span_s": 3.2
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 52,
|
||||
"peak_oldest_age_s": 1.1,
|
||||
"worker_peak_rss_mib": 80.3,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.77,
|
||||
"p95_s": 1.26,
|
||||
"max_s": 1.3,
|
||||
"throughput_per_s": 51.79,
|
||||
"span_s": 1.5
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"accepted_in_s": 0.8,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 24,
|
||||
"peak_oldest_age_s": 4.9,
|
||||
"worker_peak_rss_mib": 321.7,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 3.81,
|
||||
"p95_s": 5.4,
|
||||
"max_s": 5.4,
|
||||
"throughput_per_s": 3.97,
|
||||
"span_s": 6.1
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 430.2,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 2.11,
|
||||
"p95_s": 3.12,
|
||||
"max_s": 3.12,
|
||||
"throughput_per_s": 3.6,
|
||||
"span_s": 3.3
|
||||
}
|
||||
],
|
||||
"backpressure": {
|
||||
"MAX_QUEUED_JOBS": 30,
|
||||
"burst": 200,
|
||||
"attempt_statuses": {
|
||||
"202": 57,
|
||||
"503": 143
|
||||
},
|
||||
"accepted": 57,
|
||||
"peak_queue_depth": 48,
|
||||
"all_accepted_finished": true,
|
||||
"accepted_in_s": 0.5,
|
||||
"drained_after_s": 2.6,
|
||||
"retry_after_header_seen": true,
|
||||
"recovered_new_uploads_accepted": true,
|
||||
"no_accepted_job_lost": true,
|
||||
"ready": 57,
|
||||
"failed": 0,
|
||||
"p95_s": 1.89
|
||||
},
|
||||
"backpressure_queue_samples": [
|
||||
[
|
||||
0.2,
|
||||
0,
|
||||
0.0
|
||||
],
|
||||
[
|
||||
0.7,
|
||||
48,
|
||||
0.5
|
||||
],
|
||||
[
|
||||
1.3,
|
||||
31,
|
||||
1.0
|
||||
],
|
||||
[
|
||||
1.9,
|
||||
16,
|
||||
1.6
|
||||
],
|
||||
[
|
||||
2.4,
|
||||
0,
|
||||
0.0
|
||||
]
|
||||
]
|
||||
}
|
||||
|
After Width: | Height: | Size: 51 KiB |
|
After Width: | Height: | Size: 37 KiB |
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"MAX_QUEUED_JOBS": 30,
|
||||
"burst": 200,
|
||||
"attempt_statuses": {
|
||||
"202": 54,
|
||||
"503": 146
|
||||
},
|
||||
"accepted": 54,
|
||||
"peak_queue_depth": 48,
|
||||
"all_accepted_finished": true,
|
||||
"accepted_in_s": 0.5,
|
||||
"drained_after_s": 2.7,
|
||||
"retry_after_header_seen": true,
|
||||
"retry_after_values": [
|
||||
"5"
|
||||
],
|
||||
"transport_errors": [],
|
||||
"recovered_new_uploads_accepted": true,
|
||||
"recovered_new_uploads_ready": true,
|
||||
"no_accepted_job_lost": true,
|
||||
"ready": 54,
|
||||
"failed": 0,
|
||||
"p95_s": 1.88
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
$ python -u research/queue_overload.py --out evidence\recheck_2026-11-12
|
||||
{'small': '109 KiB', 'medium': '1961 KiB', 'large': '3100 KiB'}
|
||||
{"concurrency": 1, "image": "small", "image_kib": 109, "uploads": 80, "accepted": 80, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 80}, "transport_errors": [], "accepted_in_s": 0.5, "all_finished": true, "peak_queue_depth": 67, "peak_oldest_age_s": 2.7, "worker_peak_rss_mib": 72.5, "worker_oom_killed": "false", "ready": 80, "failed": 0, "p50_s": 1.55, "p95_s": 2.84, "max_s": 3.0, "throughput_per_s": 23.84, "span_s": 3.4}
|
||||
{"concurrency": 1, "image": "medium", "image_kib": 1961, "uploads": 24, "accepted": 24, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 24}, "transport_errors": [], "accepted_in_s": 0.5, "all_finished": true, "peak_queue_depth": 22, "peak_oldest_age_s": 5.4, "worker_peak_rss_mib": 133.2, "worker_oom_killed": "false", "ready": 24, "failed": 0, "p50_s": 2.83, "p95_s": 5.21, "max_s": 5.44, "throughput_per_s": 4.09, "span_s": 5.9}
|
||||
{"concurrency": 1, "image": "large", "image_kib": 3100, "uploads": 12, "accepted": 12, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 12}, "transport_errors": [], "accepted_in_s": 0.3, "all_finished": true, "peak_queue_depth": 11, "peak_oldest_age_s": 3.3, "worker_peak_rss_mib": 160.4, "worker_oom_killed": "false", "ready": 12, "failed": 0, "p50_s": 1.74, "p95_s": 3.35, "max_s": 3.35, "throughput_per_s": 3.38, "span_s": 3.6}
|
||||
{"concurrency": 2, "image": "small", "image_kib": 109, "uploads": 80, "accepted": 80, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 80}, "transport_errors": [], "accepted_in_s": 0.3, "all_finished": true, "peak_queue_depth": 56, "peak_oldest_age_s": 1.1, "worker_peak_rss_mib": 75.4, "worker_oom_killed": "false", "ready": 80, "failed": 0, "p50_s": 0.89, "p95_s": 1.48, "max_s": 1.55, "throughput_per_s": 44.13, "span_s": 1.8}
|
||||
{"concurrency": 2, "image": "medium", "image_kib": 1961, "uploads": 24, "accepted": 24, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 24}, "transport_errors": [], "accepted_in_s": 0.5, "all_finished": true, "peak_queue_depth": 22, "peak_oldest_age_s": 4.3, "worker_peak_rss_mib": 194.9, "worker_oom_killed": "false", "ready": 24, "failed": 0, "p50_s": 2.43, "p95_s": 4.78, "max_s": 4.8, "throughput_per_s": 4.6, "span_s": 5.2}
|
||||
{"concurrency": 2, "image": "large", "image_kib": 3100, "uploads": 12, "accepted": 12, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 12}, "transport_errors": [], "accepted_in_s": 0.4, "all_finished": true, "peak_queue_depth": 12, "peak_oldest_age_s": 2.7, "worker_peak_rss_mib": 250.9, "worker_oom_killed": "false", "ready": 12, "failed": 0, "p50_s": 1.59, "p95_s": 2.98, "max_s": 2.98, "throughput_per_s": 3.84, "span_s": 3.1}
|
||||
{"concurrency": 4, "image": "small", "image_kib": 109, "uploads": 80, "accepted": 80, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 80}, "transport_errors": [], "accepted_in_s": 0.3, "all_finished": true, "peak_queue_depth": 50, "peak_oldest_age_s": 1.1, "worker_peak_rss_mib": 79.9, "worker_oom_killed": "false", "ready": 80, "failed": 0, "p50_s": 0.7, "p95_s": 1.25, "max_s": 1.29, "throughput_per_s": 52.3, "span_s": 1.5}
|
||||
{"concurrency": 4, "image": "medium", "image_kib": 1961, "uploads": 24, "accepted": 24, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 24}, "transport_errors": [], "accepted_in_s": 0.5, "all_finished": true, "peak_queue_depth": 24, "peak_oldest_age_s": 4.8, "worker_peak_rss_mib": 318.9, "worker_oom_killed": "false", "ready": 24, "failed": 0, "p50_s": 2.59, "p95_s": 4.82, "max_s": 4.86, "throughput_per_s": 4.56, "span_s": 5.3}
|
||||
{"concurrency": 4, "image": "large", "image_kib": 3100, "uploads": 12, "accepted": 12, "resource_limits": {"memory_bytes": 805306368, "nano_cpus": 1000000000}, "attempt_statuses": {"202": 12}, "transport_errors": [], "accepted_in_s": 0.4, "all_finished": true, "peak_queue_depth": 12, "peak_oldest_age_s": 2.8, "worker_peak_rss_mib": 430.0, "worker_oom_killed": "false", "ready": 12, "failed": 0, "p50_s": 2.32, "p95_s": 3.36, "max_s": 3.36, "throughput_per_s": 3.39, "span_s": 3.5}
|
||||
{"MAX_QUEUED_JOBS": 30, "burst": 200, "attempt_statuses": {"202": 54, "503": 146}, "accepted": 54, "peak_queue_depth": 48, "all_accepted_finished": true, "accepted_in_s": 0.5, "drained_after_s": 2.7, "retry_after_header_seen": true, "retry_after_values": ["5"], "transport_errors": [], "recovered_new_uploads_accepted": true, "recovered_new_uploads_ready": true, "no_accepted_job_lost": true, "ready": 54, "failed": 0, "p95_s": 1.88}
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
FAIL: GET /api/v1/attachments/85bcfa7b-e1e8-4bdf-8b95-e1a8b2804fc5/content: expected [200], received 503
|
||||
exit=1
|
||||
@@ -0,0 +1,482 @@
|
||||
{
|
||||
"before": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:24.547575282Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:24.409530572Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-12T00:59:20.643619346Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-12T00:59:21.319524713Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_pgdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-rabbitmq-1": {
|
||||
"service": "rabbitmq",
|
||||
"started_at": "2026-11-12T00:59:21.501174229Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_rabbitmqdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-12T00:59:21.638684596Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_seaweeddata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-12T00:59:20.874723039Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_valkeydata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-worker-1": {
|
||||
"service": "worker",
|
||||
"started_at": "2026-11-12T00:59:21.145794756Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:56.213556685Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:57.500333699Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-12T00:59:52.516283057Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-12T00:59:52.920661248Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_pgdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-rabbitmq-1": {
|
||||
"service": "rabbitmq",
|
||||
"started_at": "2026-11-12T00:59:53.435646134Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_rabbitmqdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-12T00:59:53.598244344Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_seaweeddata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-12T00:59:52.145208673Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_valkeydata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-worker-1": {
|
||||
"service": "worker",
|
||||
"started_at": "2026-11-12T00:59:53.235224376Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
}
|
||||
},
|
||||
"all_services_restarted": true,
|
||||
"volumes_preserved": true,
|
||||
"repeats": [
|
||||
{
|
||||
"before": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:58:45.575826913Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:58:45.881190123Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-12T00:53:10.601145103Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-12T00:53:11.16711263Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_pgdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-rabbitmq-1": {
|
||||
"service": "rabbitmq",
|
||||
"started_at": "2026-11-12T00:53:11.484387397Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_rabbitmqdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-12T00:53:11.643485706Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_seaweeddata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-12T00:53:10.885267441Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_valkeydata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-worker-1": {
|
||||
"service": "worker",
|
||||
"started_at": "2026-11-12T00:58:45.578958104Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:58:58.27211915Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:58:57.47079238Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-12T00:58:53.874340617Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-12T00:58:54.35128221Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_pgdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-rabbitmq-1": {
|
||||
"service": "rabbitmq",
|
||||
"started_at": "2026-11-12T00:58:54.980171011Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_rabbitmqdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-12T00:58:55.259407602Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_seaweeddata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-12T00:58:54.512669953Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_valkeydata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-worker-1": {
|
||||
"service": "worker",
|
||||
"started_at": "2026-11-12T00:58:54.76173897Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
}
|
||||
},
|
||||
"all_services_restarted": true,
|
||||
"volumes_preserved": true
|
||||
},
|
||||
{
|
||||
"before": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:58:58.27211915Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:58:57.47079238Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-12T00:58:53.874340617Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-12T00:58:54.35128221Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_pgdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-rabbitmq-1": {
|
||||
"service": "rabbitmq",
|
||||
"started_at": "2026-11-12T00:58:54.980171011Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_rabbitmqdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-12T00:58:55.259407602Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_seaweeddata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-12T00:58:54.512669953Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_valkeydata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-worker-1": {
|
||||
"service": "worker",
|
||||
"started_at": "2026-11-12T00:58:54.76173897Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:24.547575282Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:24.409530572Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-12T00:59:20.643619346Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-12T00:59:21.319524713Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_pgdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-rabbitmq-1": {
|
||||
"service": "rabbitmq",
|
||||
"started_at": "2026-11-12T00:59:21.501174229Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_rabbitmqdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-12T00:59:21.638684596Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_seaweeddata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-12T00:59:20.874723039Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_valkeydata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-worker-1": {
|
||||
"service": "worker",
|
||||
"started_at": "2026-11-12T00:59:21.145794756Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
}
|
||||
},
|
||||
"all_services_restarted": true,
|
||||
"volumes_preserved": true
|
||||
},
|
||||
{
|
||||
"before": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:24.547575282Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:24.409530572Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-12T00:59:20.643619346Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-12T00:59:21.319524713Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_pgdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-rabbitmq-1": {
|
||||
"service": "rabbitmq",
|
||||
"started_at": "2026-11-12T00:59:21.501174229Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_rabbitmqdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-12T00:59:21.638684596Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_seaweeddata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-12T00:59:20.874723039Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_valkeydata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-worker-1": {
|
||||
"service": "worker",
|
||||
"started_at": "2026-11-12T00:59:21.145794756Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:56.213556685Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-12T00:59:57.500333699Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-12T00:59:52.516283057Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-12T00:59:52.920661248Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_pgdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-rabbitmq-1": {
|
||||
"service": "rabbitmq",
|
||||
"started_at": "2026-11-12T00:59:53.435646134Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_rabbitmqdata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-12T00:59:53.598244344Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_seaweeddata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-12T00:59:52.145208673Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab5_valkeydata"
|
||||
]
|
||||
},
|
||||
"mtusi-lab5-worker-1": {
|
||||
"service": "worker",
|
||||
"started_at": "2026-11-12T00:59:53.235224376Z",
|
||||
"health": "healthy",
|
||||
"volumes": []
|
||||
}
|
||||
},
|
||||
"all_services_restarted": true,
|
||||
"volumes_preserved": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
[
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 67,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 72.5,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 1.55,
|
||||
"p95_s": 2.84,
|
||||
"max_s": 3.0,
|
||||
"throughput_per_s": 23.84,
|
||||
"span_s": 3.4
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 22,
|
||||
"peak_oldest_age_s": 5.4,
|
||||
"worker_peak_rss_mib": 133.2,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.83,
|
||||
"p95_s": 5.21,
|
||||
"max_s": 5.44,
|
||||
"throughput_per_s": 4.09,
|
||||
"span_s": 5.9
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 11,
|
||||
"peak_oldest_age_s": 3.3,
|
||||
"worker_peak_rss_mib": 160.4,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 1.74,
|
||||
"p95_s": 3.35,
|
||||
"max_s": 3.35,
|
||||
"throughput_per_s": 3.38,
|
||||
"span_s": 3.6
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 56,
|
||||
"peak_oldest_age_s": 1.1,
|
||||
"worker_peak_rss_mib": 75.4,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.89,
|
||||
"p95_s": 1.48,
|
||||
"max_s": 1.55,
|
||||
"throughput_per_s": 44.13,
|
||||
"span_s": 1.8
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 22,
|
||||
"peak_oldest_age_s": 4.3,
|
||||
"worker_peak_rss_mib": 194.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.43,
|
||||
"p95_s": 4.78,
|
||||
"max_s": 4.8,
|
||||
"throughput_per_s": 4.6,
|
||||
"span_s": 5.2
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 250.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 1.59,
|
||||
"p95_s": 2.98,
|
||||
"max_s": 2.98,
|
||||
"throughput_per_s": 3.84,
|
||||
"span_s": 3.1
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 50,
|
||||
"peak_oldest_age_s": 1.1,
|
||||
"worker_peak_rss_mib": 79.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.7,
|
||||
"p95_s": 1.25,
|
||||
"max_s": 1.29,
|
||||
"throughput_per_s": 52.3,
|
||||
"span_s": 1.5
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 24,
|
||||
"peak_oldest_age_s": 4.8,
|
||||
"worker_peak_rss_mib": 318.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.59,
|
||||
"p95_s": 4.82,
|
||||
"max_s": 4.86,
|
||||
"throughput_per_s": 4.56,
|
||||
"span_s": 5.3
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.8,
|
||||
"worker_peak_rss_mib": 430.0,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 2.32,
|
||||
"p95_s": 3.36,
|
||||
"max_s": 3.36,
|
||||
"throughput_per_s": 3.39,
|
||||
"span_s": 3.5
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,331 @@
|
||||
{
|
||||
"date": "2026-11-12 03:49:47",
|
||||
"args": {
|
||||
"out": "evidence\\recheck_2026-11-12",
|
||||
"concurrency": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"n_small": 80,
|
||||
"n_medium": 24,
|
||||
"n_large": 12,
|
||||
"limit": 30,
|
||||
"burst": 200,
|
||||
"b_medium": false
|
||||
},
|
||||
"worker_limits": {
|
||||
"cpus": 1.0,
|
||||
"memory": "768m"
|
||||
},
|
||||
"image_kib": {
|
||||
"small": 109,
|
||||
"medium": 1961,
|
||||
"large": 3100
|
||||
},
|
||||
"matrix": [
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 67,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 72.5,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 1.55,
|
||||
"p95_s": 2.84,
|
||||
"max_s": 3.0,
|
||||
"throughput_per_s": 23.84,
|
||||
"span_s": 3.4
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 22,
|
||||
"peak_oldest_age_s": 5.4,
|
||||
"worker_peak_rss_mib": 133.2,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.83,
|
||||
"p95_s": 5.21,
|
||||
"max_s": 5.44,
|
||||
"throughput_per_s": 4.09,
|
||||
"span_s": 5.9
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 11,
|
||||
"peak_oldest_age_s": 3.3,
|
||||
"worker_peak_rss_mib": 160.4,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 1.74,
|
||||
"p95_s": 3.35,
|
||||
"max_s": 3.35,
|
||||
"throughput_per_s": 3.38,
|
||||
"span_s": 3.6
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 56,
|
||||
"peak_oldest_age_s": 1.1,
|
||||
"worker_peak_rss_mib": 75.4,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.89,
|
||||
"p95_s": 1.48,
|
||||
"max_s": 1.55,
|
||||
"throughput_per_s": 44.13,
|
||||
"span_s": 1.8
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 22,
|
||||
"peak_oldest_age_s": 4.3,
|
||||
"worker_peak_rss_mib": 194.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.43,
|
||||
"p95_s": 4.78,
|
||||
"max_s": 4.8,
|
||||
"throughput_per_s": 4.6,
|
||||
"span_s": 5.2
|
||||
},
|
||||
{
|
||||
"concurrency": 2,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.7,
|
||||
"worker_peak_rss_mib": 250.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 1.59,
|
||||
"p95_s": 2.98,
|
||||
"max_s": 2.98,
|
||||
"throughput_per_s": 3.84,
|
||||
"span_s": 3.1
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "small",
|
||||
"image_kib": 109,
|
||||
"uploads": 80,
|
||||
"accepted": 80,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 80
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.3,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 50,
|
||||
"peak_oldest_age_s": 1.1,
|
||||
"worker_peak_rss_mib": 79.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 80,
|
||||
"failed": 0,
|
||||
"p50_s": 0.7,
|
||||
"p95_s": 1.25,
|
||||
"max_s": 1.29,
|
||||
"throughput_per_s": 52.3,
|
||||
"span_s": 1.5
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "medium",
|
||||
"image_kib": 1961,
|
||||
"uploads": 24,
|
||||
"accepted": 24,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 24
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.5,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 24,
|
||||
"peak_oldest_age_s": 4.8,
|
||||
"worker_peak_rss_mib": 318.9,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 24,
|
||||
"failed": 0,
|
||||
"p50_s": 2.59,
|
||||
"p95_s": 4.82,
|
||||
"max_s": 4.86,
|
||||
"throughput_per_s": 4.56,
|
||||
"span_s": 5.3
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"image": "large",
|
||||
"image_kib": 3100,
|
||||
"uploads": 12,
|
||||
"accepted": 12,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"attempt_statuses": {
|
||||
"202": 12
|
||||
},
|
||||
"transport_errors": [],
|
||||
"accepted_in_s": 0.4,
|
||||
"all_finished": true,
|
||||
"peak_queue_depth": 12,
|
||||
"peak_oldest_age_s": 2.8,
|
||||
"worker_peak_rss_mib": 430.0,
|
||||
"worker_oom_killed": "false",
|
||||
"ready": 12,
|
||||
"failed": 0,
|
||||
"p50_s": 2.32,
|
||||
"p95_s": 3.36,
|
||||
"max_s": 3.36,
|
||||
"throughput_per_s": 3.39,
|
||||
"span_s": 3.5
|
||||
}
|
||||
],
|
||||
"backpressure": {
|
||||
"MAX_QUEUED_JOBS": 30,
|
||||
"burst": 200,
|
||||
"attempt_statuses": {
|
||||
"202": 54,
|
||||
"503": 146
|
||||
},
|
||||
"accepted": 54,
|
||||
"peak_queue_depth": 48,
|
||||
"all_accepted_finished": true,
|
||||
"accepted_in_s": 0.5,
|
||||
"drained_after_s": 2.7,
|
||||
"retry_after_header_seen": true,
|
||||
"retry_after_values": [
|
||||
"5"
|
||||
],
|
||||
"transport_errors": [],
|
||||
"recovered_new_uploads_accepted": true,
|
||||
"recovered_new_uploads_ready": true,
|
||||
"no_accepted_job_lost": true,
|
||||
"ready": 54,
|
||||
"failed": 0,
|
||||
"p95_s": 1.88
|
||||
},
|
||||
"backpressure_queue_samples": [
|
||||
[
|
||||
0.2,
|
||||
0,
|
||||
0.0
|
||||
],
|
||||
[
|
||||
0.7,
|
||||
48,
|
||||
0.5
|
||||
],
|
||||
[
|
||||
1.3,
|
||||
33,
|
||||
1.0
|
||||
],
|
||||
[
|
||||
1.9,
|
||||
16,
|
||||
1.4
|
||||
],
|
||||
[
|
||||
2.4,
|
||||
1,
|
||||
1.9
|
||||
]
|
||||
]
|
||||
}
|
||||
|
After Width: | Height: | Size: 51 KiB |
|
After Width: | Height: | Size: 38 KiB |
|
After Width: | Height: | Size: 95 KiB |
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 0,
|
||||
"db_rows": 923,
|
||||
"s3_objects": 4,
|
||||
"orphan_objects": [],
|
||||
"orphans_skipped_fresh": 0,
|
||||
"owned_objects_skipped": 0,
|
||||
"missing_objects": [],
|
||||
"size_mismatch": [],
|
||||
"fixed": {
|
||||
"deleted_orphans": 0,
|
||||
"marked_failed": 0
|
||||
},
|
||||
"timing_ms": {
|
||||
"db_query": 6.1,
|
||||
"s3_listing": 24.7,
|
||||
"total": 127.3
|
||||
},
|
||||
"orphan_objects_count": 0,
|
||||
"missing_objects_count": 0,
|
||||
"size_mismatch_count": 0
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 0,
|
||||
"db_rows": 481,
|
||||
"s3_objects": 4,
|
||||
"orphan_objects": [
|
||||
"att/e42b205e-2226-4de0-b4c3-9819acb5a53f/v1/crop",
|
||||
"att/e42b205e-2226-4de0-b4c3-9819acb5a53f/v1/optimized",
|
||||
"att/e42b205e-2226-4de0-b4c3-9819acb5a53f/v1/thumbnail"
|
||||
],
|
||||
"orphans_skipped_fresh": 0,
|
||||
"missing_objects": [],
|
||||
"size_mismatch": [],
|
||||
"fixed": {
|
||||
"deleted_orphans": 0,
|
||||
"marked_failed": 0
|
||||
},
|
||||
"timing_ms": {
|
||||
"db_query": 2.8,
|
||||
"s3_listing": 32.4,
|
||||
"total": 117.4
|
||||
},
|
||||
"orphan_objects_count": 3,
|
||||
"missing_objects_count": 0,
|
||||
"size_mismatch_count": 0
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ docker compose exec -T api python -m app.s3_recovery
|
||||
PASS: S3 recovery, 3670 existing objects readable
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"reason": "Recovery hook now waits for ranged GET of existing S3 objects after volume registration; API/worker processing code unchanged since matrix.",
|
||||
"source_sha256": {
|
||||
"scripts/restart.cmd": "d1de857d3544f900de64a36734950b65d272f1ef4533669df411de9b235193cf",
|
||||
"scripts/restart.sh": "cd0b99b0e981eb00049dc69b438bb21a844c2a318203660c836fd82d9e324d10",
|
||||
"src/app/s3_recovery.py": "aaabec7f578fb74f150cd80140efaae04b8f84e38358d61d056ccc930158db18"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"date": "2026-11-12",
|
||||
"base_solution_commit": "518c98fd3e6bd593bbb22b5bdd07f82ecff65720",
|
||||
"branch": "solution/bpi2302-kushnarenko",
|
||||
"python_client": "3.11.9 (tags/v3.11.9:de54cf5, Apr 2 2024, 10:12:12) [MSC v.1938 64 bit (AMD64)]",
|
||||
"source_sha256": {
|
||||
"research/requirements.txt": "0a0f9ffaa9b56d590cb38c77f9da8dbc64f2d503b69fc27b0116b9a9594b2591",
|
||||
"student-tests/README.md": "2bfdbe9f72497f6ddb6d4d1cccccedc0c9d9c044f39596a0286fec0eb975dcc0",
|
||||
"student-tests/test_dependencies.py": "e152b87441d1374ce9b69d3d441d262e47890313d2ccc798f5e2cff2e3719399",
|
||||
"student-tests/test_expiry.py": "16a374ebe3df15c50c07a9fc3bc801c6c532dde3c4185b164c6aea4605923f83",
|
||||
"student-tests/test_reconcile5.py": "7df51d06fc6da180fada10d3d6e1233c9462870ec18f6d55753a36e30dca458b",
|
||||
"Dockerfile": "f1dc711cacf00e3e2bd6b4a0616c925e357446ebda370e7e5fdb3170eaecd4cb",
|
||||
"compose.yaml": "1562ea10580ea3c58766b9696cf7b59d631ddec68f2d809a3a79d6f48a63a151",
|
||||
"infra/README.md": "33a4f7b870fd0b2bed98d46daf26d45dd7436176739672925246f8e3fe7c2278",
|
||||
"infra/haproxy/haproxy.cfg": "511fb24b32740c332cd7b6ea42d3f1cf252e1452de754ff97c8e6886f3692061",
|
||||
"requirements.txt": "9add304ba15ae74bf5d3658496b8241121f82ab748a1bd163c2763fa1cd57e65",
|
||||
"research/failover_sequence.py": "c23a0472f54538fd9d7607c017bef4f127269850ca7bf09adc57862fe2b7665b",
|
||||
"research/loadgen.py": "7e06f5a04a1ec1b187a2fd3af82b0748ed042e1e9d63e350ab2007d21826fe1c",
|
||||
"research/queue_overload.py": "b0b37aea9fd0c9b00652555724084caa2b9629e9f3e9635fc52203ca3170a6f5",
|
||||
"research/rolling_update.py": "447da4a9e2161bb232669f0558d6cb15ae3f0ef181fdf563e19cd56135b58240",
|
||||
"scripts/check_contract.py": "61e300fe5c1074360418e24b4653b12f96808247a2beaf2a64859aa360f81493",
|
||||
"scripts/init-secrets.sh": "9cbe6b6058769a820b84f61b2076028d97d487ce04fe98e42d38dddf45c97fd8",
|
||||
"scripts/restart.cmd": "b6bc654fae1b4fcf8b4765ce5ec8082d8b645e8e4cf15bfdf6904b26d860b866",
|
||||
"scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b",
|
||||
"scripts/restart.sh": "a00e3d82bc97e34deaa9fa8e7ed24f4386057502cdff7aa750680b96a82e18a7",
|
||||
"scripts/restore-replicas.sh": "88ba99a10808f211a72b111ce240f7fa7df8dbf796f4c2d81d05b1a95008b46d",
|
||||
"scripts/rolling-update.sh": "cdeaff25aaf0ae3fc030bf1d8fe8ae6300ac51741b780bfe080ce6bbac959632",
|
||||
"scripts/stop-one.cmd": "0e6d46bc7abc2f81403422f985ee9d0d2ad02e0cfd9db2db0fccc873020d8997",
|
||||
"scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811",
|
||||
"scripts/stop-one.sh": "c74672836f70c24b44afa2d531f9287e51fb87690e1d0ca5c9e4ed0abaa0b48e",
|
||||
"scripts/validate_openapi.py": "cd7a33221b1860c5d2330e52afa6cb222d375db671ae9ab379ba51a0d3a77946",
|
||||
"src/app/attachments.py": "b7ac49012e3e504e069a9a1f7de61b0a73bbf121359c897111cda719a00d60f5",
|
||||
"src/app/broker.py": "b68f297acd17bbd4d63b140cef06b89f73048da6d35fc6027717d54c5cf3733b",
|
||||
"src/app/config.py": "854a09110b8f8ce5ca634461a4a402ef37c9a2e9c32169ba388f832066e03b6f",
|
||||
"src/app/db.py": "7510882b0e8c9f6a24d680a156c285bc1a0fcf021473ca47173aa0744d845a78",
|
||||
"src/app/images.py": "3dc7434cbfbc9d4c01b95eb897c1332b2a2748dd9df88c99a7086f3e4cdf4c41",
|
||||
"src/app/main.py": "89745c1b21b87626626d82a1f5a8561441595d476eb90a49ac0d1dc7f11a4a90",
|
||||
"src/app/migrate.py": "cabc192ca4182af780636415a008d2dd4636dad7d3cf0f4a8a2b52a48aedaa7e",
|
||||
"src/app/outbox.py": "2326c21e4eb910a10bd2e4613e88b211bea1125b3f452aecc1b5ec7b4e6e0bc8",
|
||||
"src/app/passwords.py": "836fc282093058f6717eaa6374775251ed655dcd517f325dc60323a88154f261",
|
||||
"src/app/pipeline.py": "222ac87f275e4607ecdc98cdc5f68d8957fd7f68603af2d2643f3026445a3fe1",
|
||||
"src/app/reconcile.py": "c7dd35bf40e672ffe15f0e878913e17f6c6d9de3f69922b4edb38d1802330dc4",
|
||||
"src/app/replay.py": "2669b9e5fa2f6fb83dbe457f0241db166eb3e370e21d81c3e32f922060f52fa5",
|
||||
"src/app/s3.py": "091ddac8facd4c61f223111b3aaab48efff7d6ef9a30af4ac86d4352024f0727",
|
||||
"src/app/s3_bootstrap.py": "216d8121d6e7b25d95b845f4deaff18fd31eeb28484e51fe4c463c646a3af0d1",
|
||||
"src/app/serve.py": "3ebe12cf8cb2dddfc7d7234bba9bd190d868cc88d02e61fe640ba51dcf115b55",
|
||||
"src/app/sessions.py": "f975eb27c50c024f3e562f755ee97bb7bfe21cf5e01cc1d347bf107971c30670",
|
||||
"src/app/web.py": "23271be669a19bd7abfd6eaae25e25ce214718f0f623406cb9a2f6e77e365b83",
|
||||
"src/app/worker.py": "065a7e7e1c67d961452527122f5cc53c18873f0d83a8335aad2278374197a1cb",
|
||||
"student-tests/fixtures/alpha.png": "1f9d6ee26fa7beb4ba179c7c38ae97f5f31e95ad9d4108267352225e6705882f",
|
||||
"student-tests/fixtures/exif_rot6.jpg": "62386265960512fd51279e40723331c6c52f81ba2325d82afc3fb14f3b332ddc",
|
||||
"student-tests/fixtures/poison.png": "2be4bea6355e83da4733b1d7f59dc985dfe22cf5d8283eea9b20871d1f43c252",
|
||||
"student-tests/fixtures/sample.jpg": "33a614b40fe6ae25fdbd99af22e7d9d3e5dbea455b36aac750de112c034a6311",
|
||||
"student-tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"student-tests/fixtures/small.png": "001e6b4538ce86cbb0c88a34a2b88c111336b8dc8825af81e0f7d9630b731e32",
|
||||
"student-tests/test_api.py": "2635113842a5b26ec1fdd98ea42fc861edb1d10895e8be6c2eff46ed73fdba9d",
|
||||
"student-tests/test_attachments.py": "519bc20867996e03e79121ec461c94a9123faa1d11b2faedbdcf62e556fc0a05",
|
||||
"student-tests/test_idempotency.py": "4ee985a3ef26fefa816fdb136fa2b0644cca13d73fe3a226d1ba975a55698b5b",
|
||||
"student-tests/test_pipeline.py": "a322c1c72c5beeef2bb4f0436d36d2a9a005872054c48649c85be23289b84796",
|
||||
"student-tests/test_resilience5.py": "68f3f297f1068d598cedd9aa1373f63e5d135f3599e0d8bf134ea047e7def30a",
|
||||
"student-tests/test_topology.py": "58050c5b330d0b290959b95fbd541b25cd966453399fbbbb627c8df906d7ffb6"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
# Балансировщик lab3: HAProxy 3.0. Единственная публичная точка входа; API-реплики наружу не опубликованы.
|
||||
global
|
||||
log stdout format raw local0 notice
|
||||
maxconn 4096
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option dontlognull
|
||||
option forwardfor
|
||||
# connect короткий: недоступная реплика быстро заменяется другой (redispatch)
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 5s
|
||||
retries 3
|
||||
option redispatch 1
|
||||
# Only retry before a connection is established: upload POST has no idempotency key.
|
||||
retry-on conn-failure
|
||||
|
||||
# Docker DNS (127.0.0.11): адреса реплик API обновляются без перезапуска LB (scale up/down, пересоздание контейнеров).
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold nx 2s
|
||||
hold other 2s
|
||||
hold obsolete 2s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
# If the shared DB is down, live API processes still produce contract JSON,
|
||||
# validate missing credentials, and serve /health/live independently of DB.
|
||||
use_backend api_live if { nbsrv(api) eq 0 }
|
||||
default_backend api
|
||||
|
||||
# Внутренний healthcheck самого LB (контейнерный HEALTHCHECK); наружу не публикуется.
|
||||
frontend lb_health
|
||||
bind :8081
|
||||
monitor-uri /healthz
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
# Проверка готовности каждые 1 с; 2 неудачи подряд -> сервер исключён (обнаружение отказа <= ~3 с), 2 успеха -> возвращён.
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template api 1-8 api:8080
|
||||
|
||||
backend api_live
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/live
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template api 1-8 api:8080
|
||||
@@ -0,0 +1,52 @@
|
||||
-- 0001: пользователи, чаты, участники, сообщения, сессии.
|
||||
-- Источник истины для всех данных — PostgreSQL. Каждая миграция выполняется в транзакции.
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
-- Уникальность username: гонка двух регистраций решается ограничением БД, а не проверкой в приложении.
|
||||
CREATE UNIQUE INDEX users_username_key ON users (username);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
-- chat_created_at денормализован ради keyset-пагинации списка чатов пользователя без join+sort.
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
chat_created_at timestamptz NOT NULL,
|
||||
ord smallint NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_chats ON chat_members (user_id, chat_created_at, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users (id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000)
|
||||
);
|
||||
-- История чата: WHERE chat_id = ? AND (created_at, id) > (?, ?) ORDER BY created_at, id LIMIT n
|
||||
-- обслуживается этим индексом без сортировки и без сканирования всей истории.
|
||||
CREATE INDEX messages_chat_history ON messages (chat_id, created_at, id);
|
||||
|
||||
-- Серверные сессии. В БД хранится только SHA-256 токена: утечка БД не раскрывает рабочие токены.
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz
|
||||
);
|
||||
CREATE INDEX sessions_expires_at ON sessions (expires_at);
|
||||
@@ -0,0 +1,17 @@
|
||||
-- 0002: общая идемпотентность отправки сообщений (lab3).
|
||||
-- Область уникальности: пользователь + чат + ключ (метод и путь фиксированы: POST /chats/{id}/messages).
|
||||
-- Запись ключа и сообщения — одна транзакция: побочный эффект и результат фиксируются атомарно.
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
idem_key text NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
message_id uuid NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, idem_key),
|
||||
CONSTRAINT idem_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
-- сообщение вставляется после ключа в той же транзакции -> проверка внешнего ключа откладывается до COMMIT
|
||||
CONSTRAINT idem_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) ON DELETE CASCADE
|
||||
DEFERRABLE INITIALLY DEFERRED
|
||||
);
|
||||
CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,35 @@
|
||||
-- 0003: отправка сообщения одной серверной функцией (оптимизация по итогам трека 1: меньше обращений к БД на запрос).
|
||||
-- Семантика идентична версии «несколько запросов в транзакции» (migrations 0002 + код приложения):
|
||||
-- проверка членства -> просроченный ключ удаляется -> INSERT ключа (ON CONFLICT DO NOTHING) -> INSERT сообщения.
|
||||
-- Функция выполняется атомарно (один вызов = одна транзакция); гонку одинаковых ключей по-прежнему решает уникальный индекс.
|
||||
CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision)
|
||||
RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz,
|
||||
request_hash bytea)
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE
|
||||
v_new uuid := gen_random_uuid();
|
||||
v_ins uuid;
|
||||
v_created timestamptz;
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN
|
||||
RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz, NULL::bytea;
|
||||
RETURN;
|
||||
END IF;
|
||||
DELETE FROM idempotency_keys k
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key
|
||||
AND k.created_at < now() - make_interval(secs => p_ttl);
|
||||
INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id)
|
||||
VALUES (p_user, p_chat, p_key, p_hash, v_new)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING message_id INTO v_ins;
|
||||
IF v_ins IS NOT NULL THEN
|
||||
INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text)
|
||||
RETURNING created_at INTO v_created;
|
||||
RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash;
|
||||
ELSE
|
||||
RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
@@ -0,0 +1,90 @@
|
||||
-- 0004 (lab4): метаданные вложений в PostgreSQL, байты — в приватном S3-бакете.
|
||||
-- Ключ объекта назначает сервер (att/<id>/original); filename — только отображаемое имя.
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users (id),
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
status text NOT NULL DEFAULT 'ready',
|
||||
error_code text,
|
||||
s3_key text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255)
|
||||
);
|
||||
CREATE UNIQUE INDEX attachments_s3_key_key ON attachments (s3_key);
|
||||
CREATE INDEX attachments_chat ON attachments (chat_id);
|
||||
|
||||
-- Связь сообщение <-> вложения (порядок сохраняется). Вложение может быть привязано к нескольким сообщениям чата.
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id);
|
||||
|
||||
-- Отправка сообщения с вложениями: заменяет функцию из 0003 (добавлен параметр p_atts и результат msg_atts).
|
||||
DROP FUNCTION send_message(uuid, uuid, text, bytea, text, double precision);
|
||||
CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision,
|
||||
p_atts uuid[])
|
||||
RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz,
|
||||
request_hash bytea, msg_atts uuid[])
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE
|
||||
v_new uuid := gen_random_uuid();
|
||||
v_ins uuid;
|
||||
v_created timestamptz;
|
||||
v_found int;
|
||||
v_failed int;
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN
|
||||
RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz,
|
||||
NULL::bytea, NULL::uuid[];
|
||||
RETURN;
|
||||
END IF;
|
||||
IF cardinality(p_atts) > 0 THEN
|
||||
SELECT count(*), count(*) FILTER (WHERE a.status = 'failed') INTO v_found, v_failed
|
||||
FROM attachments a WHERE a.id = ANY (p_atts) AND a.chat_id = p_chat;
|
||||
IF v_found <> cardinality(p_atts) THEN
|
||||
RETURN QUERY SELECT 'attachment_not_found'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text,
|
||||
NULL::timestamptz, NULL::bytea, NULL::uuid[];
|
||||
RETURN;
|
||||
END IF;
|
||||
IF v_failed > 0 THEN
|
||||
RETURN QUERY SELECT 'attachment_failed'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text,
|
||||
NULL::timestamptz, NULL::bytea, NULL::uuid[];
|
||||
RETURN;
|
||||
END IF;
|
||||
END IF;
|
||||
DELETE FROM idempotency_keys k
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key
|
||||
AND k.created_at < now() - make_interval(secs => p_ttl);
|
||||
INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id)
|
||||
VALUES (p_user, p_chat, p_key, p_hash, v_new)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING message_id INTO v_ins;
|
||||
IF v_ins IS NOT NULL THEN
|
||||
INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text)
|
||||
RETURNING created_at INTO v_created;
|
||||
INSERT INTO message_attachments (message_id, attachment_id, position)
|
||||
SELECT v_new, a, (o - 1)::smallint FROM unnest(p_atts) WITH ORDINALITY AS t(a, o);
|
||||
RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash, p_atts;
|
||||
ELSE
|
||||
RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash,
|
||||
COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position)
|
||||
FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[])
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
@@ -0,0 +1,60 @@
|
||||
-- 0005 (lab5): асинхронная обработка вложений — состояния, варианты, outbox, журнал событий.
|
||||
-- Расширение обратно совместимо с lab4: старые строки (ready) остаются валидными.
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_key text,
|
||||
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1,
|
||||
ADD COLUMN attempt integer NOT NULL DEFAULT 0,
|
||||
ADD COLUMN job_id uuid,
|
||||
ADD COLUMN lease_owner text,
|
||||
ADD COLUMN lease_expires_at timestamptz,
|
||||
ADD COLUMN processing_started_at timestamptz,
|
||||
ADD COLUMN accepted_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
ADD COLUMN ready_at timestamptz;
|
||||
|
||||
-- очередь/backlog и поиск зависших processing без полного скана
|
||||
CREATE INDEX attachments_active ON attachments (status, accepted_at) WHERE status IN ('queued', 'processing');
|
||||
CREATE INDEX attachments_lease ON attachments (lease_expires_at) WHERE status = 'processing';
|
||||
|
||||
-- Результаты обработки. Ключ эффекта (attachment_id, pipeline_version, name): повторная запись не создаёт второй набор вариантов.
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
name text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
s3_key text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name),
|
||||
CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT variants_content_type CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp'))
|
||||
);
|
||||
|
||||
-- Transactional outbox: намерение «обработать файл» фиксируется В ТОЙ ЖЕ транзакции, что и строка вложения.
|
||||
-- Relay публикует строки в брокер (publisher confirm) и только затем ставит published_at. available_at — отложенная
|
||||
-- публикация (backoff между попытками). Доставка at-least-once; обработка идемпотентна.
|
||||
CREATE TABLE outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL DEFAULT 1,
|
||||
attempt integer NOT NULL,
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX outbox_pending ON outbox (available_at) WHERE published_at IS NULL;
|
||||
|
||||
-- Журнал событий пайплайна (аудит операторского replay, причины отказов; без секретов).
|
||||
CREATE TABLE job_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL,
|
||||
job_id uuid,
|
||||
event text NOT NULL,
|
||||
attempt integer,
|
||||
detail text,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_events_attachment ON job_events (attachment_id, id);
|
||||
@@ -0,0 +1,33 @@
|
||||
aio-pika==10.1.0
|
||||
aiobotocore==3.9.2
|
||||
aiohappyeyeballs==2.7.1
|
||||
aiohttp==3.14.3
|
||||
aioitertools==0.13.0
|
||||
aiormq==7.2.1
|
||||
aiosignal==1.4.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
argon2-cffi==23.1.0
|
||||
asyncpg==0.30.0
|
||||
attrs==26.1.0
|
||||
botocore==1.43.106
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
frozenlist==1.8.0
|
||||
h11==0.16.0
|
||||
httptools==0.8.0
|
||||
idna==3.20
|
||||
jmespath==1.1.0
|
||||
multidict==6.9.1
|
||||
pamqp==4.0.1
|
||||
pillow==12.3.0
|
||||
propcache==0.5.4
|
||||
pycparser==3.0
|
||||
python-dateutil==2.9.0.post0
|
||||
redis==5.2.1
|
||||
six==1.17.0
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.34.0
|
||||
uvloop==0.21.0; sys_platform != "win32"
|
||||
wrapt==2.5.0
|
||||
yarl==1.25.1
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4: последовательная остановка каждой API-реплики под нагрузкой (с восстановлением предыдущей).
|
||||
|
||||
Сценарий (стенд `docker compose up -d` с 2 репликами; нагрузка идёт через LB непрерывно ~70 с):
|
||||
t=10 остановить реплику A (docker stop --time 0 = мгновенный отказ процесса)
|
||||
t=25 восстановить (scripts/restore-replicas.sh)
|
||||
t=40 остановить реплику B
|
||||
t=55 восстановить
|
||||
Для каждого отказа считаем: RPS до/во время/после, число ошибочных попыток, время до прекращения ошибок
|
||||
(= обнаружение отказа балансировщиком), сохранность подтверждённых сообщений и отсутствие дублей.
|
||||
|
||||
python research/failover_sequence.py --out evidence
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import loadgen # noqa: E402
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def sh(*args):
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def api_ids():
|
||||
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab5",
|
||||
"--filter", "label=com.docker.compose.service=api").split()
|
||||
|
||||
|
||||
def scenario(rec, plan):
|
||||
"""Выполняется в отдельном потоке: действия по расписанию относительно rec.t0."""
|
||||
for at, action in plan:
|
||||
time.sleep(max(0, rec.t0 + at - time.time()))
|
||||
t = time.time() - rec.t0
|
||||
if action == "stop":
|
||||
target = api_ids()[0]
|
||||
subprocess.run(["docker", "stop", "--time", "0", target], capture_output=True)
|
||||
rec.marks.append({"t": round(t, 2), "action": "stop", "container": target})
|
||||
else:
|
||||
subprocess.run(["sh", "scripts/restore-replicas.sh"], cwd=ROOT, capture_output=True)
|
||||
rec.marks.append({"t": round(t, 2), "action": "restore"})
|
||||
|
||||
|
||||
async def main(args):
|
||||
stand = loadgen.Stand(BASE)
|
||||
await stand.setup()
|
||||
rec = loadgen.Recorder()
|
||||
rec.marks = []
|
||||
plan = [(10, "stop"), (25, "restore"), (40, "stop"), (55, "restore")]
|
||||
threading.Thread(target=scenario, args=(rec, plan), daemon=True).start()
|
||||
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
|
||||
await asyncio.sleep(2)
|
||||
check = await loadgen.verify(stand, rec)
|
||||
stops = [m for m in rec.marks if m["action"] == "stop"]
|
||||
restores = [m for m in rec.marks if m["action"] == "restore"]
|
||||
windows = []
|
||||
for i, stop in enumerate(stops):
|
||||
t0 = stop["t"]
|
||||
errs = [e for e in rec.events if e[1] == "POST" and e[2] != 201 and t0 - 0.2 <= e[0] <= t0 + 10]
|
||||
last_err = max((e[0] for e in errs), default=t0)
|
||||
restore_t = restores[i]["t"] if i < len(restores) else t0 + 15
|
||||
windows.append({"stopped_container": stop["container"], "stop_at_s": t0,
|
||||
"rps_before": loadgen.summarize(rec, t0 - 8, t0 - 0.5).get("ok_rps"),
|
||||
"rps_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("ok_rps"),
|
||||
"rps_after_restore": loadgen.summarize(rec, restore_t + 8, restore_t + 14).get("ok_rps"),
|
||||
"failed_attempts_within_10s": len(errs),
|
||||
"error_statuses": sorted({str(e[2]) for e in errs}),
|
||||
"seconds_until_errors_stop": round(last_err - t0, 2) if errs else 0.0,
|
||||
"p95_ms_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("p95_ms")})
|
||||
series = loadgen.per_second(rec)
|
||||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "marks": rec.marks, "windows": windows,
|
||||
"total": loadgen.summarize(rec), "verification": check,
|
||||
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0,
|
||||
"timeline": [{"t": t, "ok": b["ok"], "err": b["err"], "p95_ms": loadgen.pct(b["lat"], .95), "instances": len(b["instances"])}
|
||||
for t, b in sorted(series.items())]}
|
||||
with open(os.path.join(args.out, "failover_sequence.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||||
plot(report, os.path.join(args.out, "failover_sequence.png"))
|
||||
print(json.dumps({k: v for k, v in report.items() if k != "timeline"}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def plot(report, path):
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
tl = report["timeline"]
|
||||
t = [x["t"] for x in tl]
|
||||
fig, ax = plt.subplots(3, 1, figsize=(11, 8.5), sharex=True)
|
||||
ax[0].plot(t, [x["ok"] for x in tl], color="#2e7d32"); ax[0].set_ylabel("успешных POST/с"); ax[0].set_title("Пропускная способность")
|
||||
ax[1].bar(t, [x["err"] for x in tl], color="#c62828"); ax[1].set_ylabel("ошибочных попыток/с"); ax[1].set_title("Ошибки (до повтора с тем же ключом)")
|
||||
ax[2].plot(t, [x["p95_ms"] or 0 for x in tl], color="#1565c0"); ax[2].set_ylabel("p95, мс"); ax[2].set_xlabel("время, с"); ax[2].set_title("Латентность успешных POST")
|
||||
for a in ax:
|
||||
for m in report["marks"]:
|
||||
a.axvline(m["t"], color="red" if m["action"] == "stop" else "green", ls="--", lw=1)
|
||||
ax[0].text(0.01, 0.05, "красная линия — остановка реплики, зелёная — восстановление", transform=ax[0].transAxes, fontsize=9)
|
||||
fig.tight_layout(); fig.savefig(path, dpi=110)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="evidence")
|
||||
ap.add_argument("--seconds", type=int, default=70)
|
||||
ap.add_argument("--concurrency", type=int, default=16)
|
||||
asyncio.run(main(ap.parse_args()))
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Генератор нагрузки для экспериментов lab5+ (asyncio + aiohttp, closed-loop).
|
||||
|
||||
Каждый воркер выполняет цикл: POST сообщения с новым Idempotency-Key (90 %) или GET первой страницы истории (10 %).
|
||||
При ошибке соединения/5xx POST повторяется с ТЕМ ЖЕ ключом (до RETRIES раз) — именно так клиент безопасно
|
||||
повторяет неподтверждённую запись. Результат: по секундам успехи/ошибки/латентности, список подтверждённых
|
||||
сообщений (для сверки с историей), число повторов.
|
||||
"""
|
||||
import asyncio
|
||||
import base64
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import aiohttp
|
||||
|
||||
RETRIES = 3
|
||||
|
||||
|
||||
class Stand:
|
||||
"""Набор пользователей/чатов, созданных через API (через LB), для одного эксперимента."""
|
||||
|
||||
def __init__(self, base, users=20, chats=20):
|
||||
self.base, self.n_users, self.n_chats = base, users, chats
|
||||
self.users, self.chats = [], []
|
||||
|
||||
async def setup(self):
|
||||
timeout = aiohttp.ClientTimeout(total=30)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as s:
|
||||
for i in range(self.n_users):
|
||||
name = "ld_%s" % uuid.uuid4().hex[:14]
|
||||
pw = "load-test-password-" + uuid.uuid4().hex[:8]
|
||||
async with s.post(self.base + "/api/v1/users", json={"username": name, "display_name": "load %d" % i, "password": pw}) as r:
|
||||
user = await r.json()
|
||||
assert r.status == 201, user
|
||||
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
|
||||
async with s.post(self.base + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
|
||||
sess = await r.json()
|
||||
assert r.status == 201, sess
|
||||
self.users.append({"id": user["id"], "token": sess["token"]})
|
||||
for i in range(self.n_chats):
|
||||
members = random.sample(self.users, 3)
|
||||
owner = members[0]
|
||||
async with s.post(self.base + "/api/v1/chats", json={"title": "load %d" % i, "member_ids": [m["id"] for m in members]},
|
||||
headers={"Authorization": "Bearer " + owner["token"]}) as r:
|
||||
chat = await r.json()
|
||||
assert r.status == 201, chat
|
||||
self.chats.append({"id": chat["id"], "members": members})
|
||||
|
||||
|
||||
class Recorder:
|
||||
def __init__(self):
|
||||
self.t0 = time.time()
|
||||
self.events = [] # (t, kind, status, latency_ms, instance)
|
||||
self.confirmed = [] # (chat_id, message_id, text)
|
||||
self.retries = 0
|
||||
self.final_failures = 0
|
||||
self.unconfirmed = [] # (chat_id, key, text) — финально не подтверждены (могли записаться!)
|
||||
|
||||
|
||||
async def worker(stand, rec, deadline_fn, read_ratio=0.1, text_size=0, think=0.0):
|
||||
connector = aiohttp.TCPConnector(limit=2)
|
||||
timeout = aiohttp.ClientTimeout(total=5)
|
||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as s:
|
||||
n = 0
|
||||
while not deadline_fn():
|
||||
chat = random.choice(stand.chats)
|
||||
user = random.choice(chat["members"])
|
||||
hdr = {"Authorization": "Bearer " + user["token"]}
|
||||
if random.random() < read_ratio:
|
||||
t = time.perf_counter()
|
||||
try:
|
||||
async with s.get("%s/api/v1/chats/%s/messages?limit=20" % (stand.base, chat["id"]), headers=hdr) as r:
|
||||
await r.read()
|
||||
rec.events.append((time.time() - rec.t0, "GET", r.status, (time.perf_counter() - t) * 1000, r.headers.get("X-Instance-Id")))
|
||||
except Exception as exc:
|
||||
rec.events.append((time.time() - rec.t0, "GET", type(exc).__name__, (time.perf_counter() - t) * 1000, None))
|
||||
continue
|
||||
key = uuid.uuid4().hex
|
||||
text = "m-%s-%d" % (key[:8], n) + "я" * text_size
|
||||
n += 1
|
||||
for attempt in range(RETRIES + 1):
|
||||
t = time.perf_counter()
|
||||
try:
|
||||
async with s.post("%s/api/v1/chats/%s/messages" % (stand.base, chat["id"]), json={"text": text},
|
||||
headers=dict(hdr, **{"Idempotency-Key": key})) as r:
|
||||
body = await r.json(content_type=None)
|
||||
status, inst = r.status, r.headers.get("X-Instance-Id")
|
||||
except Exception as exc:
|
||||
status, inst, body = type(exc).__name__, None, None
|
||||
rec.events.append((time.time() - rec.t0, "POST", status, (time.perf_counter() - t) * 1000, inst))
|
||||
if status == 201:
|
||||
rec.confirmed.append((chat["id"], body["id"], text))
|
||||
break
|
||||
if attempt < RETRIES and (not isinstance(status, int) or status >= 500 or status == 429):
|
||||
rec.retries += 1
|
||||
await asyncio.sleep(0.05 * (attempt + 1))
|
||||
continue
|
||||
rec.final_failures += 1
|
||||
rec.unconfirmed.append((chat["id"], key, text))
|
||||
break
|
||||
if think:
|
||||
await asyncio.sleep(think)
|
||||
|
||||
|
||||
async def run_load(stand, rec, seconds, concurrency, **kw):
|
||||
end = time.time() + seconds
|
||||
await asyncio.gather(*(worker(stand, rec, lambda: time.time() > end, **kw) for _ in range(concurrency)))
|
||||
|
||||
|
||||
async def verify(stand, rec):
|
||||
"""Сверка: каждое подтверждённое сообщение читается, дублей по тексту нет, неподтверждённые встречаются не более 1 раза."""
|
||||
timeout = aiohttp.ClientTimeout(total=30)
|
||||
stored = {}
|
||||
async with aiohttp.ClientSession(timeout=timeout) as s:
|
||||
for chat in stand.chats:
|
||||
hdr = {"Authorization": "Bearer " + chat["members"][0]["token"]}
|
||||
cursor, items = None, []
|
||||
while True:
|
||||
url = "%s/api/v1/chats/%s/messages?limit=100%s" % (stand.base, chat["id"], "&cursor=" + cursor if cursor else "")
|
||||
async with s.get(url, headers=hdr) as r:
|
||||
page = await r.json()
|
||||
assert r.status == 200, page
|
||||
items += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
stored[chat["id"]] = items
|
||||
ids = {m["id"] for items in stored.values() for m in items}
|
||||
texts = [m["text"] for items in stored.values() for m in items]
|
||||
confirmed_missing = [c for c in rec.confirmed if c[1] not in ids]
|
||||
dup_texts = len(texts) - len(set(texts))
|
||||
unconfirmed_present = sum(1 for c in rec.unconfirmed if c[2] in set(texts))
|
||||
return {"stored_messages": len(ids), "confirmed_201": len(rec.confirmed), "confirmed_missing_in_history": len(confirmed_missing),
|
||||
"duplicate_texts_in_history": dup_texts, "unconfirmed_requests": len(rec.unconfirmed),
|
||||
"unconfirmed_that_were_written": unconfirmed_present, "retries": rec.retries, "final_failures": rec.final_failures}
|
||||
|
||||
|
||||
def per_second(rec, kind="POST"):
|
||||
buckets = {}
|
||||
for t, k, status, lat, inst in rec.events:
|
||||
if k != kind:
|
||||
continue
|
||||
b = buckets.setdefault(int(t), {"ok": 0, "err": 0, "lat": [], "instances": set()})
|
||||
if status == 201 or (kind == "GET" and status == 200):
|
||||
b["ok"] += 1
|
||||
b["lat"].append(lat)
|
||||
if inst:
|
||||
b["instances"].add(inst)
|
||||
else:
|
||||
b["err"] += 1
|
||||
return buckets
|
||||
|
||||
|
||||
def pct(values, p):
|
||||
if not values:
|
||||
return None
|
||||
values = sorted(values)
|
||||
return round(values[max(0, int(len(values) * p + 0.9999) - 1)], 2)
|
||||
|
||||
|
||||
def summarize(rec, t_from=None, t_to=None, kind="POST"):
|
||||
rows = [e for e in rec.events if e[1] == kind and (t_from is None or e[0] >= t_from) and (t_to is None or e[0] < t_to)]
|
||||
if not rows:
|
||||
return {}
|
||||
dur = (t_to if t_to is not None else max(e[0] for e in rows)) - (t_from if t_from is not None else min(e[0] for e in rows))
|
||||
ok = [e[3] for e in rows if e[2] == 201 or (kind == "GET" and e[2] == 200)]
|
||||
errs = {}
|
||||
for e in rows:
|
||||
if not (e[2] == 201 or (kind == "GET" and e[2] == 200)):
|
||||
errs[str(e[2])] = errs.get(str(e[2]), 0) + 1
|
||||
return {"requests": len(rows), "ok": len(ok), "errors": errs, "seconds": round(dur, 2), "ok_rps": round(len(ok) / dur, 1) if dur > 0 else None,
|
||||
"error_ratio": round((len(rows) - len(ok)) / len(rows), 4), "p50_ms": pct(ok, .5), "p95_ms": pct(ok, .95), "p99_ms": pct(ok, .99)}
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Render the measured matrix vertically so labels remain readable in an A4 report."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--input', type=Path, required=True)
|
||||
ap.add_argument('--output', type=Path, required=True)
|
||||
args = ap.parse_args()
|
||||
data = json.loads(args.input.read_text(encoding='utf-8'))
|
||||
plt.rcParams.update({'font.size': 11})
|
||||
fig, axes = plt.subplots(3, 1, figsize=(7.8, 9.2))
|
||||
sizes = ['small', 'medium', 'large']
|
||||
concs = sorted({r['concurrency'] for r in data['matrix']})
|
||||
for ax, key, title, unit in zip(axes,
|
||||
['throughput_per_s', 'p95_s', 'worker_peak_rss_mib'],
|
||||
['Пропускная способность конвейера', 'p95 времени от приёма до ready', 'Пик RSS worker'],
|
||||
['файлов/с', 'с', 'MiB']):
|
||||
for i, conc in enumerate(concs):
|
||||
values = [next(r[key] for r in data['matrix'] if r['concurrency'] == conc and r['image'] == size) for size in sizes]
|
||||
ax.bar([j + (i - 1) * .26 for j in range(3)], values, width=.25, label=f'concurrency={conc}')
|
||||
ax.set_xticks(range(3), ['320×200 PNG', '2400×1600 PNG', '6000×4000 JPEG'])
|
||||
ax.set_title(title)
|
||||
ax.set_ylabel(unit)
|
||||
ax.legend(ncol=3, fontsize=9)
|
||||
ax.grid(axis='y', alpha=.2)
|
||||
ax.set_axisbelow(True)
|
||||
fig.suptitle('ЛР5: worker 1 CPU / 768 MiB, один прогон на точку', fontsize=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(args.output, dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 1 (очередь под перегрузкой), lab5.
|
||||
|
||||
Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG,
|
||||
large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов
|
||||
(burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с),
|
||||
p50/p95/p99/max времени до ready, глубина/возраст очереди, RSS и CPU worker, ошибки HTTP и число failed.
|
||||
Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами:
|
||||
API ограничивает приём по порогу очереди и отвечает 503 + Retry-After (возможен конкурентный overshoot);
|
||||
принятые задания не теряются (все ready);
|
||||
после разгрузки новая загрузка снова принимается.
|
||||
python research/queue_overload.py --out evidence
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import aiohttp
|
||||
from PIL import Image
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
PROJECT = "mtusi-lab5"
|
||||
|
||||
|
||||
def sh(*args, env=None):
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
|
||||
env=dict(os.environ, **(env or {}))).stdout.strip()
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
|
||||
|
||||
|
||||
def healthy(service, expected=1):
|
||||
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split()
|
||||
return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids)
|
||||
|
||||
|
||||
def recreate(service, env, wait_seconds=90):
|
||||
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env)
|
||||
deadline = time.time() + wait_seconds
|
||||
while time.time() < deadline:
|
||||
if healthy(service, 1 if service == "worker" else 2):
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError("%s did not become healthy within %d seconds" % (service, wait_seconds))
|
||||
time.sleep(6 if service == "api" else 3)
|
||||
|
||||
|
||||
def make_images(root):
|
||||
import random
|
||||
random.seed(11)
|
||||
out = {}
|
||||
out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read()
|
||||
|
||||
def noisy(w, h):
|
||||
base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR)
|
||||
grad = Image.linear_gradient("L").resize((w, h)).convert("RGB")
|
||||
return Image.blend(base, grad, 0.5)
|
||||
buf = io.BytesIO()
|
||||
noisy(2400, 1600).save(buf, "PNG", compress_level=3)
|
||||
out["medium"] = buf.getvalue()
|
||||
buf = io.BytesIO()
|
||||
noisy(6000, 4000).save(buf, "JPEG", quality=88)
|
||||
out["large"] = buf.getvalue()
|
||||
return out
|
||||
|
||||
|
||||
async def register(s):
|
||||
name = "qo_%s" % uuid.uuid4().hex[:12]
|
||||
pw = "queue-test-pw-" + uuid.uuid4().hex[:6]
|
||||
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r:
|
||||
user = await r.json()
|
||||
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
|
||||
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
|
||||
token = (await r.json())["token"]
|
||||
async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
|
||||
chat = await r.json()
|
||||
return token, chat["id"]
|
||||
|
||||
|
||||
class QueueSampler(threading.Thread):
|
||||
def __init__(self):
|
||||
super().__init__(daemon=True)
|
||||
self.samples, self.stop, self.errors = [], threading.Event(), []
|
||||
self.t0 = time.time()
|
||||
|
||||
def run(self):
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')")
|
||||
n, age = out.split("|")
|
||||
self.samples.append((time.time() - self.t0, int(n), float(age)))
|
||||
except Exception as exc:
|
||||
self.errors.append(type(exc).__name__)
|
||||
return
|
||||
self.stop.wait(0.4)
|
||||
|
||||
|
||||
def worker_peak_rss():
|
||||
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()
|
||||
out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status")
|
||||
try:
|
||||
return round(int(out.split()[1]) / 1024, 1)
|
||||
except (IndexError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def worker_cpu_snapshot():
|
||||
# /proc/1/stat includes CPU time of the worker's threads. No cgroup version dependency.
|
||||
code = ("import json,os,time; s=open('/proc/1/stat').read().rsplit(')',1)[1].split(); "
|
||||
"print(json.dumps({'cpu_s':(int(s[11])+int(s[12]))/os.sysconf('SC_CLK_TCK'), "
|
||||
"'monotonic_s':time.monotonic()}))")
|
||||
return json.loads(sh("docker", "compose", "exec", "-T", "worker", "python", "-c", code))
|
||||
|
||||
|
||||
def interval_metrics(before, after, statuses):
|
||||
elapsed = after["monotonic_s"] - before["monotonic_s"]
|
||||
cpu = after["cpu_s"] - before["cpu_s"]
|
||||
attempts = sum(statuses.values())
|
||||
errors = sum(n for status, n in statuses.items() if not 200 <= int(status) < 300)
|
||||
assert elapsed > 0 and cpu >= 0 and attempts > 0
|
||||
return {"measurement_window_s": round(elapsed, 3), "worker_cpu_seconds": round(cpu, 3),
|
||||
"worker_cpu_percent_one_core": round(100 * cpu / elapsed, 2),
|
||||
"http_non_2xx_fraction": errors / attempts,
|
||||
"accepted_http_rps_over_window": round(statuses.get("202", 0) / elapsed, 2)}
|
||||
|
||||
|
||||
async def burst(s, token, chat, data, n, clients, retry_503=False):
|
||||
"""Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses)."""
|
||||
accepted, statuses, sem = [], {}, asyncio.Semaphore(clients)
|
||||
observed = {"retry_after_values": [], "transport_errors": []}
|
||||
|
||||
async def one(i):
|
||||
async with sem:
|
||||
for attempt in range(60):
|
||||
try:
|
||||
async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data,
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r:
|
||||
body = await r.json(content_type=None)
|
||||
statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1
|
||||
if r.status in (503, 429):
|
||||
observed["retry_after_values"].append(r.headers.get("Retry-After"))
|
||||
if r.status == 202:
|
||||
accepted.append(body["id"])
|
||||
return
|
||||
if r.status in (503, 429) and retry_503:
|
||||
await asyncio.sleep(float(r.headers.get("Retry-After", "1")))
|
||||
continue
|
||||
return
|
||||
except aiohttp.ClientError as exc:
|
||||
# Upload is not idempotent: an uncertain response must not trigger a hidden retry.
|
||||
observed["transport_errors"].append(type(exc).__name__)
|
||||
return
|
||||
await asyncio.gather(*(one(i) for i in range(n)))
|
||||
return accepted, statuses, observed
|
||||
|
||||
|
||||
def wait_all(ids, seconds):
|
||||
if not ids:
|
||||
raise ValueError("no accepted jobs to measure")
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids))
|
||||
if left == "0":
|
||||
return True
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
def stats_for(ids):
|
||||
rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)"
|
||||
% ",".join("'%s'" % i for i in ids)).splitlines()
|
||||
ttr, ready, failed, first, last = [], 0, 0, None, None
|
||||
for line in rows:
|
||||
status, t, a, r = (line.split("|") + ["", "", ""])[:4]
|
||||
if status == "ready" and t:
|
||||
ready += 1
|
||||
ttr.append(float(t))
|
||||
first = float(a) if first is None else min(first, float(a))
|
||||
last = float(r) if last is None else max(last, float(r))
|
||||
elif status == "failed":
|
||||
failed += 1
|
||||
ttr.sort()
|
||||
pct = lambda p: round(ttr[max(0, math.ceil(p * len(ttr)) - 1)], 2) if ttr else None # noqa: E731
|
||||
span = (last - first) if first is not None else None
|
||||
return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "p99_s": pct(0.99), "max_s": round(max(ttr), 2) if ttr else None,
|
||||
"throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None}
|
||||
|
||||
|
||||
async def part_a(args, images):
|
||||
cells = []
|
||||
sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large}
|
||||
for conc in args.concurrency:
|
||||
for size, n in sizes.items():
|
||||
recreate("worker", {"WORKER_CONCURRENCY": str(conc)})
|
||||
ids_worker = sh("docker", "compose", "ps", "-q", "worker").split()
|
||||
assert len(ids_worker) == 1, "this experiment requires exactly one worker"
|
||||
limits = json.loads(sh("docker", "inspect", "-f",
|
||||
'{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}}}', ids_worker[0]))
|
||||
assert limits == {"memory_bytes": 768 * 1024 * 1024, "nano_cpus": 1000000000}, limits
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16, force_close=True)) as s:
|
||||
token, chat = await register(s)
|
||||
cpu_before = worker_cpu_snapshot()
|
||||
sampler = QueueSampler()
|
||||
sampler.start()
|
||||
t0 = time.time()
|
||||
ids, statuses, observed = await burst(s, token, chat, images[size], n, 8)
|
||||
accepted_in = time.time() - t0
|
||||
done = wait_all(ids, 400)
|
||||
cpu_after = worker_cpu_snapshot()
|
||||
sampler.stop.set()
|
||||
sampler.join()
|
||||
cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids),
|
||||
"resource_limits": limits, "attempt_statuses": statuses, "transport_errors": observed["transport_errors"],
|
||||
"accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0),
|
||||
"peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(),
|
||||
"worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])}
|
||||
cell.update(stats_for(ids))
|
||||
cell.update(interval_metrics(cpu_before, cpu_after, statuses))
|
||||
cell["attachment_ids"] = ids
|
||||
print(json.dumps(cell, ensure_ascii=False))
|
||||
cells.append(cell)
|
||||
with open(os.path.join(args.out, "matrix_progress.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(cells, fh, ensure_ascii=False, indent=2)
|
||||
assert not sampler.errors and sampler.samples, sampler.errors
|
||||
assert len(ids) == len(set(ids)) == n and statuses == {"202": n} and not observed["transport_errors"], cell
|
||||
assert done and cell["ready"] == n and cell["failed"] == 0, cell
|
||||
assert cell["worker_oom_killed"] == "false" and 0 < cell["worker_peak_rss_mib"] < 768, cell
|
||||
return cells
|
||||
|
||||
|
||||
async def part_b(args, images):
|
||||
recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)})
|
||||
recreate("worker", {"WORKER_CONCURRENCY": "1"})
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64, force_close=True)) as s:
|
||||
token, chat = await register(s)
|
||||
cpu_before = worker_cpu_snapshot()
|
||||
sampler = QueueSampler()
|
||||
sampler.start()
|
||||
t0 = time.time()
|
||||
ids, statuses, observed = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False)
|
||||
accepted_in = time.time() - t0
|
||||
done = wait_all(ids, 300)
|
||||
cpu_after = worker_cpu_snapshot()
|
||||
drained_after = time.time() - t0
|
||||
sampler.stop.set()
|
||||
sampler.join()
|
||||
# восстановление: после разгрузки новая загрузка принимается
|
||||
extra, st2, observed2 = await burst(s, token, chat, images["small"], 3, 1)
|
||||
extra_done = wait_all(extra, 60)
|
||||
throttled = observed["retry_after_values"]
|
||||
valid_retry_after = bool(throttled) and all(v is not None and v.isdigit() and int(v) >= 1 for v in throttled)
|
||||
res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids),
|
||||
"peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done,
|
||||
"accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1),
|
||||
"retry_after_header_seen": valid_retry_after, "retry_after_values": sorted(set(throttled), key=str),
|
||||
"transport_errors": observed["transport_errors"], "recovered_new_uploads_accepted": len(extra) == 3,
|
||||
"recovered_new_uploads_ready": extra_done and stats_for(extra)["ready"] == 3,
|
||||
"no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids),
|
||||
**stats_for(ids), **interval_metrics(cpu_before, cpu_after, statuses), "attachment_ids": ids}
|
||||
print(json.dumps(res, ensure_ascii=False))
|
||||
with open(os.path.join(args.out, "backpressure_progress.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(res, fh, ensure_ascii=False, indent=2)
|
||||
assert not sampler.errors and sampler.samples, sampler.errors
|
||||
assert statuses.get("503", 0) > 0 and sum(statuses.values()) == args.burst and valid_retry_after, res
|
||||
assert set(statuses) <= {"202", "503", "429"} and not observed["transport_errors"], res
|
||||
assert done and res["ready"] == len(ids) and res["failed"] == 0 and res["no_accepted_job_lost"], res
|
||||
assert res["recovered_new_uploads_accepted"] and res["recovered_new_uploads_ready"] and not observed2["transport_errors"], res
|
||||
return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples]
|
||||
|
||||
|
||||
async def main(args):
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
images = make_images(ROOT)
|
||||
print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()})
|
||||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"},
|
||||
"image_kib": {k: round(len(v) / 1024) for k, v in images.items()}}
|
||||
try:
|
||||
report["matrix"] = await part_a(args, images)
|
||||
report["backpressure"], samples = await part_b(args, images)
|
||||
finally:
|
||||
recreate("api", {})
|
||||
recreate("worker", {})
|
||||
report["backpressure_queue_samples"] = samples
|
||||
with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||||
plot(report, os.path.join(args.out, "queue_overload.png"))
|
||||
|
||||
|
||||
def plot(report, path):
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
fig, ax = plt.subplots(1, 3, figsize=(17, 5))
|
||||
sizes = ["small", "medium", "large"]
|
||||
concs = sorted({c["concurrency"] for c in report["matrix"]})
|
||||
for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность конвейера", "готово/с"), ("p95_s", "p95 времени до ready", "с"),
|
||||
("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))):
|
||||
for i, c in enumerate(concs):
|
||||
vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes]
|
||||
ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c)
|
||||
ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"])
|
||||
ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend()
|
||||
fig.tight_layout(); fig.savefig(path, dpi=110)
|
||||
|
||||
s = report["backpressure_queue_samples"]
|
||||
fig, ax = plt.subplots(figsize=(8, 4))
|
||||
ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)")
|
||||
ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS")
|
||||
ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"])
|
||||
fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="evidence")
|
||||
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4])
|
||||
ap.add_argument("--n-small", type=int, default=80)
|
||||
ap.add_argument("--n-medium", type=int, default=24)
|
||||
ap.add_argument("--n-large", type=int, default=12)
|
||||
ap.add_argument("--limit", type=int, default=30)
|
||||
ap.add_argument("--burst", type=int, default=200)
|
||||
ap.add_argument("--b-medium", action="store_true")
|
||||
asyncio.run(main(ap.parse_args()))
|
||||
@@ -0,0 +1,3 @@
|
||||
aiohttp==3.14.3
|
||||
Pillow==12.3.0
|
||||
matplotlib==3.11.2
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4: rolling update под нагрузкой. Нагрузка идёт через LB 60 с; на t=10 запускается scripts/rolling-update.sh.
|
||||
Проверяем: ошибки клиента (после повторов с тем же ключом), потерянные/задублированные сообщения, смену instance id."""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import loadgen # noqa: E402
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
|
||||
|
||||
async def main(args):
|
||||
stand = loadgen.Stand("http://127.0.0.1:8080")
|
||||
await stand.setup()
|
||||
rec = loadgen.Recorder()
|
||||
info = {}
|
||||
|
||||
def update():
|
||||
time.sleep(max(0, rec.t0 + 10 - time.time()))
|
||||
info["start"] = round(time.time() - rec.t0, 2)
|
||||
out = subprocess.run(["sh", "scripts/rolling-update.sh", args.revision], cwd=ROOT, capture_output=True, text=True)
|
||||
info["end"] = round(time.time() - rec.t0, 2)
|
||||
info["script_exit"] = out.returncode
|
||||
info["script_output"] = out.stdout.strip().splitlines()[-6:]
|
||||
|
||||
threading.Thread(target=update, daemon=True).start()
|
||||
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
|
||||
await asyncio.sleep(1)
|
||||
check = await loadgen.verify(stand, rec)
|
||||
instances_before = {e[4] for e in rec.events if e[1] == "POST" and e[0] < info["start"] and e[4]}
|
||||
instances_during = {e[4] for e in rec.events if e[1] == "POST" and info["start"] <= e[0] <= info["end"] and e[4]}
|
||||
instances_after = {e[4] for e in rec.events if e[1] == "POST" and e[0] > info["end"] + 2 and e[4]}
|
||||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "update": info, "revision": args.revision,
|
||||
"instances_before": sorted(instances_before), "instances_during": sorted(instances_during),
|
||||
"instances_after": sorted(instances_after), "all_replicas_replaced": not (instances_before & instances_after),
|
||||
"before": loadgen.summarize(rec, 2, info["start"]), "during": loadgen.summarize(rec, info["start"], info["end"]),
|
||||
"after": loadgen.summarize(rec, info["end"] + 2, None), "total": loadgen.summarize(rec), "verification": check,
|
||||
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0 and check["final_failures"] == 0}
|
||||
with open(os.path.join(args.out, "rolling_update.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="evidence")
|
||||
ap.add_argument("--seconds", type=int, default=60)
|
||||
ap.add_argument("--concurrency", type=int, default=16)
|
||||
ap.add_argument("--revision", default="rolling-%d" % int(time.time()))
|
||||
asyncio.run(main(ap.parse_args()))
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# Создаёт локальные секреты стенда (secrets/ в .gitignore). Идемпотентно: существующие файлы не перезаписываются.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p secrets
|
||||
gen() { head -c "$1" /dev/urandom | base64 | tr -d '=+/\n'; }
|
||||
mk() { # mk <имя> <содержимое>
|
||||
if [ ! -s "secrets/$1" ]; then printf '%s' "$2" > "secrets/$1"; echo "создан secrets/$1"; fi
|
||||
}
|
||||
mk postgres_password "$(gen 24)"
|
||||
mk database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger"
|
||||
# S3 (SeaweedFS): администратор (только bootstrap) и отдельные ключи API с правами на ОДИН bucket
|
||||
mk s3_admin_access_key "adm$(gen 8)"
|
||||
mk s3_admin_secret_key "$(gen 30)"
|
||||
mk s3_access_key "api$(gen 8)"
|
||||
mk s3_secret_key "$(gen 30)"
|
||||
BUCKET="${S3_BUCKET:-messenger-attachments}"
|
||||
if [ ! -s secrets/seaweed_s3.json ]; then
|
||||
cat > secrets/seaweed_s3.json <<EOF
|
||||
{
|
||||
"identities": [
|
||||
{"name": "admin", "credentials": [{"accessKey": "$(cat secrets/s3_admin_access_key)", "secretKey": "$(cat secrets/s3_admin_secret_key)"}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
|
||||
{"name": "api", "credentials": [{"accessKey": "$(cat secrets/s3_access_key)", "secretKey": "$(cat secrets/s3_secret_key)"}],
|
||||
"actions": ["Read:$BUCKET", "Write:$BUCKET", "List:$BUCKET"]}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
echo "создан secrets/seaweed_s3.json"
|
||||
fi
|
||||
mk rabbitmq_user "mq$(gen 6)"
|
||||
mk rabbitmq_password "$(gen 24)"
|
||||
mk rabbitmq.conf "default_user = $(cat secrets/rabbitmq_user)
|
||||
default_pass = $(cat secrets/rabbitmq_password)
|
||||
"
|
||||
mk broker_url "amqp://$(cat secrets/rabbitmq_user):$(cat secrets/rabbitmq_password)@rabbitmq:5672/"
|
||||
chmod 644 secrets/* 2>/dev/null || true # читает непривилегированный пользователь контейнера (uid 10001)
|
||||
@@ -0,0 +1,9 @@
|
||||
@echo off
|
||||
rem Windows analogue of restart.sh: restart the whole stand, volumes are kept; wait until healthy.
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb >NUL 2>&1
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb >NUL 2>&1
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env exec -T api python -m app.s3_recovery
|
||||
exit /b %errorlevel%
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
# Перезапуск ВСЕГО стенда с сохранением volumes: LB, API-реплики, PostgreSQL, Valkey, S3. Данные не удаляются.
|
||||
# После рестарта ждём healthy всех сервисов (`up --wait`), чтобы проверка сразу видела готовый стенд (S3 стартует ~10 с).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb >/dev/null 2>&1
|
||||
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb >/dev/null 2>&1
|
||||
docker compose --env-file .env exec -T api python -m app.s3_recovery
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Возвращает стенд к заданному числу API-реплик (API_REPLICAS в .env, по умолчанию 2):
|
||||
# остановленные контейнеры удаляются и создаются заново; данные (volumes) не затрагиваются.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
for cid in $(docker ps -aq --filter "label=com.docker.compose.project=mtusi-lab5" --filter "label=com.docker.compose.service=api" --filter "status=exited"); do
|
||||
docker rm "$cid" >/dev/null
|
||||
done
|
||||
N="${API_REPLICAS:-$(grep -E '^API_REPLICAS=' .env | cut -d= -f2 | tr -d '\r')}"
|
||||
docker compose --env-file .env up -d --no-deps --scale api="$N" api >/dev/null 2>&1 || { echo "не удалось вернуть $N реплик" >&2; exit 1; }
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
# Rolling update API без простоя: реплики заменяются по одной.
|
||||
# 1) миграции выполняются ОДИН раз до замены (идемпотентно; должны быть обратно совместимы со старой версией);
|
||||
# 2) для каждой старой реплики: поднять новую (из нового образа) рядом -> дождаться healthy и попадания в LB ->
|
||||
# остановить старую штатно (SIGTERM -> drain 4 с -> graceful shutdown) -> удалить.
|
||||
# Использование: scripts/rolling-update.sh [REVISION]
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
REV="${1:-$(date +%s)}"
|
||||
COMPOSE="docker compose --env-file .env"
|
||||
API_FILTER="--filter label=com.docker.compose.project=mtusi-lab5 --filter label=com.docker.compose.service=api"
|
||||
|
||||
$COMPOSE build --build-arg REV="$REV" api >/dev/null
|
||||
$COMPOSE run --rm migrate >/dev/null
|
||||
OLD=$(docker ps -q $API_FILTER)
|
||||
N=$(echo "$OLD" | wc -w | tr -d ' ')
|
||||
echo "revision=$REV replicas=$N"
|
||||
for old in $OLD; do
|
||||
BEFORE_FILE=$(mktemp)
|
||||
docker ps -q $API_FILTER | sort > "$BEFORE_FILE"
|
||||
$COMPOSE up -d --no-deps --no-recreate --scale api=$((N + 1)) api >/dev/null 2>&1
|
||||
NEW=$(docker ps -q $API_FILTER | sort | comm -13 "$BEFORE_FILE" -)
|
||||
rm -f "$BEFORE_FILE"
|
||||
for i in $(seq 1 60); do
|
||||
[ "$(docker inspect -f '{{.State.Health.Status}}' "$NEW" 2>/dev/null)" = healthy ] && break
|
||||
sleep 1
|
||||
done
|
||||
sleep 3 # LB: rise=2 проверки по 1 с -> реплика в ротации
|
||||
docker stop --time 15 "$old" >/dev/null
|
||||
docker rm "$old" >/dev/null
|
||||
echo "заменена реплика $old -> $NEW"
|
||||
done
|
||||
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
rem Windows analogue of stop-one.sh: stop exactly one API replica and leave it stopped.
|
||||
set "cid="
|
||||
echo %TARGET_INSTANCE_ID%| findstr /r "^[a-f0-9][a-f0-9]*$" >NUL || exit /b 2
|
||||
for /f %%i in ('docker ps -q --filter id^=%TARGET_INSTANCE_ID% --filter label^=com.docker.compose.project^=mtusi-lab5 --filter label^=com.docker.compose.service^=api') do set "cid=%%i"
|
||||
if "%cid%"=="" exit /b 3
|
||||
docker stop --time 0 %cid% >NUL 2>&1
|
||||
exit /b %errorlevel%
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Останавливает РОВНО одну API-реплику (TARGET_INSTANCE_ID == X-Instance-Id == короткий ID контейнера) и
|
||||
# оставляет её остановленной. Не трогает БД, Valkey, LB и volumes. Возвращается быстро (< 2 с).
|
||||
# `docker stop --time 0`: SIGTERM и немедленный SIGKILL — имитация отказа процесса; политика restart
|
||||
# после ручной остановки не поднимает контейнер. Восстановление: scripts/restore-replicas.sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
''|*[!a-f0-9]*) echo "некорректный TARGET_INSTANCE_ID" >&2; exit 2 ;;
|
||||
esac
|
||||
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
|
||||
--filter "label=com.docker.compose.project=mtusi-lab5" --filter "label=com.docker.compose.service=api")
|
||||
[ -n "$cid" ] || { echo "реплика не найдена среди работающих api-контейнеров" >&2; exit 3; }
|
||||
docker stop --time 0 "$cid" >/dev/null
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Вложения (lab4): потоковая загрузка PNG/JPEG в приватный S3, метаданные в PostgreSQL, авторизованное скачивание."""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import uuid
|
||||
|
||||
from . import config, images
|
||||
from .web import ApiError, bad, format_dt, log, parse_uuid, unavailable
|
||||
|
||||
ALLOWED_TYPES = ("image/png", "image/jpeg")
|
||||
VARIANT_NAMES = ("original", "thumbnail", "crop", "optimized")
|
||||
ATT_COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.status, a.error_code, "
|
||||
"a.s3_key, a.created_at, a.updated_at, a.pipeline_version")
|
||||
VARIANT_ORDER = {"thumbnail": 0, "crop": 1, "optimized": 2}
|
||||
|
||||
|
||||
def attachment_json(row, variants=()):
|
||||
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"],
|
||||
"content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
|
||||
"variants": list(variants), "error_code": row["error_code"], "created_at": format_dt(row["created_at"]),
|
||||
"updated_at": format_dt(row["updated_at"])}
|
||||
|
||||
|
||||
def valid_filename(query):
|
||||
values = query.get("filename")
|
||||
if not values or len(values) != 1:
|
||||
raise bad("Требуется параметр filename")
|
||||
name = values[0]
|
||||
if not 1 <= len(name) <= 255 or not name.strip():
|
||||
raise bad("filename: длина 1–255 символов")
|
||||
try:
|
||||
name.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
raise bad("filename содержит некорректные символы") from None
|
||||
if "/" in name or "\\" in name or "\x00" in name or any(unicodedata.category(ch) == "Cc" for ch in name):
|
||||
raise bad("filename не должен содержать / \\ NUL и управляющие символы")
|
||||
return name
|
||||
|
||||
|
||||
class StreamResponse:
|
||||
"""Потоковый ответ: тело читается из S3 порциями и сразу отдаётся клиенту (без загрузки файла в память)."""
|
||||
|
||||
def __init__(self, content_type, length, stream):
|
||||
self.content_type, self.length, self.stream = content_type, length, stream
|
||||
|
||||
async def send(self, send, request_id, instance_id):
|
||||
headers = [(b"content-type", self.content_type.encode()), (b"content-length", str(self.length).encode()),
|
||||
(b"x-request-id", request_id.encode()), (b"x-content-type-options", b"nosniff"),
|
||||
(b"cache-control", b"private, no-store")]
|
||||
if instance_id:
|
||||
headers.append((b"x-instance-id", instance_id.encode()))
|
||||
await send({"type": "http.response.start", "status": 200, "headers": headers})
|
||||
try:
|
||||
while True:
|
||||
chunk = await self.stream.read(config.DOWNLOAD_CHUNK_BYTES)
|
||||
if not chunk:
|
||||
break
|
||||
await send({"type": "http.response.body", "body": chunk, "more_body": True})
|
||||
finally:
|
||||
self.stream.close()
|
||||
await send({"type": "http.response.body", "body": b""})
|
||||
|
||||
|
||||
class AttachmentMixin:
|
||||
"""Подмешивается в Messenger: требует self.db, self.s3, self.authenticate()."""
|
||||
|
||||
def _init_attachments(self):
|
||||
self._upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
|
||||
self._upload_waiting = 0
|
||||
|
||||
async def _receive_upload(self, req):
|
||||
"""Принимает тело в SpooledTemporaryFile (до 1 MiB в памяти, далее диск), считает SHA-256, ограничивает размер."""
|
||||
declared = req.headers.get("content-length")
|
||||
too_large = ApiError(413, "payload_too_large", "Файл больше 10 MiB", {"connection": "close"})
|
||||
if declared is not None:
|
||||
if not declared.isdigit():
|
||||
raise bad("Некорректный Content-Length")
|
||||
if int(declared) > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
|
||||
digest, size = hashlib.sha256(), 0
|
||||
try:
|
||||
while True:
|
||||
event = await req.receive()
|
||||
if event["type"] == "http.disconnect":
|
||||
raise bad("Соединение закрыто клиентом")
|
||||
chunk = event.get("body", b"")
|
||||
size += len(chunk)
|
||||
if size > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
if not event.get("more_body"):
|
||||
break
|
||||
except BaseException:
|
||||
spool.close()
|
||||
raise
|
||||
if size == 0:
|
||||
spool.close()
|
||||
raise bad("Пустой файл")
|
||||
return spool, digest.hexdigest(), size
|
||||
|
||||
async def upload_attachment(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
chat_id = await self._require_member(user_id, params)
|
||||
filename = valid_filename(req.query)
|
||||
media = req.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media not in ALLOWED_TYPES:
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: image/png или image/jpeg")
|
||||
backlog = await self.db.fetchval("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')")
|
||||
if backlog >= config.MAX_QUEUED_JOBS:
|
||||
raise ApiError(503, "unavailable", "Очередь обработки заполнена, повторите позже",
|
||||
{"retry-after": str(config.BACKLOG_RETRY_AFTER_SECONDS)})
|
||||
if self._upload_waiting >= config.UPLOAD_QUEUE:
|
||||
raise ApiError(429, "rate_limited", "Слишком много одновременных загрузок",
|
||||
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
|
||||
self._upload_waiting += 1
|
||||
try:
|
||||
await self._upload_slots.acquire() # ограничение одновременных загрузок на реплику (RAM/диск/CPU)
|
||||
finally:
|
||||
self._upload_waiting -= 1
|
||||
try:
|
||||
spool, sha256, size = await self._receive_upload(req)
|
||||
try:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
content_type, width, height = await loop.run_in_executor(self.image_pool, images.validate, spool, media)
|
||||
except images.ImageRejected as exc:
|
||||
raise ApiError(exc.status, exc.code, exc.message) from None
|
||||
att_id = uuid.uuid4()
|
||||
staging_key = "staging/%s" % att_id
|
||||
final_key = "att/%s/original" % att_id
|
||||
spool.seek(0)
|
||||
await self.s3.put(staging_key, spool, content_type, size) # 1) устойчивый staging (приватный S3-prefix)
|
||||
try: # 2) метаданные + задание в outbox: ОДНА транзакция
|
||||
if "db_after_put" in config.FAULT_INJECTION:
|
||||
raise RuntimeError("injected: сбой БД после записи в staging")
|
||||
async with self.db.conn() as c:
|
||||
async with c.transaction():
|
||||
row = await c.fetchrow(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, "
|
||||
"height, status, s3_key, staging_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'queued', $10, $11) "
|
||||
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code, "
|
||||
"s3_key, created_at, updated_at",
|
||||
att_id, uuid.UUID(chat_id), uuid.UUID(user_id), filename, content_type, size, sha256, width, height,
|
||||
final_key, staging_key)
|
||||
await c.execute(
|
||||
"INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES ($1, $2, 1, 1)",
|
||||
uuid.uuid4(), att_id)
|
||||
except BaseException:
|
||||
try:
|
||||
if "s3_delete_fails" in config.FAULT_INJECTION:
|
||||
raise OSError("injected: компенсирующее удаление не удалось")
|
||||
await self.s3.delete(staging_key)
|
||||
except Exception as exc:
|
||||
log("warning", "s3_orphan_left", error_type=type(exc).__name__)
|
||||
raise
|
||||
finally:
|
||||
spool.close()
|
||||
finally:
|
||||
self._upload_slots.release()
|
||||
self.relay.kick() # немедленная публикация; при сбое задание остаётся в outbox
|
||||
return 202, attachment_json(row)
|
||||
|
||||
async def _attachment_for(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
att_id = parse_uuid(params["attachment_id"], "attachment_id")
|
||||
row = await self.db.fetchrow(
|
||||
"SELECT " + ATT_COLUMNS + " FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 "
|
||||
"WHERE a.id = $1", att_id, user_id)
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Вложение не найдено")
|
||||
return row
|
||||
|
||||
async def _variants(self, row):
|
||||
if row["status"] != "ready":
|
||||
return []
|
||||
rows = await self.db.fetch("SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants "
|
||||
"WHERE attachment_id = $1 AND pipeline_version = $2", row["id"], row["pipeline_version"])
|
||||
rows = sorted(rows, key=lambda r: VARIANT_ORDER[r["name"]])
|
||||
return [{"name": r["name"], "content_type": r["content_type"], "size_bytes": r["size_bytes"], "width": r["width"],
|
||||
"height": r["height"], "sha256": r["sha256"]} for r in rows]
|
||||
|
||||
async def get_attachment(self, req, params):
|
||||
row = await self._attachment_for(req, params)
|
||||
return 200, attachment_json(row, await self._variants(row))
|
||||
|
||||
async def download_attachment(self, req, params):
|
||||
row = await self._attachment_for(req, params)
|
||||
variant = "original"
|
||||
if "variant" in req.query:
|
||||
if len(req.query["variant"]) != 1 or req.query["variant"][0] not in VARIANT_NAMES:
|
||||
raise bad("variant: original | thumbnail | crop | optimized")
|
||||
variant = req.query["variant"][0]
|
||||
if row["status"] != "ready":
|
||||
raise ApiError(409, "conflict", "Вложение ещё обрабатывается или обработка не удалась")
|
||||
key, content_type = row["s3_key"], row["content_type"]
|
||||
if variant != "original":
|
||||
v = await self.db.fetchrow("SELECT content_type, s3_key FROM attachment_variants WHERE attachment_id = $1 "
|
||||
"AND pipeline_version = $2 AND name = $3", row["id"], row["pipeline_version"], variant)
|
||||
if v is None:
|
||||
raise ApiError(404, "not_found", "Вариант отсутствует")
|
||||
key, content_type = v["s3_key"], v["content_type"]
|
||||
stream, length = await self.s3.open_stream(key)
|
||||
return 200, StreamResponse(content_type, length, stream)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""RabbitMQ: durable quorum-очередь, persistent-сообщения, publisher confirms, DLX для нечитаемых сообщений.
|
||||
|
||||
В брокер попадают только служебные поля задания (jobs.schema.json): ID, версия пайплайна, номер попытки, время.
|
||||
Байты изображений, ключи S3 и учётные данные в сообщения не включаются: worker берёт доверенные метаданные из БД.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import aio_pika
|
||||
from aio_pika import DeliveryMode, ExchangeType, Message
|
||||
|
||||
from . import config
|
||||
from .web import log
|
||||
|
||||
QUEUE = "attachment.process"
|
||||
DEAD_QUEUE = "attachment.process.dead"
|
||||
DEAD_EXCHANGE = "attachment.dlx"
|
||||
|
||||
|
||||
class Broker:
|
||||
def __init__(self):
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
self.queue = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def start(self, prefetch=None):
|
||||
"""Подключение; при недоступном брокере вызывающий повторяет попытку (см. ensure)."""
|
||||
self.connection = await aio_pika.connect_robust(config.BROKER_URL, timeout=config.BROKER_TIMEOUT,
|
||||
client_properties={"connection_name": config.SERVICE_NAME})
|
||||
self.channel = await self.connection.channel(publisher_confirms=True)
|
||||
if prefetch:
|
||||
await self.channel.set_qos(prefetch_count=prefetch)
|
||||
dlx = await self.channel.declare_exchange(DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True)
|
||||
dead = await self.channel.declare_queue(DEAD_QUEUE, durable=True, arguments={"x-queue-type": "quorum"})
|
||||
await dead.bind(dlx, routing_key="dead")
|
||||
self.queue = await self.channel.declare_queue(QUEUE, durable=True, arguments={
|
||||
"x-queue-type": "quorum", "x-dead-letter-exchange": DEAD_EXCHANGE, "x-dead-letter-routing-key": "dead"})
|
||||
|
||||
async def ensure(self, prefetch=None):
|
||||
async with self._lock:
|
||||
if self.channel is None or self.channel.is_closed:
|
||||
await self.start(prefetch)
|
||||
|
||||
async def stop(self):
|
||||
if self.connection:
|
||||
try:
|
||||
await asyncio.wait_for(self.connection.close(), 3)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def publish(self, job, headers=None):
|
||||
"""Публикует задание; возвращается ПОСЛЕ подтверждения брокера (publisher confirm) либо бросает исключение."""
|
||||
await self.ensure()
|
||||
body = json.dumps(job, separators=(",", ":")).encode("utf-8")
|
||||
message = Message(body, delivery_mode=DeliveryMode.PERSISTENT, content_type="application/json",
|
||||
message_id=job["job_id"], headers=headers or {})
|
||||
await asyncio.wait_for(self.channel.default_exchange.publish(message, routing_key=QUEUE, mandatory=True),
|
||||
config.BROKER_TIMEOUT)
|
||||
|
||||
async def queue_depth(self):
|
||||
await self.ensure()
|
||||
q = await self.channel.declare_queue(QUEUE, durable=True, passive=True)
|
||||
return q.declaration_result.message_count
|
||||
|
||||
async def ping(self):
|
||||
return self.channel is not None and not self.channel.is_closed
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Конфигурация из переменных окружения (см. .env.example)."""
|
||||
import os
|
||||
|
||||
|
||||
def _int(name, default):
|
||||
value = os.environ.get(name)
|
||||
return int(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _float(name, default):
|
||||
value = os.environ.get(name)
|
||||
return float(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _secret(name, default=""):
|
||||
"""Значение из переменной NAME либо из файла NAME_FILE (Docker secrets); файл читается один раз."""
|
||||
if os.environ.get(name):
|
||||
return os.environ[name]
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path and os.path.exists(path):
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return fh.read().strip()
|
||||
return default
|
||||
|
||||
|
||||
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = _int("APP_PORT", 8080)
|
||||
APP_ENV = os.environ.get("APP_ENV", "lab")
|
||||
|
||||
DATABASE_URL = _secret("DATABASE_URL")
|
||||
DB_POOL_MIN = _int("DB_POOL_MIN", 2)
|
||||
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
|
||||
DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 2.0) # ожидание свободного соединения пула
|
||||
DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5.0) # таймаут одного запроса
|
||||
DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 2.0)
|
||||
|
||||
VALKEY_URL = os.environ.get("VALKEY_URL", "") # пусто = кэш сессий выключен
|
||||
VALKEY_TIMEOUT = _float("VALKEY_TIMEOUT", 0.2)
|
||||
VALKEY_BREAKER_SECONDS = _float("VALKEY_BREAKER_SECONDS", 5.0)
|
||||
|
||||
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
|
||||
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60)
|
||||
|
||||
ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2)
|
||||
ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456) # 19 MiB, рекомендация OWASP
|
||||
ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1)
|
||||
HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4)
|
||||
|
||||
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
|
||||
MAX_INFLIGHT = _int("MAX_INFLIGHT", 64)
|
||||
LIMIT_CONCURRENCY = _int("LIMIT_CONCURRENCY", 1000)
|
||||
RETRY_AFTER_SECONDS = _int("RETRY_AFTER_SECONDS", 1)
|
||||
MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "migrations"))
|
||||
DRAIN_SECONDS = _float("DRAIN_SECONDS", 4.0) # после SIGTERM: readiness=503, сервер ещё принимает запросы
|
||||
GRACEFUL_SHUTDOWN_SECONDS = _int("GRACEFUL_SHUTDOWN_SECONDS", 8)
|
||||
KEEPALIVE_SECONDS = _int("KEEPALIVE_SECONDS", 5)
|
||||
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 24 * 3600)
|
||||
|
||||
# --- S3 / вложения (lab4) ---
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
S3_ACCESS_KEY = _secret("S3_ACCESS_KEY")
|
||||
S3_SECRET_KEY = _secret("S3_SECRET_KEY")
|
||||
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2.0)
|
||||
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 15.0)
|
||||
S3_POOL = _int("S3_POOL", 20)
|
||||
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) # 10 MiB = 10 485 760
|
||||
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000)
|
||||
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 4) # одновременных загрузок на реплику
|
||||
UPLOAD_QUEUE = _int("UPLOAD_QUEUE", 16) # ожидающих загрузок; сверх — 429
|
||||
IMAGE_THREADS = _int("IMAGE_THREADS", 2)
|
||||
DOWNLOAD_CHUNK_BYTES = _int("DOWNLOAD_CHUNK_BYTES", 65536)
|
||||
ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900) # объект без метаданных моложе этого возраста не трогаем
|
||||
FAULT_INJECTION = {x for x in os.environ.get("FAULT_INJECTION", "").split(",") if x} # db_after_put, s3_delete_fails (эксперименты)
|
||||
|
||||
# --- lab5: брокер, outbox, worker ---
|
||||
SERVICE_NAME = os.environ.get("SERVICE_NAME", "messenger-api")
|
||||
BROKER_URL = _secret("BROKER_URL")
|
||||
BROKER_TIMEOUT = _float("BROKER_TIMEOUT", 5.0)
|
||||
RELAY_BATCH = _int("RELAY_BATCH", 50)
|
||||
RELAY_INTERVAL = _float("RELAY_INTERVAL", 1.0)
|
||||
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) # prefetch == число одновременных заданий == потоков обработки
|
||||
LEASE_SECONDS = _int("LEASE_SECONDS", 60) # аренда задания; истекла -> sweeper возвращает в очередь
|
||||
JOB_TIMEOUT = _float("JOB_TIMEOUT", 45.0) # лимит времени одной обработки (меньше lease)
|
||||
MAX_ATTEMPTS = _int("MAX_ATTEMPTS", 5)
|
||||
RETRY_BASE_DELAY = _float("RETRY_BASE_DELAY", 2.0)
|
||||
RETRY_MAX_DELAY = _float("RETRY_MAX_DELAY", 60.0)
|
||||
SWEEP_INTERVAL = _float("SWEEP_INTERVAL", 5.0)
|
||||
STAGING_FAILED_RETENTION_SECONDS = _int("STAGING_FAILED_RETENTION_SECONDS", 7 * 24 * 3600)
|
||||
MAX_QUEUED_JOBS = _int("MAX_QUEUED_JOBS", 200) # backpressure: при превышении upload -> 503 + Retry-After
|
||||
BACKLOG_RETRY_AFTER_SECONDS = _int("BACKLOG_RETRY_AFTER_SECONDS", 5)
|
||||
@@ -0,0 +1,71 @@
|
||||
"""PostgreSQL: ограниченный пул asyncpg, таймауты, перевод сбоев зависимости в 503."""
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
from .web import log, unavailable
|
||||
|
||||
DEPENDENCY_ERRORS = (asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.exceptions.OperatorInterventionError,
|
||||
asyncpg.exceptions.InsufficientResourcesError, asyncio.TimeoutError, OSError)
|
||||
_RETRYABLE = (asyncpg.ConnectionDoesNotExistError, asyncpg.InterfaceError, ConnectionResetError)
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self):
|
||||
self.pool = None
|
||||
|
||||
async def start(self):
|
||||
if not config.DATABASE_URL:
|
||||
log("error", "database_url_missing")
|
||||
return
|
||||
# min_size=0: приложение стартует и без БД (readiness покажет 503), пул подключается лениво.
|
||||
self.pool = await asyncpg.create_pool(
|
||||
dsn=config.DATABASE_URL, min_size=0, max_size=config.DB_POOL_MAX,
|
||||
command_timeout=config.DB_COMMAND_TIMEOUT, timeout=config.DB_CONNECT_TIMEOUT,
|
||||
max_inactive_connection_lifetime=60,
|
||||
server_settings={"application_name": "messenger-api", "timezone": "UTC"})
|
||||
|
||||
async def stop(self):
|
||||
if self.pool:
|
||||
try:
|
||||
await asyncio.wait_for(self.pool.close(), 3)
|
||||
except Exception:
|
||||
self.pool.terminate()
|
||||
|
||||
@asynccontextmanager
|
||||
async def conn(self):
|
||||
if self.pool is None:
|
||||
raise unavailable("База данных не настроена")
|
||||
try:
|
||||
async with self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT) as connection:
|
||||
yield connection
|
||||
except DEPENDENCY_ERRORS as exc:
|
||||
log("warning", "db_unavailable", error_type=type(exc).__name__)
|
||||
raise unavailable() from exc
|
||||
|
||||
async def _run(self, method, sql, args, retry):
|
||||
for attempt in (0, 1):
|
||||
try:
|
||||
async with self.conn() as c:
|
||||
return await getattr(c, method)(sql, *args)
|
||||
except Exception as exc:
|
||||
if attempt == 0 and retry and isinstance(exc.__cause__, _RETRYABLE):
|
||||
continue # разорванное соединение пула (рестарт БД): один повтор только для чтения
|
||||
raise
|
||||
|
||||
async def fetch(self, sql, *args):
|
||||
return await self._run("fetch", sql, args, True)
|
||||
|
||||
async def fetchrow(self, sql, *args, retry=True):
|
||||
return await self._run("fetchrow", sql, args, retry)
|
||||
|
||||
async def fetchval(self, sql, *args):
|
||||
return await self._run("fetchval", sql, args, True)
|
||||
|
||||
async def execute(self, sql, *args):
|
||||
return await self._run("execute", sql, args, False)
|
||||
|
||||
async def ping(self):
|
||||
return await asyncio.wait_for(self.fetchval("SELECT 1"), 1.0)
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Проверка загружаемых изображений ДО полного декодирования: сигнатура, формат Pillow, размеры, целостность.
|
||||
|
||||
PNG/JPEG определяются по содержимому (а не по Content-Type). Лимит пикселей проверяется по заголовку,
|
||||
поэтому «decompression bomb» отклоняется без декодирования. Целостность: PNG — verify() (CRC всех чанков, без
|
||||
распаковки в память), JPEG — декодирование в режиме draft (1/8 размера => около 1/64 памяти).
|
||||
"""
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from . import config
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = None # лимит применяем сами по заголовку (ниже), без предупреждений Pillow
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
class ImageRejected(Exception):
|
||||
def __init__(self, status, code, message):
|
||||
super().__init__(message)
|
||||
self.status, self.code, self.message = status, code, message
|
||||
|
||||
|
||||
def sniff(head):
|
||||
if head.startswith(PNG_SIGNATURE):
|
||||
return "image/png"
|
||||
if head[:3] == b"\xff\xd8\xff":
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def _check_png_stream(fileobj, width, height):
|
||||
"""Целостность потока IDAT без аллокации изображения: распаковка порциями, проверка конца потока и размера данных."""
|
||||
fileobj.seek(8)
|
||||
inflater = zlib.decompressobj()
|
||||
total, channels, bit_depth, interlaced = 0, 0, 0, 0
|
||||
while True:
|
||||
head = fileobj.read(8)
|
||||
if len(head) < 8:
|
||||
raise ValueError("truncated")
|
||||
length, kind = struct.unpack(">I4s", head)
|
||||
if length > 0x7FFFFFFF:
|
||||
raise ValueError("chunk")
|
||||
if kind == b"IHDR":
|
||||
data = fileobj.read(length)
|
||||
bit_depth, color_type, _, _, interlaced = struct.unpack(">BBBBB", data[8:13])
|
||||
channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[color_type]
|
||||
fileobj.read(4)
|
||||
elif kind == b"IDAT":
|
||||
remaining = length
|
||||
while remaining:
|
||||
piece = fileobj.read(min(remaining, 65536))
|
||||
if not piece:
|
||||
raise ValueError("truncated")
|
||||
remaining -= len(piece)
|
||||
pending = piece
|
||||
while pending:
|
||||
total += len(inflater.decompress(pending, 1 << 20))
|
||||
pending = inflater.unconsumed_tail
|
||||
fileobj.read(4)
|
||||
elif kind == b"IEND":
|
||||
break
|
||||
else:
|
||||
fileobj.seek(length + 4, 1)
|
||||
if not inflater.eof:
|
||||
raise ValueError("zlib stream incomplete")
|
||||
if not interlaced:
|
||||
row = (width * channels * bit_depth + 7) // 8
|
||||
if total != height * (row + 1):
|
||||
raise ValueError("unexpected image data size")
|
||||
|
||||
|
||||
def validate(fileobj, declared_type):
|
||||
"""Возвращает (content_type, width, height) либо бросает ImageRejected (415 — не то/повреждено, 413 — много пикселей)."""
|
||||
fileobj.seek(0)
|
||||
detected = sniff(fileobj.read(16))
|
||||
if detected is None or detected != declared_type:
|
||||
raise ImageRejected(415, "unsupported_media_type", "Содержимое не соответствует PNG/JPEG или заявленному Content-Type")
|
||||
fileobj.seek(0)
|
||||
try:
|
||||
with Image.open(fileobj) as img:
|
||||
width, height = img.size
|
||||
fmt = img.format
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError):
|
||||
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено или не поддерживается") from None
|
||||
if fmt not in ("PNG", "JPEG") or width < 1 or height < 1:
|
||||
raise ImageRejected(415, "unsupported_media_type", "Неподдерживаемый формат изображения")
|
||||
if width * height > config.MAX_IMAGE_PIXELS:
|
||||
raise ImageRejected(413, "payload_too_large", "Слишком много пикселей в изображении")
|
||||
fileobj.seek(0)
|
||||
try:
|
||||
img = Image.open(fileobj)
|
||||
if fmt == "PNG":
|
||||
img.verify()
|
||||
_check_png_stream(fileobj, width, height)
|
||||
else:
|
||||
img.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
||||
img.load()
|
||||
img.close()
|
||||
except Exception: # любая ошибка декодера = повреждённый файл
|
||||
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено") from None
|
||||
return detected, width, height
|
||||
@@ -0,0 +1,417 @@
|
||||
"""ASGI-приложение мессенджера (lab2): REST API, PostgreSQL, сессии Bearer, кэш Valkey."""
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config, passwords
|
||||
from .attachments import AttachmentMixin, StreamResponse
|
||||
from .db import DEPENDENCY_ERRORS, Database
|
||||
from .broker import Broker
|
||||
from .outbox import Relay
|
||||
from .s3 import S3
|
||||
from .sessions import SessionCache, Sessions
|
||||
from .web import (ApiError, Request, bad, dt_to_us, encode_cursor, format_dt, log, new_request_id, page_params,
|
||||
parse_json_object, parse_uuid, respond, unavailable, us_to_dt, valid_text, USERNAME_RE)
|
||||
|
||||
ROUTES = [
|
||||
("health_live", re.compile(r"/health/live"), {"GET"}),
|
||||
("health_ready", re.compile(r"/health/ready"), {"GET"}),
|
||||
("users", re.compile(r"/api/v1/users"), {"POST"}),
|
||||
("users_me", re.compile(r"/api/v1/users/me"), {"GET"}),
|
||||
("create_session", re.compile(r"/api/v1/auth/sessions"), {"POST"}),
|
||||
("session_current", re.compile(r"/api/v1/auth/sessions/current"), {"DELETE"}),
|
||||
("chats", re.compile(r"/api/v1/chats"), {"GET", "POST"}),
|
||||
("chat", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)"), {"GET"}),
|
||||
("messages", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/messages"), {"GET", "POST"}),
|
||||
("upload_attachment", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/attachments"), {"POST"}),
|
||||
("get_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)"), {"GET"}),
|
||||
("download_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)/content"), {"GET"}),
|
||||
]
|
||||
ROUTE_LABELS = {
|
||||
"health_live": "/health/live", "health_ready": "/health/ready", "users": "/api/v1/users",
|
||||
"users_me": "/api/v1/users/me", "create_session": "/api/v1/auth/sessions",
|
||||
"session_current": "/api/v1/auth/sessions/current", "chats": "/api/v1/chats",
|
||||
"chat": "/api/v1/chats/{chat_id}", "messages": "/api/v1/chats/{chat_id}/messages",
|
||||
"upload_attachment": "/api/v1/chats/{chat_id}/attachments", "get_attachment": "/api/v1/attachments/{attachment_id}",
|
||||
"download_attachment": "/api/v1/attachments/{attachment_id}/content",
|
||||
}
|
||||
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
BEARER = 'Bearer realm="messenger"'
|
||||
BASIC = 'Basic realm="messenger", charset="UTF-8"'
|
||||
# Идентификатор реплики: имя хоста контейнера (12 hex, == короткий ID контейнера) -> легко сопоставить с docker ps.
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or (os.environ.get("HOSTNAME") or uuid.uuid4().hex)[:12]
|
||||
|
||||
MSG_ATTS = ("COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) FROM message_attachments ma "
|
||||
"WHERE ma.message_id = messages.id), '{}'::uuid[]) AS attachment_ids")
|
||||
CHAT_SELECT = ("SELECT c.id, c.title, c.created_at, "
|
||||
"(SELECT array_agg(m2.user_id ORDER BY m2.ord) FROM chat_members m2 WHERE m2.chat_id = c.id) AS member_ids ")
|
||||
|
||||
|
||||
def user_json(row):
|
||||
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"],
|
||||
"created_at": format_dt(row["created_at"])}
|
||||
|
||||
|
||||
def chat_json(row):
|
||||
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in row["member_ids"]],
|
||||
"created_at": format_dt(row["created_at"])}
|
||||
|
||||
|
||||
def message_json(row):
|
||||
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]),
|
||||
"text": row["text"], "created_at": format_dt(row["created_at"]),
|
||||
"attachment_ids": [str(x) for x in (row["attachment_ids"] or [])]}
|
||||
|
||||
|
||||
def unauthorized(challenge=BEARER, message="Требуется действительная сессия"):
|
||||
return ApiError(401, "unauthorized", message, {"www-authenticate": challenge})
|
||||
|
||||
|
||||
class Messenger(AttachmentMixin):
|
||||
def __init__(self):
|
||||
self.db = Database()
|
||||
self.s3 = S3()
|
||||
self.broker = Broker()
|
||||
self.relay = Relay(self.db, self.broker)
|
||||
self.image_pool = ThreadPoolExecutor(max_workers=config.IMAGE_THREADS, thread_name_prefix="image")
|
||||
self._init_attachments()
|
||||
self.cache = SessionCache()
|
||||
self.sessions = Sessions(self.db, self.cache)
|
||||
self.inflight = 0
|
||||
self.draining = False
|
||||
|
||||
# --- аутентификация ------------------------------------------------------
|
||||
@staticmethod
|
||||
def bearer_token(req):
|
||||
header = req.headers.get("authorization", "")
|
||||
scheme, _, token = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not TOKEN_RE.fullmatch(token.strip()):
|
||||
raise unauthorized()
|
||||
return token.strip()
|
||||
|
||||
async def authenticate(self, req):
|
||||
token = self.bearer_token(req)
|
||||
user_id = await self.sessions.resolve(token)
|
||||
if user_id is None:
|
||||
raise unauthorized()
|
||||
req.user_id, req.token = user_id, token
|
||||
return user_id
|
||||
|
||||
# --- обработчики -----------------------------------------------------------
|
||||
async def health_live(self, req, params):
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
async def health_ready(self, req, params):
|
||||
# Критичная зависимость — PostgreSQL. Valkey — необязательный кэш: его отказ не останавливает API.
|
||||
if self.draining:
|
||||
raise unavailable("Сервис останавливается")
|
||||
try:
|
||||
await self.db.ping()
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception:
|
||||
raise unavailable() from None
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
async def users(self, req, params):
|
||||
req.require_json_media_type()
|
||||
data = parse_json_object(await req.body(), {"username", "display_name", "password"},
|
||||
{"username", "display_name", "password"})
|
||||
username = data["username"]
|
||||
if not isinstance(username, str) or not USERNAME_RE.fullmatch(username):
|
||||
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
|
||||
display_name = valid_text(data["display_name"], "display_name", 100)
|
||||
password = valid_text(data["password"], "password", 128, allow_empty=True)
|
||||
if len(password) < 12:
|
||||
raise bad("password должен содержать от 12 до 128 символов")
|
||||
pw_hash = await passwords.hash_password(password)
|
||||
row = await self.db.fetchrow(
|
||||
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) "
|
||||
"ON CONFLICT (username) DO NOTHING RETURNING id, username, display_name, created_at",
|
||||
uuid.uuid4(), username, display_name, pw_hash)
|
||||
if row is None:
|
||||
raise ApiError(409, "conflict", "username уже занят")
|
||||
return 201, user_json(row)
|
||||
|
||||
async def users_me(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
row = await self.db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id)
|
||||
if row is None:
|
||||
raise unauthorized()
|
||||
return 200, user_json(row)
|
||||
|
||||
async def create_session(self, req, params):
|
||||
header = req.headers.get("authorization", "")
|
||||
scheme, _, value = header.partition(" ")
|
||||
if scheme.lower() != "basic" or not value.strip():
|
||||
raise unauthorized(BASIC, "Неверные учётные данные")
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
username, sep, password = decoded.partition(":")
|
||||
if not sep:
|
||||
raise ValueError
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
raise unauthorized(BASIC, "Неверные учётные данные") from None
|
||||
row = None
|
||||
if USERNAME_RE.fullmatch(username):
|
||||
row = await self.db.fetchrow("SELECT id, username, display_name, created_at, password_hash FROM users "
|
||||
"WHERE username = $1", username)
|
||||
ok = await passwords.verify_password(row["password_hash"] if row else None, password)
|
||||
if not ok:
|
||||
raise unauthorized(BASIC, "Неверные учётные данные")
|
||||
token, expires_at = await self.sessions.create(row["id"])
|
||||
return 201, {"token": token, "token_type": "Bearer", "expires_at": format_dt(expires_at), "user": user_json(row)}
|
||||
|
||||
async def session_current(self, req, params):
|
||||
await self.authenticate(req)
|
||||
await self.sessions.revoke(req.token)
|
||||
return 204, None
|
||||
|
||||
async def chats(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
if req.method == "GET":
|
||||
limit, after = page_params(req.query, "c", user_id)
|
||||
if after is None:
|
||||
rows = await self.db.fetch(
|
||||
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
|
||||
"ORDER BY m.chat_created_at, m.chat_id LIMIT $2", user_id, limit + 1)
|
||||
else:
|
||||
rows = await self.db.fetch(
|
||||
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
|
||||
"AND (m.chat_created_at, m.chat_id) > ($2, $3) ORDER BY m.chat_created_at, m.chat_id LIMIT $4",
|
||||
user_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
cursor = encode_cursor("c", user_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
|
||||
return 200, {"items": [chat_json(r) for r in rows], "next_cursor": cursor}
|
||||
req.require_json_media_type()
|
||||
data = parse_json_object(await req.body(), {"title", "member_ids"}, {"title", "member_ids"})
|
||||
title = valid_text(data["title"], "title", 100)
|
||||
members = data["member_ids"]
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
raise bad("member_ids должен содержать от 1 до 100 элементов")
|
||||
ids = [parse_uuid(m, "member_id") for m in members]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise bad("member_ids содержит дубликаты")
|
||||
if user_id not in ids:
|
||||
raise bad("Создатель должен быть в member_ids")
|
||||
chat_id = uuid.uuid4()
|
||||
uuids = [uuid.UUID(i) for i in ids]
|
||||
try:
|
||||
async with self.db.conn() as c:
|
||||
async with c.transaction(): # чат и участники создаются атомарно
|
||||
# Существование участников проверяет внешний ключ chat_members.user_id: при неизвестном
|
||||
# участнике вставка падает и откатывается вся транзакция, включая уже созданный чат.
|
||||
created = await c.fetchval("INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at",
|
||||
chat_id, title)
|
||||
await c.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) "
|
||||
"SELECT $1, u, $2, o FROM unnest($3::uuid[]) WITH ORDINALITY AS t(u, o)",
|
||||
chat_id, created, uuids)
|
||||
except asyncpg.ForeignKeyViolationError:
|
||||
raise bad("Неизвестный участник") from None
|
||||
return 201, {"id": str(chat_id), "title": title, "member_ids": ids, "created_at": format_dt(created)}
|
||||
|
||||
async def _require_member(self, user_id, params):
|
||||
chat_id = parse_uuid(params["chat_id"], "chat_id")
|
||||
ok = await self.db.fetchval("SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user_id)
|
||||
if not ok:
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
return chat_id
|
||||
|
||||
async def chat(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
chat_id = parse_uuid(params["chat_id"], "chat_id")
|
||||
row = await self.db.fetchrow(
|
||||
CHAT_SELECT + "FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 WHERE c.id = $1",
|
||||
chat_id, user_id)
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
return 200, chat_json(row)
|
||||
|
||||
async def messages(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
if req.method == "POST":
|
||||
row = await self._post_message(req, parse_uuid(params["chat_id"], "chat_id"), user_id)
|
||||
return 201, message_json(row)
|
||||
chat_id = await self._require_member(user_id, params)
|
||||
if req.method == "GET":
|
||||
limit, after = page_params(req.query, "m", chat_id)
|
||||
if after is None:
|
||||
rows = await self.db.fetch(
|
||||
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
|
||||
"ORDER BY created_at, id LIMIT $2", chat_id, limit + 1)
|
||||
else:
|
||||
rows = await self.db.fetch(
|
||||
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
|
||||
"AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4",
|
||||
chat_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
cursor = encode_cursor("m", chat_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
|
||||
return 200, {"items": [message_json(r) for r in rows], "next_cursor": cursor}
|
||||
|
||||
async def _post_message(self, req, chat_id, user_id):
|
||||
# Сначала разбор запроса; ошибка валидации (400/415) возвращается только участнику чата — постороннему всегда 404.
|
||||
try:
|
||||
key = req.headers.get("idempotency-key", "")
|
||||
if not IDEMPOTENCY_KEY_RE.fullmatch(key):
|
||||
raise bad("Требуется заголовок Idempotency-Key ([A-Za-z0-9._:-]{1,128})")
|
||||
req.require_json_media_type()
|
||||
data = parse_json_object(await req.body(), {"text"}, {"text", "attachment_ids"})
|
||||
raw_atts = data.get("attachment_ids", [])
|
||||
if not isinstance(raw_atts, list) or len(raw_atts) > 10:
|
||||
raise bad("attachment_ids: массив до 10 элементов")
|
||||
atts = [parse_uuid(a, "attachment_id") for a in raw_atts]
|
||||
if len(set(atts)) != len(atts):
|
||||
raise bad("attachment_ids содержит дубликаты")
|
||||
text = valid_text(data["text"], "text", 4000, allow_empty=bool(atts))
|
||||
except ApiError:
|
||||
await self._require_member(user_id, {"chat_id": chat_id})
|
||||
raise
|
||||
# Нормализованное представление запроса: порядок ключей JSON не важен, отсутствие attachment_ids == [].
|
||||
request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": atts}, sort_keys=True,
|
||||
ensure_ascii=False).encode("utf-8")).digest()
|
||||
row = await self.db.fetchrow("SELECT * FROM send_message($1, $2, $3, $4, $5, $6, $7)", uuid.UUID(chat_id),
|
||||
uuid.UUID(user_id), key, request_hash, text, float(config.IDEMPOTENCY_TTL_SECONDS),
|
||||
[uuid.UUID(a) for a in atts], retry=False)
|
||||
if row["outcome"] == "not_member":
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
if row["outcome"] == "attachment_not_found":
|
||||
raise ApiError(404, "not_found", "Вложение не найдено")
|
||||
if row["outcome"] == "attachment_failed":
|
||||
raise ApiError(409, "conflict", "Вложение не удалось обработать")
|
||||
if bytes(row["request_hash"]) != request_hash:
|
||||
raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим телом запроса")
|
||||
return {"id": row["msg_id"], "chat_id": row["msg_chat"], "sender_id": row["msg_sender"], "text": row["msg_text"],
|
||||
"created_at": row["msg_created"], "attachment_ids": row["msg_atts"]}
|
||||
|
||||
# --- ASGI ----------------------------------------------------------------------
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] == "lifespan":
|
||||
return await self.lifespan(receive, send)
|
||||
if scope["type"] != "http":
|
||||
return
|
||||
started = time.perf_counter()
|
||||
request_id = new_request_id()
|
||||
route_name, status, headers = "unknown", 500, {}
|
||||
try:
|
||||
req = Request(scope, receive, request_id)
|
||||
handler, params, route_name = self.route(req.method, req.path)
|
||||
if not route_name.startswith("health") and self.inflight >= config.MAX_INFLIGHT:
|
||||
raise ApiError(429, "rate_limited", "Слишком много одновременных запросов",
|
||||
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
|
||||
self.inflight += 1
|
||||
try:
|
||||
await asyncio.sleep(0)
|
||||
status, body = await handler(req, params)
|
||||
finally:
|
||||
self.inflight -= 1
|
||||
except ApiError as exc:
|
||||
status, headers = exc.status, exc.headers
|
||||
body = {"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}
|
||||
except DEPENDENCY_ERRORS as exc:
|
||||
log("warning", "dependency_error", request_id=request_id, error_type=type(exc).__name__)
|
||||
status, headers = 503, {"retry-after": str(config.RETRY_AFTER_SECONDS)}
|
||||
body = {"error": {"code": "unavailable", "message": "Зависимость временно недоступна", "request_id": request_id}}
|
||||
except Exception as exc:
|
||||
log("error", "unhandled_exception", request_id=request_id, error_type=type(exc).__name__)
|
||||
status = 500
|
||||
body = {"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}
|
||||
if isinstance(body, StreamResponse):
|
||||
try:
|
||||
await body.send(send, request_id, INSTANCE_ID)
|
||||
except Exception as exc: # клиент оборвал загрузку/ошибка S3 посреди потока
|
||||
log("warning", "stream_aborted", request_id=request_id, error_type=type(exc).__name__)
|
||||
else:
|
||||
await respond(send, status, body, request_id, headers, INSTANCE_ID)
|
||||
if os.environ.get("ACCESS_LOG", "1") != "0":
|
||||
log("info", "request", request_id=request_id, method=scope.get("method"),
|
||||
route=ROUTE_LABELS.get(route_name, "unmatched"), status=status,
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 3))
|
||||
|
||||
async def janitor(self):
|
||||
"""Удаляет давно просроченные ключи идемпотентности и сессии (безопасно запускать на каждой реплике)."""
|
||||
import random
|
||||
await asyncio.sleep(random.uniform(5, 30))
|
||||
while True:
|
||||
try:
|
||||
await self.db.execute("DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys "
|
||||
"WHERE created_at < now() - make_interval(secs => $1) LIMIT 5000)",
|
||||
float(config.IDEMPOTENCY_TTL_SECONDS) + 3600)
|
||||
await self.db.execute("DELETE FROM sessions WHERE ctid IN (SELECT ctid FROM sessions "
|
||||
"WHERE expires_at < now() - interval '1 day' LIMIT 5000)")
|
||||
except Exception as exc:
|
||||
log("warning", "janitor_failed", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
def route(self, method, path):
|
||||
for name, pattern, methods in ROUTES:
|
||||
m = pattern.fullmatch(path)
|
||||
if m:
|
||||
if method not in methods:
|
||||
raise ApiError(405, "invalid_request", "Метод не поддерживается", {"allow": ", ".join(sorted(methods))})
|
||||
return getattr(self, name), m.groupdict(), name
|
||||
raise ApiError(404, "not_found", "Ресурс не найден")
|
||||
|
||||
async def lifespan(self, receive, send):
|
||||
while True:
|
||||
event = await receive()
|
||||
if event["type"] == "lifespan.startup":
|
||||
await self.db.start()
|
||||
await self.cache.start()
|
||||
await self.s3.start()
|
||||
self._relay_task = asyncio.create_task(self.relay.run())
|
||||
self._janitor = asyncio.create_task(self.janitor())
|
||||
log("info", "startup", env=config.APP_ENV, instance=INSTANCE_ID, revision=os.environ.get("APP_REVISION", "dev"), cache=bool(config.VALKEY_URL),
|
||||
session_ttl=config.SESSION_TTL_SECONDS, pool_max=config.DB_POOL_MAX)
|
||||
await send({"type": "lifespan.startup.complete"})
|
||||
elif event["type"] == "lifespan.shutdown":
|
||||
self.draining = True
|
||||
self._janitor.cancel()
|
||||
await self.cache.stop()
|
||||
self.relay.stop()
|
||||
self._relay_task.cancel()
|
||||
await self.broker.stop()
|
||||
await self.s3.stop()
|
||||
await self.db.stop()
|
||||
log("info", "shutdown")
|
||||
await send({"type": "lifespan.shutdown.complete"})
|
||||
return
|
||||
|
||||
|
||||
async def overloaded_app(scope, receive, send):
|
||||
"""Ответ uvicorn при превышении --limit-concurrency в контрактном формате."""
|
||||
if scope["type"] != "http":
|
||||
return
|
||||
request_id = new_request_id()
|
||||
body = {"error": {"code": "unavailable", "message": "Сервер перегружен: слишком много соединений",
|
||||
"request_id": request_id}}
|
||||
await respond(send, 503, body, request_id, {"retry-after": str(config.RETRY_AFTER_SECONDS), "connection": "close"}, INSTANCE_ID)
|
||||
|
||||
|
||||
def _install_overload_response():
|
||||
for name in ("uvicorn.protocols.http.httptools_impl", "uvicorn.protocols.http.h11_impl"):
|
||||
try:
|
||||
module = __import__(name, fromlist=["service_unavailable"])
|
||||
except ImportError:
|
||||
continue
|
||||
module.service_unavailable = overloaded_app
|
||||
|
||||
|
||||
def create_app():
|
||||
_install_overload_response()
|
||||
return Messenger()
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Применение версионируемых SQL-миграций: python -m app.migrate
|
||||
|
||||
Свойства:
|
||||
* повторный запуск ничего не делает (таблица schema_migrations);
|
||||
* одновременный запуск нескольких экземпляров безопасен (advisory lock);
|
||||
* каждая миграция — одна транзакция вместе с записью о применении;
|
||||
* ждёт готовности PostgreSQL до MIGRATE_WAIT_SECONDS.
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
from .web import log
|
||||
|
||||
LOCK_ID = 7352001
|
||||
|
||||
|
||||
async def connect(deadline):
|
||||
while True:
|
||||
try:
|
||||
return await asyncpg.connect(config.DATABASE_URL, timeout=3)
|
||||
except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
log("info", "waiting_for_database", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
async def main():
|
||||
wait = float(os.environ.get("MIGRATE_WAIT_SECONDS", "60"))
|
||||
conn = await connect(time.monotonic() + wait)
|
||||
try:
|
||||
await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID)
|
||||
await conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
"version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())")
|
||||
applied = {r["version"]: r["checksum"] for r in await conn.fetch("SELECT version, checksum FROM schema_migrations")}
|
||||
directory = os.path.abspath(config.MIGRATIONS_DIR)
|
||||
files = sorted(f for f in os.listdir(directory) if f.endswith(".sql"))
|
||||
count = 0
|
||||
for name in files:
|
||||
sql = open(os.path.join(directory, name), encoding="utf-8").read()
|
||||
checksum = hashlib.sha256(sql.encode("utf-8")).hexdigest()
|
||||
if name in applied:
|
||||
if applied[name] != checksum:
|
||||
log("error", "migration_modified", version=name)
|
||||
sys.exit(2)
|
||||
continue
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", name, checksum)
|
||||
count += 1
|
||||
log("info", "migration_applied", version=name)
|
||||
log("info", "migrations_done", applied=count, total=len(files))
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Transactional outbox: relay публикует задания из PostgreSQL в брокер.
|
||||
|
||||
Гарантия: задание, зафиксированное в БД (COMMIT), будет опубликовано даже если процесс упал между COMMIT и publish.
|
||||
Relay запускается в каждой реплике API и в worker; строки берутся `FOR UPDATE SKIP LOCKED`, поэтому одну строку
|
||||
публикует один relay. Если процесс упадёт после подтверждения брокера, но до отметки published_at, задание будет
|
||||
опубликовано повторно — доставка at-least-once, дубликаты безвредны (обработка идемпотентна).
|
||||
"""
|
||||
import asyncio
|
||||
from datetime import timezone
|
||||
|
||||
from . import config
|
||||
from .web import format_dt, log
|
||||
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
|
||||
|
||||
def job_message(row):
|
||||
return {"schema_version": 1, "job_id": str(row["job_id"]), "type": JOB_TYPE, "attachment_id": str(row["attachment_id"]),
|
||||
"pipeline_version": row["pipeline_version"], "attempt": row["attempt"],
|
||||
"created_at": format_dt(row["created_at"].astimezone(timezone.utc))}
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, db, broker):
|
||||
self.db, self.broker = db, broker
|
||||
self._kick = asyncio.Event()
|
||||
self._stop = False
|
||||
self.last_error = None
|
||||
|
||||
def kick(self):
|
||||
self._kick.set()
|
||||
|
||||
def stop(self):
|
||||
self._stop = True
|
||||
self._kick.set()
|
||||
|
||||
async def relay_once(self):
|
||||
published = 0
|
||||
async with self.db.conn() as c:
|
||||
async with c.transaction():
|
||||
rows = await c.fetch(
|
||||
"SELECT id, job_id, attachment_id, pipeline_version, attempt, created_at FROM outbox "
|
||||
"WHERE published_at IS NULL AND available_at <= now() ORDER BY id LIMIT $1 FOR UPDATE SKIP LOCKED",
|
||||
config.RELAY_BATCH)
|
||||
for row in rows:
|
||||
await self.broker.publish(job_message(row), headers=self.headers_for(row))
|
||||
await c.execute("UPDATE outbox SET published_at = now() WHERE id = $1", row["id"])
|
||||
published += 1
|
||||
return published
|
||||
|
||||
def headers_for(self, row):
|
||||
return {}
|
||||
|
||||
async def run(self):
|
||||
delay = config.RELAY_INTERVAL
|
||||
while not self._stop:
|
||||
try:
|
||||
n = await self.relay_once()
|
||||
self.last_error = None
|
||||
delay = config.RELAY_INTERVAL
|
||||
if n >= config.RELAY_BATCH:
|
||||
continue # возможно, есть ещё
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if self.last_error != type(exc).__name__:
|
||||
log("warning", "outbox_relay_failed", error_type=type(exc).__name__)
|
||||
self.last_error = type(exc).__name__
|
||||
delay = min(delay * 2, 10.0) # брокер/БД недоступны: пауза с ростом до 10 с, задания остаются в outbox
|
||||
try:
|
||||
await asyncio.wait_for(self._kick.wait(), delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
self._kick.clear()
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Хеширование паролей: Argon2id (библиотека argon2-cffi), вычисления вне event loop."""
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
|
||||
from . import config
|
||||
|
||||
_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB,
|
||||
parallelism=config.ARGON2_PARALLELISM)
|
||||
_pool = ThreadPoolExecutor(max_workers=config.HASH_CONCURRENCY, thread_name_prefix="argon2")
|
||||
# Для неизвестного username проверяем фиктивный хеш: время ответа не выдаёт существование пользователя.
|
||||
_DUMMY = _hasher.hash("dummy-password-for-timing-equalisation")
|
||||
|
||||
|
||||
async def hash_password(password):
|
||||
return await asyncio.get_running_loop().run_in_executor(_pool, _hasher.hash, password)
|
||||
|
||||
|
||||
def _verify(stored, password):
|
||||
try:
|
||||
return _hasher.verify(stored, password)
|
||||
except (VerifyMismatchError, VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
async def verify_password(stored, password):
|
||||
"""stored=None -> проверка фиктивного хеша, результат всегда False."""
|
||||
ok = await asyncio.get_running_loop().run_in_executor(_pool, _verify, stored or _DUMMY, password)
|
||||
return ok and stored is not None
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Обработка изображений (pipeline_version=1): thumbnail, crop, optimized.
|
||||
|
||||
Правила (contracts/README.md):
|
||||
* thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, БЕЗ увеличения, EXIF orientation применяется, alpha — на белый
|
||||
фон; размеры округляются вниз, минимум 1 px (320x200 -> 256x160); качество 85, метаданные не сохраняются;
|
||||
* crop: JPEG, центрированный квадрат 128x128 (маленькие исходники допускается увеличивать), качество 85;
|
||||
* optimized: WebP, вписан в 1280x1280 без увеличения, quality=80 method=4, EXIF/ICC не сохраняются (ориентация применена).
|
||||
Память: JPEG декодируется через draft (масштабирование при декодировании), число пикселей ограничено MAX_IMAGE_PIXELS.
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import math
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from . import config
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
PIPELINE_VERSION = 1
|
||||
JPEG_QUALITY = 85
|
||||
WEBP_QUALITY = 80
|
||||
|
||||
|
||||
class PermanentError(Exception):
|
||||
"""Постоянная ошибка входных данных: повтор не поможет (-> failed, error_code)."""
|
||||
|
||||
def __init__(self, code):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _flatten(img):
|
||||
"""RGBA/LA/P с прозрачностью -> RGB на белом фоне."""
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return background
|
||||
return img.convert("RGB")
|
||||
|
||||
|
||||
def _fit_size(width, height, box, upscale=False):
|
||||
scale = min(box / width, box / height)
|
||||
if not upscale:
|
||||
scale = min(scale, 1.0)
|
||||
return max(1, math.floor(width * scale)), max(1, math.floor(height * scale))
|
||||
|
||||
|
||||
def _encode(img, fmt, **params):
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, fmt, **params)
|
||||
data = buf.getvalue()
|
||||
return data, hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def process(fileobj):
|
||||
"""Возвращает список вариантов: dict(name, content_type, data, sha256, width, height). Бросает PermanentError."""
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
src = Image.open(fileobj)
|
||||
fmt = src.format
|
||||
width, height = src.size
|
||||
if fmt not in ("PNG", "JPEG"):
|
||||
raise PermanentError("invalid_image")
|
||||
if width * height > config.MAX_IMAGE_PIXELS:
|
||||
raise PermanentError("image_too_large")
|
||||
if fmt == "JPEG" and width > 2560 and height > 2560:
|
||||
# экономия памяти: JPEG декодируется сразу в масштабе 1/2 (результат всё равно вписывается в 1280x1280)
|
||||
src.draft("RGB", (width // 2, height // 2))
|
||||
src.load()
|
||||
img = ImageOps.exif_transpose(src)
|
||||
except PermanentError:
|
||||
raise
|
||||
except MemoryError:
|
||||
raise
|
||||
except Exception:
|
||||
raise PermanentError("invalid_image") from None
|
||||
base = _flatten(img)
|
||||
out = []
|
||||
# thumbnail
|
||||
tw, th = _fit_size(base.width, base.height, 256)
|
||||
thumb = base.resize((tw, th), Image.LANCZOS) if (tw, th) != base.size else base
|
||||
data, sha = _encode(thumb, "JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
out.append({"name": "thumbnail", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": tw, "height": th})
|
||||
# crop 128x128 (центрированный квадрат)
|
||||
side = min(base.width, base.height)
|
||||
left, top = (base.width - side) // 2, (base.height - side) // 2
|
||||
crop = base.crop((left, top, left + side, top + side)).resize((128, 128), Image.LANCZOS)
|
||||
data, sha = _encode(crop, "JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
out.append({"name": "crop", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": 128, "height": 128})
|
||||
# optimized WebP (прозрачность сохраняется)
|
||||
opt_src = img.convert("RGBA") if (img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info)) else base
|
||||
ow, oh = _fit_size(opt_src.width, opt_src.height, 1280)
|
||||
opt = opt_src.resize((ow, oh), Image.LANCZOS) if (ow, oh) != opt_src.size else opt_src
|
||||
data, sha = _encode(opt, "WEBP", quality=WEBP_QUALITY, method=4)
|
||||
out.append({"name": "optimized", "content_type": "image/webp", "data": data, "sha256": sha, "width": ow, "height": oh})
|
||||
return out
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Сверка S3 и PostgreSQL (reconcile): python -m app.reconcile [--apply] [--grace СЕКУНДЫ] [--prefix att/]
|
||||
|
||||
Находит и (с --apply) безопасно исправляет расхождения двух хранилищ:
|
||||
* orphan_object — объект в S3 без строки в attachments (сбой между записью в S3 и COMMIT метаданных, прерванная загрузка);
|
||||
удаляется ТОЛЬКО если старше grace (по умолчанию ORPHAN_GRACE_SECONDS=900 с): свежий объект может быть
|
||||
загрузкой, метаданные которой ещё не зафиксированы;
|
||||
* missing_object — строка attachments (ready) без объекта в S3 (потеря/удаление объекта): строка НЕ удаляется,
|
||||
статус -> failed, error_code=object_missing (сообщения со ссылкой остаются, скачивание даёт 409);
|
||||
* size_mismatch — размер объекта не совпадает с size_bytes: статус -> failed, error_code=size_mismatch.
|
||||
Режим по умолчанию — dry-run (ничего не меняет). Повторный запуск идемпотентен. Используемые (привязанные к сообщению)
|
||||
вложения и свежие объекты не удаляются никогда: удаляются только объекты, у которых нет строки в БД.
|
||||
Выход: JSON-отчёт со счётчиками и временем обхода.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
from .s3 import S3
|
||||
|
||||
|
||||
def object_owner(key):
|
||||
"""Recognize server-generated prefixes, including effects not committed yet."""
|
||||
parts = key.split("/")
|
||||
if (parts[0] == "att" and len(parts) >= 3) or (parts[0] == "staging" and len(parts) == 2):
|
||||
try:
|
||||
return uuid.UUID(parts[1])
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
async def reconcile(apply, grace, prefix):
|
||||
started = time.perf_counter()
|
||||
s3 = S3()
|
||||
await s3.start()
|
||||
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
|
||||
try:
|
||||
t0 = time.perf_counter()
|
||||
rows = await conn.fetch("SELECT id, s3_key, staging_key, size_bytes, status, created_at FROM attachments")
|
||||
variants = await conn.fetch(
|
||||
"SELECT a.id, v.s3_key, v.size_bytes, a.status, a.created_at "
|
||||
"FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id")
|
||||
db_ms = (time.perf_counter() - t0) * 1000
|
||||
by_key = {r["s3_key"]: dict(r, check=True) for r in [*rows, *variants]}
|
||||
# Staging may already be removed after ready, or retained for failed/replay.
|
||||
by_key.update({r["staging_key"]: dict(r, check=False) for r in rows if r["staging_key"]})
|
||||
owners = {r["id"] for r in rows}
|
||||
t0 = time.perf_counter()
|
||||
objects = {}
|
||||
async for key, size, modified in s3.list_keys(prefix):
|
||||
objects[key] = (size, modified)
|
||||
list_ms = (time.perf_counter() - t0) * 1000
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=grace)
|
||||
report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects),
|
||||
"orphan_objects": [], "orphans_skipped_fresh": 0, "owned_objects_skipped": 0,
|
||||
"missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
|
||||
for key, (size, modified) in objects.items():
|
||||
row = by_key.get(key)
|
||||
if row is None:
|
||||
if object_owner(key) in owners:
|
||||
# Worker can write S3 variants before committing their metadata.
|
||||
report["owned_objects_skipped"] += 1
|
||||
elif modified < cutoff:
|
||||
report["orphan_objects"].append(key)
|
||||
else:
|
||||
report["orphans_skipped_fresh"] += 1
|
||||
elif row["check"] and row["status"] == "ready" and row["size_bytes"] != size:
|
||||
report["size_mismatch"].append(str(row["id"]))
|
||||
for key, row in by_key.items():
|
||||
if row["check"] and key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
|
||||
report["missing_objects"].append(str(row["id"]))
|
||||
for field in ("missing_objects", "size_mismatch"):
|
||||
report[field] = sorted(set(report[field]))
|
||||
if apply:
|
||||
for key in report["orphan_objects"]:
|
||||
# повторная проверка перед удалением: за время обхода строка могла появиться
|
||||
referenced = await conn.fetchval(
|
||||
"SELECT EXISTS (SELECT 1 FROM attachments WHERE s3_key = $1 OR staging_key = $1 OR id = $2) "
|
||||
"OR EXISTS (SELECT 1 FROM attachment_variants WHERE s3_key = $1)", key, object_owner(key))
|
||||
if not referenced:
|
||||
await s3.delete(key)
|
||||
report["fixed"]["deleted_orphans"] += 1
|
||||
for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")):
|
||||
for att_id in ids:
|
||||
n = await conn.execute("UPDATE attachments SET status = 'failed', error_code = $2, updated_at = now() "
|
||||
"WHERE id = $1 AND status = 'ready'", att_id, code)
|
||||
report["fixed"]["marked_failed"] += int(n.split()[-1])
|
||||
report["timing_ms"] = {"db_query": round(db_ms, 1), "s3_listing": round(list_ms, 1),
|
||||
"total": round((time.perf_counter() - started) * 1000, 1)}
|
||||
return report
|
||||
finally:
|
||||
await conn.close()
|
||||
await s3.stop()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--apply", action="store_true", help="исправлять (по умолчанию dry-run)")
|
||||
ap.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
|
||||
ap.add_argument("--prefix", default="att/")
|
||||
ap.add_argument("--max-listed", type=int, default=20, help="сколько идентификаторов печатать в отчёте")
|
||||
args = ap.parse_args()
|
||||
report = asyncio.run(reconcile(args.apply, args.grace, args.prefix))
|
||||
for k in ("orphan_objects", "missing_objects", "size_mismatch"):
|
||||
report[k + "_count"] = len(report[k])
|
||||
report[k] = report[k][:args.max_listed]
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Операторский replay и реестр неудавшихся заданий: python -m app.replay [list | replay <attachment_id> | replay-all]
|
||||
|
||||
list — реестр failed-вложений: id, error_code, число попыток, возраст, наличие staging;
|
||||
replay <attachment_id> — возвращает ТОЛЬКО failed-вложение в queued: новая попытка (attempt сбрасывается на 1), новый job_id,
|
||||
запись в job_events (аудит), задание в outbox. Если у вложения уже есть варианты (ready/успешный
|
||||
эффект), повторного исполнения не будет: вложение не в статусе failed -> replay отклоняется.
|
||||
Эффекты защищены ключом (attachment_id, pipeline_version, name): даже повторное выполнение не создаёт второй набор вариантов.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
async def list_failed(conn):
|
||||
rows = await conn.fetch(
|
||||
"SELECT id, error_code, attempt, staging_key IS NOT NULL AS has_staging, updated_at FROM attachments "
|
||||
"WHERE status = 'failed' ORDER BY updated_at DESC LIMIT 200")
|
||||
return [{"attachment_id": str(r["id"]), "error_code": r["error_code"], "attempt": r["attempt"],
|
||||
"staging_present": r["has_staging"], "failed_at": r["updated_at"].isoformat()} for r in rows]
|
||||
|
||||
|
||||
async def replay(conn, att_id):
|
||||
async with conn.transaction():
|
||||
row = await conn.fetchrow("SELECT id, status, pipeline_version, staging_key FROM attachments WHERE id = $1 FOR UPDATE", att_id)
|
||||
if row is None:
|
||||
return {"attachment_id": str(att_id), "result": "not_found"}
|
||||
if row["status"] != "failed":
|
||||
return {"attachment_id": str(att_id), "result": "skipped", "reason": "status=%s (replay только для failed)" % row["status"]}
|
||||
if not row["staging_key"]:
|
||||
return {"attachment_id": str(att_id), "result": "skipped", "reason": "staging удалён по retention, исходник недоступен"}
|
||||
job_id = uuid.uuid4()
|
||||
await conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, attempt = 0, job_id = $2, lease_owner = NULL, "
|
||||
"lease_expires_at = NULL, updated_at = now() WHERE id = $1", att_id, job_id)
|
||||
await conn.execute("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES ($1, $2, $3, 1)",
|
||||
job_id, att_id, row["pipeline_version"])
|
||||
await conn.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) VALUES ($1, $2, 'operator_replay', 1, 'failed -> queued')",
|
||||
att_id, job_id)
|
||||
return {"attachment_id": str(att_id), "result": "requeued", "job_id": str(job_id)}
|
||||
|
||||
|
||||
async def main_async(args):
|
||||
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
|
||||
try:
|
||||
if args.command == "list":
|
||||
return await list_failed(conn)
|
||||
if args.command == "replay":
|
||||
return await replay(conn, uuid.UUID(args.attachment_id))
|
||||
return [await replay(conn, uuid.UUID(f["attachment_id"])) for f in await list_failed(conn)]
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = ap.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("list")
|
||||
r = sub.add_parser("replay")
|
||||
r.add_argument("attachment_id")
|
||||
sub.add_parser("replay-all")
|
||||
print(json.dumps(asyncio.run(main_async(ap.parse_args())), ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Приватное S3-хранилище (MinIO) через aiobotocore: потоковая загрузка/скачивание, таймауты, перевод сбоев в 503."""
|
||||
import asyncio
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from . import config
|
||||
from .web import log, unavailable
|
||||
|
||||
S3_ERRORS = (BotoCoreError, ClientError, asyncio.TimeoutError, OSError)
|
||||
|
||||
|
||||
class S3:
|
||||
def __init__(self):
|
||||
self._ctx = None
|
||||
self.client = None
|
||||
|
||||
async def start(self):
|
||||
if not config.S3_ENDPOINT:
|
||||
log("error", "s3_endpoint_missing")
|
||||
return
|
||||
session = get_session()
|
||||
self._ctx = session.create_client(
|
||||
"s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY, aws_secret_access_key=config.S3_SECRET_KEY,
|
||||
config=AioConfig(connect_timeout=config.S3_CONNECT_TIMEOUT, read_timeout=config.S3_READ_TIMEOUT,
|
||||
retries={"max_attempts": 2, "mode": "standard"}, max_pool_connections=config.S3_POOL,
|
||||
s3={"addressing_style": "path"}))
|
||||
self.client = await self._ctx.__aenter__()
|
||||
|
||||
async def stop(self):
|
||||
if self._ctx:
|
||||
await self._ctx.__aexit__(None, None, None)
|
||||
|
||||
def _check(self):
|
||||
if self.client is None:
|
||||
raise unavailable("Хранилище файлов не настроено")
|
||||
|
||||
async def put(self, key, fileobj, content_type, size):
|
||||
self._check()
|
||||
try:
|
||||
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentType=content_type,
|
||||
ContentLength=size)
|
||||
except S3_ERRORS as exc:
|
||||
log("warning", "s3_put_failed", error_type=type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
|
||||
async def put_bytes(self, key, data, content_type):
|
||||
self._check()
|
||||
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=data, ContentType=content_type)
|
||||
|
||||
async def open_stream(self, key):
|
||||
"""Возвращает (stream, content_length). Вызывающий обязан закрыть stream."""
|
||||
self._check()
|
||||
try:
|
||||
resp = await self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except S3_ERRORS as exc:
|
||||
log("warning", "s3_get_failed", error_type=type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
return resp["Body"], resp["ContentLength"]
|
||||
|
||||
async def delete(self, key):
|
||||
self._check()
|
||||
await self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
|
||||
async def head(self, key):
|
||||
"""Метаданные объекта или None, если его нет."""
|
||||
self._check()
|
||||
try:
|
||||
return await self.client.head_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"):
|
||||
return None
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
except S3_ERRORS as exc:
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
|
||||
async def ping(self):
|
||||
self._check()
|
||||
await asyncio.wait_for(self.client.head_bucket(Bucket=config.S3_BUCKET), 2.0)
|
||||
|
||||
async def list_keys(self, prefix):
|
||||
"""Асинхронный итератор (key, size, last_modified) по префиксу — для сверки (reconcile)."""
|
||||
self._check()
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
async for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix):
|
||||
for obj in page.get("Contents", []):
|
||||
yield obj["Key"], obj["Size"], obj["LastModified"]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Идемпотентный bootstrap S3: python -m app.s3_bootstrap (one-shot сервис s3-init при каждом `docker compose up`).
|
||||
|
||||
1) bucket создаётся администратором, если его ещё нет (повтор ничего не меняет);
|
||||
2) проверяется, что анонимный запрос к bucket отклоняется (private);
|
||||
3) проверяется, что отдельный ключ API (права только на этот bucket) может писать/читать/удалять и НЕ может создавать bucket.
|
||||
Ключи читаются из файлов секретов; в вывод не попадают. Код выхода != 0, если любая проверка не пройдена.
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
def secret(name):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return fh.read().strip()
|
||||
|
||||
|
||||
def client(session, access, secret_key):
|
||||
return session.create_client("s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, aws_access_key_id=access,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=AioConfig(connect_timeout=3, read_timeout=10, retries={"max_attempts": 1},
|
||||
s3={"addressing_style": "path"}))
|
||||
|
||||
|
||||
async def main():
|
||||
bucket = config.S3_BUCKET
|
||||
session = get_session()
|
||||
deadline = time.time() + 60
|
||||
async with client(session, secret("S3_ADMIN_ACCESS_KEY"), secret("S3_ADMIN_SECRET_KEY")) as admin:
|
||||
while True:
|
||||
try:
|
||||
await admin.list_buckets()
|
||||
break
|
||||
except Exception as exc:
|
||||
if time.time() > deadline:
|
||||
print("S3 недоступен:", type(exc).__name__)
|
||||
return 1
|
||||
await asyncio.sleep(1)
|
||||
try:
|
||||
await admin.head_bucket(Bucket=bucket)
|
||||
print("bucket %s уже существует" % bucket)
|
||||
except ClientError:
|
||||
await admin.create_bucket(Bucket=bucket)
|
||||
print("bucket %s создан" % bucket)
|
||||
# приватность: запрос без подписи отклоняется
|
||||
try:
|
||||
urllib.request.urlopen("%s/%s/" % (config.S3_ENDPOINT, bucket), timeout=5)
|
||||
print("ОШИБКА: bucket доступен анонимно")
|
||||
return 2
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code not in (401, 403):
|
||||
print("ОШИБКА: анонимный доступ вернул %d, ожидался 401/403" % exc.code)
|
||||
return 2
|
||||
print("анонимный доступ запрещён (HTTP %d)" % exc.code)
|
||||
# ключ API: рабочие права на bucket, нет прав администратора
|
||||
async with client(session, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) as api:
|
||||
key = "bootstrap-check/%d" % int(time.time())
|
||||
await api.put_object(Bucket=bucket, Key=key, Body=b"ok")
|
||||
got = await (await api.get_object(Bucket=bucket, Key=key))["Body"].read()
|
||||
await api.delete_object(Bucket=bucket, Key=key)
|
||||
if got != b"ok":
|
||||
print("ОШИБКА: чтение/запись ключом API")
|
||||
return 3
|
||||
try:
|
||||
await api.create_bucket(Bucket="not-allowed-" + str(int(time.time())))
|
||||
print("ОШИБКА: ключ API смог создать bucket (избыточные права)")
|
||||
return 4
|
||||
except ClientError as exc:
|
||||
print("ключ API ограничен одним bucket (create_bucket отклонён: %s)" % exc.response["Error"]["Code"])
|
||||
print("bootstrap завершён")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Read-only operator probe after a full restart: LIST and ranged GET of existing objects.
|
||||
|
||||
An open S3 socket / HEAD bucket does not prove that SeaweedFS volume locations
|
||||
have been registered. Read one byte per nonempty object, including staging.
|
||||
This is a bounded recovery hook, not an expensive per-request health check.
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from . import config
|
||||
from .s3 import S3
|
||||
|
||||
|
||||
async def probe(s3):
|
||||
count = 0
|
||||
batch = []
|
||||
|
||||
async def read(key, size):
|
||||
params = {"Bucket": config.S3_BUCKET, "Key": key}
|
||||
if size:
|
||||
params["Range"] = "bytes=0-0"
|
||||
response = await s3.client.get_object(**params)
|
||||
async with response["Body"] as stream:
|
||||
data = await stream.read()
|
||||
if len(data) != min(size, 1):
|
||||
raise ValueError("unexpected ranged read size")
|
||||
|
||||
async def flush():
|
||||
results = await asyncio.gather(*(read(key, size) for key, size in batch), return_exceptions=True)
|
||||
for result in results:
|
||||
if isinstance(result, BaseException):
|
||||
raise result
|
||||
|
||||
await s3.ping()
|
||||
async for key, size, _ in s3.list_keys(""):
|
||||
batch.append((key, size))
|
||||
count += 1
|
||||
if len(batch) == 8:
|
||||
await flush()
|
||||
batch.clear()
|
||||
if batch:
|
||||
await flush()
|
||||
return count
|
||||
|
||||
|
||||
async def main():
|
||||
s3 = S3()
|
||||
await s3.start()
|
||||
try:
|
||||
async with asyncio.timeout(45):
|
||||
while True:
|
||||
try:
|
||||
count = await probe(s3)
|
||||
print(f"PASS: S3 recovery, {count} existing objects readable")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
# Do not print SDK requests/credentials or object contents.
|
||||
print(f"Waiting for S3 data volumes: {type(exc).__name__}", flush=True)
|
||||
await asyncio.sleep(1)
|
||||
except TimeoutError:
|
||||
print("FAIL: S3 data recovery exceeded 45 seconds")
|
||||
return 1
|
||||
finally:
|
||||
await s3.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Точка входа: uvicorn с управляемым завершением (drain).
|
||||
|
||||
SIGTERM -> readiness сразу 503 -> DRAIN_SECONDS продолжаем обслуживать запросы, пока балансировщик
|
||||
исключает реплику -> uvicorn прекращает приём и дожидается текущих запросов (timeout_graceful_shutdown).
|
||||
Так плановая остановка/rolling update не даёт ошибок клиентам.
|
||||
"""
|
||||
import signal
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from . import config
|
||||
from .main import app
|
||||
from .web import log
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
_draining = False
|
||||
|
||||
def handle_exit(self, sig, frame):
|
||||
if sig == signal.SIGTERM and not self._draining and config.DRAIN_SECONDS > 0:
|
||||
self._draining = True
|
||||
app.draining = True
|
||||
log("info", "draining", seconds=config.DRAIN_SECONDS)
|
||||
timer = threading.Timer(config.DRAIN_SECONDS, super().handle_exit, args=(sig, frame))
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
return
|
||||
super().handle_exit(sig, frame)
|
||||
|
||||
|
||||
def main():
|
||||
cfg = uvicorn.Config(app, host=config.APP_HOST, port=config.APP_PORT, workers=1, access_log=False,
|
||||
log_level="warning", timeout_graceful_shutdown=config.GRACEFUL_SHUTDOWN_SECONDS,
|
||||
timeout_keep_alive=config.KEEPALIVE_SECONDS, limit_concurrency=config.LIMIT_CONCURRENCY)
|
||||
DrainingServer(cfg).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Сессии: PostgreSQL — источник истины; Valkey — кэш с TTL (read-through).
|
||||
|
||||
Политики:
|
||||
* токен — 256 бит из secrets; в БД и Valkey хранится только SHA-256 токена;
|
||||
* expiry: expires_at в БД (по часам БД). В кэше TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS);
|
||||
* инвалидация: logout помечает revoked_at в БД и удаляет ключ в Valkey;
|
||||
* Valkey недоступен -> кэш обходится (чтение из БД), анонимный запрос никогда не считается доверенным;
|
||||
circuit breaker не даёт каждому запросу ждать таймаут;
|
||||
* окно устаревания: если Valkey был недоступен во время logout, ключ может прожить до SESSION_CACHE_TTL_SECONDS.
|
||||
"""
|
||||
import hashlib
|
||||
import secrets
|
||||
import time
|
||||
|
||||
from . import config
|
||||
from .web import log
|
||||
|
||||
try:
|
||||
import redis.asyncio as aioredis
|
||||
except ImportError: # кэш опционален
|
||||
aioredis = None
|
||||
|
||||
|
||||
def token_hash(token):
|
||||
return hashlib.sha256(token.encode("ascii")).digest()
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self):
|
||||
self.client = None
|
||||
self.down_until = 0.0
|
||||
self.stats = {"hit": 0, "miss": 0, "error": 0}
|
||||
|
||||
async def start(self):
|
||||
if config.VALKEY_URL and aioredis:
|
||||
self.client = aioredis.Redis.from_url(config.VALKEY_URL, socket_timeout=config.VALKEY_TIMEOUT,
|
||||
socket_connect_timeout=config.VALKEY_TIMEOUT)
|
||||
|
||||
async def stop(self):
|
||||
if self.client:
|
||||
await self.client.aclose()
|
||||
|
||||
def _usable(self):
|
||||
return self.client is not None and time.monotonic() >= self.down_until
|
||||
|
||||
def _fail(self, exc):
|
||||
self.stats["error"] += 1
|
||||
if time.monotonic() >= self.down_until:
|
||||
log("warning", "session_cache_unavailable", error_type=type(exc).__name__)
|
||||
self.down_until = time.monotonic() + config.VALKEY_BREAKER_SECONDS
|
||||
|
||||
async def get(self, key):
|
||||
if not self._usable():
|
||||
return None
|
||||
try:
|
||||
value = await self.client.get(b"s:" + key)
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
return None
|
||||
self.stats["hit" if value is not None else "miss"] += 1
|
||||
return value
|
||||
|
||||
async def put(self, key, value, ttl):
|
||||
if not self._usable() or ttl <= 0:
|
||||
return
|
||||
try:
|
||||
await self.client.set(b"s:" + key, value, ex=max(1, int(ttl)))
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
|
||||
async def delete(self, key):
|
||||
if not self._usable():
|
||||
return
|
||||
try:
|
||||
await self.client.delete(b"s:" + key)
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
|
||||
async def ping(self):
|
||||
if self.client is None:
|
||||
return None
|
||||
try:
|
||||
return bool(await self.client.ping())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class Sessions:
|
||||
def __init__(self, db, cache):
|
||||
self.db, self.cache = db, cache
|
||||
|
||||
@staticmethod
|
||||
def _cache_value(user_id, expires_at):
|
||||
return ("%s|%d" % (user_id, int(expires_at.timestamp() * 1000))).encode()
|
||||
|
||||
async def create(self, user_id):
|
||||
token = secrets.token_urlsafe(32) # 256 бит энтропии
|
||||
digest = token_hash(token)
|
||||
row = await self.db.fetchrow(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) "
|
||||
"VALUES ($1, $2, now() + make_interval(secs => $3)) RETURNING expires_at",
|
||||
digest, user_id, float(config.SESSION_TTL_SECONDS))
|
||||
remaining = row["expires_at"].timestamp() - time.time()
|
||||
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
|
||||
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
|
||||
return token, row["expires_at"]
|
||||
|
||||
async def resolve(self, token):
|
||||
"""token -> user_id (str) либо None, если сессии нет/истекла/отозвана."""
|
||||
digest = token_hash(token)
|
||||
cached = await self.cache.get(digest)
|
||||
if cached is not None:
|
||||
user_id, _, expires_ms = cached.decode().partition("|")
|
||||
if int(expires_ms) > time.time() * 1000:
|
||||
return user_id
|
||||
await self.cache.delete(digest)
|
||||
row = await self.db.fetchrow(
|
||||
"SELECT user_id, expires_at FROM sessions "
|
||||
"WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()", digest)
|
||||
if row is None:
|
||||
return None
|
||||
user_id = str(row["user_id"])
|
||||
remaining = row["expires_at"].timestamp() - time.time()
|
||||
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
|
||||
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
|
||||
return user_id
|
||||
|
||||
async def revoke(self, token):
|
||||
digest = token_hash(token)
|
||||
await self.db.execute("UPDATE sessions SET revoked_at = now() WHERE token_hash = $1 AND revoked_at IS NULL", digest)
|
||||
await self.cache.delete(digest)
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Общий HTTP-слой: ошибки, запрос, валидация, курсоры, ответы, журнал."""
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from . import config
|
||||
|
||||
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
LIMIT_RE = re.compile(r"[0-9]{1,9}")
|
||||
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
super().__init__(message)
|
||||
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
|
||||
|
||||
|
||||
def bad(message):
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unavailable(message="Зависимость временно недоступна"):
|
||||
return ApiError(503, "unavailable", message, {"retry-after": str(config.RETRY_AFTER_SECONDS)})
|
||||
|
||||
|
||||
def log(level, msg, **fields):
|
||||
record = {"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "level": level, "msg": msg}
|
||||
record.update(fields)
|
||||
sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
# --- время -------------------------------------------------------------------
|
||||
def format_dt(value):
|
||||
value = value.astimezone(timezone.utc)
|
||||
return value.strftime("%Y-%m-%dT%H:%M:%S.") + "%06dZ" % value.microsecond
|
||||
|
||||
|
||||
def dt_to_us(value):
|
||||
return (value - EPOCH) // timedelta(microseconds=1)
|
||||
|
||||
|
||||
def us_to_dt(us):
|
||||
return EPOCH + timedelta(microseconds=us)
|
||||
|
||||
|
||||
# --- валидация ---------------------------------------------------------------
|
||||
def parse_uuid(value, what="id"):
|
||||
if not isinstance(value, str) or not UUID_RE.fullmatch(value):
|
||||
raise bad("Некорректный %s" % what)
|
||||
return value.lower()
|
||||
|
||||
|
||||
def valid_text(value, field, maximum, allow_empty=False):
|
||||
if not isinstance(value, str):
|
||||
raise bad("Поле %s должно быть строкой" % field)
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
raise bad("Поле %s содержит некорректные символы" % field) from None
|
||||
if "\x00" in value:
|
||||
raise bad("Поле %s содержит NUL" % field)
|
||||
if len(value) > maximum:
|
||||
raise bad("Поле %s слишком длинное" % field)
|
||||
if not allow_empty and not value.strip():
|
||||
raise bad("Поле %s не может быть пустым" % field)
|
||||
return value
|
||||
|
||||
|
||||
def _reject_constant(name):
|
||||
raise ValueError(name)
|
||||
|
||||
|
||||
def parse_json_object(raw, required, allowed):
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant)
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise bad("Тело запроса — некорректный JSON") from None
|
||||
if not isinstance(data, dict):
|
||||
raise bad("Ожидается JSON-объект")
|
||||
if set(data) - allowed:
|
||||
raise bad("Лишние поля в запросе")
|
||||
if any(key not in data for key in required):
|
||||
raise bad("Не хватает обязательных полей")
|
||||
return data
|
||||
|
||||
|
||||
def encode_cursor(kind, resource, key):
|
||||
payload = json.dumps({"k": kind, "r": resource, "t": key[0], "i": key[1]}, separators=(",", ":"))
|
||||
return base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(text, kind, resource):
|
||||
try:
|
||||
data = json.loads(base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)).decode())
|
||||
if (not isinstance(data, dict) or data.get("k") != kind or data.get("r") != resource
|
||||
or not isinstance(data.get("t"), int) or isinstance(data.get("t"), bool)
|
||||
or not isinstance(data.get("i"), str) or not UUID_RE.fullmatch(data["i"])
|
||||
or not -62135596800000000 < data["t"] < 253402300799000000):
|
||||
raise ValueError
|
||||
return data["t"], data["i"]
|
||||
except (ValueError, UnicodeError):
|
||||
raise bad("Некорректный cursor") from None
|
||||
|
||||
|
||||
def page_params(query, kind, resource):
|
||||
limit, cursor = 50, None
|
||||
if "limit" in query:
|
||||
if len(query["limit"]) != 1 or not LIMIT_RE.fullmatch(query["limit"][0]):
|
||||
raise bad("limit должен быть целым числом 1–100")
|
||||
limit = int(query["limit"][0])
|
||||
if not 1 <= limit <= 100:
|
||||
raise bad("limit должен быть целым числом 1–100")
|
||||
if "cursor" in query:
|
||||
if len(query["cursor"]) != 1 or not query["cursor"][0]:
|
||||
raise bad("Некорректный cursor")
|
||||
cursor = decode_cursor(query["cursor"][0], kind, resource)
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def parse_query(raw):
|
||||
result = {}
|
||||
if raw:
|
||||
for key, value in parse_qsl(raw.decode("latin-1"), keep_blank_values=True):
|
||||
result.setdefault(key, []).append(value)
|
||||
return result
|
||||
|
||||
|
||||
class Request:
|
||||
def __init__(self, scope, receive, request_id):
|
||||
self.method = scope["method"]
|
||||
self.path = scope["path"]
|
||||
self.query = parse_query(scope.get("query_string", b""))
|
||||
self.headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
|
||||
self.receive = receive
|
||||
self.request_id = request_id
|
||||
self.user_id = None
|
||||
|
||||
async def body(self):
|
||||
declared = self.headers.get("content-length")
|
||||
if declared is not None:
|
||||
if not declared.isdigit():
|
||||
raise bad("Некорректный Content-Length")
|
||||
if int(declared) > config.MAX_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
|
||||
chunks, size = [], 0
|
||||
while True:
|
||||
event = await self.receive()
|
||||
if event["type"] == "http.disconnect":
|
||||
raise bad("Соединение закрыто клиентом")
|
||||
chunk = event.get("body", b"")
|
||||
size += len(chunk)
|
||||
if size > config.MAX_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
|
||||
chunks.append(chunk)
|
||||
if not event.get("more_body"):
|
||||
return b"".join(chunks)
|
||||
|
||||
def require_json_media_type(self):
|
||||
media = self.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json")
|
||||
|
||||
|
||||
async def respond(send, status, body, request_id, extra=None, instance_id=None):
|
||||
payload = b"" if status == 204 else json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
headers = [(b"x-request-id", request_id.encode())]
|
||||
if status != 204:
|
||||
headers += [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())]
|
||||
else:
|
||||
headers.append((b"content-length", b"0"))
|
||||
if instance_id:
|
||||
headers.append((b"x-instance-id", instance_id.encode()))
|
||||
headers += [(k.encode(), v.encode()) for k, v in (extra or {}).items()]
|
||||
await send({"type": "http.response.start", "status": status, "headers": headers})
|
||||
await send({"type": "http.response.body", "body": payload})
|
||||
|
||||
|
||||
def new_request_id():
|
||||
return uuid.uuid4().hex
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Worker вложений: python -m app.worker
|
||||
|
||||
Конвейер задания (jobs.schema.json, состояния — contracts/attachment-states.md):
|
||||
claim (queued -> processing, lease) -> чтение staging -> варианты (Pillow, в потоках) -> запись в конечный S3-prefix
|
||||
-> ОДНА транзакция: варианты + ready -> ack -> очистка staging.
|
||||
Ack отправляется только после устойчивого эффекта (COMMIT результата). Повторная доставка уже готовой задачи — no-op с ack.
|
||||
Временные ошибки: возврат в queued с backoff (экспоненциальный + jitter) через outbox; после MAX_ATTEMPTS — failed/retry_exhausted.
|
||||
Постоянные ошибки (невалидное изображение): failed с error_code сразу. Зависшие processing возвращает sweeper по истечении lease.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import signal
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import aio_pika
|
||||
|
||||
from . import config, pipeline
|
||||
from .broker import QUEUE, Broker
|
||||
from .db import Database
|
||||
from .outbox import Relay
|
||||
from .s3 import S3
|
||||
from .web import log
|
||||
|
||||
WORKER_ID = "%s-%s" % ((os.environ.get("HOSTNAME") or "worker")[:12], uuid.uuid4().hex[:6])
|
||||
|
||||
|
||||
def backoff_seconds(attempt):
|
||||
"""Экспоненциальная задержка с полным jitter: случайное значение в [0, min(cap, base * 2^(attempt-1))]."""
|
||||
ceiling = min(config.RETRY_MAX_DELAY, config.RETRY_BASE_DELAY * (2 ** max(0, attempt - 1)))
|
||||
return random.uniform(ceiling * 0.5, ceiling)
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self):
|
||||
self.db, self.s3, self.broker = Database(), S3(), Broker()
|
||||
self.relay = Relay(self.db, self.broker)
|
||||
self.pool = ThreadPoolExecutor(max_workers=config.WORKER_CONCURRENCY, thread_name_prefix="img")
|
||||
self.busy = 0
|
||||
self.stopping = False
|
||||
self.tasks = []
|
||||
|
||||
# --- claim / finalize ---------------------------------------------------------
|
||||
async def claim(self, job):
|
||||
return await self.db.fetchrow(
|
||||
"UPDATE attachments SET status = 'processing', lease_owner = $2, processing_started_at = now(), "
|
||||
"lease_expires_at = now() + make_interval(secs => $3), attempt = $4, job_id = $5, updated_at = now() "
|
||||
"WHERE id = $1 AND status = 'queued' AND pipeline_version <= $6 "
|
||||
"RETURNING id, chat_id, staging_key, s3_key, content_type, size_bytes, attempt",
|
||||
uuid.UUID(job["attachment_id"]), WORKER_ID, float(config.LEASE_SECONDS), job["attempt"],
|
||||
uuid.UUID(job["job_id"]), job["pipeline_version"], retry=False)
|
||||
|
||||
async def fail(self, att_id, job, code, event="failed"):
|
||||
await self.db.execute(
|
||||
"WITH u AS (UPDATE attachments SET status = 'failed', error_code = $2, lease_owner = NULL, lease_expires_at = NULL, "
|
||||
"updated_at = now() WHERE id = $1 AND status = 'processing' AND lease_owner = $3 RETURNING id) "
|
||||
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $4, $5, $6, $2 FROM u",
|
||||
att_id, code, WORKER_ID, uuid.UUID(job["job_id"]), event, job["attempt"])
|
||||
|
||||
async def reschedule(self, att_id, job, reason):
|
||||
"""Временная ошибка: processing -> queued, новая запись outbox с отложенной публикацией (backoff + jitter)."""
|
||||
nxt = job["attempt"] + 1
|
||||
if nxt > config.MAX_ATTEMPTS:
|
||||
log("warning", "job_retry_exhausted", attachment_id=str(att_id), attempts=job["attempt"])
|
||||
await self.fail(att_id, job, "retry_exhausted", "retry_exhausted")
|
||||
return
|
||||
delay = backoff_seconds(job["attempt"])
|
||||
await self.db.execute(
|
||||
"WITH u AS (UPDATE attachments SET status = 'queued', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() "
|
||||
"WHERE id = $1 AND status = 'processing' AND lease_owner = $2 RETURNING id, pipeline_version), "
|
||||
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, available_at) "
|
||||
"SELECT $3, id, pipeline_version, $4, now() + make_interval(secs => $5) FROM u RETURNING 1) "
|
||||
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $3, 'retry_scheduled', $4, $6 FROM u",
|
||||
att_id, WORKER_ID, uuid.UUID(job["job_id"]), nxt, delay, reason)
|
||||
log("info", "job_rescheduled", attachment_id=str(att_id), next_attempt=nxt, delay_seconds=round(delay, 2), reason=reason)
|
||||
|
||||
async def publish_result(self, row, job, variants):
|
||||
"""Одна транзакция: варианты (идемпотентно) + ready. Только владелец lease может опубликовать результат."""
|
||||
async with self.db.conn() as c:
|
||||
async with c.transaction():
|
||||
owner = await c.fetchval("SELECT 1 FROM attachments WHERE id = $1 AND status = 'processing' AND lease_owner = $2 "
|
||||
"FOR UPDATE", row["id"], WORKER_ID)
|
||||
if owner is None:
|
||||
return False # lease потерян (sweeper вернул задачу): результат не публикуем, чужой повтор доделает
|
||||
for v in variants:
|
||||
await c.execute(
|
||||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, "
|
||||
"height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) "
|
||||
"ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
|
||||
row["id"], job["pipeline_version"], v["name"], v["content_type"], len(v["data"]), v["width"], v["height"],
|
||||
v["sha256"], "att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]))
|
||||
await c.execute("UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = now(), updated_at = now(), "
|
||||
"lease_owner = NULL, lease_expires_at = NULL WHERE id = $1", row["id"])
|
||||
await c.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt) VALUES ($1, $2, 'ready', $3)",
|
||||
row["id"], uuid.UUID(job["job_id"]), job["attempt"])
|
||||
return True
|
||||
|
||||
# --- обработка ----------------------------------------------------------------
|
||||
async def process(self, job):
|
||||
att_id = uuid.UUID(job["attachment_id"])
|
||||
row = await self.claim(job)
|
||||
if row is None:
|
||||
state = await self.db.fetchval("SELECT status FROM attachments WHERE id = $1", att_id)
|
||||
log("info", "job_noop", attachment_id=job["attachment_id"], status=state or "unknown")
|
||||
return # ready/failed/processing у другого владельца/неизвестное вложение: идемпотентный no-op, ack
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
await self.fault("after_claim")
|
||||
stream, _ = await self.s3.open_stream(row["staging_key"])
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
|
||||
try:
|
||||
try:
|
||||
while True:
|
||||
chunk = await stream.read(256 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
spool.write(chunk)
|
||||
finally:
|
||||
stream.close()
|
||||
loop = asyncio.get_running_loop()
|
||||
variants = await asyncio.wait_for(loop.run_in_executor(self.pool, pipeline.process, spool), config.JOB_TIMEOUT)
|
||||
finally:
|
||||
spool.close()
|
||||
# оригинал и варианты в конечный prefix: ключи детерминированы, повторная запись перезаписывает те же байты
|
||||
await self.copy_original(row)
|
||||
for v in variants:
|
||||
await self.s3.put_bytes("att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]), v["data"], v["content_type"])
|
||||
await self.fault("after_s3_before_commit")
|
||||
if not await self.publish_result(row, job, variants):
|
||||
log("warning", "lease_lost", attachment_id=job["attachment_id"])
|
||||
return
|
||||
await self.fault("after_commit_before_ack")
|
||||
log("info", "job_ready", attachment_id=job["attachment_id"], attempt=job["attempt"],
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 1))
|
||||
try:
|
||||
await self.s3.delete(row["staging_key"]) # staging безопасно удалять после COMMIT результата
|
||||
except Exception as exc:
|
||||
log("warning", "staging_cleanup_failed", error_type=type(exc).__name__)
|
||||
except pipeline.PermanentError as exc:
|
||||
await self.fail(att_id, job, exc.code)
|
||||
log("warning", "job_failed", attachment_id=job["attachment_id"], error_code=exc.code)
|
||||
except MemoryError:
|
||||
await self.fail(att_id, job, "processing_failed")
|
||||
log("error", "job_failed_oom", attachment_id=job["attachment_id"])
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except InjectedCrash:
|
||||
raise
|
||||
except Exception as exc: # S3/БД/таймаут/неожиданное: временная ошибка
|
||||
await self.reschedule(att_id, job, type(exc).__name__)
|
||||
|
||||
async def copy_original(self, row):
|
||||
stream, size = await self.s3.open_stream(row["staging_key"])
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
|
||||
try:
|
||||
while True:
|
||||
chunk = await stream.read(256 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
spool.write(chunk)
|
||||
stream.close()
|
||||
spool.seek(0)
|
||||
await self.s3.put(row["s3_key"], spool, row["content_type"], size)
|
||||
finally:
|
||||
spool.close()
|
||||
|
||||
async def fault(self, point):
|
||||
"""Точки внедрения сбоев для экспериментов (FAULT_INJECTION=crash:<точка> => процесс умирает мгновенно)."""
|
||||
if "crash_once:" + point in config.FAULT_INJECTION:
|
||||
marker = "/tmp/fault-%s.done" % point # сохраняется при рестарте того же контейнера -> сбой ровно один раз
|
||||
if not os.path.exists(marker):
|
||||
open(marker, "w").close()
|
||||
log("error", "fault_crash_once", point=point)
|
||||
os._exit(137)
|
||||
if "crash:" + point in config.FAULT_INJECTION:
|
||||
log("error", "fault_crash", point=point)
|
||||
os._exit(137)
|
||||
if "fail:" + point in config.FAULT_INJECTION:
|
||||
raise OSError("injected failure at " + point)
|
||||
|
||||
async def on_message(self, message):
|
||||
try:
|
||||
job = json.loads(message.body)
|
||||
if (job.get("schema_version") != 1 or job.get("type") != "attachment.process.v1"
|
||||
or not isinstance(job.get("attempt"), int) or not isinstance(job.get("pipeline_version"), int)):
|
||||
raise ValueError("bad job")
|
||||
uuid.UUID(job["attachment_id"]), uuid.UUID(job["job_id"])
|
||||
except Exception:
|
||||
log("error", "job_unreadable")
|
||||
await message.reject(requeue=False) # -> DLQ
|
||||
return
|
||||
self.busy += 1
|
||||
try:
|
||||
await self.process(job)
|
||||
await message.ack()
|
||||
except InjectedCrash:
|
||||
raise
|
||||
except Exception as exc:
|
||||
log("error", "job_unhandled", error_type=type(exc).__name__)
|
||||
await message.nack(requeue=True)
|
||||
finally:
|
||||
self.busy -= 1
|
||||
|
||||
# --- sweeper -------------------------------------------------------------------
|
||||
async def sweep_once(self):
|
||||
"""Истёкший lease -> queued (новая попытка) или failed/retry_exhausted; чистка старого staging."""
|
||||
rows = await self.db.fetch(
|
||||
"WITH exp AS (SELECT id FROM attachments WHERE status = 'processing' AND lease_expires_at < now() FOR UPDATE SKIP LOCKED), "
|
||||
"u AS (UPDATE attachments a SET status = CASE WHEN a.attempt >= $1 THEN 'failed' ELSE 'queued' END, "
|
||||
"error_code = CASE WHEN a.attempt >= $1 THEN 'retry_exhausted' ELSE NULL END, lease_owner = NULL, lease_expires_at = NULL, "
|
||||
"updated_at = now() FROM exp WHERE a.id = exp.id RETURNING a.id, a.status, a.pipeline_version, a.attempt, a.job_id), "
|
||||
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) SELECT COALESCE(job_id, gen_random_uuid()), id, "
|
||||
"pipeline_version, attempt + 1 FROM u WHERE status = 'queued' RETURNING 1), "
|
||||
"e AS (INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, job_id, 'lease_expired', attempt, status FROM u RETURNING 1) "
|
||||
"SELECT id, status FROM u", config.MAX_ATTEMPTS)
|
||||
for r in rows:
|
||||
log("warning", "lease_expired", attachment_id=str(r["id"]), new_status=r["status"])
|
||||
if rows:
|
||||
self.relay.kick()
|
||||
|
||||
async def sweeper(self):
|
||||
while not self.stopping:
|
||||
try:
|
||||
await self.sweep_once()
|
||||
except Exception as exc:
|
||||
log("warning", "sweeper_failed", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(config.SWEEP_INTERVAL)
|
||||
|
||||
async def staging_janitor(self):
|
||||
"""Удаляет staging для ready (если не удалён после COMMIT) и для failed старше retention; текущую работу не трогает."""
|
||||
while not self.stopping:
|
||||
await asyncio.sleep(max(30, config.SWEEP_INTERVAL * 6))
|
||||
try:
|
||||
rows = await self.db.fetch(
|
||||
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (status = 'ready' OR "
|
||||
"(status = 'failed' AND updated_at < now() - make_interval(secs => $1))) LIMIT 200",
|
||||
float(config.STAGING_FAILED_RETENTION_SECONDS))
|
||||
for r in rows:
|
||||
await self.s3.delete(r["staging_key"])
|
||||
await self.db.execute("UPDATE attachments SET staging_key = NULL WHERE id = $1", r["id"])
|
||||
except Exception as exc:
|
||||
log("warning", "staging_janitor_failed", error_type=type(exc).__name__)
|
||||
|
||||
# --- жизненный цикл ------------------------------------------------------------
|
||||
async def connect_broker(self):
|
||||
while not self.stopping:
|
||||
try:
|
||||
await self.broker.ensure(prefetch=config.WORKER_CONCURRENCY)
|
||||
return
|
||||
except Exception as exc:
|
||||
log("warning", "broker_unavailable", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def consume(self):
|
||||
while not self.stopping:
|
||||
await self.connect_broker()
|
||||
try:
|
||||
queue = await self.broker.channel.declare_queue(QUEUE, durable=True, passive=True)
|
||||
tag = await queue.consume(self.on_message)
|
||||
log("info", "consuming", worker=WORKER_ID, prefetch=config.WORKER_CONCURRENCY)
|
||||
while not self.stopping and not self.broker.channel.is_closed:
|
||||
await asyncio.sleep(1)
|
||||
if self.stopping:
|
||||
await queue.cancel(tag)
|
||||
return
|
||||
except Exception as exc:
|
||||
log("warning", "consume_failed", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def run(self):
|
||||
await self.db.start()
|
||||
await self.s3.start()
|
||||
log("info", "worker_started", worker=WORKER_ID, concurrency=config.WORKER_CONCURRENCY, lease=config.LEASE_SECONDS,
|
||||
max_attempts=config.MAX_ATTEMPTS)
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
try:
|
||||
loop.add_signal_handler(sig, self.request_stop)
|
||||
except NotImplementedError:
|
||||
signal.signal(sig, lambda *_: self.request_stop())
|
||||
self.tasks = [asyncio.create_task(self.consume()), asyncio.create_task(self.relay.run()),
|
||||
asyncio.create_task(self.sweeper()), asyncio.create_task(self.staging_janitor())]
|
||||
beat = 0.0
|
||||
while not self.stopping:
|
||||
if time.monotonic() - beat > 5: # heartbeat для HEALTHCHECK контейнера
|
||||
beat = time.monotonic()
|
||||
try:
|
||||
with open("/tmp/worker.alive", "w") as fh:
|
||||
fh.write(str(time.time()))
|
||||
except OSError:
|
||||
pass
|
||||
await asyncio.sleep(0.5)
|
||||
# graceful: перестаём брать новые задания, ждём текущие до GRACEFUL_SHUTDOWN_SECONDS
|
||||
deadline = time.monotonic() + config.GRACEFUL_SHUTDOWN_SECONDS
|
||||
while self.busy and time.monotonic() < deadline:
|
||||
await asyncio.sleep(0.2)
|
||||
self.relay.stop()
|
||||
for t in self.tasks:
|
||||
t.cancel()
|
||||
await self.broker.stop()
|
||||
await self.s3.stop()
|
||||
await self.db.stop()
|
||||
log("info", "worker_stopped", unfinished=self.busy)
|
||||
|
||||
def request_stop(self):
|
||||
log("info", "worker_stopping")
|
||||
self.stopping = True
|
||||
|
||||
|
||||
class InjectedCrash(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
asyncio.run(Worker().run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,51 @@
|
||||
# Собственные проверки ЛР5
|
||||
|
||||
Поднимите отдельный учебный стенд по [REPORT.md](../REPORT.md) и дождитесь готовности всех сервисов. Команды выполняются из корня репозитория. На Windows установите `$env:PYTHONUTF8='1'`. Для пиксельных проверок необходим Pillow: `python -m pip install Pillow==12.3.0`.
|
||||
|
||||
```sh
|
||||
python -m unittest discover -s student-tests -p 'test_api.py' -v
|
||||
python -m unittest discover -s student-tests -p 'test_idempotency.py' -v
|
||||
python -m unittest discover -s student-tests -p 'test_attachments.py' -v
|
||||
python -m unittest discover -s student-tests -p 'test_topology.py' -v
|
||||
python -m unittest discover -s student-tests -p 'test_pipeline.py' -v
|
||||
```
|
||||
|
||||
Это 47 тестов API, сессий, идемпотентности, файлов, топологии и вариантов изображений. В pipeline проверяются EXIF, alpha, JPEG/WebP, размеры, неизменность оригинала, ACL, failed, redelivery и очистка staging.
|
||||
|
||||
Следующие 13 тестов меняют состояние сервисов, пересоздают контейнеры или внедряют сбои. Запускайте последовательно; volumes сохраняются. Reconciler проверяется только на созданных тестом объектах и префиксах.
|
||||
|
||||
```sh
|
||||
python student-tests/test_reconcile5.py -v
|
||||
python student-tests/test_resilience5.py -v
|
||||
python student-tests/test_expiry.py -v
|
||||
python student-tests/test_dependencies.py -v
|
||||
```
|
||||
|
||||
Проверка reconciler подтверждает, что варианты и незавершённые записи worker не удаляются; staging принятого задания защищён, а unowned orphan удаляется только после grace. Семь сценариев resilience5 воспроизводят остановку worker, рестарт API/broker, недоступность broker, crash между COMMIT/publish, crash worker до COMMIT/ack, retry exhaustion и безопасный replay. Тесты TTL и отказа PostgreSQL/Valkey перенесены из ЛР4; перед проверкой ошибки БД ожидается срабатывание healthcheck балансировщика.
|
||||
|
||||
Публичные проверки и повторяемый bootstrap:
|
||||
|
||||
```sh
|
||||
python scripts/check_contract.py
|
||||
python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
|
||||
sh scripts/restore-replicas.sh
|
||||
docker compose --env-file .env run --rm -T s3-init
|
||||
docker compose --env-file .env run --rm -T s3-init
|
||||
```
|
||||
|
||||
В Windows используйте `scripts/restart.cmd`, `scripts/stop-one.cmd`; после failover восстановите реплику командой `docker compose --env-file .env start api`. Полная проверка OpenAPI: установите `requirements-dev.txt`, затем `python scripts/validate_openapi.py`.
|
||||
|
||||
## Исследование очереди
|
||||
|
||||
```sh
|
||||
python -m pip install -r research/requirements.txt
|
||||
python research/queue_overload.py --out evidence/recheck_2026-11-12
|
||||
```
|
||||
|
||||
Скрипт проверяет одну worker-реплику с фактическими лимитами 1 CPU / 768 MiB, затем сравнивает concurrency/prefetch 1, 2, 4 для трёх размеров изображений. В отдельном burst проверяются реальные 503, Retry-After, завершение принятых заданий и новые загрузки после разгрузки. Порог очереди является мягким: одновременный приём допускает превышение порога; достигнутый максимум сохраняется в JSON.
|
||||
|
||||
Эксперимент временно пересоздаёт API/worker и восстанавливает настройки в finally. Синтетические данные сохраняются. Промежуточные результаты пишутся после каждой точки; Docker-ошибки, потеря измерений, failed/OOM и несоответствие результата ожиданиям завершают скрипт с ошибкой.
|
||||
|
||||
Доказательства текущей проверки: [checks_2026-11-12.txt](../evidence/checks_2026-11-12.txt), [JSON и графики](../evidence/recheck_2026-11-12/).
|
||||
|
After Width: | Height: | Size: 344 B |
|
After Width: | Height: | Size: 1.6 KiB |
|
After Width: | Height: | Size: 151 B |
|
After Width: | Height: | Size: 24 KiB |
|
After Width: | Height: | Size: 109 KiB |
|
After Width: | Height: | Size: 132 B |
@@ -0,0 +1,208 @@
|
||||
"""Собственные тесты lab2 против запущенного стенда (docker compose up -d).
|
||||
|
||||
python -m unittest discover -s student-tests -v # BASE_URL по умолчанию http://127.0.0.1:8080
|
||||
Тесты используют только синтетические данные и ничего не удаляют. Проверки expiry и недоступности
|
||||
зависимостей лежат отдельно (test_expiry.py, test_dependencies.py): они управляют контейнерами.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote
|
||||
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
ROOT = os.path.join(os.path.dirname(__file__), "..")
|
||||
|
||||
|
||||
def call(method, path, body=None, token=None, headers=None, raw=None):
|
||||
h = dict(headers or {})
|
||||
if method == "POST" and path.endswith("/messages") and headers is None:
|
||||
h["Idempotency-Key"] = uuid.uuid4().hex # lab3: ключ обязателен; тесты без ключа — в test_idempotency.py
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
h.setdefault("Content-Type", "application/json")
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
status, hdrs, payload = resp.status, resp.headers, resp.read()
|
||||
except urllib.error.HTTPError as err:
|
||||
status, hdrs, payload = err.code, err.headers, err.read()
|
||||
return status, hdrs, (json.loads(payload) if payload else None)
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
return {"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode()}
|
||||
|
||||
|
||||
def register(prefix="u"):
|
||||
name = "%s_%s" % (prefix, uuid.uuid4().hex[:12])
|
||||
password = "Pa55:word-" + uuid.uuid4().hex # двоеточие в пароле допустимо
|
||||
status, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "Имя " + name, "password": password})
|
||||
assert status == 201, user
|
||||
status, _, session = call("POST", "/api/v1/auth/sessions", headers=basic(name, password))
|
||||
assert status == 201, session
|
||||
return {"user": user, "password": password, "token": session["token"], "session": session}
|
||||
|
||||
|
||||
def psql(sql):
|
||||
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger",
|
||||
"-d", "messenger", "-Atc", sql], cwd=ROOT, capture_output=True, text=True, check=True)
|
||||
return out.stdout.strip()
|
||||
|
||||
|
||||
class AuthTest(unittest.TestCase):
|
||||
def assertError(self, result, status, code):
|
||||
self.assertEqual(result[0], status, result[2])
|
||||
self.assertEqual(result[2]["error"]["code"], code)
|
||||
self.assertEqual(result[2]["error"]["request_id"], result[1]["X-Request-Id"])
|
||||
|
||||
def test_register_validation(self):
|
||||
base = {"username": "val_" + uuid.uuid4().hex[:8], "display_name": "x", "password": "p" * 12}
|
||||
for patch in ({"password": "short"}, {"password": "p" * 129}, {"password": 12345678901234}, {"extra": 1},
|
||||
{"username": "BAD"}, {"display_name": " "}):
|
||||
body = dict(base, **patch)
|
||||
self.assertError(call("POST", "/api/v1/users", body), 400, "invalid_request")
|
||||
missing = {k: v for k, v in base.items() if k != "password"}
|
||||
self.assertError(call("POST", "/api/v1/users", missing), 400, "invalid_request")
|
||||
self.assertEqual(call("POST", "/api/v1/users", dict(base, password="я" * 12))[0], 201) # 12 code points
|
||||
|
||||
def test_password_is_not_exposed_or_stored_plain(self):
|
||||
actor = register()
|
||||
self.assertEqual(set(actor["user"]), {"id", "username", "display_name", "created_at"})
|
||||
stored = psql("SELECT password_hash FROM users WHERE username = '%s'" % actor["user"]["username"])
|
||||
self.assertTrue(stored.startswith("$argon2id$"), stored[:12])
|
||||
self.assertNotIn(actor["password"], stored)
|
||||
|
||||
def test_wrong_password_and_unknown_user_are_indistinguishable(self):
|
||||
actor = register()
|
||||
wrong = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], "wrong-password-123"))
|
||||
unknown = call("POST", "/api/v1/auth/sessions", headers=basic("nobody_" + uuid.uuid4().hex[:8], "wrong-password-123"))
|
||||
for r in (wrong, unknown):
|
||||
self.assertError(r, 401, "unauthorized")
|
||||
self.assertEqual(r[1]["WWW-Authenticate"], 'Basic realm="messenger", charset="UTF-8"')
|
||||
self.assertEqual(wrong[2]["error"]["message"], unknown[2]["error"]["message"])
|
||||
for bad in ({}, {"Authorization": "Basic !!!"}, {"Authorization": "Basic " + base64.b64encode(b"nocolon").decode()},
|
||||
{"Authorization": "Bearer abc"}):
|
||||
self.assertEqual(call("POST", "/api/v1/auth/sessions", headers=bad)[0], 401)
|
||||
|
||||
def test_bearer_rules(self):
|
||||
actor = register()
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[2], actor["user"])
|
||||
for headers in ({}, basic(actor["user"]["username"], actor["password"]), {"X-User-Id": actor["user"]["id"]},
|
||||
{"Authorization": "Bearer " + "A" * 43}, {"Authorization": "Bearer short"}):
|
||||
r = call("GET", "/api/v1/users/me", headers=headers)
|
||||
self.assertError(r, 401, "unauthorized")
|
||||
self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"')
|
||||
other = register()
|
||||
me = call("GET", "/api/v1/users/me", token=actor["token"], headers={"X-User-Id": other["user"]["id"]})[2]
|
||||
self.assertEqual(me["id"], actor["user"]["id"]) # X-User-Id игнорируется
|
||||
|
||||
def test_session_shape_entropy_and_logout(self):
|
||||
actor = register()
|
||||
s = actor["session"]
|
||||
self.assertEqual(s["token_type"], "Bearer")
|
||||
self.assertGreaterEqual(len(s["token"]), 43) # 32 байта в base64url = 256 бит
|
||||
self.assertNotIn(actor["user"]["id"], s["token"])
|
||||
self.assertNotEqual(register()["token"], s["token"])
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"])[0], 204)
|
||||
self.assertError(call("GET", "/api/v1/users/me", token=s["token"]), 401, "unauthorized")
|
||||
self.assertError(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"]), 401, "unauthorized")
|
||||
|
||||
def test_logout_revokes_in_database_and_cache(self):
|
||||
actor = register()
|
||||
call("GET", "/api/v1/users/me", token=actor["token"]) # прогреваем кэш
|
||||
call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])
|
||||
revoked = psql("SELECT count(*) FROM sessions s JOIN users u ON u.id = s.user_id "
|
||||
"WHERE u.username = '%s' AND s.revoked_at IS NOT NULL" % actor["user"]["username"])
|
||||
self.assertEqual(revoked, "1")
|
||||
|
||||
def test_token_only_stored_as_hash(self):
|
||||
actor = register()
|
||||
count = psql("SELECT count(*) FROM sessions WHERE token_hash = '\\x00'::bytea OR encode(token_hash,'escape') = '%s'" % actor["token"])
|
||||
self.assertEqual(count, "0")
|
||||
|
||||
def test_concurrent_registration_of_same_username(self):
|
||||
name = "race_" + uuid.uuid4().hex[:10]
|
||||
body = {"username": name, "display_name": "x", "password": "racing-password-1"}
|
||||
with ThreadPoolExecutor(12) as pool:
|
||||
codes = sorted(pool.map(lambda _: call("POST", "/api/v1/users", body)[0], range(12)))
|
||||
self.assertEqual(codes, [201] + [409] * 11)
|
||||
self.assertEqual(psql("SELECT count(*) FROM users WHERE username = '%s'" % name), "1")
|
||||
|
||||
|
||||
class DataTest(unittest.TestCase):
|
||||
def test_chat_creation_is_atomic(self):
|
||||
a = register()
|
||||
before = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)")
|
||||
ghost = str(uuid.uuid4())
|
||||
r = call("POST", "/api/v1/chats", {"title": "откат", "member_ids": [a["user"]["id"], ghost]}, token=a["token"])
|
||||
self.assertEqual(r[0], 400)
|
||||
after = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)")
|
||||
self.assertEqual(before, after, "после неудачного создания не должно остаться чата или участников")
|
||||
ok = call("POST", "/api/v1/chats", {"title": "ок", "member_ids": [a["user"]["id"]]}, token=a["token"])
|
||||
self.assertEqual(ok[0], 201)
|
||||
|
||||
def test_foreign_chat_empty_message_bad_json(self):
|
||||
a, b, eve = register(), register(), register()
|
||||
chat = call("POST", "/api/v1/chats", {"title": "ЛС", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2]
|
||||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
self.assertEqual(call("POST", path, {"text": "чужое"}, eve["token"])[0], 404)
|
||||
self.assertEqual(call("GET", path, token=eve["token"])[0], 404)
|
||||
self.assertEqual(call("POST", path, {"text": " "}, a["token"])[0], 400)
|
||||
self.assertEqual(call("POST", path, raw=b"{oops", token=a["token"], headers={"Content-Type": "application/json"})[0], 400)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "sender_id": b["user"]["id"]}, a["token"])[0], 400)
|
||||
sent = call("POST", path, {"text": "Привет 👋"}, a["token"])[2]
|
||||
self.assertEqual(sent["sender_id"], a["user"]["id"])
|
||||
|
||||
def test_multi_page_history_and_cursor_binding(self):
|
||||
a, b = register(), register()
|
||||
chat = call("POST", "/api/v1/chats", {"title": "стр", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2]
|
||||
other = call("POST", "/api/v1/chats", {"title": "др", "member_ids": [a["user"]["id"]]}, a["token"])[2]
|
||||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
sent = [call("POST", path, {"text": "m%d" % i}, (a if i % 2 else b)["token"])[2] for i in range(23)]
|
||||
walked, cursor, pages = [], None, 0
|
||||
while True:
|
||||
q = "?limit=%d" % (5 if pages % 2 == 0 else 8) + ("&cursor=" + quote(cursor, safe="") if cursor else "")
|
||||
page = call("GET", path + q, token=b["token"])[2]
|
||||
walked += page["items"]
|
||||
pages += 1
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
self.assertEqual(walked, sent)
|
||||
self.assertGreaterEqual(pages, 3)
|
||||
first = call("GET", path + "?limit=1", token=a["token"])[2]["next_cursor"]
|
||||
r = call("GET", "/api/v1/chats/%s/messages?cursor=%s" % (other["id"], quote(first, safe="")), token=a["token"])
|
||||
self.assertEqual(r[0], 400)
|
||||
|
||||
def test_parallel_writes_from_many_clients(self):
|
||||
users = [register() for _ in range(5)]
|
||||
chat = call("POST", "/api/v1/chats", {"title": "гонки", "member_ids": [u["user"]["id"] for u in users]}, users[0]["token"])[2]
|
||||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
|
||||
def client(u):
|
||||
return [call("POST", path, {"text": "%s-%d" % (u["user"]["username"], i)}, u["token"])[0] for i in range(20)]
|
||||
|
||||
with ThreadPoolExecutor(5) as pool:
|
||||
codes = [c for part in pool.map(client, users) for c in part]
|
||||
self.assertEqual(codes, [201] * 100)
|
||||
total, cursor = [], None
|
||||
while True:
|
||||
page = call("GET", path + "?limit=100" + ("&cursor=" + quote(cursor, safe="") if cursor else ""), token=users[0]["token"])[2]
|
||||
total += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
self.assertEqual(len({m["id"] for m in total}), 100)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Собственные тесты вложений lab5 (негативные сценарии и права). Запуск:
|
||||
python -m unittest discover -s student-tests -p "test_attachments.py" -v (BASE_URL по умолчанию http://127.0.0.1:8080)
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import BASE, ROOT, call, psql, register # noqa: E402
|
||||
|
||||
FIX = os.path.join(os.path.dirname(__file__), "fixtures")
|
||||
PNG = open(os.path.join(FIX, "sample.png"), "rb").read()
|
||||
JPG = open(os.path.join(FIX, "sample.jpg"), "rb").read()
|
||||
|
||||
|
||||
def upload(token, chat_id, data, ctype="image/png", filename="sample.png", raw_filename=None):
|
||||
qs = raw_filename if raw_filename is not None else "filename=" + filename
|
||||
req = urllib.request.Request("%s/api/v1/chats/%s/attachments?%s" % (BASE, chat_id, qs), data=data, method="POST",
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, r.headers, json.loads(r.read())
|
||||
except urllib.error.HTTPError as err:
|
||||
body = err.read()
|
||||
return err.code, err.headers, (json.loads(body) if body else None)
|
||||
|
||||
|
||||
def download(token, att_id, suffix="/content"):
|
||||
req = urllib.request.Request("%s/api/v1/attachments/%s%s" % (BASE, att_id, suffix),
|
||||
headers={"Authorization": "Bearer " + token if token else ""})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, r.headers, r.read()
|
||||
except urllib.error.HTTPError as err:
|
||||
return err.code, err.headers, err.read()
|
||||
|
||||
|
||||
def wait_ready(token, att_id, seconds=60):
|
||||
"""lab5: загрузка возвращает 202 queued; готовность определяется опросом метаданных."""
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
code, _, body = download(token, att_id, "")
|
||||
att = json.loads(body)
|
||||
if att["status"] == "ready":
|
||||
return att
|
||||
assert att["status"] != "failed", att
|
||||
time.sleep(0.3)
|
||||
raise AssertionError("not ready")
|
||||
|
||||
|
||||
def png_chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def bomb_png(width=20000, height=20000):
|
||||
"""Корректный по структуре PNG, заявляющий 400 млн пикселей, без данных изображения (отклоняется по заголовку)."""
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", ihdr) + png_chunk(b"IDAT", zlib.compress(b"\x00" * 16)) + png_chunk(b"IEND", b"")
|
||||
|
||||
|
||||
class AttachmentTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.a, self.b, self.eve = register("att"), register("att"), register("att")
|
||||
self.chat = call("POST", "/api/v1/chats", {"title": "файлы", "member_ids": [self.a["user"]["id"], self.b["user"]["id"]]}, self.a["token"])[2]
|
||||
self.other = call("POST", "/api/v1/chats", {"title": "другой", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2]
|
||||
|
||||
def assertError(self, result, status, code=None):
|
||||
self.assertEqual(result[0], status, result[2])
|
||||
if code:
|
||||
self.assertEqual(result[2]["error"]["code"], code)
|
||||
|
||||
def test_png_and_jpeg_roundtrip_with_checksum(self):
|
||||
for data, ctype, name in ((PNG, "image/png", "фото.png"), (JPG, "image/jpeg", "photo.jpg")):
|
||||
status, headers, att = upload(self.a["token"], self.chat["id"], data, ctype, urllib.parse.quote(name))
|
||||
self.assertEqual(status, 202, att)
|
||||
self.assertEqual((att["status"], att["variants"], att["error_code"]), ("queued", [], None))
|
||||
wait_ready(self.b["token"], att["id"])
|
||||
self.assertEqual((att["size_bytes"], att["sha256"], att["content_type"]), (len(data), hashlib.sha256(data).hexdigest(), ctype))
|
||||
self.assertEqual((att["chat_id"], att["owner_id"], att["filename"]), (self.chat["id"], self.a["user"]["id"], name))
|
||||
code, hdr, body = download(self.b["token"], att["id"]) # другой участник чата
|
||||
self.assertEqual((code, body, hdr.get_content_type()), (200, data, ctype))
|
||||
self.assertEqual(download(self.b["token"], att["id"], "")[0], 200)
|
||||
|
||||
def test_too_large_is_413_and_nothing_is_stored(self):
|
||||
before = psql("SELECT count(*) FROM attachments")
|
||||
r = upload(self.a["token"], self.chat["id"], PNG[:8] + b"\x00" * (10 * 1024 * 1024 + 1))
|
||||
self.assertError(r, 413, "payload_too_large")
|
||||
self.assertEqual(psql("SELECT count(*) FROM attachments"), before)
|
||||
|
||||
def test_corrupted_and_spoofed_files_are_415_not_500(self):
|
||||
cases = {
|
||||
"truncated png": (PNG[: len(PNG) // 2], "image/png"),
|
||||
"garbage png": (b"\x89PNG\r\n\x1a\n" + b"\x00" * 200, "image/png"),
|
||||
"random bytes": (os.urandom(500), "image/png"),
|
||||
"truncated jpeg": (JPG[: len(JPG) // 2], "image/jpeg"),
|
||||
"garbage jpeg": (b"\xff\xd8\xff\xe0" + b"\x00" * 200, "image/jpeg"),
|
||||
"png as jpeg": (PNG, "image/jpeg"),
|
||||
"jpeg as png": (JPG, "image/png"),
|
||||
"text as png": (b"hello, i am not an image", "image/png"),
|
||||
"gif": (b"GIF89a" + b"\x00" * 50, "image/png"),
|
||||
}
|
||||
for name, (data, ctype) in cases.items():
|
||||
with self.subTest(name):
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], data, ctype), 415, "unsupported_media_type")
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], PNG, "text/plain"), 415)
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], PNG, "multipart/form-data; boundary=x"), 415)
|
||||
|
||||
def test_decompression_bomb_header_is_413(self):
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], bomb_png()), 413, "payload_too_large")
|
||||
# 6000x6600 = 39.6 млн пикселей < лимита: по размерам допустимо, но данных изображения нет -> файл повреждён (415)
|
||||
self.assertEqual(upload(self.a["token"], self.chat["id"], bomb_png(6000, 6600))[0], 415)
|
||||
|
||||
def test_dangerous_filenames_are_400(self):
|
||||
bad = ["a%2Fb.png", "..%5Cx.png", "x%00.png", "x%0A.png", "x%7F.png", "", "%20%20", "n" * 256]
|
||||
for name in bad:
|
||||
with self.subTest(name):
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + name), 400, "invalid_request")
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename=""), 400)
|
||||
self.assertEqual(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + "n" * 255)[0], 202)
|
||||
|
||||
def test_empty_body_is_400(self):
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], b""), 400)
|
||||
|
||||
def test_foreign_user_cannot_upload_read_or_download(self):
|
||||
self.assertError(upload(self.eve["token"], self.chat["id"], PNG), 404)
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
wait_ready(self.a["token"], att["id"])
|
||||
for suffix in ("", "/content"):
|
||||
self.assertEqual(download(self.eve["token"], att["id"], suffix)[0], 404)
|
||||
self.assertEqual(download(None, att["id"], suffix)[0], 401)
|
||||
self.assertEqual(download(self.a["token"], str(uuid.uuid4()))[0], 404)
|
||||
self.assertEqual(download(self.a["token"], "not-a-uuid")[0], 400)
|
||||
self.assertEqual(download(self.a["token"], att["id"], "/content?variant=thumbnail")[0], 200) # после ready вариант доступен
|
||||
self.assertEqual(download(self.a["token"], att["id"], "/content?variant=bogus")[0], 400)
|
||||
|
||||
def test_message_with_attachments_and_chat_binding(self):
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
foreign_chat_att = upload(self.a["token"], self.other["id"], JPG, "image/jpeg")[2]
|
||||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
sent = call("POST", path, {"text": "", "attachment_ids": [att["id"]]}, self.a["token"])
|
||||
self.assertEqual(sent[0], 201, sent[2])
|
||||
self.assertEqual(sent[2]["attachment_ids"], [att["id"]])
|
||||
self.assertEqual(call("POST", path, {"text": "ещё раз то же вложение", "attachment_ids": [att["id"]]}, self.b["token"])[0], 201)
|
||||
listed = call("GET", path + "?limit=10", token=self.b["token"])[2]["items"]
|
||||
self.assertEqual([m["attachment_ids"] for m in listed], [[att["id"]], [att["id"]]])
|
||||
# вложение другого чата / несуществующее -> 404; дубликаты, >10, пустое сообщение без вложений -> 400
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [foreign_chat_att["id"]]}, self.a["token"])[0], 404)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, self.a["token"])[0], 404)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"], att["id"]]}, self.a["token"])[0], 400)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, self.a["token"])[0], 400)
|
||||
self.assertEqual(call("POST", path, {"text": " ", "attachment_ids": []}, self.a["token"])[0], 400)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": "nope"}, self.a["token"])[0], 400)
|
||||
# чужой пользователь не может сослаться на вложение
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"]]}, self.eve["token"])[0], 404)
|
||||
|
||||
def test_idempotent_replay_with_attachments(self):
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
key = uuid.uuid4().hex
|
||||
h = {"Idempotency-Key": key}
|
||||
first = call("POST", path, {"text": "t", "attachment_ids": [att["id"]]}, self.a["token"], headers=h)
|
||||
again = call("POST", path, {"attachment_ids": [att["id"]], "text": "t"}, self.a["token"], headers=h) # другой порядок ключей
|
||||
self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2]))
|
||||
self.assertEqual(call("POST", path, {"text": "t"}, self.a["token"], headers=h)[0], 409) # другое тело
|
||||
text_key = uuid.uuid4().hex
|
||||
m1 = call("POST", path, {"text": "без вложений"}, self.a["token"], headers={"Idempotency-Key": text_key})
|
||||
m2 = call("POST", path, {"text": "без вложений", "attachment_ids": []}, self.a["token"], headers={"Idempotency-Key": text_key})
|
||||
self.assertEqual(m1[2], m2[2]) # отсутствие == []
|
||||
self.assertEqual(m1[2]["attachment_ids"], [])
|
||||
|
||||
def test_aborted_upload_leaves_nothing(self):
|
||||
before = (psql("SELECT count(*) FROM attachments"))
|
||||
host, port = BASE.split("//")[1].split(":")
|
||||
with socket.create_connection((host, int(port))) as s:
|
||||
s.sendall(("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer %s\r\n"
|
||||
"Content-Type: image/png\r\nContent-Length: 200000\r\n\r\n" % (self.chat["id"], self.a["token"])).encode() + PNG[:5000])
|
||||
import time
|
||||
time.sleep(1)
|
||||
self.assertEqual(psql("SELECT count(*) FROM attachments"), before)
|
||||
|
||||
def test_bucket_is_private_outside_the_api(self):
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
wait_ready(self.a["token"], att["id"])
|
||||
key = psql("SELECT s3_key FROM attachments WHERE id = '%s'" % att["id"])
|
||||
import subprocess
|
||||
code = ("import urllib.request,urllib.error,sys\n"
|
||||
"try:\n urllib.request.urlopen('http://seaweedfs:8333/messenger-attachments/%s', timeout=5); print('OPEN')\n"
|
||||
"except urllib.error.HTTPError as e: print(e.code)\n" % key)
|
||||
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code], cwd=ROOT,
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
self.assertEqual(out, "403", "анонимный GET объекта напрямую из S3 должен быть запрещён")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Поведение при недоступности зависимостей: Valkey (кэш) и PostgreSQL (источник истины).
|
||||
|
||||
python student-tests/test_dependencies.py
|
||||
Контейнеры останавливаются и запускаются обратно (volumes сохраняются). Ожидания:
|
||||
* Valkey остановлен -> кэш обходится, API работает (чтение сессии из PostgreSQL), logout работает;
|
||||
* PostgreSQL остановлен -> 503 в контрактном формате с Retry-After (не 500, не «анонимный» доступ),
|
||||
/health/ready = 503, запрос без токена по-прежнему 401; после запуска БД API восстанавливается сам.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, basic, call, register # noqa: E402
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, capture_output=True, text=True)
|
||||
|
||||
|
||||
def wait(predicate, seconds=60):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if predicate():
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError("условие не выполнено за %d с" % seconds)
|
||||
|
||||
|
||||
class DependencyTest(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
compose("start", "postgres", "valkey")
|
||||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||||
|
||||
def test_valkey_down_is_bypassed(self):
|
||||
actor = register("dep")
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
|
||||
compose("stop", "valkey")
|
||||
started = time.time()
|
||||
for _ in range(5):
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
|
||||
self.assertLess(time.time() - started, 3, "circuit breaker не должен давать каждому запросу ждать таймаут")
|
||||
# вход и выход работают без кэша
|
||||
s = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
|
||||
self.assertEqual(s[0], 201)
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s[2]["token"])[0], 204)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=s[2]["token"])[0], 401) # отзыв виден без кэша
|
||||
self.assertEqual(call("GET", "/health/ready")[0], 200) # кэш не критичен для readiness
|
||||
|
||||
def test_postgres_down_gives_bounded_503_never_anonymous_access(self):
|
||||
actor = register("dep")
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # токен в кэше
|
||||
compose("stop", "postgres")
|
||||
# Wait past HAProxy's fall=2 health checks: cover the all-unready state,
|
||||
# not just the short interval while the load balancer still routes normally.
|
||||
time.sleep(5)
|
||||
for method, path, body, token in (("GET", "/api/v1/users/me", None, actor["token"]),
|
||||
("GET", "/api/v1/chats", None, actor["token"]),
|
||||
("POST", "/api/v1/chats", {"title": "x", "member_ids": [actor["user"]["id"]]}, actor["token"]),
|
||||
("GET", "/api/v1/users/me", None, "Z" * 43)): # неизвестный токен: кэш пуст -> БД -> 503
|
||||
r = call(method, path, body, token)
|
||||
self.assertEqual(r[0], 503, (path, r))
|
||||
self.assertEqual(r[2]["error"]["code"], "unavailable")
|
||||
self.assertGreaterEqual(int(r[1]["Retry-After"]), 1)
|
||||
self.assertEqual(r[2]["error"]["request_id"], r[1]["X-Request-Id"])
|
||||
r = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
|
||||
self.assertEqual(r[0], 503)
|
||||
self.assertEqual(call("GET", "/health/ready")[0], 503)
|
||||
self.assertEqual(call("GET", "/health/live")[0], 200)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) # без токена — по-прежнему 401
|
||||
compose("start", "postgres")
|
||||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # сессия пережила простой БД
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Проверка срока жизни сессии при коротком SESSION_TTL_SECONDS (по умолчанию 2).
|
||||
|
||||
python student-tests/test_expiry.py
|
||||
Скрипт пересоздаёт только контейнер api с SESSION_TTL_SECONDS=2, проверяет expiry и возвращает настройки
|
||||
по умолчанию. Данные и volumes не затрагиваются.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, call, register # noqa: E402
|
||||
|
||||
TTL = 2
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True,
|
||||
capture_output=True, text=True, env=dict(os.environ, **(env or {})))
|
||||
|
||||
|
||||
def wait_ready(seconds=60):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if call("GET", "/health/ready")[0] == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError("api не стал ready")
|
||||
|
||||
|
||||
class ExpiryTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "api", env={"SESSION_TTL_SECONDS": str(TTL)})
|
||||
wait_ready()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "api")
|
||||
wait_ready()
|
||||
|
||||
def test_session_expires_after_ttl(self):
|
||||
actor = register("exp")
|
||||
session = actor["session"]
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # валидна, кэш прогрет
|
||||
time.sleep(TTL + 1.5)
|
||||
r = call("GET", "/api/v1/users/me", token=actor["token"])
|
||||
self.assertEqual(r[0], 401, "после TTL токен должен быть отвергнут (и кэшем, и БД)")
|
||||
self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"')
|
||||
self.assertEqual(r[2]["error"]["code"], "unauthorized")
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])[0], 401)
|
||||
self.assertIn("expires_at", session)
|
||||
|
||||
def test_expiry_is_not_extended_by_use(self):
|
||||
actor = register("exp")
|
||||
for _ in range(3):
|
||||
call("GET", "/api/v1/users/me", token=actor["token"])
|
||||
time.sleep(0.5)
|
||||
time.sleep(1.5)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 401)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Идемпотентность отправки сообщений (lab3) и распределение по репликам. Запуск:
|
||||
python student-tests/test_idempotency.py (BASE_URL по умолчанию http://127.0.0.1:8080, через LB)
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import call, psql, register # noqa: E402
|
||||
|
||||
|
||||
def chat_with(owner, *others):
|
||||
ids = [owner["user"]["id"]] + [o["user"]["id"] for o in others]
|
||||
return call("POST", "/api/v1/chats", {"title": "идемпотентность", "member_ids": ids}, owner["token"])[2]
|
||||
|
||||
|
||||
def history(token, chat_id):
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
page = call("GET", "/api/v1/chats/%s/messages?limit=100%s" % (chat_id, "&cursor=" + quote(cursor, safe="") if cursor else ""), token=token)[2]
|
||||
items += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
return items
|
||||
|
||||
|
||||
class IdempotencyTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.a, self.b, self.eve = register(), register(), register()
|
||||
self.chat = chat_with(self.a, self.b)
|
||||
self.path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
|
||||
def post(self, actor, body, key, path=None):
|
||||
return call("POST", path or self.path, body, actor["token"], headers={"Idempotency-Key": key})
|
||||
|
||||
def test_key_validation(self):
|
||||
for headers in ({}, {"Idempotency-Key": ""}, {"Idempotency-Key": "a b"},
|
||||
{"Idempotency-Key": "x" * 129}, {"Idempotency-Key": "a/b"}):
|
||||
h = dict(headers)
|
||||
h.setdefault("Content-Type", "application/json")
|
||||
r = call("POST", self.path, raw=b'{"text":"x"}', token=self.a["token"], headers=h)
|
||||
self.assertEqual(r[0], 400, headers)
|
||||
self.assertEqual(r[2]["error"]["code"], "invalid_request")
|
||||
self.assertEqual(self.post(self.a, {"text": "ok"}, "A-z_0.9:-" + "k" * 100)[0], 201)
|
||||
|
||||
def test_replay_returns_same_message_and_creates_no_duplicate(self):
|
||||
key = uuid.uuid4().hex
|
||||
first = self.post(self.a, {"text": "один раз"}, key)
|
||||
second = self.post(self.a, {"text": "один раз"}, key)
|
||||
self.assertEqual((first[0], second[0]), (201, 201))
|
||||
self.assertEqual(first[2], second[2])
|
||||
self.assertEqual(len([m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "один раз"]), 1)
|
||||
|
||||
def test_different_body_same_key_is_409(self):
|
||||
key = uuid.uuid4().hex
|
||||
self.post(self.a, {"text": "первое"}, key)
|
||||
r = self.post(self.a, {"text": "другое"}, key)
|
||||
self.assertEqual(r[0], 409)
|
||||
self.assertEqual(r[2]["error"]["code"], "conflict")
|
||||
|
||||
def test_scope_is_user_and_chat(self):
|
||||
key = uuid.uuid4().hex
|
||||
m1 = self.post(self.a, {"text": "общий ключ"}, key)[2]
|
||||
m2 = self.post(self.b, {"text": "общий ключ"}, key)[2] # другой пользователь
|
||||
other = chat_with(self.a)
|
||||
m3 = self.post(self.a, {"text": "общий ключ"}, key, "/api/v1/chats/%s/messages" % other["id"])[2] # другой чат
|
||||
self.assertEqual(len({m1["id"], m2["id"], m3["id"]}), 3)
|
||||
|
||||
def test_replay_rechecks_session_and_membership(self):
|
||||
key = uuid.uuid4().hex
|
||||
self.post(self.a, {"text": "защищено"}, key)
|
||||
self.assertEqual(self.post(self.eve, {"text": "защищено"}, key)[0], 404) # посторонний
|
||||
r = call("POST", self.path, {"text": "защищено"}, "Z" * 43, headers={"Idempotency-Key": key})
|
||||
self.assertEqual(r[0], 401) # недействительный токен
|
||||
logout = register()
|
||||
chat = chat_with(logout)
|
||||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
self.post(logout, {"text": "x"}, key, path)
|
||||
call("DELETE", "/api/v1/auth/sessions/current", token=logout["token"])
|
||||
self.assertEqual(call("POST", path, {"text": "x"}, logout["token"], headers={"Idempotency-Key": key})[0], 401)
|
||||
|
||||
def test_semantically_equal_body_is_replayed(self):
|
||||
key = uuid.uuid4().hex
|
||||
first = self.post(self.a, {"text": "эквивалент"}, key)
|
||||
raw = call("POST", self.path, raw=b'{ "text" : "\u044d\u043a\u0432\u0438\u0432\u0430\u043b\u0435\u043d\u0442" }',
|
||||
token=self.a["token"], headers={"Idempotency-Key": key, "Content-Type": "application/json"})
|
||||
self.assertEqual((raw[0], raw[2]), (201, first[2]))
|
||||
|
||||
def test_concurrent_same_key_creates_exactly_one_message_across_replicas(self):
|
||||
key = uuid.uuid4().hex
|
||||
with ThreadPoolExecutor(24) as pool:
|
||||
results = list(pool.map(lambda _: self.post(self.a, {"text": "гонка ключа"}, key), range(24)))
|
||||
self.assertEqual({r[0] for r in results}, {201})
|
||||
self.assertEqual(len({r[2]["id"] for r in results}), 1)
|
||||
instances = {r[1]["X-Instance-Id"] for r in results}
|
||||
self.assertGreaterEqual(len(instances), 2, "повторы должны обслуживаться разными репликами: %s" % instances)
|
||||
found = [m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "гонка ключа"]
|
||||
self.assertEqual(len(found), 1)
|
||||
|
||||
def test_expired_key_is_reusable_after_ttl(self):
|
||||
key = uuid.uuid4().hex
|
||||
first = self.post(self.a, {"text": "ttl"}, key)[2]
|
||||
psql("UPDATE idempotency_keys SET created_at = now() - interval '25 hours' WHERE idem_key = '%s'" % key)
|
||||
again = self.post(self.a, {"text": "ttl"}, key)
|
||||
self.assertEqual(again[0], 201)
|
||||
self.assertNotEqual(again[2]["id"], first["id"], "после 24 ч ключ считается новым")
|
||||
|
||||
def test_session_is_shared_between_replicas_without_relogin(self):
|
||||
seen = set()
|
||||
for _ in range(30):
|
||||
r = call("GET", "/api/v1/users/me", token=self.a["token"])
|
||||
self.assertEqual(r[0], 200)
|
||||
seen.add(r[1]["X-Instance-Id"])
|
||||
self.assertGreaterEqual(len(seen), 2, seen)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Тесты асинхронного конвейера вложений lab5 (контракт вариантов, состояния, права). Запуск:
|
||||
python -m unittest discover -s student-tests -p "test_pipeline.py" -v
|
||||
Пиксельные проверки выполняются, если установлен Pillow (иначе пропускаются); размеры проверяются по заголовкам файлов.
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import call, register # noqa: E402
|
||||
from test_attachments import FIX, JPG, PNG, download, upload # noqa: E402
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError: # pragma: no cover
|
||||
Image = None
|
||||
|
||||
|
||||
def fixture(name):
|
||||
return open(os.path.join(FIX, name), "rb").read()
|
||||
|
||||
|
||||
def jpeg_size(data):
|
||||
i = 2
|
||||
while i < len(data):
|
||||
while data[i] == 0xFF:
|
||||
i += 1
|
||||
marker = data[i]
|
||||
i += 1
|
||||
if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7:
|
||||
continue
|
||||
length = int.from_bytes(data[i:i + 2], "big")
|
||||
if marker in (0xC0, 0xC1, 0xC2):
|
||||
return int.from_bytes(data[i + 5:i + 7], "big"), int.from_bytes(data[i + 3:i + 5], "big")
|
||||
i += length
|
||||
raise AssertionError("JPEG dimensions not found")
|
||||
|
||||
|
||||
def webp_size(data):
|
||||
assert data[:4] == b"RIFF" and data[8:12] == b"WEBP", "not WebP"
|
||||
kind = data[12:16]
|
||||
if kind == b"VP8X":
|
||||
return 1 + int.from_bytes(data[24:27], "little"), 1 + int.from_bytes(data[27:30], "little")
|
||||
if kind == b"VP8L":
|
||||
bits = int.from_bytes(data[21:25], "little")
|
||||
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
|
||||
if kind == b"VP8 ":
|
||||
return struct.unpack("<HH", data[26:30])[0] & 0x3FFF, struct.unpack("<HH", data[26:30])[1] & 0x3FFF
|
||||
raise AssertionError("unknown WebP chunk")
|
||||
|
||||
|
||||
def wait_status(token, att_id, wanted=("ready",), seconds=60):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
code, _, body = download(token, att_id, "")
|
||||
att = __import__("json").loads(body)
|
||||
if att["status"] in wanted:
|
||||
return att
|
||||
time.sleep(0.3)
|
||||
raise AssertionError("вложение не достигло %s за %d с: %s" % (wanted, seconds, att["status"]))
|
||||
|
||||
|
||||
class PipelineTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.a, self.b, self.eve = register("pipe"), register("pipe"), register("pipe")
|
||||
self.chat = call("POST", "/api/v1/chats", {"title": "конвейер", "member_ids": [self.a["user"]["id"], self.b["user"]["id"]]}, self.a["token"])[2]
|
||||
|
||||
def ready(self, data, ctype="image/png", name="x.png"):
|
||||
status, _, att = upload(self.a["token"], self.chat["id"], data, ctype, name)
|
||||
self.assertEqual(status, 202, att)
|
||||
self.assertEqual((att["status"], att["variants"], att["error_code"]), ("queued", [], None))
|
||||
return wait_status(self.b["token"], att["id"])
|
||||
|
||||
def variants(self, att):
|
||||
return {v["name"]: v for v in att["variants"]}
|
||||
|
||||
def fetch(self, att, variant):
|
||||
code, hdr, body = download(self.b["token"], att["id"], "/content?variant=" + variant)
|
||||
self.assertEqual(code, 200)
|
||||
return hdr.get_content_type(), body
|
||||
|
||||
def test_sample_png_produces_all_variants_per_contract(self):
|
||||
att = self.ready(PNG)
|
||||
v = self.variants(att)
|
||||
self.assertEqual(set(v), {"thumbnail", "crop", "optimized"})
|
||||
self.assertEqual((att["sha256"], att["size_bytes"]), (hashlib.sha256(PNG).hexdigest(), len(PNG))) # описывают оригинал
|
||||
self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"], v["thumbnail"]["content_type"]), (256, 160, "image/jpeg"))
|
||||
self.assertEqual((v["crop"]["width"], v["crop"]["height"], v["crop"]["content_type"]), (128, 128, "image/jpeg"))
|
||||
self.assertEqual(v["optimized"]["content_type"], "image/webp")
|
||||
self.assertEqual((v["optimized"]["width"], v["optimized"]["height"]), (320, 200)) # без увеличения
|
||||
ctype, thumb = self.fetch(att, "thumbnail")
|
||||
self.assertEqual((ctype, jpeg_size(thumb)), ("image/jpeg", (256, 160)))
|
||||
self.assertEqual((v["thumbnail"]["size_bytes"], v["thumbnail"]["sha256"]), (len(thumb), hashlib.sha256(thumb).hexdigest()))
|
||||
_, crop = self.fetch(att, "crop")
|
||||
self.assertEqual(jpeg_size(crop), (128, 128))
|
||||
ctype, opt = self.fetch(att, "optimized")
|
||||
self.assertEqual((ctype, webp_size(opt)), ("image/webp", (320, 200)))
|
||||
code, hdr, original = download(self.b["token"], att["id"])
|
||||
self.assertEqual((code, original, hdr.get_content_type()), (200, PNG, "image/png")) # оригинал неизменен
|
||||
|
||||
def test_exif_orientation_is_applied(self):
|
||||
att = self.ready(fixture("exif_rot6.jpg"), "image/jpeg", "rot.jpg")
|
||||
v = self.variants(att)
|
||||
# 300x100 с Orientation=6 -> 100x300 -> вписывается в 256: 85x256 (округление вниз)
|
||||
self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), (85, 256))
|
||||
self.assertEqual(jpeg_size(self.fetch(att, "thumbnail")[1]), (85, 256))
|
||||
|
||||
def test_small_image_is_not_upscaled_but_crop_is_128(self):
|
||||
att = self.ready(fixture("small.png"), "image/png", "small.png")
|
||||
v = self.variants(att)
|
||||
self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), (50, 30))
|
||||
self.assertEqual((v["crop"]["width"], v["crop"]["height"]), (128, 128))
|
||||
self.assertEqual((v["optimized"]["width"], v["optimized"]["height"]), (50, 30))
|
||||
|
||||
@unittest.skipIf(Image is None, "нужен Pillow")
|
||||
def test_alpha_is_composited_on_white(self):
|
||||
att = self.ready(fixture("alpha.png"), "image/png", "alpha.png")
|
||||
thumb = Image.open(io.BytesIO(self.fetch(att, "thumbnail")[1])).convert("RGB")
|
||||
left, right = thumb.getpixel((10, thumb.height // 2)), thumb.getpixel((thumb.width - 10, thumb.height // 2))
|
||||
self.assertTrue(all(c > 235 for c in left), "прозрачная область должна стать белой: %s" % (left,))
|
||||
self.assertTrue(right[0] > 200 and right[1] < 60, "непрозрачная красная область сохранена: %s" % (right,))
|
||||
|
||||
def test_message_can_reference_attachment_before_ready_and_failed_cannot(self):
|
||||
status, _, att = upload(self.a["token"], self.chat["id"], PNG)
|
||||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
sent = call("POST", path, {"text": "", "attachment_ids": [att["id"]]}, self.a["token"])
|
||||
self.assertEqual((status, sent[0], sent[2]["attachment_ids"]), (202, 201, [att["id"]]))
|
||||
poison = upload(self.a["token"], self.chat["id"], fixture("poison.png"))
|
||||
self.assertEqual(poison[0], 202, poison[2])
|
||||
failed = wait_status(self.b["token"], poison[2]["id"], ("failed",))
|
||||
self.assertEqual((failed["status"], failed["error_code"], failed["variants"]), ("failed", "invalid_image", []))
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [failed["id"]]}, self.a["token"])[0], 409)
|
||||
self.assertEqual(download(self.b["token"], failed["id"], "/content")[0], 409)
|
||||
self.assertEqual(download(self.b["token"], failed["id"], "/content?variant=thumbnail")[0], 409)
|
||||
|
||||
def test_content_is_409_while_not_ready_and_original_is_untouched(self):
|
||||
status, _, att = upload(self.a["token"], self.chat["id"], PNG)
|
||||
code = download(self.b["token"], att["id"], "/content")[0]
|
||||
self.assertIn(code, (200, 409)) # быстрый worker мог успеть: 200 допустим, но не 404/500
|
||||
done = wait_status(self.b["token"], att["id"])
|
||||
self.assertEqual(download(self.b["token"], done["id"], "/content")[2], PNG)
|
||||
|
||||
def test_other_variants_visible_only_to_chat_members(self):
|
||||
att = self.ready(PNG)
|
||||
for variant in ("thumbnail", "crop", "optimized", "original"):
|
||||
self.assertEqual(download(self.eve["token"], att["id"], "/content?variant=" + variant)[0], 404)
|
||||
self.assertEqual(download(self.b["token"], att["id"], "/content?variant=bogus")[0], 400)
|
||||
|
||||
def test_ready_job_redelivery_does_not_duplicate_variants(self):
|
||||
from test_api import psql
|
||||
att = self.ready(PNG)
|
||||
before = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"])
|
||||
self.assertEqual(before, "3")
|
||||
psql("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES (gen_random_uuid(), '%s', 1, 2)" % att["id"])
|
||||
time.sleep(4)
|
||||
self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3")
|
||||
self.assertEqual(wait_status(self.b["token"], att["id"])["status"], "ready")
|
||||
|
||||
def test_staging_is_cleaned_after_ready_and_kept_for_failed(self):
|
||||
import subprocess
|
||||
from test_api import ROOT, psql
|
||||
att = self.ready(PNG)
|
||||
poison = upload(self.a["token"], self.chat["id"], fixture("poison.png"))[2]
|
||||
wait_status(self.b["token"], poison["id"], ("failed",))
|
||||
|
||||
def exists(att_id):
|
||||
code = "\n".join(["import asyncio", "from app.s3 import S3", "async def m():",
|
||||
" s = S3(); await s.start(); print((await s.head('staging/%s')) is not None); await s.stop()" % att_id,
|
||||
"asyncio.run(m())"])
|
||||
return subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code], cwd=ROOT,
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
self.assertEqual(exists(att["id"]), "False", "staging готового вложения должен быть удалён после COMMIT результата")
|
||||
self.assertEqual(exists(poison["id"]), "True", "staging неудавшегося вложения хранится для разбора/replay (retention)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Reconciliation must preserve variants, in-flight effects and durable staging."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, call, register
|
||||
from test_attachments import PNG, download, upload
|
||||
from test_pipeline import wait_status
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
|
||||
check=True, capture_output=True, text=True, encoding='utf-8').stdout
|
||||
|
||||
|
||||
def reconcile(prefix, grace=0):
|
||||
return json.loads(compose('run', '--rm', '-T', 'api', 'python', '-m', 'app.reconcile',
|
||||
'--apply', '--grace', str(grace), '--prefix', prefix))
|
||||
|
||||
|
||||
def s3_action(action, key):
|
||||
code = ('import asyncio\nfrom app import config\nfrom app.s3 import S3\nasync def m():\n'
|
||||
' s=S3(); await s.start()\n'
|
||||
+ {'put': ' await s.put_bytes(%r, b"temporary", "application/octet-stream")\n',
|
||||
'head': ' print((await s.head(%r)) is not None)\n',
|
||||
'delete': ' await s.delete(%r)\n'}[action] % key
|
||||
+ ' await s.stop()\nasyncio.run(m())')
|
||||
return compose('exec', '-T', 'api', 'python', '-c', code).strip()
|
||||
|
||||
|
||||
class ReconcilePipelineTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.actor = register('reconcile')
|
||||
self.token = self.actor['token']
|
||||
self.chat = call('POST', '/api/v1/chats', {'title': 'reconcile', 'member_ids': [self.actor['user']['id']]}, self.token)[2]
|
||||
|
||||
def accept(self):
|
||||
status, _, att = upload(self.token, self.chat['id'], PNG)
|
||||
self.assertEqual(status, 202)
|
||||
return att
|
||||
|
||||
def test_ready_variants_and_uncommitted_side_effect_are_preserved(self):
|
||||
att = wait_status(self.token, self.accept()['id'])
|
||||
key = 'att/%s/v1/uncommitted' % att['id']
|
||||
s3_action('put', key)
|
||||
try:
|
||||
report = reconcile('att/%s/' % att['id'])
|
||||
self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0})
|
||||
self.assertEqual(report['orphan_objects_count'], 0)
|
||||
self.assertEqual(report['owned_objects_skipped'], 1)
|
||||
self.assertEqual(s3_action('head', key), 'True')
|
||||
self.assertEqual(download(self.token, att['id'])[2], PNG)
|
||||
for variant in att['variants']:
|
||||
code, _, data = download(self.token, att['id'], '/content?variant=' + variant['name'])
|
||||
self.assertEqual(code, 200)
|
||||
self.assertEqual(hashlib.sha256(data).hexdigest(), variant['sha256'])
|
||||
finally:
|
||||
s3_action('delete', key)
|
||||
|
||||
def test_staging_is_protected_and_unowned_orphan_obeys_grace(self):
|
||||
compose('stop', 'worker')
|
||||
att = None
|
||||
orphan = 'staging/' + str(uuid.uuid4())
|
||||
try:
|
||||
att = self.accept()
|
||||
report = reconcile('staging/' + att['id'])
|
||||
self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0})
|
||||
self.assertEqual(s3_action('head', 'staging/' + att['id']), 'True')
|
||||
s3_action('put', orphan)
|
||||
fresh = reconcile(orphan, grace=900)
|
||||
self.assertEqual(fresh['orphans_skipped_fresh'], 1)
|
||||
self.assertEqual(fresh['fixed']['deleted_orphans'], 0)
|
||||
self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 1)
|
||||
self.assertEqual(s3_action('head', orphan), 'False')
|
||||
self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 0)
|
||||
finally:
|
||||
s3_action('delete', orphan)
|
||||
compose('start', 'worker')
|
||||
self.assertEqual(wait_status(self.token, att['id'])['status'], 'ready')
|
||||
self.assertEqual(download(self.token, att['id'])[2], PNG)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Отказы конвейера lab5 (окна отказа из contracts/attachment-states.md). Тесты управляют контейнерами (volumes сохраняются):
|
||||
python student-tests/test_resilience5.py # ~5-7 минут
|
||||
|
||||
Окна: (1) worker остановлен -> upload -> рестарт API -> worker поднят; (2) рестарт брокера с volume; (3) брокер недоступен
|
||||
при приёме (outbox); (4) crash API между COMMIT и publish; (5) crash worker после записи в S3 до COMMIT результата;
|
||||
(6) crash после COMMIT до ack; (7) исчерпание попыток и операторский replay.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, call, psql, register # noqa: E402
|
||||
from test_attachments import PNG, download, upload # noqa: E402
|
||||
from test_pipeline import wait_status # noqa: E402
|
||||
|
||||
|
||||
def compose(*args, env=None, check=True):
|
||||
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True, text=True,
|
||||
env=dict(os.environ, **(env or {})), check=check).stdout
|
||||
|
||||
|
||||
def wait(predicate, seconds=90, message="условие"):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if predicate():
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise AssertionError("%s не выполнено за %d с" % (message, seconds))
|
||||
|
||||
|
||||
def healthy(service):
|
||||
ids = compose("ps", "-q", service).split()
|
||||
return bool(ids) and all(subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", i], capture_output=True, text=True).stdout.strip() == "healthy" for i in ids)
|
||||
|
||||
|
||||
def api_ready():
|
||||
return call("GET", "/health/ready")[0] == 200
|
||||
|
||||
|
||||
def recreate_worker(env=None):
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "worker", env=env)
|
||||
wait(lambda: healthy("worker"), 90, "worker healthy")
|
||||
|
||||
|
||||
class ResilienceTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
recreate_worker()
|
||||
compose("up", "-d", "--no-deps", "rabbitmq", "api")
|
||||
|
||||
def setUp(self):
|
||||
compose("start", "rabbitmq", "postgres", "seaweedfs", "valkey")
|
||||
wait(lambda: healthy("rabbitmq") and healthy("postgres"), 90, "rabbitmq/postgres healthy")
|
||||
wait(api_ready, 60, "api ready")
|
||||
self.a = register("res")
|
||||
self.chat = call("POST", "/api/v1/chats", {"title": "отказы", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2]
|
||||
|
||||
def accept(self, data=PNG):
|
||||
status, _, att = upload(self.a["token"], self.chat["id"], data)
|
||||
self.assertEqual(status, 202, att)
|
||||
return att
|
||||
|
||||
def done(self, att, seconds=90):
|
||||
ready = wait_status(self.a["token"], att["id"], ("ready",), seconds)
|
||||
self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3") # без дублей
|
||||
code, _, body = download(self.a["token"], att["id"])
|
||||
self.assertEqual((code, body), (200, PNG))
|
||||
return ready
|
||||
|
||||
def test_1_worker_down_upload_api_restart_worker_up(self):
|
||||
compose("stop", "worker")
|
||||
att = self.accept()
|
||||
sent = call("POST", "/api/v1/chats/%s/messages" % self.chat["id"], {"text": "", "attachment_ids": [att["id"]]}, self.a["token"])
|
||||
self.assertEqual(sent[0], 201)
|
||||
time.sleep(3)
|
||||
self.assertEqual(wait_status(self.a["token"], att["id"], ("queued",), 5)["status"], "queued") # не обрабатывается без worker
|
||||
compose("restart", "api")
|
||||
wait(api_ready, 60, "api ready")
|
||||
compose("start", "worker")
|
||||
self.done(att)
|
||||
|
||||
def test_2_broker_restart_keeps_accepted_jobs(self):
|
||||
compose("stop", "worker")
|
||||
att = self.accept()
|
||||
time.sleep(2)
|
||||
compose("restart", "rabbitmq")
|
||||
wait(lambda: healthy("rabbitmq"), 90, "rabbitmq healthy")
|
||||
compose("start", "worker")
|
||||
self.done(att)
|
||||
|
||||
def test_3_broker_down_during_acceptance_outbox_delivers_later(self):
|
||||
compose("stop", "rabbitmq")
|
||||
att = self.accept() # 202 даже без брокера: намерение зафиксировано в outbox
|
||||
self.assertEqual(psql("SELECT count(*) FROM outbox WHERE attachment_id = '%s' AND published_at IS NULL" % att["id"]), "1")
|
||||
time.sleep(3)
|
||||
self.assertEqual(wait_status(self.a["token"], att["id"], ("queued",), 5)["status"], "queued")
|
||||
compose("start", "rabbitmq")
|
||||
self.done(att)
|
||||
self.assertEqual(psql("SELECT count(*) FROM outbox WHERE attachment_id = '%s' AND published_at IS NULL" % att["id"]), "0")
|
||||
|
||||
def test_4_api_crash_between_commit_and_publish(self):
|
||||
compose("stop", "rabbitmq")
|
||||
att = self.accept() # COMMIT выполнен, publish невозможен
|
||||
for cid in compose("ps", "-q", "api").split():
|
||||
subprocess.run(["docker", "kill", cid], capture_output=True) # внезапная смерть всех реплик API
|
||||
compose("start", "rabbitmq")
|
||||
wait(lambda: healthy("rabbitmq"), 90, "rabbitmq healthy")
|
||||
compose("up", "-d", "--no-deps", "api")
|
||||
wait(api_ready, 90, "api ready")
|
||||
self.done(att) # relay (в worker/новом API) довёл задание без ручного вмешательства
|
||||
|
||||
def test_5_worker_crash_after_s3_before_commit_lease_recovery(self):
|
||||
recreate_worker({"WORKER_FAULT_INJECTION": "crash_once:after_s3_before_commit", "LEASE_SECONDS": "8"})
|
||||
att = self.accept()
|
||||
ready = self.done(att, 120)
|
||||
events = psql("SELECT string_agg(event, ',' ORDER BY id) FROM job_events WHERE attachment_id = '%s'" % att["id"])
|
||||
self.assertIn("lease_expired", events)
|
||||
self.assertIn("ready", events)
|
||||
self.assertEqual(ready["status"], "ready")
|
||||
|
||||
def test_6_worker_crash_after_commit_before_ack_redelivery_is_noop(self):
|
||||
recreate_worker({"WORKER_FAULT_INJECTION": "crash_once:after_commit_before_ack"})
|
||||
att = self.accept()
|
||||
self.done(att, 120)
|
||||
time.sleep(5) # повторная доставка: no-op с ack, вариантов по-прежнему 3
|
||||
self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3")
|
||||
self.assertEqual(psql("SELECT status FROM attachments WHERE id = '%s'" % att["id"]), "ready")
|
||||
|
||||
def test_7_retry_exhaustion_registry_and_operator_replay(self):
|
||||
recreate_worker({"WORKER_FAULT_INJECTION": "fail:after_claim", "MAX_ATTEMPTS": "3", "RETRY_BASE_DELAY": "0.3"})
|
||||
att = self.accept()
|
||||
failed = wait_status(self.a["token"], att["id"], ("failed",), 60)
|
||||
self.assertEqual((failed["error_code"], failed["variants"]), ("retry_exhausted", []))
|
||||
events = psql("SELECT string_agg(event || ':' || coalesce(attempt::text,''), ',' ORDER BY id) FROM job_events WHERE attachment_id = '%s'" % att["id"])
|
||||
self.assertEqual(events.count("retry_scheduled"), 2, events)
|
||||
registry = json.loads(compose("run", "--rm", "-T", "worker", "python", "-m", "app.replay", "list"))
|
||||
self.assertIn(att["id"], [r["attachment_id"] for r in registry])
|
||||
recreate_worker() # причина устранена
|
||||
result = json.loads(compose("run", "--rm", "-T", "worker", "python", "-m", "app.replay", "replay", att["id"]))
|
||||
self.assertEqual(result["result"], "requeued")
|
||||
self.done(att)
|
||||
again = json.loads(compose("run", "--rm", "-T", "worker", "python", "-m", "app.replay", "replay", att["id"]))
|
||||
self.assertEqual(again["result"], "skipped") # replay не исполняет дубль уже успешной работы
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Проверка топологии стенда (уровень 4): число реплик, отсутствие публикации портов API, лимиты ресурсов, healthcheck,
|
||||
отсутствие container_name, единая публичная точка входа. Запуск: python student-tests/test_topology.py"""
|
||||
import json
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
PROJECT = "mtusi-lab5"
|
||||
|
||||
|
||||
def sh(*args):
|
||||
return subprocess.run(args, capture_output=True, text=True, encoding="utf-8").stdout.strip()
|
||||
|
||||
|
||||
def containers(service):
|
||||
ids = sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
|
||||
"--filter", "label=com.docker.compose.service=" + service).split()
|
||||
return [json.loads(sh("docker", "inspect", i))[0] for i in ids]
|
||||
|
||||
|
||||
class TopologyTest(unittest.TestCase):
|
||||
def test_at_least_two_api_replicas_without_published_ports(self):
|
||||
api = containers("api")
|
||||
self.assertGreaterEqual(len(api), 2)
|
||||
for c in api:
|
||||
self.assertFalse([p for p in c["HostConfig"]["PortBindings"] or {}], "реплика API публикует порт в обход LB")
|
||||
self.assertEqual(c["State"]["Health"]["Status"], "healthy")
|
||||
|
||||
def test_only_lb_is_published_and_only_on_localhost(self):
|
||||
published = {}
|
||||
for service in ("api", "postgres", "valkey", "lb", "seaweedfs"):
|
||||
for c in containers(service):
|
||||
for port, binds in (c["HostConfig"]["PortBindings"] or {}).items():
|
||||
published.setdefault(service, []).extend((port, b["HostIp"], b["HostPort"]) for b in binds)
|
||||
self.assertEqual(list(published), ["lb"], published)
|
||||
self.assertEqual(published["lb"], [("8080/tcp", "127.0.0.1", "8080")])
|
||||
|
||||
def test_resource_limits_are_set(self):
|
||||
for service in ("api", "postgres", "valkey", "lb", "seaweedfs"):
|
||||
for c in containers(service):
|
||||
self.assertGreater(c["HostConfig"]["Memory"], 0, service)
|
||||
self.assertGreater(c["HostConfig"]["NanoCpus"], 0, service)
|
||||
|
||||
def test_no_fixed_container_names(self):
|
||||
for c in containers("api"):
|
||||
labels = c["Config"]["Labels"]
|
||||
self.assertEqual(c["Name"].lstrip("/"), "%s-api-%s" % (PROJECT, labels["com.docker.compose.container-number"]))
|
||||
|
||||
def test_api_runs_as_non_root(self):
|
||||
for c in containers("api"):
|
||||
self.assertEqual(c["Config"]["User"], "10001:10001")
|
||||
|
||||
def test_stop_grace_period_covers_drain(self):
|
||||
for c in containers("api"):
|
||||
self.assertGreaterEqual(c["Config"].get("StopTimeout", 10), 15)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||