48 changed files with 2458 additions and 14 deletions
+1
View File
@@ -17,4 +17,5 @@ backups
__pycache__
node_modules
evidence
output
.DS_Store
+6
View File
@@ -0,0 +1,6 @@
* text=auto
*.sh text eol=lf
*.py text eol=lf
*.yaml text eol=lf
*.sql text eol=lf
*.cmd text eol=crlf
+24
View File
@@ -0,0 +1,24 @@
name: Application grade 3
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- run: python scripts/bootstrap.py
- run: docker compose build api
- run: docker compose up -d --no-build
- run: python scripts/wait_ready.py
- run: python scripts/verify.py
- uses: actions/upload-artifact@v4
if: always()
with:
name: grade3-evidence
path: evidence/
- if: always()
run: docker compose down
+1
View File
@@ -30,3 +30,4 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
/output/
+16
View File
@@ -0,0 +1,16 @@
FROM python:3.13.7-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app
COPY src ./src
COPY migrations ./migrations
COPY contracts ./contracts
COPY tests ./tests
COPY student-tests ./student-tests
COPY scripts ./scripts
COPY lab.json requirements-dev.txt ./
USER app
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=4s --start-period=10s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3)"]
CMD ["python", "-m", "messenger.server"]
+111 -14
View File
@@ -1,37 +1,134 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: **Осокин Глеб, БВТ2302**, Gitea `glebossokin`.
- Целевая оценка: **3**. Уровни 4/5 и исследовательские треки не заявлены.
- Ветка решения: [`solution/BVT2302-Osokin`](https://gitea.privcloud.tech/glebossokin/mtusi2026lab4/src/branch/solution/BVT2302-Osokin). Запрос на слияние направляется в `mtusi2026/mtusi2026lab4:main`.
- Дата полного прогона: **16.11.2026**.
- Перенос: собственная [ЛР3](https://gitea.privcloud.tech/glebossokin/mtusi2026lab3/src/branch/solution/BVT2302-Osokin), исходники приложения, миграции 001/002, Dockerfile, конфигурация HAProxy/Redis, проверка общей идемпотентности. Rolling update и исследования нагрузки предыдущей работы не переносились.
- Помощь ИИ: OpenAI Codex — реализация, подготовка отчёта и запуск проверок. Наблюдения получены на реально работающем Docker-стенде. Библиотеки: psycopg, Redis client, boto3, Pillow; первичные источники приведены ниже.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Git, Python 3.9+ для публичных проверок, Docker Engine с Compose v2+ и Linux containers. На Windows использованы Python 3.14.3 и Docker Desktop. Свободный порт **127.0.0.1:8084**; остальные порты не публикуются. При первой сборке нужен доступ к Docker Hub и PyPI.
```powershell
git clone --branch solution/BVT2302-Osokin https://gitea.privcloud.tech/glebossokin/mtusi2026lab4.git
cd mtusi2026lab4
python -X utf8 scripts/bootstrap.py
docker compose --env-file .env config --quiet
docker compose build api
docker compose --env-file .env up -d --no-build
python -X utf8 scripts/wait_ready.py
python -X utf8 scripts/verify.py
```
На Linux заменить `python -X utf8` на `python3`. У hooks `scripts/restart.sh` и `scripts/stop-one.sh` executable bit сохранён в Git; при необходимости выполнить `chmod +x scripts/restart.sh scripts/stop-one.sh`. Если Docker Desktop ещё не настроен на Linux engine, выбрать его командой `docker context use desktop-linux`.
`bootstrap.py` создаёт локальные `.env` и `secrets/`: случайные пароль PostgreSQL, ключ HMAC cursor и credentials MinIO. Существующие значения сохраняются. Эти файлы исключены из Git и Docker build context; их содержимое не печатается. Не менять secrets при повторном использовании прежних volumes.
`migrate` применяет SQL в одной транзакции с блокировкой и проверкой контрольных сумм. `bucket` создаёт приватный bucket перед API; anonymous/public policy не устанавливается. API запускаются после миграций, bootstrap bucket и готовности Redis. HAProxy проверяет readiness обеих реплик. После запуска LB нужно дождаться `wait_ready.py`.
Отдельные публичные проверки на Windows:
```powershell
python -X utf8 scripts/check_contract.py
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8084
python -X utf8 tests/resilience.py --mode persistence --base-url http://127.0.0.1:8084 --action-script scripts/restart.cmd
python -X utf8 tests/resilience.py --mode failover --base-url http://127.0.0.1:8084 --action-script scripts/stop-one.cmd
python -X utf8 scripts/labctl.py restore
```
Это прямые эквиваленты `make check`, `make test BASE_URL=http://127.0.0.1:8084` и `make test-persistence/test-failover` с соответствующим `ACTION_SCRIPT`. На Linux передавать `.sh` hooks. Контракт и публичные тесты не изменялись. Полный `verify.py` сохраняет результаты в `evidence/`, останавливает/восстанавливает API и MinIO, перезапускает работающий стенд. Все данные синтетические. После ручного failover реплика остаётся остановленной до `labctl.py restore`.
Для другого host-порта изменить `APP_PORT` в `.env`, задать `BASE_URL` для `wait_ready.py`, `verify.py`, `grade3.py` и явно передать его публичным тестам. Безопасная остановка с сохранением данных: `docker compose down`. **Намеренное удаление всех учебных данных этого Compose-проекта:** `docker compose down --volumes`. При проверках отказов volumes не удаляются. У ЛР4 отдельное имя проекта `mtusi-lab4-osokin`.
Закреплённые версии образов: Python `3.13.7-slim-bookworm`, PostgreSQL `17.6-bookworm`, Redis `7.4.5-bookworm`, HAProxy `3.0.16-alpine`, MinIO `RELEASE.2025-04-22T22-12-26Z`. Зависимости в `requirements.txt`, в том числе psycopg 3.2.10, psycopg-pool 3.2.6, Redis client 6.4.0, boto3 1.40.6, Pillow 11.3.0. Фактическое окружение и digest инфраструктурных образов: [environment.json](evidence/environment.json).
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
Клиент → 127.0.0.1:8084 → HAProxy → API × 2
├─ PostgreSQL → pgdata
├─ Redis → redisdata (сессии, AOF always)
└─ MinIO → s3data (приватный bucket)
```
PostgreSQL — источник истины для пользователей, чатов, участников, сообщений, результатов идемпотентных запросов и метаданных вложений. Общий Redis хранит серверные сессии с абсолютным expiry и AOF `appendfsync always`; restart не продлевает TTL. У API нет volume с оригиналами или обязательного локального состояния. Все зависимости находятся в общем `compose.yaml`, backend-сеть внутренняя. Публичный порт есть только у LB. Реальные процессы, порты, лимиты и mounts: [processes.txt](evidence/processes.txt).
Сохранён минимум ЛР3: Bearer-сессии, scrypt, миграции PostgreSQL, keyset pagination по `(created_at,id)`, общие данные, `X-Instance-Id` и общий `Idempotency-Key` минимум на 24 часа. Advisory transaction lock защищает одновременные повторы: создание сообщения и сохранение результата атомарны. Отсутствующий `attachment_ids` и `[]` дают одинаковый fingerprint. Повтор заново проверяет сессию и участие в чате.
Новое в ЛР4:
- `003_attachments.sql`: Attachment с UUID, chat/owner, отображаемым filename, внутренним object_key, MIME, размером, SHA-256 и временем. В Message добавлен массив UUID вложений с пустым default, старые текстовые сообщения валидны. Приложение проверяет существование всех вложений именно этого чата внутри транзакции создания Message. Удаление вложений в обязательный API не входит.
- Binary POST принимает PNG/JPEG. До чтения body проверяются сессия, доступ к чату, filename и Content-Length ≤10 MiB. Transfer-Encoding и неправильная длина отклоняются. Чтение ограничено; временных файлов нет. Pillow открывает header лениво: формат и ≤40 млн пикселей проверяются до decode, MIME сверяется с содержимым; далее `verify()` и `load()` проверяют изображение. Filename 1–255 code points, без `/`, `\`, NUL и управляющих символов.
- Сервер независимо от filename генерирует случайный S3 key `originals/<uuid>`, считает lowercase SHA-256 и отправляет неизменённые bytes в приватный bucket. После успешного PUT выполняются INSERT metadata и COMMIT. Только после этого возвращается **201 ready**, `variants=[]`, `error_code=null`.
- Metadata/content читаются только участником чата через JOIN с chat_members. Посторонний/неизвестный объект — 404. Download через авторизованный API, без redirect, с истинным MIME и проверкой размера/hash. `variant=original` поддерживается, других вариантов нет. Публичный API не раскрывает object key и credentials.
Общей распределённой транзакции PostgreSQL/S3 нет. Если PUT не состоялся, metadata не создаются и 201 не выдаётся. После PUT до COMMIT возможен orphan. Потерянный HTTP-ответ может скрывать уже завершённую операцию; самостоятельный повтор upload может создать второй объект. Автоматическая компенсация, сверка и очистка с grace period не реализованы — это дополнительный уровень 4/5.
`/health/live` проверяет процесс. `/health/ready` проверяет PostgreSQL, Redis и доступность bucket MinIO, поскольку S3 участвует в обязательном синхронном пути. Ошибки S3 преобразуются в 503 общего формата. При потере S3 HAProxy может исключить обе реплики; это консервативная readiness. В таком сценарии ответ самого API проверяется из контейнера без публикации порта в обход LB.
| Переменные `.env.example` | Использование |
| --- | --- |
| APP_HOST / APP_PORT / APP_ENV | API внутри слушает 0.0.0.0:8080; APP_PORT задаёт host-порт LB (bootstrap: 8084); APP_ENV=lab |
| SESSION_TTL_SECONDS | TTL сессии 3600 s |
| SESSION_STORE / CACHE_URL_FILE | В Compose выбран redis вместо postgres из примера; адрес из secret-файла |
| DATABASE_URL_FILE | DSN из /run/secrets/database_url |
| API_REPLICAS | scale API, default 2 |
| IDEMPOTENCY_TTL_SECONDS | минимум 86400 s |
| S3_ENDPOINT / S3_REGION / S3_BUCKET | http://s3:9000 / us-east-1 / messenger-attachments |
| S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE | credentials через Compose secrets |
| MAX_UPLOAD_BYTES / MAX_IMAGE_PIXELS | 10485760 / 40000000; повышать сверх контракта нельзя |
Дополнительно CURSOR_KEY_FILE задаёт общий ключ HMAC. У каждой API 4 workers и очередь до 32 соединений. Используются credentials учебного MinIO без отдельной ограниченной S3-роли уровня 4.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Целостность skeleton | `python -X utf8 scripts/check_contract.py` | PASS | [check.txt](evidence/check.txt) |
| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8084` | PASS: базовый API, upload ready, checksum и ACL | [smoke.txt](evidence/smoke.txt) |
| Минимум ЛР3: две реплики, общая сессия, идемпотентность | `docker compose exec -T api python student-tests/test_idempotency.py` | 16 одновременных повторов через 2 процесса создали 1 Message | [idempotency.txt](evidence/idempotency.txt) |
| PostgreSQL metadata, private S3 bytes, persistent volumes | `scripts/labctl.py state` и private probe в grade3.py | API без volume оригиналов, три data volumes; anonymous S3 GET=403 | [processes.txt](evidence/processes.txt), [private-and-formats.json](evidence/private-and-formats.json) |
| Реальные PNG/JPEG и byte-identical download | `python -X utf8 student-tests/grade3.py` | Оба формата приняты, оригиналы скачаны без изменения | [private-and-formats.json](evidence/private-and-formats.json) |
| Лимиты bytes/pixels до чтения/decode, filename | `docker compose exec -T api python student-tests/test_limits.py` | Точно 10 MiB и 40 млн пикселей приняты; превышение отклонено до read/decode; опасные имена отклонены | [limits.txt](evidence/limits.txt) |
| Upload → остановка именно загрузившей API → download другой | `python -X utf8 student-tests/grade3.py` | Разные X-Instance-Id, uploader exited, оригинал и Message доступны второму участнику | [upload-failover.json](evidence/upload-failover.json) |
| Чужой пользователь | Public smoke и grade3.py | Metadata/content возвращают 404 | [upload-failover.json](evidence/upload-failover.json), [smoke.txt](evidence/smoke.txt) |
| Restart всего стенда | Публичный persistence с `scripts/restart.cmd` | PASS с прежним токеном, metadata/bytes и сообщениями; StartedAt изменён, volumes прежние | [persistence.txt](evidence/persistence.txt), [restart.json](evidence/restart.json) |
| Публичный failover с записью | Failover с `scripts/stop-one.cmd` | PASS, 99 записей, 4 переходные ошибки, восстановление 5.198 s, без потерь подтверждённых записей и дублей | [failover.txt](evidence/failover.txt), [stop-one.json](evidence/stop-one.json) |
| Недоступность S3 | `python -X utf8 student-tests/grade3.py` | live/metadata 200, ready/upload/content 503; после start прежний файл доступен | [s3-outage.json](evidence/s3-outage.json) |
| Стенд восстановлен | Финальный public smoke полного verify | PASS | [smoke-after.txt](evidence/smoke-after.txt) |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Публичные `tests/`, обязательные `contracts/`, README и COURSE сохранены. Проверки выполнены на реальных контейнерах. Уровни 4/5 не заявлены. Результаты полного прогона — в `evidence/`.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Исследования производительности уровня 4/5 не проводились. Функциональные проверки выполнялись с синтетическими данными. Граничные изображения проверены последовательно: PNG 10 MiB с сохранением trailing bytes и PNG 8000×5000. Это проверка допустимого входа, не исследование параллельной загрузки.
Docker VM: 8 CPU, примерно 7.7 GiB RAM, Linux x86_64, Engine 29.8.2; host Windows. Лимиты: каждая API 1 CPU / 768 MiB; PostgreSQL 2 CPU / 512 MiB; Redis 0.5 CPU / 128 MiB; HAProxy 0.5 CPU / 128 MiB; MinIO 1 CPU / 512 MiB. Версии/digest/kernel — в environment.json. Перед отказами выполнены ожидание readiness и smoke, отдельного нагрузочного прогрева не было.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Остановить uploader API | Скачать оригинал через другую | X-Instance-Id различается, bytes совпадают, attached Message прочитан | Без потери подтверждённого файла; затем явный restore |
| Остановить MinIO | Не вернуть ложный 201, корректно сообщить недоступность | ready/upload/content=503; live/metadata=200; после start прежний токен скачал тот же файл | 20.312 s на весь сценарий с Docker CLI, проверками и восстановлением LB; original сохранён |
| Restart API, PostgreSQL, Redis, MinIO, LB | Сохранить сессию, metadata, bytes и сообщения | Public persistence PASS; StartedAt всех работающих сервисов изменён, volumes те же | Данные не потеряны, публичный тест дождался готовности в своём бюджете 90 s |
| Остановить API во время 15 s записи | Восстановить запись ≤10 s, сохранить подтверждения | Public failover PASS, 99 успешных записей, 4 переходные ошибки | 5.198 s; потерь подтверждённых сообщений/дублей нет |
Public failover последовательно отправляет сообщения с паузой 0.1 s; неопределённый POST повторяет с прежним ключом. Эти результаты не выдаются за RPS-бенчмарк, p50/p95/p99 или пик производительности. 20.312 s для S3 включает команды Docker и проверки, а не только время восстановления S3.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один Docker-host и одиночные PostgreSQL/Redis/LB/MinIO: потерю host или volume две API не перекрывают.
- Между S3 PUT и DB COMMIT возможен orphan. Автоматическая компенсация, reconciler и очистка с grace period не реализованы.
- Body буферизуется до 10 MiB, decode ограничен 40 млн пикселей. Нет эксперимента ≥10 одновременных uploads и измерения пика памяти; RAM-лимит контейнера не гарантирует успех любой комбинации больших конкурентных decode.
- API использует root credentials учебного MinIO. Отдельных ограниченных S3 credentials уровня 4 нет. S3/консоль не опубликованы на host в соответствии с общей изоляцией стенда.
- Presigned upload, thumbnails, worker, варианты изображений и исследовательские треки не реализованы. Дополнительная полная матрица негативных тестов уровня 4 не заявлена.
- HTTP допускается только для локального учебного стенда. Это обязательный минимум на 3 без заявления о production-ready/HA.
## Источники
- [Задание ЛР4](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4), локальные COURSE/CONTRIBUTING, OpenAPI, contracts/README и tests/README.
- [Собственная ЛР3](https://gitea.privcloud.tech/glebossokin/mtusi2026lab3/src/branch/solution/BVT2302-Osokin).
- [Pillow Image: open/verify/load и pixel limits](https://pillow.readthedocs.io/en/stable/reference/Image.html).
- [Boto3 S3 PutObject](https://docs.aws.amazon.com/boto3/latest/reference/services/s3/client/put_object.html).
- [MinIO](https://github.com/minio/minio).
+130
View File
@@ -0,0 +1,130 @@
name: mtusi-lab4-osokin
x-app: &app
image: mtusi-messenger:lab4
build: .
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
SESSION_STORE: redis
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_KEY_FILE: /run/secrets/cursor_key
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
WORKERS: 4
secrets: [database_url, cache_url, cursor_key, s3_access_key_id, s3_secret_access_key]
mem_limit: 768m
cpus: 1
dns_opt: ['timeout:1', 'attempts:1']
networks: [backend]
services:
s3:
image: minio/minio:RELEASE.2025-04-22T22-12-26Z
command: [server, /data, --console-address, ':9001']
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_access_key_id
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_secret_access_key
secrets: [s3_access_key_id, s3_secret_access_key]
volumes: [s3data:/data]
healthcheck:
test: [CMD, curl, -f, http://localhost:9000/minio/health/live]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: 1
networks: [backend]
db:
image: postgres:17.6-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
secrets: [db_password]
volumes: [pgdata:/var/lib/postgresql/data]
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: 2
networks: [backend]
redis:
image: redis:7.4.5-bookworm
command: [redis-server, /usr/local/etc/redis/redis.conf]
volumes:
- redisdata:/data
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 128m
cpus: 0.5
networks: [backend]
migrate:
<<: *app
command: [python, -m, messenger.migrate]
healthcheck: {disable: true}
depends_on:
db: {condition: service_healthy}
bucket:
<<: *app
command: [python, -m, messenger.bucket]
healthcheck: {disable: true}
depends_on:
s3: {condition: service_healthy}
api:
<<: *app
scale: ${API_REPLICAS:-2}
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
bucket: {condition: service_completed_successfully}
stop_grace_period: 10s
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
interval: 2s
timeout: 5s
start_period: 5s
retries: 2
lb:
image: haproxy:3.0.16-alpine
ports: ['127.0.0.1:${APP_PORT:-8080}:8080']
volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
depends_on:
api: {condition: service_healthy}
healthcheck:
test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
interval: 2s
timeout: 3s
retries: 5
stop_grace_period: 10s
mem_limit: 128m
cpus: 0.5
networks: [frontend, backend]
volumes:
s3data:
pgdata:
redisdata:
networks:
frontend:
backend:
internal: true
secrets:
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
db_password: {file: ./secrets/db_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
cursor_key: {file: ./secrets/cursor_key}
+7
View File
@@ -0,0 +1,7 @@
$ python student-tests/grade3.py
Container mtusi-lab4-osokin-api-1 Running
Container mtusi-lab4-osokin-api-2 Starting
Container mtusi-lab4-osokin-api-2 Started
PASS: uploading API stopped; another API returned identical original and attached message.
PASS: real PNG/JPEG originals and anonymous S3 403.
PASS: unavailable S3 yields 503; restart restores the same original and existing session.
+2
View File
@@ -0,0 +1,2 @@
$ python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+1
View File
@@ -0,0 +1 @@
$ docker compose --env-file .env config --quiet
+28
View File
@@ -0,0 +1,28 @@
{
"verified_at_utc": "2026-11-16T12:17:45.593148+00:00",
"host_os": "Windows-10-10.0.19045-SP0",
"client_python": "3.14.3",
"docker": {
"ServerVersion": "29.8.2",
"OperatingSystem": "Docker Desktop",
"KernelVersion": "6.18.54.1-microsoft-standard-WSL2",
"NCPU": 8,
"MemTotal": 8275030016,
"Architecture": "x86_64"
},
"images": {
"postgres:17.6-bookworm": [
"postgres@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3"
],
"redis:7.4.5-bookworm": [
"redis@sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb"
],
"haproxy:3.0.16-alpine": [
"haproxy@sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf"
],
"minio/minio:RELEASE.2025-04-22T22-12-26Z": [
"minio/minio@sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e"
]
},
"app_python": "Python 3.13.7"
}
+4
View File
@@ -0,0 +1,4 @@
$ python tests/resilience.py --mode failover --base-url http://127.0.0.1:8084 --action-script scripts/stop-one.cmd
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 4, "first_recovery_seconds": 5.198}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
$ docker compose exec -T api python student-tests/test_idempotency.py
{"result": "PASS", "parallel_requests": 16, "api_instances": ["2026dc4e851d-ef8f1bfddac8", "63b31780a94f-17a31e0697e3"], "unique_messages": 1, "scope_conflict_logout": "PASS"}
+3
View File
@@ -0,0 +1,3 @@
$ docker compose exec -T api python student-tests/test_limits.py
PASS: 10 MiB and 40,000,000 pixels accepted; excess bytes rejected before read;
excess dimensions rejected before decode; unsafe filenames rejected; Unicode filename accepted.
+3
View File
@@ -0,0 +1,3 @@
$ python tests/resilience.py --mode persistence --base-url http://127.0.0.1:8084 --action-script scripts/restart.cmd
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+15
View File
@@ -0,0 +1,15 @@
{
"anonymous_get": 403,
"real_images": [
{
"format": "PNG",
"bytes": 95,
"sha256": "506fcf4116edf75bba71fe37e2317dbe2c2fed5c3ed12505f37b3d088d5572c5"
},
{
"format": "JPEG",
"bytes": 645,
"sha256": "7cf106893f3c292cd0c385ed007e48e74919288e09e4674ab3c6efb1ff800ef7"
}
]
}
+155
View File
@@ -0,0 +1,155 @@
$ python scripts/labctl.py state
[
{
"id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4",
"name": "mtusi-lab4-osokin-api-1",
"service": "api",
"hostname": "2026dc4e851d",
"pid": 12852,
"started_at": "2026-11-16T12:17:20.776941959Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
},
{
"id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b",
"name": "mtusi-lab4-osokin-api-2",
"service": "api",
"hostname": "63b31780a94f",
"pid": 11211,
"started_at": "2026-11-16T12:16:40.901560588Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
},
{
"id": "6f7fa2489ec73b82edd15bd154a13fc5b6782a419b080588035554640799f0f5",
"name": "mtusi-lab4-osokin-bucket-1",
"service": "bucket",
"hostname": "6f7fa2489ec7",
"pid": 0,
"started_at": "2026-11-16T12:13:32.160735673Z",
"status": "exited",
"health": null,
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {},
"mounts": []
},
{
"id": "7a3000f4e2e3053f9be9a1e14172ad606d5cca86c2c1b6e2f6c10720c01444e1",
"name": "mtusi-lab4-osokin-db-1",
"service": "db",
"hostname": "7a3000f4e2e3",
"pid": 11171,
"started_at": "2026-11-16T12:16:40.543566533Z",
"status": "running",
"health": "healthy",
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
"memory_limit": 536870912,
"cpu_limit": 2.0,
"published_ports": {
"5432/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_pgdata",
"destination": "/var/lib/postgresql/data"
}
]
},
{
"id": "f69eb31bce350053a70a069732156ba5fea9360300676dbe567c017d6c1c9688",
"name": "mtusi-lab4-osokin-lb-1",
"service": "lb",
"hostname": "f69eb31bce35",
"pid": 11295,
"started_at": "2026-11-16T12:16:41.324174766Z",
"status": "running",
"health": "healthy",
"image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf",
"memory_limit": 134217728,
"cpu_limit": 0.5,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8084"
}
]
},
"mounts": []
},
{
"id": "e1f68174df2ed177dc6a002b23228d3859bdcf83b0734d9e6f7bdc8fcce4c81e",
"name": "mtusi-lab4-osokin-migrate-1",
"service": "migrate",
"hostname": "e1f68174df2e",
"pid": 0,
"started_at": "2026-11-16T12:13:31.670346003Z",
"status": "exited",
"health": null,
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {},
"mounts": []
},
{
"id": "ae03f444cf66bca154f467aea2333362aeea77fed94f819369fb5176a00ae922",
"name": "mtusi-lab4-osokin-redis-1",
"service": "redis",
"hostname": "ae03f444cf66",
"pid": 11162,
"started_at": "2026-11-16T12:16:40.390083828Z",
"status": "running",
"health": "healthy",
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
"memory_limit": 134217728,
"cpu_limit": 0.5,
"published_ports": {
"6379/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_redisdata",
"destination": "/data"
}
]
},
{
"id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900",
"name": "mtusi-lab4-osokin-s3-1",
"service": "s3",
"hostname": "c49b92fdbaae",
"pid": 11084,
"started_at": "2026-11-16T12:16:40.108761513Z",
"status": "running",
"health": "healthy",
"image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e",
"memory_limit": 536870912,
"cpu_limit": 1.0,
"published_ports": {
"9000/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_s3data",
"destination": "/data"
}
]
}
]
+250
View File
@@ -0,0 +1,250 @@
{
"before": [
{
"id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4",
"name": "mtusi-lab4-osokin-api-1",
"service": "api",
"hostname": "2026dc4e851d",
"pid": 3801,
"started_at": "2026-11-16T12:13:39.391673918Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
},
{
"id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b",
"name": "mtusi-lab4-osokin-api-2",
"service": "api",
"hostname": "63b31780a94f",
"pid": 8365,
"started_at": "2026-11-16T12:15:31.242158531Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
},
{
"id": "7a3000f4e2e3053f9be9a1e14172ad606d5cca86c2c1b6e2f6c10720c01444e1",
"name": "mtusi-lab4-osokin-db-1",
"service": "db",
"hostname": "7a3000f4e2e3",
"pid": 811,
"started_at": "2026-11-16T12:12:04.793750358Z",
"status": "running",
"health": "healthy",
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
"memory_limit": 536870912,
"cpu_limit": 2.0,
"published_ports": {
"5432/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_pgdata",
"destination": "/var/lib/postgresql/data"
}
]
},
{
"id": "f69eb31bce350053a70a069732156ba5fea9360300676dbe567c017d6c1c9688",
"name": "mtusi-lab4-osokin-lb-1",
"service": "lb",
"hostname": "f69eb31bce35",
"pid": 4289,
"started_at": "2026-11-16T12:13:52.556515025Z",
"status": "running",
"health": "healthy",
"image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf",
"memory_limit": 134217728,
"cpu_limit": 0.5,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8084"
}
]
},
"mounts": []
},
{
"id": "ae03f444cf66bca154f467aea2333362aeea77fed94f819369fb5176a00ae922",
"name": "mtusi-lab4-osokin-redis-1",
"service": "redis",
"hostname": "ae03f444cf66",
"pid": 959,
"started_at": "2026-11-16T12:12:07.441668202Z",
"status": "running",
"health": "healthy",
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
"memory_limit": 134217728,
"cpu_limit": 0.5,
"published_ports": {
"6379/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_redisdata",
"destination": "/data"
}
]
},
{
"id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900",
"name": "mtusi-lab4-osokin-s3-1",
"service": "s3",
"hostname": "c49b92fdbaae",
"pid": 10343,
"started_at": "2026-11-16T12:16:24.37991011Z",
"status": "running",
"health": "healthy",
"image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e",
"memory_limit": 536870912,
"cpu_limit": 1.0,
"published_ports": {
"9000/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_s3data",
"destination": "/data"
}
]
}
],
"after": [
{
"id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4",
"name": "mtusi-lab4-osokin-api-1",
"service": "api",
"hostname": "2026dc4e851d",
"pid": 11286,
"started_at": "2026-11-16T12:16:41.122280354Z",
"status": "running",
"health": "starting",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
},
{
"id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b",
"name": "mtusi-lab4-osokin-api-2",
"service": "api",
"hostname": "63b31780a94f",
"pid": 11211,
"started_at": "2026-11-16T12:16:40.901560588Z",
"status": "running",
"health": "starting",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
},
{
"id": "7a3000f4e2e3053f9be9a1e14172ad606d5cca86c2c1b6e2f6c10720c01444e1",
"name": "mtusi-lab4-osokin-db-1",
"service": "db",
"hostname": "7a3000f4e2e3",
"pid": 11171,
"started_at": "2026-11-16T12:16:40.543566533Z",
"status": "running",
"health": "healthy",
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
"memory_limit": 536870912,
"cpu_limit": 2.0,
"published_ports": {
"5432/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_pgdata",
"destination": "/var/lib/postgresql/data"
}
]
},
{
"id": "f69eb31bce350053a70a069732156ba5fea9360300676dbe567c017d6c1c9688",
"name": "mtusi-lab4-osokin-lb-1",
"service": "lb",
"hostname": "f69eb31bce35",
"pid": 11295,
"started_at": "2026-11-16T12:16:41.324174766Z",
"status": "running",
"health": "starting",
"image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf",
"memory_limit": 134217728,
"cpu_limit": 0.5,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8084"
}
]
},
"mounts": []
},
{
"id": "ae03f444cf66bca154f467aea2333362aeea77fed94f819369fb5176a00ae922",
"name": "mtusi-lab4-osokin-redis-1",
"service": "redis",
"hostname": "ae03f444cf66",
"pid": 11162,
"started_at": "2026-11-16T12:16:40.390083828Z",
"status": "running",
"health": "healthy",
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
"memory_limit": 134217728,
"cpu_limit": 0.5,
"published_ports": {
"6379/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_redisdata",
"destination": "/data"
}
]
},
{
"id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900",
"name": "mtusi-lab4-osokin-s3-1",
"service": "s3",
"hostname": "c49b92fdbaae",
"pid": 11084,
"started_at": "2026-11-16T12:16:40.108761513Z",
"status": "running",
"health": "starting",
"image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e",
"memory_limit": 536870912,
"cpu_limit": 1.0,
"published_ports": {
"9000/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_s3data",
"destination": "/data"
}
]
}
]
}
+4
View File
@@ -0,0 +1,4 @@
$ python scripts/labctl.py restore
Container mtusi-lab4-osokin-api-2 Running
Container mtusi-lab4-osokin-api-1 Starting
Container mtusi-lab4-osokin-api-1 Started
+59
View File
@@ -0,0 +1,59 @@
{
"before": [
{
"id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900",
"name": "mtusi-lab4-osokin-s3-1",
"service": "s3",
"hostname": "c49b92fdbaae",
"pid": 891,
"started_at": "2026-11-16T12:12:06.319676602Z",
"status": "running",
"health": "healthy",
"image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e",
"memory_limit": 536870912,
"cpu_limit": 1.0,
"published_ports": {
"9000/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_s3data",
"destination": "/data"
}
]
}
],
"after": [
{
"id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900",
"name": "mtusi-lab4-osokin-s3-1",
"service": "s3",
"hostname": "c49b92fdbaae",
"pid": 10343,
"started_at": "2026-11-16T12:16:24.37991011Z",
"status": "running",
"health": "starting",
"image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e",
"memory_limit": 536870912,
"cpu_limit": 1.0,
"published_ports": {
"9000/tcp": null
},
"mounts": [
{
"name": "mtusi-lab4-osokin_s3data",
"destination": "/data"
}
]
}
],
"observation": {
"live": 200,
"ready": 503,
"upload": 503,
"content": 503,
"metadata": 200
},
"recovered_within_seconds": 20.312,
"original_preserved": true
}
+2
View File
@@ -0,0 +1,2 @@
$ python tests/smoke.py --base-url http://127.0.0.1:8084
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+2
View File
@@ -0,0 +1,2 @@
$ python tests/smoke.py --base-url http://127.0.0.1:8084
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+74
View File
@@ -0,0 +1,74 @@
{
"target_instance": "2026dc4e851d-9723d8bbbf19",
"before": [
{
"id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4",
"name": "mtusi-lab4-osokin-api-1",
"service": "api",
"hostname": "2026dc4e851d",
"pid": 11286,
"started_at": "2026-11-16T12:16:41.122280354Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
},
{
"id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b",
"name": "mtusi-lab4-osokin-api-2",
"service": "api",
"hostname": "63b31780a94f",
"pid": 11211,
"started_at": "2026-11-16T12:16:40.901560588Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
}
],
"after": [
{
"id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4",
"name": "mtusi-lab4-osokin-api-1",
"service": "api",
"hostname": "2026dc4e851d",
"pid": 0,
"started_at": "2026-11-16T12:16:41.122280354Z",
"status": "exited",
"health": "unhealthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {},
"mounts": []
},
{
"id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b",
"name": "mtusi-lab4-osokin-api-2",
"service": "api",
"hostname": "63b31780a94f",
"pid": 11211,
"started_at": "2026-11-16T12:16:40.901560588Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
}
],
"hook_seconds": 2.982078799999954
}
+80
View File
@@ -0,0 +1,80 @@
{
"uploader": "63b31780a94f-17a31e0697e3",
"downloader": "2026dc4e851d-ef8f1bfddac8",
"before": [
{
"id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4",
"name": "mtusi-lab4-osokin-api-1",
"service": "api",
"hostname": "2026dc4e851d",
"pid": 3801,
"started_at": "2026-11-16T12:13:39.391673918Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
},
{
"id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b",
"name": "mtusi-lab4-osokin-api-2",
"service": "api",
"hostname": "63b31780a94f",
"pid": 3747,
"started_at": "2026-11-16T12:13:38.666975081Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
}
],
"after": [
{
"id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4",
"name": "mtusi-lab4-osokin-api-1",
"service": "api",
"hostname": "2026dc4e851d",
"pid": 3801,
"started_at": "2026-11-16T12:13:39.391673918Z",
"status": "running",
"health": "healthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {
"8080/tcp": null
},
"mounts": []
},
{
"id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b",
"name": "mtusi-lab4-osokin-api-2",
"service": "api",
"hostname": "63b31780a94f",
"pid": 0,
"started_at": "2026-11-16T12:13:38.666975081Z",
"status": "exited",
"health": "unhealthy",
"image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d",
"memory_limit": 805306368,
"cpu_limit": 1.0,
"published_ports": {},
"mounts": []
}
],
"bytes": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"same_original": true,
"attached_message_read": true,
"stranger_metadata": 404,
"stranger_content": 404
}
+1
View File
@@ -0,0 +1 @@
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+38
View File
@@ -0,0 +1,38 @@
global
maxconn 256
log stdout format raw local0 warning
defaults
mode http
log global
timeout connect 500ms
timeout client 8s
timeout server 8s
timeout http-request 3s
timeout queue 1s
timeout check 1s
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 2
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
hold timeout 1s
hold refused 1s
hold other 1s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fastinter 1s downinter 1s fall 2 rise 1
server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
+10
View File
@@ -0,0 +1,10 @@
bind 0.0.0.0
protected-mode no
port 6379
dir /data
appendonly yes
appendfsync always
save ""
maxmemory 64mb
maxmemory-policy noeviction
# Only the private Compose network can reach Redis; no host port is published.
+30
View File
@@ -0,0 +1,30 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX chat_members_user ON chat_members(user_id, chat_id);
CREATE INDEX chats_order ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_history ON messages(chat_id, created_at, id);
+12
View File
@@ -0,0 +1,12 @@
-- Additive migration: old lab2 tables and rows are unchanged.
-- This table is only for POST /chats/{chat_id}/messages; method/path scope is implicit.
CREATE TABLE message_requests (
actor_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
fingerprint text NOT NULL,
response jsonb NOT NULL,
expires_at timestamptz NOT NULL,
PRIMARY KEY (actor_id, chat_id, key)
);
CREATE INDEX message_requests_expiry ON message_requests(expires_at);
+19
View File
@@ -0,0 +1,19 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL,
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
object_key text NOT NULL UNIQUE,
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX attachments_chat ON attachments(chat_id);
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (
char_length(text) <= 4000 AND (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0)
);
ALTER TABLE messages ADD CONSTRAINT messages_attachment_count CHECK (cardinality(attachment_ids) <= 10);
+7
View File
@@ -0,0 +1,7 @@
psycopg==3.2.10
psycopg-binary==3.2.10
psycopg-pool==3.2.6
redis==6.4.0
typing_extensions==4.15.0
boto3==1.40.6
Pillow==11.3.0
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env python3
"""Create local synthetic credentials once; never print them or replace existing files."""
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
folder = root / 'secrets'
folder.mkdir(exist_ok=True)
password_file = folder / 'db_password'
if not password_file.exists():
password_file.write_text(secrets.token_hex(24), encoding='utf-8')
password = password_file.read_text(encoding='utf-8').strip()
values = {
'database_url': 'postgresql://messenger:' + password + '@db:5432/messenger',
'cache_url': 'redis://redis:6379/0',
'cursor_key': secrets.token_hex(32),
's3_access_key_id': 'lab4-' + secrets.token_hex(8),
's3_secret_access_key': secrets.token_hex(24),
}
for name, value in values.items():
target = folder / name
if not target.exists():
target.write_text(value, encoding='utf-8')
if not (root / '.env').exists():
(root / '.env').write_text('APP_HOST=0.0.0.0\nAPP_PORT=8084\nAPP_ENV=lab\nSESSION_TTL_SECONDS=3600\nSESSION_STORE=redis\nAPI_REPLICAS=2\n', encoding='utf-8')
print('Local .env and secrets ready (existing values preserved).')
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env python3
"""Operate only the lab4 Compose project; restart hooks never delete volumes."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8').strip()
def compose(*args):
return command('docker', 'compose', '--env-file', '.env', *args)
def containers(service=None, all_states=False):
ids = compose('ps', '-q', *(['--all'] if all_states else []), *([service] if service else [])).split()
if not ids:
return []
result = []
for item in json.loads(command('docker', 'inspect', *ids)):
result.append(dict(id=item['Id'], name=item['Name'].lstrip('/'),
service=item['Config']['Labels']['com.docker.compose.service'],
hostname=item['Config']['Hostname'], pid=item['State']['Pid'],
started_at=item['State']['StartedAt'], status=item['State']['Status'],
health=item['State'].get('Health', {}).get('Status'), image=item['Image'],
memory_limit=item['HostConfig']['Memory'],
cpu_limit=item['HostConfig']['NanoCpus'] / 1e9,
published_ports=item['NetworkSettings']['Ports'],
mounts=[dict(name=m.get('Name'), destination=m['Destination'])
for m in item['Mounts'] if m['Type'] == 'volume']))
return sorted(result, key=lambda x: x['name'])
def save(name, data):
folder = ROOT / 'evidence'
folder.mkdir(exist_ok=True)
(folder / name).write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
def restore():
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
items = containers('api')
if len(items) == 2 and all(x['health'] == 'healthy' for x in items):
time.sleep(3)
return
time.sleep(.5)
raise RuntimeError('Two API replicas did not become healthy')
def stop_one():
target = os.environ.get('TARGET_INSTANCE_ID')
before = containers('api')
assert len(before) >= 2, 'Need two live replicas'
selected = [x for x in before if target and target.startswith(x['hostname'] + '-')]
assert len(selected) == 1, 'TARGET_INSTANCE_ID must identify one live API process'
started = time.monotonic()
command('docker', 'stop', '--time', '8', selected[0]['id'])
after = containers('api', True)
assert next(x for x in after if x['id'] == selected[0]['id'])['status'] == 'exited'
save('stop-one.json', dict(target_instance=target, before=before, after=after,
hook_seconds=time.monotonic() - started))
print('Selected API stopped; another replica remains running.')
def restart():
before = containers()
compose('restart', 'api', 'db', 'redis', 's3', 'lb')
after = containers()
for old in before:
new = next(x for x in after if x['id'] == old['id'])
assert old['started_at'] != new['started_at'], old['name'] + ' did not restart'
assert old['mounts'] == new['mounts'], 'Persistent volumes changed'
save('restart.json', dict(before=before, after=after))
print('API, PostgreSQL, Redis, MinIO and LB restarted; same volumes preserved.')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('action', choices=['restart', 'stop-one', 'restore', 'state'])
action = parser.parse_args().action
if action == 'state':
print(json.dumps(containers(all_states=True), indent=2))
else:
{'restart': restart, 'stop-one': stop_one, 'restore': restore}[action]()
+4
View File
@@ -0,0 +1,4 @@
@echo off
cd /d "%~dp0\.."
python -X utf8 scripts\labctl.py restart
exit /b %errorlevel%
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/labctl.py restart
+4
View File
@@ -0,0 +1,4 @@
@echo off
cd /d "%~dp0\.."
python -X utf8 scripts\labctl.py stop-one
exit /b %errorlevel%
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/labctl.py stop-one
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Run the grade 3 checks and keep small sanitized, reproducible evidence files."""
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import platform
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / 'evidence'
BASE = os.getenv('BASE_URL', 'http://127.0.0.1:8084')
ENV = dict(os.environ, BASE_URL=BASE, PYTHONUTF8='1', PYTHONIOENCODING='utf-8')
def run(name, args):
print('Running', name, flush=True)
proc = subprocess.run(args, cwd=ROOT, env=ENV, text=True, encoding='utf-8',
errors='replace', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = '$ ' + ' '.join(args).replace(sys.executable, 'python') + '\n' + proc.stdout
output = '\n'.join(line.rstrip() for line in output.splitlines()).rstrip() + '\n'
(EVIDENCE / (name + '.txt')).write_text(output, encoding='utf-8')
print(proc.stdout, end='', flush=True)
if proc.returncode:
raise SystemExit('Check failed: ' + name)
def main():
EVIDENCE.mkdir(exist_ok=True)
run('check', [sys.executable, 'scripts/check_contract.py'])
run('compose', ['docker', 'compose', '--env-file', '.env', 'config', '--quiet'])
run('smoke', [sys.executable, 'tests/smoke.py', '--base-url', BASE])
run('idempotency', ['docker', 'compose', 'exec', '-T', 'api', 'python', 'student-tests/test_idempotency.py'])
run('limits', ['docker', 'compose', 'exec', '-T', 'api', 'python', 'student-tests/test_limits.py'])
run('attachments-grade3', [sys.executable, 'student-tests/grade3.py'])
run('persistence', [sys.executable, 'tests/resilience.py', '--mode', 'persistence', '--base-url', BASE,
'--action-script', 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'])
run('failover', [sys.executable, 'tests/resilience.py', '--mode', 'failover', '--base-url', BASE,
'--action-script', 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh'])
run('restore', [sys.executable, 'scripts/labctl.py', 'restore'])
run('smoke-after', [sys.executable, 'tests/smoke.py', '--base-url', BASE])
run('processes', [sys.executable, 'scripts/labctl.py', 'state'])
info = json.loads(subprocess.check_output(['docker', 'info', '--format', '{{json .}}'], env=ENV, text=True))
environment = dict(verified_at_utc=datetime.now(timezone.utc).isoformat(), host_os=platform.platform(),
client_python=platform.python_version(),
docker={k: info[k] for k in ('ServerVersion', 'OperatingSystem', 'KernelVersion',
'NCPU', 'MemTotal', 'Architecture')}, images={})
environment['app_python'] = subprocess.check_output(
['docker', 'compose', 'exec', '-T', 'api', 'python', '--version'], text=True, env=ENV).strip()
for name in ('postgres:17.6-bookworm', 'redis:7.4.5-bookworm', 'haproxy:3.0.16-alpine',
'minio/minio:RELEASE.2025-04-22T22-12-26Z'):
environment['images'][name] = json.loads(subprocess.check_output(
['docker', 'image', 'inspect', name, '--format', '{{json .RepoDigests}}'], text=True, env=ENV))
(EVIDENCE / 'environment.json').write_text(json.dumps(environment, indent=2) + '\n', encoding='utf-8')
print('PASS: all required grade 3 scenarios completed.')
if __name__ == '__main__':
main()
+19
View File
@@ -0,0 +1,19 @@
"""Wait for the public entrypoint after starting the Compose stack."""
import os
import time
import urllib.error
import urllib.request
base = os.getenv('BASE_URL', 'http://127.0.0.1:8084').rstrip('/')
deadline = time.monotonic() + 90
while True:
try:
with urllib.request.urlopen(base + '/health/ready', timeout=3) as response:
if response.status == 200:
print('READY:', base)
break
except (urllib.error.URLError, OSError):
pass
if time.monotonic() >= deadline:
raise SystemExit('Public API did not become ready within 90 seconds')
time.sleep(.5)
+1
View File
@@ -0,0 +1 @@
"""Lab 1: bounded, single-process in-memory messenger."""
+102
View File
@@ -0,0 +1,102 @@
"""Synchronous original-only uploads. PostgreSQL holds metadata; S3 holds bytes."""
import hashlib
from io import BytesIO
import os
import unicodedata
import uuid
import warnings
import boto3
from botocore.config import Config as S3Config
from PIL import Image, UnidentifiedImageError
from .core import APIError, public_row, secret_file, string, uuid_value
def s3_client():
return boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'],
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=secret_file('S3_ACCESS_KEY_ID_FILE'),
aws_secret_access_key=secret_file('S3_SECRET_ACCESS_KEY_FILE'),
config=S3Config(connect_timeout=1, read_timeout=2,
retries={'total_max_attempts': 1},
s3={'addressing_style': 'path'},
request_checksum_calculation='when_required',
response_checksum_validation='when_required'))
def filename_value(value):
value = string(value, 255)
if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value):
raise APIError(400, 'Invalid filename')
return value
def validate_image(raw, declared_type, max_pixels):
try:
with warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
# open parses the header lazily. Check dimensions before load/verify.
with Image.open(BytesIO(raw), formats=['PNG', 'JPEG']) as image:
if image.width * image.height > max_pixels:
raise APIError(413, 'Image exceeds pixel limit')
actual_type = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}[image.format]
if actual_type != declared_type:
raise APIError(415, 'Image type differs from Content-Type')
image.verify()
# JPEG verify does not decode entropy data; load catches truncated files.
with Image.open(BytesIO(raw), formats=['PNG', 'JPEG']) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise APIError(413, 'Image exceeds pixel limit') from None
except (UnidentifiedImageError, OSError, SyntaxError, ValueError):
raise APIError(415, 'Invalid PNG/JPEG') from None
return actual_type
def metadata(row):
result = public_row({k: v for k, v in row.items() if k != 'object_key'})
return dict(result, status='ready', variants=[], error_code=None, updated_at=result['created_at'])
class Attachments:
def __init__(self, store):
self.store = store
self.s3 = s3_client()
self.bucket = os.environ['S3_BUCKET']
def ready(self):
self.s3.head_bucket(Bucket=self.bucket)
def upload(self, actor, chat_id, filename, content_type, raw):
actual = validate_image(raw, content_type, self.store.config.max_image_pixels)
identifier = uuid.uuid4()
key = 'originals/' + uuid.uuid4().hex
digest = hashlib.sha256(raw).hexdigest()
# No shared transaction with S3: failures after PUT may leave an orphan.
# Grade 3 intentionally has no automatic reconciliation/cleanup mechanism.
self.s3.put_object(Bucket=self.bucket, Key=key, Body=raw, ContentType=actual)
with self.store.pool.connection() as conn:
row = conn.execute('INSERT INTO attachments '
'(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) '
'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *',
(identifier, chat_id, actor, filename, key, actual, len(raw), digest)).fetchone()
return metadata(row)
def get(self, actor, identifier):
identifier = uuid_value(identifier)
with self.store.pool.connection() as conn:
row = conn.execute('SELECT a.* FROM attachments a JOIN chat_members m '
'ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',
(identifier, actor)).fetchone()
if row is None:
raise APIError(404, 'Object not found')
return row
def content(self, row):
response = self.s3.get_object(Bucket=self.bucket, Key=row['object_key'])
with response['Body'] as stream:
raw = stream.read(self.store.config.max_upload_bytes + 1)
if len(raw) != row['size_bytes'] or hashlib.sha256(raw).hexdigest() != row['sha256']:
raise APIError(503, 'Original temporarily unavailable')
return raw
+21
View File
@@ -0,0 +1,21 @@
"""Create a private bucket on first deployment. Existing buckets need no bootstrap."""
import os
from .attachments import s3_client
def main():
s3 = s3_client()
try:
bucket = os.environ['S3_BUCKET']
# This Compose project owns a new volume/bucket. For subsequent runs,
# list names first; do not modify existing policies or objects.
names = [item['Name'] for item in s3.list_buckets()['Buckets']]
if bucket not in names:
s3.create_bucket(Bucket=bucket)
print('Private bucket available; no anonymous policy is installed.')
finally:
s3.close()
if __name__ == '__main__':
main()
+356
View File
@@ -0,0 +1,356 @@
"""Validation and persistent state. HTTP 201 is emitted after transaction exit/COMMIT."""
import base64
from dataclasses import dataclass, fields
from datetime import datetime, timezone
import hashlib
import hmac
import json
import os
from pathlib import Path
import re
import secrets
import uuid
import psycopg
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from psycopg_pool import ConnectionPool
import redis
from redis.backoff import NoBackoff
from redis.retry import Retry
class APIError(Exception):
def __init__(self, status, message):
self.status, self.message = status, message
@dataclass
class Config:
app_host: str = '0.0.0.0'
app_port: int = 8080
workers: int = 16
request_queue: int = 32
socket_timeout: int = 2
max_body_bytes: int = 65536
session_ttl_seconds: int = 3600
idempotency_ttl_seconds: int = 86400
max_upload_bytes: int = 10485760
max_image_pixels: int = 40000000
@classmethod
def from_env(cls):
values = {}
for field in fields(cls):
raw = os.environ.get(field.name.upper(), field.default)
values[field.name] = str(raw) if field.type is str else int(raw)
if field.type is int and values[field.name] < 1:
raise ValueError(field.name + ' must be positive')
result = cls(**values)
if result.app_port > 65535 or result.workers > 32 or result.request_queue > 128:
raise ValueError('port/workers/queue outside bounds')
if result.max_body_bytes > 1048576 or result.socket_timeout > 5:
raise ValueError('body <= 1 MiB and socket timeout <= 5 required')
if os.getenv('SESSION_STORE', 'redis') != 'redis':
raise ValueError('This solution uses Redis sessions')
if result.idempotency_ttl_seconds < 86400:
raise ValueError('Idempotency retention must be at least 24 hours')
if result.max_upload_bytes > 10485760 or result.max_image_pixels > 40000000:
raise ValueError('Upload limits exceed the contract')
return result
def secret_file(name):
return Path(os.environ[name]).read_text(encoding='utf-8').strip()
def db_connect_kwargs():
return dict(connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1000 '
'-c idle_in_transaction_session_timeout=5000', row_factory=dict_row)
def uuid_value(value):
if not isinstance(value, str) or not re.fullmatch(
r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value):
raise APIError(400, 'Invalid UUID')
return str(uuid.UUID(value))
def string(value, maximum):
if not isinstance(value, str) or not value.strip() or len(value) > maximum or '\x00' in value:
raise APIError(400, 'Invalid string')
try:
value.encode('utf-8')
except UnicodeError:
raise APIError(400, 'Invalid Unicode') from None
return value
def object_fields(body, expected):
if not isinstance(body, dict) or set(body) != set(expected):
raise APIError(400, 'Invalid object fields')
def password_value(value):
if not isinstance(value, str) or not value.strip() or not 12 <= len(value) <= 128:
raise APIError(400, 'Invalid password length')
try:
value.encode('utf-8')
except UnicodeError:
raise APIError(400, 'Invalid Unicode') from None
return value
def password_hash(value, salt=None):
salt = salt or secrets.token_bytes(16)
digest = hashlib.scrypt(value.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32)
return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex()
def password_matches(value, encoded):
# Only this versioned format is generated by registration.
salt = bytes.fromhex(encoded.split('$')[4])
return hmac.compare_digest(password_hash(value, salt), encoded)
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def public_row(row):
return {key: ([str(item) for item in value] if key == 'attachment_ids' else
str(value) if isinstance(value, uuid.UUID) else stamp(value)
if isinstance(value, datetime) else value)
for key, value in row.items() if key != 'password_hash'}
USER_COLUMNS = 'id, username, display_name, created_at'
MESSAGE_COLUMNS = 'id, chat_id, sender_id, text, created_at, attachment_ids'
# Shared by the real history endpoint and EXPLAIN evidence; all values are parameters.
HISTORY_SQL = ('SELECT ' + MESSAGE_COLUMNS + ' FROM messages WHERE chat_id = %s '
'AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s')
class Store:
def __init__(self, config):
self.config = config
from .attachments import Attachments
self.attachments = Attachments(self)
self.secret = bytes.fromhex(secret_file('CURSOR_KEY_FILE'))
if len(self.secret) < 32:
raise ValueError('Cursor key must be at least 256 bits')
self.pool = ConnectionPool(secret_file('DATABASE_URL_FILE'), min_size=1, max_size=8,
timeout=1, max_waiting=16, kwargs=db_connect_kwargs(),
check=ConnectionPool.check_connection, open=True)
self.cache_pool = redis.ConnectionPool.from_url(
secret_file('CACHE_URL_FILE'), max_connections=8, socket_connect_timeout=.5,
socket_timeout=.5, decode_responses=True, retry=Retry(NoBackoff(), 0))
self.cache = redis.Redis(connection_pool=self.cache_pool)
self.dummy_hash = password_hash('dummy-password-for-timing')
def close(self):
self.attachments.s3.close()
self.pool.close(timeout=2)
self.cache_pool.disconnect()
def ready(self):
with self.pool.connection() as conn:
conn.execute('SELECT 1')
self.cache.ping()
self.attachments.ready()
def create_user(self, body):
object_fields(body, ('username', 'display_name', 'password'))
username = string(body['username'], 32)
if not re.fullmatch(r'[a-z0-9_]{3,32}', username):
raise APIError(400, 'Invalid username')
display = string(body['display_name'], 100)
hashed = password_hash(password_value(body['password']))
try:
with self.pool.connection() as conn:
row = conn.execute('INSERT INTO users(id, username, display_name, password_hash) '
'VALUES (%s,%s,%s,%s) RETURNING ' + USER_COLUMNS,
(uuid.uuid4(), username, display, hashed)).fetchone()
except psycopg.errors.UniqueViolation:
raise APIError(409, 'Username already exists') from None
return public_row(row)
def login(self, authorization):
try:
scheme, encoded = authorization.split(' ', 1)
if scheme.lower() != 'basic' or len(encoded) > 1024:
raise ValueError()
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
if not re.fullmatch(r'[a-z0-9_]{3,32}', username):
raise ValueError()
password_value(password)
except (AttributeError, ValueError, UnicodeError, APIError):
raise APIError(401, 'Invalid credentials') from None
with self.pool.connection() as conn:
row = conn.execute('SELECT * FROM users WHERE username = %s', (username,)).fetchone()
matched = password_matches(password, row['password_hash'] if row else self.dummy_hash)
if not matched or row is None:
raise APIError(401, 'Invalid credentials')
token = secrets.token_urlsafe(32)
expires_ms = int(datetime.now(timezone.utc).timestamp() * 1000) + self.config.session_ttl_seconds * 1000
self.cache.set(self.session_key(token), str(row['id']), pxat=expires_ms)
return dict(token=token, token_type='Bearer',
expires_at=stamp(datetime.fromtimestamp(expires_ms / 1000, timezone.utc)),
user=public_row(row))
@staticmethod
def session_key(token):
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
def auth(self, header):
try:
scheme, token = header.split(' ', 1)
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
raise ValueError()
except (ValueError, AttributeError):
raise APIError(401, 'Invalid session') from None
key = self.session_key(token)
actor = self.cache.get(key)
if actor is None:
raise APIError(401, 'Invalid session')
return actor, key
def me(self, actor):
with self.pool.connection() as conn:
row = conn.execute('SELECT ' + USER_COLUMNS + ' FROM users WHERE id = %s', (actor,)).fetchone()
if row is None:
raise APIError(401, 'Invalid session')
return public_row(row)
def chat_access(self, actor, identifier):
identifier = uuid_value(identifier)
with self.pool.connection() as conn:
row = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s',
(identifier, actor)).fetchone()
if row is None:
raise APIError(404, 'Object not found')
return identifier
@staticmethod
def chat_row(conn, row):
members = conn.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position',
(row['id'],)).fetchall()
return dict(public_row(row), member_ids=[str(x['user_id']) for x in members])
def chat(self, chat_id):
with self.pool.connection() as conn:
row = conn.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone()
return self.chat_row(conn, row)
def create_chat(self, actor, body):
object_fields(body, ('title', 'member_ids'))
title = string(body['title'], 100)
members = body['member_ids']
if not isinstance(members, list) or not 1 <= len(members) <= 100:
raise APIError(400, 'Invalid members')
members = [uuid_value(x) for x in members]
if len(members) != len(set(members)) or actor not in members:
raise APIError(400, 'Invalid members')
try:
with self.pool.connection() as conn:
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *',
(uuid.uuid4(), title)).fetchone()
with conn.cursor() as cur:
cur.executemany('INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)',
[(row['id'], member, pos) for pos, member in enumerate(members)])
except psycopg.errors.ForeignKeyViolation:
raise APIError(400, 'Invalid members') from None
return dict(public_row(row), member_ids=members)
def create_message(self, actor, chat_id, body, key):
if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
raise APIError(400, 'One valid Idempotency-Key is required')
if not isinstance(body, dict) or set(body) not in ({'text'}, {'text', 'attachment_ids'}):
raise APIError(400, 'Invalid object fields')
ids = body.get('attachment_ids', [])
if not isinstance(ids, list) or len(ids) > 10:
raise APIError(400, 'Invalid attachment list')
ids = [uuid_value(item) for item in ids]
if len(ids) != len(set(ids)):
raise APIError(400, 'Duplicate attachment')
text = body['text']
if not isinstance(text, str) or len(text) > 4000 or '\x00' in text or (not text.strip() and not ids):
raise APIError(400, 'Invalid message text')
try:
text.encode('utf-8')
except UnicodeError:
raise APIError(400, 'Invalid Unicode') from None
digest = hashlib.sha256(json.dumps({'text': text, 'attachment_ids': ids},
ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
scope = json.dumps([actor, 'POST', '/api/v1/chats/' + chat_id + '/messages', key])
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], 'big', signed=True)
with self.pool.connection() as conn:
# Transaction lock spans SELECT, message INSERT, result INSERT and COMMIT.
# All replicas use PostgreSQL; a timeout rolls back and may be retried.
conn.execute('SELECT pg_advisory_xact_lock(%s)', (lock,))
previous = conn.execute('SELECT fingerprint, response, expires_at > clock_timestamp() AS valid '
'FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s',
(actor, chat_id, key)).fetchone()
if previous and previous['valid']:
if previous['fingerprint'] != digest:
raise APIError(409, 'Idempotency-Key already used with another body')
return previous['response']
if previous:
conn.execute('DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s',
(actor, chat_id, key))
if ids:
found = conn.execute('SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])',
(chat_id, ids)).fetchall()
if len(found) != len(ids):
raise APIError(404, 'Object not found')
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) '
'VALUES (%s,%s,%s,%s,%s::uuid[]) RETURNING ' + MESSAGE_COLUMNS,
(uuid.uuid4(), chat_id, actor, text, ids)).fetchone()
result = public_row(row)
conn.execute('INSERT INTO message_requests(actor_id,chat_id,key,fingerprint,response,expires_at) '
"VALUES (%s,%s,%s,%s,%s,clock_timestamp() + %s * interval '1 second')",
(actor, chat_id, key, digest, Jsonb(result), self.config.idempotency_ttl_seconds))
return result
def encode_cursor(self, scope, key):
payload = json.dumps([1, scope, *key], separators=(',', ':')).encode()
return base64.urlsafe_b64encode(payload + hmac.digest(self.secret, payload, 'sha256')).rstrip(b'=').decode()
def decode_cursor(self, cursor, scope):
try:
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', cursor):
raise ValueError()
raw = base64.b64decode(cursor + '=' * (-len(cursor) % 4), altchars=b'-_', validate=True)
if base64.urlsafe_b64encode(raw).rstrip(b'=').decode() != cursor:
raise ValueError()
payload, signature = raw[:-32], raw[-32:]
if not hmac.compare_digest(signature, hmac.digest(self.secret, payload, 'sha256')):
raise ValueError()
version, resource, timestamp, identifier = json.loads(payload)
if version != 1 or resource != scope or not isinstance(timestamp, str) or not timestamp.endswith('Z'):
raise ValueError()
return datetime.fromisoformat(timestamp.replace('Z', '+00:00')), uuid_value(identifier)
except (ValueError, TypeError, UnicodeError, APIError):
raise APIError(400, 'Invalid cursor') from None
def page(self, actor, chat_id, query):
if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
raise APIError(400, 'Invalid query')
limit = query.get('limit', ['50'])[0]
if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100:
raise APIError(400, 'Invalid limit')
limit = int(limit)
scope = 'messages:' + chat_id if chat_id else 'chats:' + actor
key = self.decode_cursor(query['cursor'][0], scope) if 'cursor' in query else (
datetime.min.replace(tzinfo=timezone.utc), '00000000-0000-0000-0000-000000000000')
with self.pool.connection() as conn:
if chat_id:
rows = conn.execute(HISTORY_SQL, (chat_id, *key, limit + 1)).fetchall()
items = [public_row(row) for row in rows[:limit]]
else:
rows = conn.execute('SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id '
'WHERE m.user_id=%s AND (c.created_at,c.id) > (%s,%s) '
'ORDER BY c.created_at,c.id LIMIT %s', (actor, *key, limit + 1)).fetchall()
items = [self.chat_row(conn, row) for row in rows[:limit]]
cursor = self.encode_cursor(scope, (items[-1]['created_at'], items[-1]['id'])) if len(rows) > limit else None
return dict(items=items, next_cursor=cursor)
+30
View File
@@ -0,0 +1,30 @@
"""Transactional, versioned migrations; repeat application is a no-op."""
import hashlib
from pathlib import Path
import psycopg
from .core import secret_file, db_connect_kwargs
def main():
with psycopg.connect(secret_file('DATABASE_URL_FILE'), **db_connect_kwargs()) as conn:
conn.execute('SELECT pg_advisory_xact_lock(2026, 2)')
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
'(name text PRIMARY KEY, sha256 text NOT NULL, '
'applied_at timestamptz NOT NULL DEFAULT clock_timestamp())')
for path in sorted((Path(__file__).resolve().parents[2] / 'migrations').glob('*.sql')):
content = path.read_bytes()
digest = hashlib.sha256(content).hexdigest()
row = conn.execute('SELECT sha256 FROM schema_migrations WHERE name=%s', (path.name,)).fetchone()
if row:
if row['sha256'] != digest:
raise RuntimeError('Applied migration checksum mismatch')
print('Already applied:', path.name)
continue
conn.execute(content.decode('utf-8'), prepare=False)
conn.execute('INSERT INTO schema_migrations(name,sha256) VALUES (%s,%s)', (path.name, digest))
print('Applied:', path.name)
print('Migrations committed.')
if __name__ == '__main__':
main()
+376
View File
@@ -0,0 +1,376 @@
"""Bounded HTTP worker pool; one request per connection; JSON logs omit bodies."""
from collections import Counter
from http.server import BaseHTTPRequestHandler, HTTPServer
import http.client
import json
import logging
import os
import queue
import signal
import socket
import threading
import time
from urllib.parse import parse_qs, urlsplit
import uuid
import psycopg
from psycopg_pool import PoolTimeout, TooManyRequests
import redis
from botocore.exceptions import BotoCoreError, ClientError
from .core import APIError, Config, Store
from .attachments import filename_value, metadata
CODES = {400: "invalid_request", 401: "unauthorized", 404: "not_found", 409: "conflict",
413: "payload_too_large", 415: "unsupported_media_type", 429: "rate_limited",
503: "unavailable", 500: "internal_error"}
def error_body(status, message, request_id):
return {"error": {"code": CODES.get(status, "invalid_request"),
"message": message, "request_id": request_id}}
def log_event(**data):
logging.info(json.dumps(data, ensure_ascii=True))
class Server(HTTPServer):
allow_reuse_address = True
def __init__(self, config):
self.config, self.store = config, Store(config)
self.instance_id = os.environ.get('HOSTNAME', 'api') + '-' + uuid.uuid4().hex[:12]
self.jobs = queue.Queue(maxsize=config.request_queue)
self.draining = threading.Event()
self.stats_lock = threading.Lock()
self.statuses = Counter()
self.active = 0
self.clients = set()
self.started = time.monotonic()
self.request_queue_size = config.request_queue
super().__init__((config.app_host, config.app_port), Handler)
self.threads = [threading.Thread(target=self.worker, name="http-" + str(n), daemon=True)
for n in range(config.workers)]
for thread in self.threads:
thread.start()
def get_request(self):
client, address = super().get_request()
client.settimeout(self.config.socket_timeout)
return client, address
def record(self, status):
with self.stats_lock:
self.statuses[str(status)] += 1
def process_request(self, request, address):
try:
if self.draining.is_set():
raise queue.Full()
self.jobs.put_nowait((request, address))
except queue.Full:
# Transport admission before HTTP parsing: bounded time, no extra thread.
identifier = str(uuid.uuid4())
body = json.dumps(error_body(503, "Request queue full or shutting down", identifier)).encode()
headers = ("HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
"Connection: close\r\nRetry-After: 1\r\nX-Request-Id: " + identifier +
"\r\nX-Instance-Id: " + self.instance_id +
"\r\nContent-Length: " + str(len(body)) + "\r\n\r\n").encode()
try:
request.settimeout(.05)
# Wait for the first request bytes: closing before a local client
# finishes sendall can otherwise discard the 503 with a TCP reset.
try:
request.recv(16384)
except OSError:
pass
request.sendall(headers + body)
request.shutdown(socket.SHUT_WR)
request.settimeout(.01)
for _ in range(5):
if not request.recv(16384):
break
except OSError:
pass
self.shutdown_request(request)
self.record(503)
log_event(event="queue_rejected", status=503, request_id=identifier)
def worker(self):
while not self.draining.is_set() or not self.jobs.empty():
try:
request, address = self.jobs.get(timeout=.1)
except queue.Empty:
continue
with self.stats_lock:
self.active += 1
self.clients.add(request)
try:
self.finish_request(request, address)
except OSError:
pass
except Exception:
log_event(event="worker_error")
finally:
self.shutdown_request(request)
with self.stats_lock:
self.active -= 1
self.clients.discard(request)
self.jobs.task_done()
def server_close(self):
self.draining.set()
super().server_close()
# Stop listening, reject queued work, let accepted in-flight work COMMIT
# and send its response before forcibly closing any remaining socket.
deadline = time.monotonic() + 6
for thread in self.threads:
thread.join(timeout=max(0, deadline - time.monotonic()))
with self.stats_lock:
clients = list(self.clients)
for client in clients:
try:
client.shutdown(socket.SHUT_RDWR)
except OSError:
pass
while True:
try:
request, _ = self.jobs.get_nowait()
except queue.Empty:
break
self.shutdown_request(request)
self.jobs.task_done()
self.store.close()
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "Messenger"
sys_version = ""
def setup(self):
super().setup()
self.request_id = str(uuid.uuid4())
self.received = time.monotonic()
def log_message(self, *_):
pass
def parse_request(self):
original = self.rfile
class HeaderBudget:
remaining = 16384
def readline(self, limit=-1):
line = original.readline(min(limit, self.remaining + 1) if limit >= 0 else self.remaining + 1)
self.remaining -= len(line)
if self.remaining < 0:
raise http.client.LineTooLong("header section")
return line
self.rfile = HeaderBudget()
try:
return super().parse_request()
finally:
self.rfile = original
def send_error(self, code, message=None, explain=None):
self.reply(code if code in CODES else 400, error_body(
code if code in CODES else 400, "Invalid HTTP request", self.request_id))
def reply(self, status, body):
if isinstance(body, tuple):
raw, content_type = body
else:
raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
content_type = 'application/json'
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(raw)))
self.send_header("X-Request-Id", self.request_id)
self.send_header("X-Instance-Id", self.server.instance_id)
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("Connection", "close")
if status == 401:
challenge = ('Basic realm="messenger", charset="UTF-8"'
if urlsplit(self.path).path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
self.send_header('WWW-Authenticate', challenge)
if status in (429, 503):
self.send_header("Retry-After", "1")
self.end_headers()
self.close_connection = True
self.wfile.write(raw)
self.server.record(status)
# No URL query, header, username, text, or exception detail in logs.
log_event(event="request", method=self.command, status=status,
request_id=self.request_id, elapsed_ms=round(1000 * (time.monotonic() - self.received), 3))
def handle_expect_100(self):
# Do not acknowledge a large unauthorized request before checking identity.
return True
def read_body(self, maximum, content_types):
if self.headers.get_all("Transfer-Encoding"):
raise APIError(400, "Transfer-Encoding is unsupported")
sizes = self.headers.get_all("Content-Length", [])
if len(sizes) != 1 or not sizes[0].isascii() or not sizes[0].isdigit() or len(sizes[0]) > 10:
raise APIError(400, "One valid Content-Length is required")
size = int(sizes[0])
if size > maximum:
raise APIError(413, "Request body exceeds byte limit")
if len(self.headers.get_all('Content-Type', [])) != 1 or self.headers.get_content_type() not in content_types:
raise APIError(415, "Unsupported Content-Type")
if self.headers.get("Expect", "").lower() == "100-continue":
self.send_response_only(100)
self.end_headers()
raw = self.rfile.read(size)
if len(raw) != size:
raise APIError(400, "Incomplete request body")
return raw
def body(self):
raw = self.read_body(self.server.config.max_body_bytes, ('application/json',))
def pairs(values):
result = {}
for key, value in values:
if key in result:
raise ValueError("duplicate key")
result[key] = value
return result
def constant(_):
raise ValueError("non JSON constant")
try:
return json.loads(raw.decode("utf-8"), object_pairs_hook=pairs, parse_constant=constant)
except (ValueError, UnicodeError, RecursionError):
raise APIError(400, "Invalid JSON") from None
def dispatch(self):
try:
url = urlsplit(self.path)
except ValueError:
raise APIError(400, "Invalid request target") from None
path, method = url.path, self.command
store = self.server.store
if self.server.draining.is_set():
raise APIError(503, "Shutting down")
if method == 'GET' and path == '/health/live':
return 200, {'status': 'ok'}
if method == 'GET' and path == '/health/ready':
store.ready()
return 200, {'status': 'ok'}
if path == '/api/v1/users' and method == 'POST':
return 201, store.create_user(self.body())
headers = self.headers.get_all('Authorization', [])
authorization = headers[0] if len(headers) == 1 else None
if path == '/api/v1/auth/sessions' and method == 'POST':
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
raise APIError(400, 'Login takes no request body')
return 201, store.login(authorization)
actor, session_key = store.auth(authorization)
parts = path.strip('/').split('/')
if len(parts) in (4, 5) and parts[:3] == ['api', 'v1', 'attachments'] and method == 'GET':
row = store.attachments.get(actor, parts[3])
if len(parts) == 4:
if url.query:
raise APIError(400, 'Invalid query')
return 200, metadata(row)
if parts[4] == 'content':
query = self.query(url)
if set(query) - {'variant'}:
raise APIError(400, 'Invalid query')
if query.get('variant', ['original'])[0] != 'original':
raise APIError(404, 'Object not found')
return 200, (store.attachments.content(row), row['content_type'])
if method == 'DELETE' and path == '/api/v1/auth/sessions/current':
store.cache.delete(session_key)
return 204, None
if method == 'GET' and path == '/api/v1/users/me':
return 200, store.me(actor)
chat_id = None
if len(parts) >= 4 and parts[:3] == ["api", "v1", "chats"]:
chat_id = store.chat_access(actor, parts[3])
if method == "POST" and path == "/api/v1/chats":
return 201, store.create_chat(actor, self.body())
if method == 'POST' and chat_id and len(parts) == 5 and parts[4] == 'attachments':
query = self.query(url)
if set(query) != {'filename'}:
raise APIError(400, 'One filename is required')
filename = filename_value(query['filename'][0])
raw = self.read_body(self.server.config.max_upload_bytes, ('image/png', 'image/jpeg'))
return 201, store.attachments.upload(actor, chat_id, filename, self.headers.get_content_type(), raw)
if method == "POST" and chat_id and len(parts) == 5 and parts[4] == "messages":
keys = self.headers.get_all('Idempotency-Key', [])
key = keys[0] if len(keys) == 1 else None
return 201, store.create_message(actor, chat_id, self.body(), key)
if method == "GET":
if chat_id and len(parts) == 4:
return 200, store.chat(chat_id)
if path == "/api/v1/chats" or (chat_id and len(parts) == 5 and parts[4] == "messages"):
return 200, store.page(actor, chat_id, self.query(url))
raise APIError(404, "Object not found")
@staticmethod
def query(url):
try:
query = parse_qs(url.query, keep_blank_values=True, strict_parsing=True,
max_num_fields=8, errors='strict')
if any(len(values) != 1 for values in query.values()):
raise ValueError()
return query
except (ValueError, UnicodeError):
raise APIError(400, 'Invalid query') from None
def handle_api(self):
try:
status, result = self.dispatch()
except APIError as error:
status = error.status
result = error_body(status, error.message, self.request_id)
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled,
psycopg.errors.LockNotAvailable, PoolTimeout, TooManyRequests, redis.RedisError,
BotoCoreError, ClientError):
status, result = 503, error_body(503, 'Dependency temporarily unavailable', self.request_id)
except (TimeoutError, socket.timeout):
status, result = 400, error_body(400, "Request read timeout", self.request_id)
except Exception:
status, result = 500, error_body(500, "Internal error", self.request_id)
log_event(event="internal_error", request_id=self.request_id)
try:
self.reply(status, result)
except OSError:
pass
do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_HEAD = do_OPTIONS = handle_api
def main():
logging.basicConfig(level=logging.INFO, format="%(message)s")
# Pool diagnostics may include connection details; application emits sanitized errors.
logging.getLogger('psycopg.pool').disabled = True
config = Config.from_env()
server = Server(config)
def stop(signum, _):
if not server.draining.is_set():
server.draining.set()
log_event(event="shutdown_started", signal=signum)
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
log_event(event="started", instance_id=server.instance_id, port=config.app_port, workers=config.workers)
try:
server.serve_forever(poll_interval=.1)
finally:
server.server_close()
log_event(event="shutdown_complete")
if __name__ == "__main__":
main()
+136
View File
@@ -0,0 +1,136 @@
"""Required original-file, replica-switch, bucket and S3-outage demonstrations."""
import hashlib
from io import BytesIO
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT / 'tests'), str(ROOT / 'scripts')]
from client import Client, check
from labctl import containers, restore, save
def compose(*args, payload=None):
process = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
input=payload, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, encoding='utf-8')
if process.returncode:
raise RuntimeError('Compose action failed: ' + args[0])
return process.stdout.strip()
def wait_ready(c):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call('GET', '/health/ready', schema='Health')
return
except AssertionError:
time.sleep(.5)
raise RuntimeError('Readiness did not recover')
def inside_probe():
"""Called in one API container with existing synthetic actors over stdin."""
sys.path.insert(0, str(ROOT / 'src'))
from messenger.core import Config, Store
from PIL import Image
data = json.load(sys.stdin)
c = Client('http://localhost:8080')
alice, bob, eve, chat, att = (data[x] for x in ('alice', 'bob', 'eve', 'chat', 'att'))
if data['mode'] == 'private':
store = Store(Config.from_env())
try:
row = store.attachments.get(alice['id'], att['id'])
url = os.environ['S3_ENDPOINT'] + '/' + os.environ['S3_BUCKET'] + '/' + row['object_key']
try:
urllib.request.urlopen(url, timeout=3)
raise AssertionError('Anonymous object read succeeded')
except urllib.error.HTTPError as error:
check(error.code == 403, 'Expected anonymous S3 403')
images = []
for fmt, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]:
buf = BytesIO()
Image.new('RGB', (32, 24), 'blue').save(buf, format=fmt)
raw = buf.getvalue()
uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=real.' + fmt.lower(),
201, actor=alice, raw=raw, headers={'Content-Type': mime}, schema='Attachment')['json']
downloaded = c.call('GET', '/api/v1/attachments/' + uploaded['id'] + '/content', actor=bob)
check(downloaded['bytes'] == raw, 'Original changed')
images.append(dict(format=fmt, bytes=len(raw), sha256=uploaded['sha256']))
print(json.dumps(dict(anonymous_get=403, real_images=images)))
finally:
store.close()
else:
c.call('GET', '/health/live', schema='Health')
c.call('GET', '/health/ready', 503)
raw = (ROOT / 'tests/fixtures/sample.png').read_bytes()
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=outage.png',
503, actor=alice, raw=raw, headers={'Content-Type': 'image/png'})
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')
print(json.dumps(dict(live=200, ready=503, upload=503, content=503, metadata=200)))
def main():
c = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8084'))
alice, bob, eve, chat = c.bootstrap()
raw = (ROOT / 'tests/fixtures/sample.png').read_bytes()
response = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
201, actor=alice, raw=raw, headers={'Content-Type': 'image/png'}, schema='Attachment')
att, uploader = response['json'], response['headers']['X-Instance-Id']
message = c.message(alice, chat, '', attachments=[att['id']])['json']
before = containers('api')
target = next(x for x in before if uploader.startswith(x['hostname'] + '-'))
try:
subprocess.run(['docker', 'stop', '--time', '8', target['id']], check=True, stdout=subprocess.PIPE)
after = containers('api', True)
check(next(x for x in after if x['id'] == target['id'])['status'] == 'exited', 'Uploader not stopped')
wait_ready(c)
downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
reader = downloaded['headers']['X-Instance-Id']
check(reader != uploader and downloaded['bytes'] == raw, 'Replica-switch original mismatch')
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob, schema='MessagePage')['json']
check(message in page['items'], 'Attached message missing from history')
for suffix in ('', '/content'):
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve)
result = dict(uploader=uploader, downloader=reader, before=before, after=after,
bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest(),
same_original=True, attached_message_read=True, stranger_metadata=404, stranger_content=404)
save('upload-failover.json', result)
print('PASS: uploading API stopped; another API returned identical original and attached message.')
finally:
restore()
data = dict(alice=alice, bob=bob, eve=eve, chat=chat, att=att, mode='private')
result = compose('exec', '-T', 'api', 'python', 'student-tests/grade3.py', '--inside',
payload=json.dumps(data))
save('private-and-formats.json', json.loads(result))
print('PASS: real PNG/JPEG originals and anonymous S3 403.')
before = containers('s3')
started = time.monotonic()
try:
compose('stop', '--timeout', '8', 's3')
check(not containers('s3'), 'MinIO still running')
data['mode'] = 'outage'
result = compose('exec', '-T', 'api', 'python', 'student-tests/grade3.py', '--inside',
payload=json.dumps(data))
observation = json.loads(result)
finally:
compose('start', 's3')
wait_ready(c)
downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
check(downloaded['bytes'] == raw, 'Original changed after S3 outage')
save('s3-outage.json', dict(before=before, after=containers('s3'), observation=observation,
recovered_within_seconds=round(time.monotonic() - started, 3),
original_preserved=True))
print('PASS: unavailable S3 yields 503; restart restores the same original and existing session.')
if __name__ == '__main__':
inside_probe() if '--inside' in sys.argv else main()
+55
View File
@@ -0,0 +1,55 @@
"""Run inside API network: same session and simultaneous requests at distinct processes."""
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import socket
import sys
import threading
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
def main():
clients = [Client('http://' + ip + ':8080') for ip in sorted(set(socket.gethostbyname_ex('api')[2]))]
check(len(clients) == 2, 'Expected two API addresses in Docker DNS')
c = clients[0]
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages'
key = uuid.uuid4().hex
barrier = threading.Barrier(16)
def send(n):
barrier.wait(timeout=10)
return clients[n % 2].call('POST', path, 201, actor=alice,
body={'attachment_ids': [], 'text': 'Параллельный повтор'} if n % 2 else
{'text': 'Параллельный повтор'},
headers={'Idempotency-Key': key}, schema='Message')
with ThreadPoolExecutor(max_workers=16) as pool:
responses = list(pool.map(send, range(16)))
messages = [x['json'] for x in responses]
instances = sorted({x['headers']['X-Instance-Id'] for x in responses})
check(len(instances) == 2, 'Requests did not reach distinct API processes')
check(all(x == messages[0] for x in messages), 'Concurrent responses differ')
history = clients[1].call('GET', path, actor=bob, schema='MessagePage')['json']['items']
check(history == [messages[0]], 'Concurrent requests created duplicates')
clients[1].call('POST', path, 409, actor=alice, body={'text': 'Другой body'}, headers={'Idempotency-Key': key})
# Scope is user + chat; an independent user and another chat may reuse the key.
other = clients[1].message(bob, chat, 'Другой пользователь', key=key)['json']
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id']]}, schema='Chat')['json']
third = clients[1].message(alice, chat2, 'Другой чат', key=key)['json']
check(len({messages[0]['id'], other['id'], third['id']}) == 3, 'Incorrect idempotency scope')
for invalid in ('', 'contains space', 'x' * 129):
c.call('POST', path, 400, actor=alice, body={'text': 'Invalid key'}, headers={'Idempotency-Key': invalid})
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
clients[1].call('POST', path, 401, actor=alice, body={'text': 'Параллельный повтор'},
headers={'Idempotency-Key': key})
print(json.dumps(dict(result='PASS', parallel_requests=16, api_instances=instances,
unique_messages=1, scope_conflict_logout='PASS'), ensure_ascii=False))
if __name__ == '__main__':
main()
+71
View File
@@ -0,0 +1,71 @@
"""Check the upload limits required by grade 3 before body reading/decoding."""
from email.message import Message
from io import BytesIO
from pathlib import Path
import struct
import sys
from types import SimpleNamespace
import zlib
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
from messenger.attachments import filename_value, validate_image
from messenger.core import APIError
from messenger.server import Handler
def rejected(status, action):
try:
action()
except APIError as error:
check(error.status == status, 'Unexpected rejection code')
else:
raise AssertionError('Expected rejection')
class NeverRead:
def read(self, *_):
raise AssertionError('Oversized body was read')
def main():
headers = Message()
headers['Content-Length'] = '10485761'
headers['Content-Type'] = 'image/png'
handler = SimpleNamespace(headers=headers, rfile=NeverRead())
rejected(413, lambda: Handler.read_body(handler, 10485760, ('image/png',)))
original = (ROOT / 'tests/fixtures/sample.png').read_bytes()
# Header-only dimension change: rejection must happen before decoding pixels.
header = original[12:16] + struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0)
huge = original[:12] + header + struct.pack('!I', zlib.crc32(header)) + original[33:]
rejected(413, lambda: validate_image(huge, 'image/png', 40000000))
for name in ('../x.png', 'a\\x.png', 'a\0.png', 'a\n.png'):
rejected(400, lambda: filename_value(name))
check(filename_value('фото.png') == 'фото.png', 'Unicode display filename changed')
c = Client('http://lb:8080')
alice, bob, _, chat = c.bootstrap()
# Exact byte limit is legal; trailing PNG bytes must be returned unchanged.
raw = original + b'\0' * (10485760 - len(original))
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=limit.png',
201, actor=alice, raw=raw, headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == raw,
'10 MiB original changed')
del raw
# Exact pixel limit: valid image, checked in one request without concurrency.
buffer = BytesIO()
with Image.new('RGB', (8000, 5000), 'black') as image:
image.save(buffer, format='PNG')
raw = buffer.getvalue()
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=pixels.png',
201, actor=alice, raw=raw, headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == raw,
'40 million pixel original changed')
print('PASS: 10 MiB and 40,000,000 pixels accepted; excess bytes rejected before read;')
print('excess dimensions rejected before decode; unsafe filenames rejected; Unicode filename accepted.')
if __name__ == '__main__':
main()