From a96711a51a93d61d3b6ada15950be3b83553ba10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 1 Oct 2026 21:17:29 +0300 Subject: [PATCH 1/7] chore: keep lab plans and notes local --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b5a71896..f101a4f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ # Local/private state +/docs/plans/ +/LAB_NOTES.md .env .env.* !.env.example -- 2.54.0 From af7213900bfbba3abdc655f42d717d0488f10e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 13:28:26 +0300 Subject: [PATCH 2/7] feat: Bootstrap durable RabbitMQ queue (U1) --- .env.example | 1 + compose.broker.yaml | 36 +++++++++++++++++++++++++++++ evidence/broker-restart.txt | 22 ++++++++++++++++++ infra/README.md | 12 ++++++++++ scripts/bootstrap_broker.py | 44 ++++++++++++++++++++++++++++++++++++ scripts/init_local_broker.py | 38 +++++++++++++++++++++++++++++++ 6 files changed, 153 insertions(+) create mode 100644 compose.broker.yaml create mode 100644 evidence/broker-restart.txt create mode 100644 scripts/bootstrap_broker.py create mode 100644 scripts/init_local_broker.py diff --git a/.env.example b/.env.example index 119384b5..ca76d513 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,7 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url +BROKER_QUEUE=attachments.process WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 diff --git a/compose.broker.yaml b/compose.broker.yaml new file mode 100644 index 00000000..8fcedf52 --- /dev/null +++ b/compose.broker.yaml @@ -0,0 +1,36 @@ +# Independent broker bootstrap fragment. Merge these roles into compose.yaml +# when the lab4 application base is ready. +services: + broker: + image: rabbitmq:4.1.8-management + hostname: broker + restart: unless-stopped + volumes: + - broker_data:/var/lib/rabbitmq + - ./secrets/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 24 + networks: [backend] + + broker-init: + image: python:3.12.14-alpine3.24 + restart: "no" + depends_on: + broker: + condition: service_healthy + environment: + BROKER_URL_FILE: /run/secrets/broker_url + volumes: + - ./scripts/bootstrap_broker.py:/app/bootstrap_broker.py:ro + - ./secrets/broker_url:/run/secrets/broker_url:ro + command: ["python", "/app/bootstrap_broker.py"] + networks: [backend] + +networks: + backend: + +volumes: + broker_data: diff --git a/evidence/broker-restart.txt b/evidence/broker-restart.txt new file mode 100644 index 00000000..d4178ea5 --- /dev/null +++ b/evidence/broker-restart.txt @@ -0,0 +1,22 @@ +Lab5 broker-only bootstrap check, 2026-10-02, local Colima Docker. + +Commands (from repository root; local ignored secrets generated first): + python3 scripts/init_local_broker.py + docker compose -f compose.broker.yaml config --quiet + docker compose -f compose.broker.yaml up -d + docker compose -f compose.broker.yaml ps -a + docker compose -f compose.broker.yaml restart broker + docker compose -f compose.broker.yaml exec -T broker rabbitmqctl list_queues name durable type + +Observed after first bootstrap: + broker: Up (healthy) + broker-init: Exited (0) + Durable quorum queue attachments.process is ready + +Observed after broker restart, before another bootstrap: + name durable type + attachments.process true quorum + +No broker ports were published. This verifies queue metadata survives a +container restart with broker_data; it does not verify message persistence, +publisher confirms, worker acknowledgements, or full lab5 readiness. diff --git a/infra/README.md b/infra/README.md index 5a39e862..4d32be2f 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,3 +5,15 @@ В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. + +## Локальный RabbitMQ + +```sh +python3 scripts/init_local_broker.py +docker compose -f compose.broker.yaml up -d +docker compose -f compose.broker.yaml ps -a +``` + +Скрипт создаёт игнорируемые Git файлы `secrets/rabbitmq.conf` и `secrets/broker_url` с локальным паролем, не печатая его. `broker-init` повторяемо объявляет `attachments.process` как durable quorum queue; API publisher и worker должны использовать тот же `BROKER_QUEUE` и файл `BROKER_URL_FILE`. Порт RabbitMQ наружу не публикуется. Volume `broker_data` переживает `docker compose restart broker`; `down -v` намеренно удаляет учебные данные. + +`compose.broker.yaml` — самостоятельный фрагмент для подготовки брокера. При переносе реализации lab4 включите его сервисы в общий `compose.yaml` и добавьте отдельные publisher/worker; до этого полный стенд лабораторной 5 не запустится. Один узел RabbitMQ с quorum queue обеспечивает сохранность при рестарте с volume, но не отказоустойчивость при потере узла или диска. diff --git a/scripts/bootstrap_broker.py b/scripts/bootstrap_broker.py new file mode 100644 index 00000000..25d27806 --- /dev/null +++ b/scripts/bootstrap_broker.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Idempotently declare the durable processing queue via RabbitMQ management API.""" + +import base64 +import json +import os +import time +from urllib.error import HTTPError, URLError +from urllib.parse import quote, unquote, urlsplit +from urllib.request import Request, urlopen + + +broker_url = open(os.environ.get("BROKER_URL_FILE", "/run/secrets/broker_url"), encoding="utf-8").read().strip() +parsed = urlsplit(broker_url) +if parsed.scheme != "amqp" or not parsed.username or not parsed.password: + raise SystemExit("Invalid broker URL file") +credentials = f"{unquote(parsed.username)}:{unquote(parsed.password)}".encode() +headers = { + "Authorization": "Basic " + base64.b64encode(credentials).decode("ascii"), + "Content-Type": "application/json", +} +url = "http://broker:15672/api/queues/%2F/" + quote("attachments.process", safe="") +body = json.dumps( + {"durable": True, "auto_delete": False, "arguments": {"x-queue-type": "quorum"}} +).encode() + +for attempt in range(30): + try: + with urlopen(Request(url, data=body, headers=headers, method="PUT"), timeout=3): + pass + with urlopen(Request(url, headers=headers), timeout=3) as response: + queue = json.load(response) + if queue.get("type") != "quorum" or not queue.get("durable"): + raise SystemExit("Processing queue is not durable quorum") + print("Durable quorum queue attachments.process is ready") + break + except HTTPError as exc: + if exc.code in (400, 401, 403, 406): + raise SystemExit(f"RabbitMQ queue declaration failed (HTTP {exc.code})") from None + except URLError: + pass + if attempt == 29: + raise SystemExit("RabbitMQ management API did not become ready") + time.sleep(1) diff --git a/scripts/init_local_broker.py b/scripts/init_local_broker.py new file mode 100644 index 00000000..e24bf49f --- /dev/null +++ b/scripts/init_local_broker.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create local RabbitMQ credentials without writing them to tracked files.""" + +from pathlib import Path +from secrets import token_hex +from urllib.parse import quote + + +root = Path(__file__).resolve().parents[1] +secret_dir = root / "secrets" +secret_dir.mkdir(mode=0o700, exist_ok=True) +secret_dir.chmod(0o700) +config = secret_dir / "rabbitmq.conf" +url_file = secret_dir / "broker_url" +username = "lab5_app" + +if config.exists(): + values = dict( + line.split(" = ", 1) + for line in config.read_text(encoding="utf-8").splitlines() + if " = " in line + ) + if values.get("default_user") != username or not values.get("default_pass"): + raise SystemExit("Existing RabbitMQ config cannot be reused") + password = values["default_pass"] +else: + password = token_hex(24) + config.write_text( + f"default_user = {username}\ndefault_pass = {password}\n", + encoding="utf-8", + ) +config.chmod(0o600) +url_file.write_text( + f"amqp://{username}:{quote(password, safe='')}@broker:5672/%2F\n", + encoding="utf-8", +) +url_file.chmod(0o600) +print("Local broker credentials are ready in ignored secrets/") -- 2.54.0 From b23ecc111f705906ca6fedd24d3859d1c63929c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:35:35 +0300 Subject: [PATCH 3/7] feat: Accept durable queued uploads and publish outbox jobs (U2) Carry forward the lab4 API and storage baseline, stage verified originals in MinIO, and atomically commit attachment, job, and outbox records before 202. Publish identifier-only jobs with broker confirms and reclaimable outbox leases. Clean abandoned and completed staging objects after database checks. --- .gitignore | 1 + Dockerfile | 12 ++ compose.yaml | 225 +++++++++++++++++++ infra/README.md | 14 +- infra/bootstrap-s3.sh | 16 ++ infra/haproxy.cfg | 26 +++ infra/minio-api-policy.json | 15 ++ infra/minio-maintenance-policy.json | 15 ++ migrations/001_initial.sql | 36 ++++ migrations/002_sessions.sql | 7 + migrations/003_message_replays.sql | 11 + migrations/004_attachments.sql | 32 +++ migrations/005_attachment_owner.sql | 2 + migrations/006_pipeline.sql | 74 +++++++ requirements.txt | 10 + scripts/migrate.py | 30 +++ scripts/restart.sh | 5 + scripts/setup-local-secrets.py | 42 ++++ scripts/stop-one.sh | 13 ++ src/api.py | 117 ++++++++++ src/attachments.py | 171 +++++++++++++++ src/auth.py | 43 ++++ src/db.py | 38 ++++ src/main.py | 67 ++++++ src/models.py | 68 ++++++ src/pipeline_jobs.py | 278 ++++++++++++++++++++++++ src/publisher.py | 70 ++++++ src/server.py | 14 ++ src/staging_cleanup.py | 104 +++++++++ src/store.py | 297 ++++++++++++++++++++++++++ student-tests/test_outbox_payload.py | 35 +++ student-tests/test_staging_cleanup.py | 46 ++++ student-tests/test_upload_handoff.py | 61 ++++++ student-tests/test_upload_queued.py | 26 +++ 34 files changed, 2017 insertions(+), 4 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/bootstrap-s3.sh create mode 100644 infra/haproxy.cfg create mode 100644 infra/minio-api-policy.json create mode 100644 infra/minio-maintenance-policy.json create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_sessions.sql create mode 100644 migrations/003_message_replays.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_attachment_owner.sql create mode 100644 migrations/006_pipeline.sql create mode 100644 requirements.txt create mode 100644 scripts/migrate.py create mode 100755 scripts/restart.sh create mode 100644 scripts/setup-local-secrets.py create mode 100755 scripts/stop-one.sh create mode 100644 src/api.py create mode 100644 src/attachments.py create mode 100644 src/auth.py create mode 100644 src/db.py create mode 100644 src/main.py create mode 100644 src/models.py create mode 100644 src/pipeline_jobs.py create mode 100644 src/publisher.py create mode 100644 src/server.py create mode 100644 src/staging_cleanup.py create mode 100644 src/store.py create mode 100644 student-tests/test_outbox_payload.py create mode 100644 student-tests/test_staging_cleanup.py create mode 100644 student-tests/test_upload_handoff.py create mode 100644 student-tests/test_upload_queued.py diff --git a/.gitignore b/.gitignore index f101a4f8..860fedda 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .env .env.* !.env.example +compose.local.yaml secrets/ certs/private/ *.key diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1ecad6dc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --system --uid 10001 --home /app app +COPY --chown=app:app src/ src/ +COPY --chown=app:app migrations/ migrations/ +COPY --chown=app:app scripts/migrate.py scripts/migrate.py +USER app +EXPOSE 8080 +CMD ["python", "-m", "src.server"] diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..8bdc4328 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,225 @@ +name: mtusi2026lab5 + +services: + postgres: + image: postgres:17.6-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + + migrate: + build: . + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + restart: "no" + + s3-volume-init: + image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 + user: "0:0" + entrypoint: ["/bin/sh", "-c"] + command: ["chown -R 100:100 /data"] + volumes: [s3_data:/data] + restart: "no" + + s3: + image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + secrets: [minio_root_user, minio_root_password] + volumes: [s3_data:/data] + depends_on: + s3-volume-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9000/minio/health/ready"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 384m + + s3-bootstrap: + image: bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158 + user: "0:0" + entrypoint: ["/bin/sh", "/bootstrap.sh"] + command: [] + volumes: + - ./infra/bootstrap-s3.sh:/bootstrap.sh:ro + - ./infra/minio-api-policy.json:/policies/minio-api-policy.json:ro + - ./infra/minio-maintenance-policy.json:/policies/minio-maintenance-policy.json:ro + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + - s3_maintenance_key_id + - s3_maintenance_secret_key + depends_on: + s3: + condition: service_healthy + restart: "no" + + broker: + image: rabbitmq:4.1.8-management + hostname: broker + restart: unless-stopped + volumes: + - broker_data:/var/lib/rabbitmq + - ./secrets/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 24 + + broker-init: + image: python:3.12.14-alpine3.24 + restart: "no" + depends_on: + broker: + condition: service_healthy + environment: + BROKER_URL_FILE: /run/secrets/broker_url + volumes: + - ./scripts/bootstrap_broker.py:/app/bootstrap_broker.py:ro + secrets: [broker_url] + command: ["python", "/app/bootstrap_broker.py"] + + api: + build: . + scale: 2 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + stop_grace_period: 10s + cpus: 0.5 + mem_limit: 256m + + publisher: + build: . + command: ["python", "-m", "src.publisher"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_QUEUE: attachments.process + secrets: [database_url, broker_url] + depends_on: + migrate: + condition: service_completed_successfully + broker-init: + condition: service_completed_successfully + restart: unless-stopped + + staging-cleanup: + build: . + command: ["python", "-m", "src.staging_cleanup"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: s3:9000 + S3_MAINTENANCE_KEY_ID_FILE: /run/secrets/s3_maintenance_key_id + S3_MAINTENANCE_SECRET_KEY_FILE: /run/secrets/s3_maintenance_secret_key + STAGING_GRACE_SECONDS: ${STAGING_GRACE_SECONDS:-3600} + STAGING_CLEANUP_INTERVAL_SECONDS: ${STAGING_CLEANUP_INTERVAL_SECONDS:-60} + secrets: [database_url, s3_maintenance_key_id, s3_maintenance_secret_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + restart: unless-stopped + + worker: + build: . + command: ["python", "-m", "src.worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_QUEUE: attachments.process + S3_ENDPOINT: s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + broker-init: + condition: service_completed_successfully + restart: unless-stopped + cpus: 0.5 + mem_limit: 768m + + lb: + image: haproxy:3.0.11-alpine + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + cpus: 0.5 + mem_limit: 128m + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_maintenance_key_id: + file: ./secrets/s3_maintenance_key_id + s3_maintenance_secret_key: + file: ./secrets/s3_maintenance_secret_key + broker_url: + file: ./secrets/broker_url + +volumes: + postgres_data: + s3_data: + broker_data: diff --git a/infra/README.md b/infra/README.md index 4d32be2f..3fb2fe04 100644 --- a/infra/README.md +++ b/infra/README.md @@ -9,11 +9,17 @@ ## Локальный RabbitMQ ```sh +python3 scripts/setup-local-secrets.py python3 scripts/init_local_broker.py -docker compose -f compose.broker.yaml up -d -docker compose -f compose.broker.yaml ps -a +cp .env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build ``` -Скрипт создаёт игнорируемые Git файлы `secrets/rabbitmq.conf` и `secrets/broker_url` с локальным паролем, не печатая его. `broker-init` повторяемо объявляет `attachments.process` как durable quorum queue; API publisher и worker должны использовать тот же `BROKER_QUEUE` и файл `BROKER_URL_FILE`. Порт RabbitMQ наружу не публикуется. Volume `broker_data` переживает `docker compose restart broker`; `down -v` намеренно удаляет учебные данные. +Скрипты создают игнорируемые Git файлы в `secrets/`, не печатая пароли. `broker-init` повторяемо объявляет `attachments.process` как durable quorum queue; publisher и worker используют тот же `BROKER_QUEUE` и файл `BROKER_URL_FILE`. Порт RabbitMQ наружу не публикуется. Volume `broker_data` переживает `docker compose restart broker`; `down -v` намеренно удаляет учебные данные. -`compose.broker.yaml` — самостоятельный фрагмент для подготовки брокера. При переносе реализации lab4 включите его сервисы в общий `compose.yaml` и добавьте отдельные publisher/worker; до этого полный стенд лабораторной 5 не запустится. Один узел RabbitMQ с quorum queue обеспечивает сохранность при рестарте с volume, но не отказоустойчивость при потере узла или диска. +`compose.yaml` содержит все роли; `compose.broker.yaml` оставлен для отдельной проверки брокера. Один узел RabbitMQ с quorum queue обеспечивает сохранность при рестарте с volume, но не отказоустойчивость при потере узла или диска. + +`staging-cleanup` раз в минуту сверяет объекты `staging/` с PostgreSQL. Он удаляет только staging уже готовой текущей версии после записи исходного варианта либо непривязанные объекты старше часа, если нет действующей аренды загрузки. `queued` и `processing` сохраняются. Просроченные записи аренды также удаляются после часа ожидания. Настройки: `STAGING_GRACE_SECONDS` и `STAGING_CLEANUP_INTERVAL_SECONDS`. Сервис получает отдельные maintenance credentials с правом просмотра bucket; секреты остаются в игнорируемом `secrets/`. + +При недоступности Docker DNS локальный игнорируемый `compose.local.yaml` может задавать `build.extra_hosts` для Python package host. Это настройка конкретного стенда; основной Compose остаётся переносимым. diff --git a/infra/bootstrap-s3.sh b/infra/bootstrap-s3.sh new file mode 100644 index 00000000..becaf218 --- /dev/null +++ b/infra/bootstrap-s3.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +read_secret() { + tr -d '\n' < "/run/secrets/$1" +} + +mc alias set local http://s3:9000 "$(read_secret minio_root_user)" "$(read_secret minio_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin policy create local messenger-api /policies/minio-api-policy.json >/dev/null +mc admin policy create local messenger-maintenance /policies/minio-maintenance-policy.json >/dev/null +mc admin user add local "$(read_secret s3_access_key_id)" "$(read_secret s3_secret_access_key)" >/dev/null +mc admin policy attach local messenger-api --user "$(read_secret s3_access_key_id)" >/dev/null +mc admin user add local "$(read_secret s3_maintenance_key_id)" "$(read_secret s3_maintenance_secret_key)" >/dev/null +mc admin policy attach local messenger-maintenance --user "$(read_secret s3_maintenance_key_id)" >/dev/null diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..40952ab9 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,26 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + timeout connect 2s + timeout client 15s + timeout server 15s + +resolvers docker + nameserver docker 127.0.0.11:53 + resolve_retries 2 + timeout retry 1s + hold valid 1s + +frontend public + bind *:8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + server-template api 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none diff --git a/infra/minio-api-policy.json b/infra/minio-api-policy.json new file mode 100644 index 00000000..54be94d5 --- /dev/null +++ b/infra/minio-api-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/infra/minio-maintenance-policy.json b/infra/minio-maintenance-policy.json new file mode 100644 index 00000000..6beb6bd2 --- /dev/null +++ b/infra/minio-maintenance-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..dbfb0065 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX chats_history_idx ON chats (created_at, id); + +CREATE TABLE memberships ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 99), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX memberships_user_idx ON memberships (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES memberships(chat_id, user_id) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_sessions.sql b/migrations/002_sessions.sql new file mode 100644 index 00000000..5d2f364a --- /dev/null +++ b/migrations/002_sessions.sql @@ -0,0 +1,7 @@ +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_user_idx ON sessions (user_id); diff --git a/migrations/003_message_replays.sql b/migrations/003_message_replays.sql new file mode 100644 index 00000000..e5a86270 --- /dev/null +++ b/migrations/003_message_replays.sql @@ -0,0 +1,11 @@ +CREATE TABLE message_replays ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + request_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid REFERENCES messages(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, request_key) +); + +CREATE INDEX message_replays_expiry_idx ON message_replays (created_at); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..576f30b1 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,32 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (id, chat_id) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at); + +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000); +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE, + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/005_attachment_owner.sql b/migrations/005_attachment_owner.sql new file mode 100644 index 00000000..6db42908 --- /dev/null +++ b/migrations/005_attachment_owner.sql @@ -0,0 +1,2 @@ +ALTER TABLE attachments ADD COLUMN owner_id uuid NOT NULL REFERENCES users(id); +ALTER TABLE attachments ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(); diff --git a/migrations/006_pipeline.sql b/migrations/006_pipeline.sql new file mode 100644 index 00000000..f332a6fc --- /dev/null +++ b/migrations/006_pipeline.sql @@ -0,0 +1,74 @@ +-- Lab5: PostgreSQL is the source of truth; RabbitMQ carries only job IDs. +ALTER TABLE attachments + ADD COLUMN staging_key text UNIQUE, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version > 0), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt > 0), + status text NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + lease_token uuid, + lease_expires_at timestamptz, + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (attachment_id, pipeline_version), + CHECK ((status = 'processing') = + (lease_token IS NOT NULL AND lease_expires_at IS NOT NULL)) +); + +CREATE INDEX attachment_jobs_due_idx + ON attachment_jobs (next_attempt_at, id) WHERE status = 'queued'; +CREATE INDEX attachment_jobs_expired_idx + ON attachment_jobs (lease_expires_at, id) WHERE status = 'processing'; + +-- One logical publication per managed attempt. A crash after broker confirm +-- but before published_at may republish; consumers must tolerate that duplicate. +CREATE TABLE attachment_outbox ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + attempt integer NOT NULL CHECK (attempt > 0), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + claim_token uuid, + claim_expires_at timestamptz, + published_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (job_id, attempt), + CHECK ((claim_token IS NULL) = (claim_expires_at IS NULL)) +); + +CREATE INDEX attachment_outbox_pending_idx + ON attachment_outbox (available_at, id) WHERE published_at IS NULL; + +-- Internal 'original' is stored alongside public thumbnail/crop/optimized +-- so every physical effect has the same versioned identity and checksum. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL, + pipeline_version integer NOT NULL, + name text NOT NULL CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + object_key text NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name), + FOREIGN KEY (attachment_id, pipeline_version) + REFERENCES attachment_jobs (attachment_id, pipeline_version) ON DELETE CASCADE +); + +-- An API upload reserves its staging key before S3 PUT. Cleanup only removes +-- objects without a committed job and without an unexpired upload marker. +CREATE TABLE attachment_uploads ( + id uuid PRIMARY KEY, + staging_key text NOT NULL UNIQUE, + lease_expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachment_uploads_expired_idx ON attachment_uploads (lease_expires_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..a9a509e2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.118.3 +pydantic==2.12.4 +uvicorn==0.38.0 +psycopg[binary,pool]==3.2.12 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +minio==7.2.20 +Pillow==11.3.0 +urllib3==2.8.0 +pika==1.4.4 diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..e346d4f1 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,30 @@ +"""Apply numbered SQL migrations once, atomically and under a DB lock.""" + +from pathlib import Path +import sys + +import psycopg + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.db import database_url + + +def main() -> None: + migration_dir = Path(__file__).resolve().parents[1] / "migrations" + with psycopg.connect(database_url(), connect_timeout=5, options="-c statement_timeout=30000") as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(2026, 2)") + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())" + ) + for path in sorted(migration_dir.glob("[0-9][0-9][0-9]_*.sql")): + version = int(path.name.split("_", 1)[0]) + if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone(): + continue + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,)) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..d8a434a0 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env stop api lb publisher worker staging-cleanup broker postgres s3 +docker compose --env-file .env up -d --wait --scale api=2 diff --git a/scripts/setup-local-secrets.py b/scripts/setup-local-secrets.py new file mode 100644 index 00000000..168168d4 --- /dev/null +++ b/scripts/setup-local-secrets.py @@ -0,0 +1,42 @@ +"""Create local Compose secrets without printing or overwriting existing values.""" + +import os +import secrets +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] / "secrets" + + +def write(name: str, value: str) -> None: + fd = os.open(ROOT / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as out: + out.write(value) + + +def main() -> None: + ROOT.mkdir(mode=0o700, exist_ok=True) + db_names = {"postgres_password", "database_url"} + s3_names = { + "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key", + "s3_maintenance_key_id", "s3_maintenance_secret_key", + } + for names in (db_names, s3_names): + existing = {name for name in names if (ROOT / name).exists()} + if existing and existing != names: + raise SystemExit(f"Incomplete secret group: {', '.join(sorted(names - existing))}") + if not (ROOT / "postgres_password").exists(): + password = secrets.token_urlsafe(32) + write("postgres_password", password) + write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") + if not (ROOT / "minio_root_user").exists(): + write("minio_root_user", "lab5admin") + write("minio_root_password", secrets.token_urlsafe(32)) + write("s3_access_key_id", secrets.token_hex(10)) + write("s3_secret_access_key", secrets.token_urlsafe(32)) + write("s3_maintenance_key_id", secrets.token_hex(10)) + write("s3_maintenance_secret_key", secrets.token_urlsafe(32)) + + +if __name__ == "__main__": + main() diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..99c65a25 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +for container in $(docker compose --env-file .env ps -q api); do + instance=$(docker exec "$container" python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).headers['X-Instance-Id'])") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop -t 5 "$container" >/dev/null + exit 0 + fi +done +echo "target API instance not found" >&2 +exit 1 diff --git a/src/api.py b/src/api.py new file mode 100644 index 00000000..69bc46a8 --- /dev/null +++ b/src/api.py @@ -0,0 +1,117 @@ +"""Messenger HTTP routes.""" + +import re +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response + +from .auth import basic_credentials, bearer_token +from .attachments import storage_ready, upload +from .attachment_read import download, get_view +from .models import CreateChat, CreateMessage, CreateUser +from .store import store + +router = APIRouter() + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +def ready(): + result = store.ready() + storage_ready() + return result + + +@router.post("/api/v1/users", status_code=201) +def register_user(body: CreateUser): + return store.register(body.username, body.display_name, body.password) + + +@router.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request, response: Response): + if await request.body(): + raise HTTPException(400) + username, password = basic_credentials(request.headers.get("authorization")) + response.headers["Cache-Control"] = "no-store" + return store.create_session(username, password) + + +def current_user(request: Request) -> dict: + return store.authenticated(bearer_token(request.headers.get("authorization"))) + + +def parse_limit(value: str) -> int: + try: + number = int(value) if value.isdecimal() else 0 + except ValueError: + raise HTTPException(400) from None + if not 1 <= number <= 100: + raise HTTPException(400) + return number + + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session(request: Request): + store.revoke_session(bearer_token(request.headers.get("authorization"))) + return Response(status_code=204) + + +@router.get("/api/v1/users/me") +def user_me(actor: dict = Depends(current_user)): + return actor + + +@router.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, actor: dict = Depends(current_user)): + return store.create_chat(actor["id"], body.title, body.member_ids) + + +@router.get("/api/v1/chats") +def list_chats( + limit: str = Query("50"), cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user) +): + return store.list_chats(actor["id"], parse_limit(limit), cursor) + + +@router.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: UUID, actor: dict = Depends(current_user)): + return store.chat(actor["id"], str(chat_id)) + + +@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(chat_id: UUID, request: Request, filename: str = Query(...), actor: dict = Depends(current_user)): + return await upload(request, str(chat_id), actor["id"], filename) + + +@router.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: UUID, actor: dict = Depends(current_user)): + return get_view(actor["id"], attachment_id) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +def get_attachment_content( + attachment_id: UUID, variant: str = Query("original"), actor: dict = Depends(current_user) +): + return download(actor["id"], attachment_id, variant) + + +@router.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message(chat_id: UUID, body: CreateMessage, request: Request, actor: dict = Depends(current_user)): + key = request.headers.get("Idempotency-Key") + if key is None or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise HTTPException(400) + return store.create_message(actor["id"], str(chat_id), body.text, key, body.attachment_ids) + + +@router.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: UUID, + limit: str = Query("50"), + cursor: str | None = Query(None, min_length=1), + actor: dict = Depends(current_user), +): + return store.list_messages(actor["id"], str(chat_id), parse_limit(limit), cursor) diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..818c5a53 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,171 @@ +"""Private S3 access shared by attachment routes and maintenance jobs.""" + +import asyncio +import hashlib +import logging +import os +import unicodedata +from functools import lru_cache +from pathlib import Path +from tempfile import SpooledTemporaryFile +from uuid import UUID, uuid4 + +from fastapi import HTTPException, Request +from minio import Minio +from minio.error import S3Error +from PIL import Image, UnidentifiedImageError +from urllib3 import PoolManager, Retry, Timeout + +from .pipeline_jobs import queue_attachment, renew_upload, reserve_upload +from .store import connection, store, timestamp + + +BUCKET = "messenger-attachments" +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +logger = logging.getLogger(__name__) + + +def secret(name: str) -> str: + return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip() + + +@lru_cache(maxsize=1) +def storage() -> Minio: + return Minio( + os.environ.get("S3_ENDPOINT", "s3:9000"), + access_key=secret("S3_ACCESS_KEY_ID"), + secret_key=secret("S3_SECRET_ACCESS_KEY"), + secure=False, + region="us-east-1", + http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)), + ) + + +def storage_ready() -> None: + try: + storage().stat_object(BUCKET, "health/ready-probe") + except S3Error as exc: + if exc.code != "NoSuchKey": + raise HTTPException(503) from None + except Exception: + raise HTTPException(503) from None + + +def attachment_view(row: dict) -> dict: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), + "filename": row["filename"], "content_type": row["content_type"], + "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], + "variants": [], "error_code": row["error_code"], + "created_at": timestamp(row["created_at"]), "updated_at": timestamp(row["updated_at"]), + } + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or filename in {".", ".."}: + raise HTTPException(400) + if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename): + raise HTTPException(400) + + +def validate_image(stream, declared_type: str) -> str: + try: + with Image.open(stream) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) + if actual_type is None or actual_type != declared_type: + raise HTTPException(415) + if image.width * image.height > MAX_PIXELS: + raise HTTPException(413) + image.verify() + stream.seek(0) + with Image.open(stream) as image: + image.load() + return actual_type + except Image.DecompressionBombError: + raise HTTPException(413) from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise HTTPException(415) from None + + +def verify_staging(client: Minio, key: str, size: int, sha256: str, content_type: str) -> None: + """Confirm complete durable bytes; an S3 ETag is not a SHA-256 checksum.""" + info = client.stat_object(BUCKET, key) + if info.size != size or info.content_type.split(";", 1)[0] != content_type: + raise ValueError("staging metadata mismatch") + obj = client.get_object(BUCKET, key) + try: + digest = hashlib.sha256() + received = 0 + for chunk in obj.stream(64 * 1024): + received += len(chunk) + if received > size: + raise ValueError("staging size mismatch") + digest.update(chunk) + if received != size or digest.hexdigest() != sha256: + raise ValueError("staging checksum mismatch") + finally: + obj.close() + obj.release_conn() + + +def persist_attachment(chat_id: str, actor_id: str, filename: str, content_type: str, size: int, sha256: str, stream) -> dict: + attachment_id = uuid4() + staging_key = f"staging/{attachment_id}" + object_key = f"originals/{attachment_id}" + with connection() as conn: + reserve_upload(conn, attachment_id, staging_key) + + try: + stream.seek(0) + client = storage() + client.put_object(BUCKET, staging_key, stream, size, content_type=content_type) + with connection() as conn: + if not renew_upload(conn, attachment_id): + raise HTTPException(503) + verify_staging(client, staging_key, size, sha256, content_type) + with connection() as conn: + row, _job_id = queue_attachment( + conn, attachment_id=attachment_id, chat_id=UUID(chat_id), owner_id=UUID(actor_id), + filename=filename, staging_key=staging_key, object_key=object_key, + content_type=content_type, size_bytes=size, sha256=sha256, + ) + except HTTPException: + raise + except Exception as exc: + # A lost COMMIT acknowledgement may leave a valid queued job. Keep the + # staging object; cleanup must check PostgreSQL before deleting it. + logger.warning("Attachment handoff failed for %s (%s)", attachment_id, type(exc).__name__) + raise HTTPException(503) from None + return attachment_view(row) + + +async def upload(request: Request, chat_id: str, actor_id: str, filename: str) -> dict: + await asyncio.to_thread(store.chat, actor_id, chat_id) + validate_filename(filename) + declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if declared_type not in {"image/png", "image/jpeg"}: + raise HTTPException(415) + length = request.headers.get("content-length") + if length is not None: + try: + if int(length) > MAX_BYTES: + raise HTTPException(413) + except ValueError: + raise HTTPException(400) from None + + with SpooledTemporaryFile(max_size=1024 * 1024) as stream: + digest = hashlib.sha256() + size = 0 + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise HTTPException(413) + stream.write(chunk) + digest.update(chunk) + stream.seek(0) + validate_image(stream, declared_type) + return await asyncio.to_thread( + persist_attachment, chat_id, actor_id, filename, declared_type, size, digest.hexdigest(), stream + ) diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..5714b173 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,43 @@ +"""Parse HTTP credentials without exposing them in errors or logs.""" + +import base64 +import binascii +import hashlib +import re + +from fastapi import HTTPException + + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +USERNAME = re.compile(r"[a-z0-9_]{3,32}\Z") + + +def unauthorized(challenge: str) -> HTTPException: + return HTTPException(401, headers={"WWW-Authenticate": challenge}) + + +def basic_credentials(header: str | None) -> tuple[str, str]: + if not header or not header.startswith("Basic "): + raise unauthorized(BASIC_CHALLENGE) + try: + raw = base64.b64decode(header[6:], validate=True).decode("utf-8") + username, password = raw.split(":", 1) + except (binascii.Error, UnicodeError, ValueError): + raise unauthorized(BASIC_CHALLENGE) from None + if not USERNAME.fullmatch(username) or not 12 <= len(password) <= 128: + raise unauthorized(BASIC_CHALLENGE) + return username, password + + +def bearer_token(header: str | None) -> str: + if not header or not header.startswith("Bearer "): + raise unauthorized(BEARER_CHALLENGE) + token = header[7:] + if not 32 <= len(token) <= 256 or not token.isascii() or not re.fullmatch(r"[A-Za-z0-9_-]+", token): + raise unauthorized(BEARER_CHALLENGE) + return token + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..832ff074 --- /dev/null +++ b/src/db.py @@ -0,0 +1,38 @@ +"""Bounded PostgreSQL pool shared by API storage and health checks.""" + +import os +import atexit +from functools import lru_cache +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from psycopg_pool import PoolTimeout + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("DATABASE_URL", "postgresql:///messenger") + + +@lru_cache(maxsize=1) +def pool() -> ConnectionPool: + result = ConnectionPool( + database_url(), min_size=0, max_size=8, timeout=5, + kwargs={"connect_timeout": 5, "options": "-c statement_timeout=5000", "row_factory": dict_row}, + open=True, + ) + atexit.register(result.close) + return result + + +def ready() -> bool: + try: + with pool().connection() as conn: + conn.execute("SELECT 1 FROM users LIMIT 0") + return True + except (psycopg.Error, PoolTimeout, OSError): + return False diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..26dea4b3 --- /dev/null +++ b/src/main.py @@ -0,0 +1,67 @@ +"""FastAPI entry point and contract-wide response handling.""" + +from uuid import uuid4 + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from .api import router + +app = FastAPI() +app.include_router(router) +INSTANCE_ID = uuid4().hex + +ERRORS = { + 400: ("invalid_request", "Неверный запрос"), + 401: ("unauthorized", "Требуется действительный пользователь"), + 404: ("not_found", "Объект не найден"), + 409: ("conflict", "Конфликт данных"), + 413: ("payload_too_large", "Слишком большой запрос"), + 415: ("unsupported_media_type", "Неподдерживаемый тип данных"), + 429: ("rate_limited", "Слишком много запросов"), + 503: ("unavailable", "Сервис недоступен"), +} + + +def error_response(request: Request, status: int, extra_headers: dict | None = None) -> JSONResponse: + code, message = ERRORS.get(status, ("internal_error", "Внутренняя ошибка")) + request_id = request.state.request_id + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID, **(extra_headers or {})}, + ) + + +@app.middleware("http") +async def request_contract(request: Request, call_next): + request.state.request_id = str(uuid4()) + is_upload = request.url.path.startswith("/api/v1/chats/") and request.url.path.endswith("/attachments") + if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions" and not is_upload: + media_type, _, charset = request.headers.get("content-type", "").lower().partition(";") + if media_type.strip() != "application/json" or (charset and charset.strip() != "charset=utf-8"): + return error_response(request, 415) + try: + (await request.body()).decode("utf-8") + except UnicodeDecodeError: + return error_response(request, 400) + response = await call_next(request) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(RequestValidationError) +async def validation_error(request: Request, exc: RequestValidationError): + return error_response(request, 400) + + +@app.exception_handler(HTTPException) +async def http_error(request: Request, exc: HTTPException): + return error_response(request, exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def unexpected_error(request: Request, exc: Exception): + return error_response(request, 500) diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..06b16beb --- /dev/null +++ b/src/models.py @@ -0,0 +1,68 @@ +"""Request bodies from the lab 1 contract.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class StrictBody(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateUser(StrictBody): + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def nonblank_name(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank display name") + return value + + @field_validator("password") + @classmethod + def nonblank_password(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank password") + return value + + +class CreateChat(StrictBody): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def nonblank_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank title") + return value + + @field_validator("member_ids") + @classmethod + def valid_members(cls, value: list[str]) -> list[str]: + members = [str(UUID(member_id)) for member_id in value] + if len(set(members)) != len(members): + raise ValueError("duplicate member") + return members + + +class CreateMessage(StrictBody): + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def valid_attachments(cls, value: list[str]) -> list[str]: + ids = [str(UUID(item)) for item in value] + if len(set(ids)) != len(ids): + raise ValueError("duplicate attachment") + return ids + + @model_validator(mode="after") + def text_or_attachment(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("message needs text or attachment") + return self diff --git a/src/pipeline_jobs.py b/src/pipeline_jobs.py new file mode 100644 index 00000000..5fa477f7 --- /dev/null +++ b/src/pipeline_jobs.py @@ -0,0 +1,278 @@ +"""Atomic PostgreSQL state changes for the attachment processing pipeline. + +Callers own the connection and must commit before replying 202 or acking AMQP. +S3 writes and RabbitMQ publication happen outside these transactions. +""" + +from datetime import timezone +from uuid import uuid4 + + +REQUIRED_VARIANTS = {"original", "thumbnail", "crop", "optimized"} + + +def job_payload(row): + """Broker message per jobs.schema.json: identifiers only, never file bytes.""" + return { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "created_at": row["created_at"].astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), + } + + +def reserve_upload(conn, attachment_id, staging_key, lease_seconds=300): + """Record an active staging PUT before it starts.""" + if lease_seconds < 1: + raise ValueError("invalid upload lease") + with conn.transaction(): + conn.execute( + "INSERT INTO attachment_uploads (id,staging_key,lease_expires_at) " + "VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s))", + (attachment_id, staging_key, lease_seconds), + ) + + +def renew_upload(conn, attachment_id, lease_seconds=300): + """Keep a slow, active upload protected from orphan cleanup.""" + if lease_seconds < 1: + raise ValueError("invalid upload lease") + with conn.transaction(): + return conn.execute( + "UPDATE attachment_uploads SET lease_expires_at=clock_timestamp()+make_interval(secs => %s) " + "WHERE id=%s AND lease_expires_at>clock_timestamp() RETURNING id", + (lease_seconds, attachment_id), + ).fetchone() is not None + + +def queue_attachment(conn, *, attachment_id, chat_id, owner_id, filename, + staging_key, object_key, content_type, size_bytes, sha256, + pipeline_version=1): + """Commit attachment, job, and outbox after the full S3 staging PUT.""" + job_id = uuid4() + with conn.transaction(): + marker = conn.execute( + "SELECT id FROM attachment_uploads WHERE id=%s AND staging_key=%s " + "AND lease_expires_at>clock_timestamp() FOR UPDATE", + (attachment_id, staging_key), + ).fetchone() + if marker is None: + raise ValueError("upload reservation is missing") + row = conn.execute( + "INSERT INTO attachments " + "(id,chat_id,owner_id,filename,object_key,staging_key,content_type,size_bytes,sha256,status,pipeline_version) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s) " + "RETURNING id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,error_code,created_at,updated_at", + (attachment_id, chat_id, owner_id, filename, object_key, staging_key, + content_type, size_bytes, sha256, pipeline_version), + ).fetchone() + conn.execute( + "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) VALUES (%s,%s,%s)", + (job_id, attachment_id, pipeline_version), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)", + (job_id,), + ) + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (attachment_id,)) + return row, job_id + + +def claim_outbox(conn, limit=10, lease_seconds=30): + """Lease pending publications; publisher confirms must call confirm_outbox.""" + if not 1 <= limit <= 100 or lease_seconds < 1: + raise ValueError("invalid outbox lease") + token = uuid4() + with conn.transaction(): + rows = conn.execute( + "WITH due AS (" + " SELECT id FROM attachment_outbox" + " WHERE published_at IS NULL AND available_at<=clock_timestamp()" + " AND (claim_expires_at IS NULL OR claim_expires_at %s)" + " FROM due,attachment_jobs j" + " WHERE o.id=due.id AND j.id=o.job_id" + " RETURNING o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at", + (limit, token, lease_seconds), + ).fetchall() + return token, rows + + +def confirm_outbox(conn, outbox_id, token): + """Mark published only after RabbitMQ publisher confirm.""" + with conn.transaction(): + return conn.execute( + "UPDATE attachment_outbox SET published_at=clock_timestamp()," + "claim_token=NULL,claim_expires_at=NULL " + "WHERE id=%s AND claim_token=%s AND published_at IS NULL RETURNING id", + (outbox_id, token), + ).fetchone() is not None + + +def claim_job(conn, job_id, delivery_attempt, lease_seconds=60): + """A stale/redelivered message cannot steal an active or finished job.""" + if lease_seconds < 1: + raise ValueError("invalid job lease") + token = uuid4() + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.staging_key,a.object_key,a.sha256,a.content_type,a.size_bytes," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j", + (job_id,), + ).fetchone() + if (job is None or job["status"] != "queued" or + job["pipeline_version"] != job["current_pipeline_version"] or + job["attempt"] != delivery_attempt or + job["next_attempt_at"] > conn.execute("SELECT clock_timestamp() AS now").fetchone()["now"]): + return None + conn.execute( + "UPDATE attachment_jobs SET status='processing',lease_token=%s," + "lease_expires_at=clock_timestamp()+make_interval(secs => %s)," + "updated_at=clock_timestamp() WHERE id=%s", + (token, lease_seconds, job_id), + ) + conn.execute( + "UPDATE attachments SET status='processing',updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s AND status='queued'", + (job["attachment_id"], job["pipeline_version"]), + ) + return {**job, "lease_token": token} + + +def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent): + if max_attempts < 1 or delay_seconds < 0: + raise ValueError("invalid retry configuration") + failed = permanent or job["attempt"] >= max_attempts + if failed: + code = error_code if permanent else "retry_exhausted" + conn.execute( + "UPDATE attachment_jobs SET status='failed',lease_token=NULL,lease_expires_at=NULL," + "error_code=%s,updated_at=clock_timestamp() WHERE id=%s", + (code, job["id"]), + ) + conn.execute( + "UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (code, job["attachment_id"], job["pipeline_version"]), + ) + else: + attempt = job["attempt"] + 1 + due = conn.execute( + "UPDATE attachment_jobs SET status='queued',attempt=%s,lease_token=NULL,lease_expires_at=NULL," + "next_attempt_at=clock_timestamp()+make_interval(secs => %s)," + "error_code=%s,updated_at=clock_timestamp() WHERE id=%s RETURNING next_attempt_at", + (attempt, delay_seconds, error_code, job["id"]), + ).fetchone()["next_attempt_at"] + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt,available_at) VALUES (%s,%s,%s)", + (job["id"], attempt, due), + ) + conn.execute( + "UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (job["attachment_id"], job["pipeline_version"]), + ) + return "failed" if failed else "queued" + + +def finish_attempt(conn, job_id, token, *, max_attempts, delay_seconds=0, + error_code="processing_failed", permanent=False): + """Persist retry or terminal failure before the worker acks delivery.""" + if error_code not in {"invalid_image", "processing_failed", "storage_unavailable", "worker_timeout"}: + raise ValueError("unsafe error code") + with conn.transaction(): + job = conn.execute( + "SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job_id,) + ).fetchone() + if job is None or job["status"] != "processing" or job["lease_token"] != token: + raise ValueError("job lease is no longer owned") + return _reschedule_locked( + conn, job, max_attempts=max_attempts, delay_seconds=delay_seconds, + error_code=error_code, permanent=permanent, + ) + + +def recover_expired(conn, *, max_attempts, delay_seconds=0, limit=20): + """Requeue dead workers; old delivery attempts become harmless duplicates.""" + if not 1 <= limit <= 100: + raise ValueError("invalid recovery batch") + with conn.transaction(): + jobs = conn.execute( + "SELECT * FROM attachment_jobs WHERE status='processing' " + "AND lease_expires_at<=clock_timestamp() " + "ORDER BY lease_expires_at,id LIMIT %s FOR UPDATE SKIP LOCKED", + (limit,), + ).fetchall() + return [ + (job["id"], _reschedule_locked( + conn, job, max_attempts=max_attempts, delay_seconds=delay_seconds, + error_code="worker_timeout", permanent=False, + )) + for job in jobs + ] + + +def complete_job(conn, job_id, token, variants): + """Record every versioned effect, then expose ready atomically. + + Returns True when this version is still current. A duplicate ready job is + a harmless no-op; an old version never changes the current attachment. + """ + if set(variants) != REQUIRED_VARIANTS: + raise ValueError("incomplete variant set") + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.object_key,a.sha256,a.content_type,a.size_bytes," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j", + (job_id,), + ).fetchone() + if job is None: + raise ValueError("unknown job") + if job["status"] == "ready": + return job["pipeline_version"] == job["current_pipeline_version"] + if job["status"] != "processing" or job["lease_token"] != token: + raise ValueError("job lease is no longer owned") + original = variants["original"] + if (original["object_key"] != job["object_key"] or + original["sha256"] != job["sha256"] or + original["content_type"] != job["content_type"] or + original["size_bytes"] != job["size_bytes"]): + raise ValueError("original does not match accepted upload") + for name, value in variants.items(): + fields = (value["object_key"], value["content_type"], value["size_bytes"], + value["width"], value["height"], value["sha256"]) + conn.execute( + "INSERT INTO attachment_variants " + "(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,width,height,sha256) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + (job["attachment_id"], job["pipeline_version"], name, *fields), + ) + stored = conn.execute( + "SELECT object_key,content_type,size_bytes,width,height,sha256 " + "FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s", + (job["attachment_id"], job["pipeline_version"], name), + ).fetchone() + if stored is None or tuple(stored[key] for key in + ("object_key", "content_type", "size_bytes", "width", "height", "sha256")) != fields: + raise ValueError("versioned variant conflicts with a previous effect") + conn.execute( + "UPDATE attachment_jobs SET status='ready',lease_token=NULL,lease_expires_at=NULL," + "error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", + (job_id,), + ) + current = conn.execute( + "UPDATE attachments SET status='ready',error_code=NULL," + "updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s RETURNING id", + (job["attachment_id"], job["pipeline_version"]), + ).fetchone() + return current is not None diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 00000000..53fdfb8a --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,70 @@ +"""Publish PostgreSQL outbox jobs with RabbitMQ confirms.""" + +import json +import logging +import os +import time +from pathlib import Path + +import pika +import psycopg +from psycopg_pool import PoolTimeout + +from .db import pool +from .pipeline_jobs import claim_outbox, confirm_outbox, job_payload + + +logger = logging.getLogger(__name__) +QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") + + +def publish_batch(channel): + with pool().connection() as conn: + token, rows = claim_outbox(conn, limit=10, lease_seconds=30) + for row in rows: + body = json.dumps(job_payload(row), separators=(",", ":")).encode("utf-8") + channel.basic_publish( + exchange="", routing_key=QUEUE, body=body, mandatory=True, + properties=pika.BasicProperties( + content_type="application/json", delivery_mode=2, + message_id=str(row["job_id"]), type="attachment.process.v1", + ), + ) + # basic_publish returns only after a broker confirm in confirm mode. + with pool().connection() as conn: + if not confirm_outbox(conn, row["id"], token): + logger.warning("Outbox claim expired before confirm for row %s", row["id"]) + return len(rows) + + +def main(): + logging.basicConfig(level=logging.INFO) + logging.getLogger("pika").setLevel(logging.ERROR) + url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() + params = pika.URLParameters(url) + params.heartbeat = 30 + params.socket_timeout = 5 + params.blocked_connection_timeout = 10 + while True: + broker = None + try: + broker = pika.BlockingConnection(params) + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.confirm_delivery() + while broker.is_open: + if not publish_batch(channel): + broker.process_data_events(time_limit=1) + except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Publisher reconnecting after %s", type(exc).__name__) + time.sleep(2) + finally: + if broker is not None and broker.is_open: + try: + broker.close() + except (pika.exceptions.AMQPError, OSError): + pass + + +if __name__ == "__main__": + main() diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..20fb9c9c --- /dev/null +++ b/src/server.py @@ -0,0 +1,14 @@ +"""Run the ASGI worker as the container's main process.""" + +import os + +import uvicorn + + +if __name__ == "__main__": + uvicorn.run( + "src.main:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + workers=1, + ) diff --git a/src/staging_cleanup.py b/src/staging_cleanup.py new file mode 100644 index 00000000..72d70b36 --- /dev/null +++ b/src/staging_cleanup.py @@ -0,0 +1,104 @@ +"""Remove abandoned staging objects after a grace period or a committed result.""" + +import logging +import os +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import UUID + +from minio import Minio + +from .db import pool + + +logger = logging.getLogger(__name__) +BUCKET = "messenger-attachments" + + +def storage(): + def secret(name): + return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip() + + return Minio( + os.environ.get("S3_ENDPOINT", "s3:9000"), + access_key=secret("S3_MAINTENANCE_KEY_ID"), + secret_key=secret("S3_MAINTENANCE_SECRET_KEY"), + secure=False, + region="us-east-1", + ) + + +def clean_once(client, grace_seconds=3600): + """Return removed count; lock DB state through each S3 delete.""" + if grace_seconds < 1: + raise ValueError("invalid staging grace period") + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + removed = 0 + for obj in client.list_objects(BUCKET, prefix="staging/", recursive=True): + key = obj.object_name + suffix = key.removeprefix("staging/") + try: + if key != f"staging/{UUID(suffix)}": + continue + except ValueError: + continue + with pool().connection() as conn, conn.transaction(): + attachment = conn.execute( + "SELECT a.status,EXISTS(" + "SELECT 1 FROM attachment_variants v WHERE v.attachment_id=a.id " + "AND v.pipeline_version=a.pipeline_version AND v.name='original' " + "AND v.object_key=a.object_key) AS original_ready " + "FROM attachments a WHERE a.staging_key=%s FOR UPDATE OF a", + (key,), + ).fetchone() + if attachment: + if attachment["status"] != "ready" or not attachment["original_ready"]: + continue + else: + if obj.last_modified is None or obj.last_modified > cutoff: + continue + marker = conn.execute( + "SELECT lease_expires_at>clock_timestamp() AS active " + "FROM attachment_uploads WHERE staging_key=%s FOR UPDATE", + (key,), + ).fetchone() + if marker and marker["active"]: + continue + client.remove_object(BUCKET, key) + if not attachment: + conn.execute( + "DELETE FROM attachment_uploads WHERE staging_key=%s " + "AND lease_expires_at<=clock_timestamp()", + (key,), + ) + removed += 1 + with pool().connection() as conn, conn.transaction(): + conn.execute( + "DELETE FROM attachment_uploads u WHERE u.lease_expires_at<" + "clock_timestamp()-make_interval(secs => %s) " + "AND NOT EXISTS (SELECT 1 FROM attachments a WHERE a.staging_key=u.staging_key)", + (grace_seconds,), + ) + return removed + + +def main(): + logging.basicConfig(level=logging.INFO) + client = storage() + grace = int(os.environ.get("STAGING_GRACE_SECONDS", "3600")) + interval = int(os.environ.get("STAGING_CLEANUP_INTERVAL_SECONDS", "60")) + if interval < 1: + raise ValueError("invalid cleanup interval") + while True: + try: + removed = clean_once(client, grace) + if removed: + logger.info("Removed %s staging objects", removed) + except Exception as exc: + logger.warning("Staging cleanup retrying after %s", type(exc).__name__) + time.sleep(interval) + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..298109a7 --- /dev/null +++ b/src/store.py @@ -0,0 +1,297 @@ +"""PostgreSQL-backed users, chats and message history.""" + +import base64 +import binascii +import hashlib +import json +import os +import secrets +from contextlib import contextmanager +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +import psycopg +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from fastapi import HTTPException +from psycopg.errors import UniqueViolation +from psycopg_pool import PoolTimeout + +from . import db +from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, token_hash, unauthorized + +password_hasher = PasswordHasher() + + +def timestamp(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def encode_cursor(resource: str, item: dict) -> str: + value = [resource, item["created_at"], item["id"]] + return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).decode().rstrip("=") + + +def decode_cursor(cursor: str | None, resource: str) -> tuple[datetime, UUID] | None: + if cursor is None: + return None + try: + raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True) + value = json.loads(raw) + if not isinstance(value, list) or len(value) != 3 or value[0] != resource: + raise ValueError + date_text, item_id = value[1:] + if not isinstance(date_text, str) or not date_text.endswith("Z") or not isinstance(item_id, str): + raise ValueError + date = datetime.fromisoformat(date_text[:-1] + "+00:00") + if date.utcoffset().total_seconds() != 0: + raise ValueError + return date, UUID(item_id) + except (binascii.Error, UnicodeError, ValueError, TypeError, AttributeError): + raise HTTPException(400) from None + + +@contextmanager +def connection(): + try: + with db.pool().connection() as conn: + yield conn + except (psycopg.Error, PoolTimeout, OSError): + raise HTTPException(503) from None + + +def user_view(row: dict) -> dict: + return { + "id": str(row["id"]), "username": row["username"], + "display_name": row["display_name"], "created_at": timestamp(row["created_at"]), + } + + +def chat_view(row: dict) -> dict: + return { + "id": str(row["id"]), "title": row["title"], + "member_ids": [str(member_id) for member_id in row["member_ids"]], + "created_at": timestamp(row["created_at"]), + } + + +def message_view(row: dict) -> dict: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), "text": row["text"], + "created_at": timestamp(row["created_at"]), + "attachment_ids": [str(attachment_id) for attachment_id in row.get("attachment_ids", [])], + } + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM memberships m WHERE m.chat_id=c.id ORDER BY m.position) AS member_ids""" + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id=m.id ORDER BY a.position) AS attachment_ids""" + + +class Store: + def ready(self) -> dict: + if not db.ready(): + raise HTTPException(503) + return {"status": "ok"} + + def register(self, username: str, display_name: str, password: str) -> dict: + password_hash = password_hasher.hash(password) + with connection() as conn: + try: + row = conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,%s,%s,%s) " + "RETURNING id,username,display_name,created_at", + (uuid4(), username, display_name, password_hash), + ).fetchone() + except UniqueViolation: + raise HTTPException(409) from None + return user_view(row) + + def user(self, user_id: str | None) -> dict: + if not user_id: + raise HTTPException(401) + try: + user_uuid = UUID(user_id) + except (ValueError, TypeError): + raise HTTPException(401) from None + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at FROM users WHERE id=%s", (user_uuid,) + ).fetchone() + if row is None: + raise HTTPException(401) + return user_view(row) + + def create_session(self, username: str, password: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s", + (username,), + ).fetchone() + try: + valid = row is not None and password_hasher.verify(row["password_hash"], password) + except (VerificationError, InvalidHashError): + valid = False + if not valid: + raise unauthorized(BASIC_CHALLENGE) + token = secrets.token_urlsafe(32) + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + expiry = conn.execute( + "INSERT INTO sessions (token_hash,user_id,expires_at) " + "VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s)) RETURNING expires_at", + (token_hash(token), row["id"], ttl), + ).fetchone()["expires_at"] + return {"token": token, "token_type": "Bearer", "expires_at": timestamp(expiry), "user": user_view(row)} + + def authenticated(self, token: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT u.id,u.username,u.display_name,u.created_at FROM sessions s " + "JOIN users u ON u.id=s.user_id " + "WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + return user_view(row) + + def revoke_session(self, token: str) -> None: + with connection() as conn: + row = conn.execute( + "DELETE FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp() RETURNING token_hash", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + + def create_chat(self, actor_id: str, title: str, member_ids: list[str]) -> dict: + if actor_id not in member_ids: + raise HTTPException(400) + members = [UUID(member_id) for member_id in member_ids] + with connection() as conn: + count = conn.execute("SELECT count(*) AS n FROM users WHERE id=ANY(%s)", (members,)).fetchone()["n"] + if count != len(members): + raise HTTPException(400) + row = conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), title)).fetchone() + for position, member_id in enumerate(members): + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,%s)", + (row["id"], member_id, position), + ) + return chat_view({**row, "member_ids": members}) + + def chat(self, actor_id: str, chat_id: str) -> dict: + with connection() as conn: + row = conn.execute( + f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id " + "WHERE access.user_id=%s AND c.id=%s", + (UUID(actor_id), UUID(chat_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return chat_view(row) + + def list_chats(self, actor_id: str, limit: int, cursor: str | None) -> dict: + after = decode_cursor(cursor, f"chats:{actor_id}") + query = (f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id WHERE access.user_id=%s") + params = [UUID(actor_id)] + if after: + query += " AND (c.created_at,c.id)>(%s,%s)" + params.extend(after) + query += " ORDER BY c.created_at,c.id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + rows = conn.execute(query, params).fetchall() + items = [chat_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None} + + def create_message(self, actor_id: str, chat_id: str, text: str, key: str, attachment_ids: list[str] | None = None) -> dict: + ids = [UUID(item) for item in attachment_ids or []] + if len(ids) > 10 or len(set(ids)) != len(ids) or (not text.strip() and not ids): + raise HTTPException(400) + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + request_hash = hashlib.sha256(json.dumps( + {"text": text, "attachment_ids": [str(item) for item in ids]}, ensure_ascii=False, sort_keys=True, + ).encode()).digest() + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + conn.execute( + "DELETE FROM message_replays WHERE user_id=%s AND chat_id=%s AND request_key=%s " + "AND created_at dict: + after = decode_cursor(cursor, f"messages:{chat_id}") + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + query = f"SELECT {MESSAGE_COLUMNS} FROM messages m WHERE m.chat_id=%s" + params = [chat_uuid] + if after: + query += " AND (m.created_at,m.id)>(%s,%s)" + params.extend(after) + query += " ORDER BY m.created_at,m.id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + rows = conn.execute(query, params).fetchall() + items = [message_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"messages:{chat_id}", items[-1]) if len(rows) > limit else None} + + +store = Store() diff --git a/student-tests/test_outbox_payload.py b/student-tests/test_outbox_payload.py new file mode 100644 index 00000000..0fe6b3fd --- /dev/null +++ b/student-tests/test_outbox_payload.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Keep broker messages within the public job schema and free of file data.""" + +import json +import sys +import unittest +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "tests")) +from client import validate # noqa: E402 +from src.pipeline_jobs import job_payload # noqa: E402 + + +class OutboxPayloadTest(unittest.TestCase): + def test_job_message_has_only_contract_fields(self): + schema = json.loads((ROOT / "contracts/jobs.schema.json").read_text(encoding="utf-8")) + row = { + "job_id": uuid4(), "attachment_id": uuid4(), "pipeline_version": 2, + "attempt": 3, "created_at": datetime.now(timezone.utc), + "object_key": "staging/must-not-leak", "bytes": b"must-not-leak", + } + message = job_payload(row) + validate(message, schema) + self.assertEqual(set(message), set(schema["required"])) + self.assertNotIn("staging", json.dumps(message)) + self.assertNotIn("bytes", json.dumps(message)) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_staging_cleanup.py b/student-tests/test_staging_cleanup.py new file mode 100644 index 00000000..4dddb284 --- /dev/null +++ b/student-tests/test_staging_cleanup.py @@ -0,0 +1,46 @@ +"""Run inside the cleanup image against an isolated pair of staging keys.""" + +import time +import sys +from io import BytesIO +from pathlib import Path +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.db import pool +from src.pipeline_jobs import reserve_upload +from src.staging_cleanup import BUCKET, clean_once, storage + + +client = storage() +orphan_id, active_id = uuid4(), uuid4() +orphan_key, active_key = f"staging/{orphan_id}", f"staging/{active_id}" + + +def exists(key): + return any(obj.object_name == key for obj in client.list_objects(BUCKET, prefix=key)) + + +try: + for key in (orphan_key, active_key): + client.put_object(BUCKET, key, BytesIO(b"test"), 4, content_type="image/png") + with pool().connection() as conn: + reserve_upload(conn, active_id, active_key, lease_seconds=60) + time.sleep(2) + clean_once(client, grace_seconds=1) + assert not exists(orphan_key), "old orphan was not removed" + assert exists(active_key), "active upload was removed" + + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_uploads SET lease_expires_at=clock_timestamp()-interval '1 second' WHERE id=%s", + (active_id,), + ) + clean_once(client, grace_seconds=1) + assert not exists(active_key), "expired orphan was not removed" + print("PASS: cleanup removes old orphans and preserves active uploads") +finally: + for key in (orphan_key, active_key): + client.remove_object(BUCKET, key) + with pool().connection() as conn: + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (active_id,)) diff --git a/student-tests/test_upload_handoff.py b/student-tests/test_upload_handoff.py new file mode 100644 index 00000000..8a261823 --- /dev/null +++ b/student-tests/test_upload_handoff.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Exercise 202 handoff while RabbitMQ and worker are unavailable.""" + +import subprocess +import sys +import time +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + + +def compose(*args): + subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, + ) + + +def ready_after_restart(client): + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not recover after restart") + + +def main(): + args = arguments("Lab5 durable upload handoff with stopped broker and worker") + client = Client(args.base_url, args.ca_file) + compose("stop", "worker", "publisher", "broker") + try: + alice, bob, _eve, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=bob) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment cannot be linked") + + compose("up", "-d", "broker") + compose("restart", "api") + ready_after_restart(client) + check( + client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=bob, schema="Attachment")["json"]["status"] == "queued", + "accepted job disappeared before publisher restart", + ) + compose("up", "-d", "publisher", "worker") + client.wait_ready(bob, attachment) + result = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob) + check(result["bytes"] == original, "accepted original changed after recovery") + finally: + compose("up", "-d", "broker", "publisher", "worker") + print("PASS: accepted upload survived broker/API outage and reached ready") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_upload_queued.py b/student-tests/test_upload_queued.py new file mode 100644 index 00000000..ec6521df --- /dev/null +++ b/student-tests/test_upload_queued.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Public U2 check while publisher and worker are stopped.""" + +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def main(): + args = arguments("Lab5 durable queued upload") + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + attachment, _content = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload did not enter queued state") + client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=bob) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment cannot be linked") + print("PASS: 202 queued upload is readable and linkable; content is pending") + + +if __name__ == "__main__": + run(main) -- 2.54.0 From 03899d790bb8e7047d2f2fccc6bfa10957b0208a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:39:08 +0300 Subject: [PATCH 4/7] Process versioned image variants with leased worker (U3) --- src/attachment_read.py | 72 +++++++++ src/image_variants.py | 83 ++++++++++ src/worker.py | 216 +++++++++++++++++++++++++++ student-tests/test_image_pipeline.py | 74 +++++++++ student-tests/test_variants_api.py | 47 ++++++ 5 files changed, 492 insertions(+) create mode 100644 src/attachment_read.py create mode 100644 src/image_variants.py create mode 100644 src/worker.py create mode 100644 student-tests/test_image_pipeline.py create mode 100644 student-tests/test_variants_api.py diff --git a/src/attachment_read.py b/src/attachment_read.py new file mode 100644 index 00000000..34f91a69 --- /dev/null +++ b/src/attachment_read.py @@ -0,0 +1,72 @@ +"""Authorize metadata and stream versioned variants from the private bucket.""" + +import logging +from uuid import UUID + +from fastapi import HTTPException +from fastapi.responses import StreamingResponse + +from .attachments import BUCKET, attachment_view, storage +from .store import connection + + +logger = logging.getLogger(__name__) + + +def metadata(actor_id: str, attachment_id: UUID) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a " + "JOIN memberships m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s", + (attachment_id, UUID(actor_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return row + + +def download(actor_id: str, attachment_id: UUID, variant: str) -> StreamingResponse: + row = metadata(actor_id, attachment_id) + if row["status"] != "ready": + raise HTTPException(409) + with connection() as conn: + selected = conn.execute( + "SELECT object_key,content_type,size_bytes FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s AND name=%s", + (attachment_id, row["pipeline_version"], variant), + ).fetchone() + if selected is None: + raise HTTPException(404) + try: + obj = storage().get_object(BUCKET, selected["object_key"]) + except Exception as exc: + logger.warning("S3 read failed for attachment %s (%s)", attachment_id, type(exc).__name__) + raise HTTPException(503) from None + + def chunks(): + try: + yield from obj.stream(64 * 1024) + finally: + obj.close() + obj.release_conn() + + return StreamingResponse( + chunks(), media_type=selected["content_type"], + headers={"Content-Length": str(selected["size_bytes"])} + ) + + +def get_view(actor_id: str, attachment_id: UUID) -> dict: + row = metadata(actor_id, attachment_id) + view = attachment_view(row) + if row["status"] == "ready": + with connection() as conn: + variants = conn.execute( + "SELECT name,content_type,size_bytes,width,height,sha256 " + "FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s " + "AND name <> 'original' ORDER BY name", + (attachment_id, row["pipeline_version"]), + ).fetchall() + view["variants"] = [dict(variant) for variant in variants] + return view diff --git a/src/image_variants.py b/src/image_variants.py new file mode 100644 index 00000000..72a8cd33 --- /dev/null +++ b/src/image_variants.py @@ -0,0 +1,83 @@ +"""Deterministic, bounded image variants for one accepted PNG or JPEG.""" + +import hashlib +from io import BytesIO + +from PIL import Image, ImageOps, UnidentifiedImageError + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +class InvalidImage(ValueError): + pass + + +def _fit(image: Image.Image, bound: int) -> Image.Image: + factor = min(1, bound / max(image.size)) + size = (max(1, int(image.width * factor)), max(1, int(image.height * factor))) + return image.resize(size, Image.Resampling.LANCZOS) if size != image.size else image + + +def _jpeg(image: Image.Image) -> bytes: + output = BytesIO() + image.save(output, format="JPEG", quality=85, optimize=True) + return output.getvalue() + + +def _variant(data: bytes, content_type: str, image: Image.Image) -> dict: + return { + "data": data, + "content_type": content_type, + "size_bytes": len(data), + "width": image.width, + "height": image.height, + "sha256": hashlib.sha256(data).hexdigest(), + } + + +def render(data: bytes, content_type: str) -> dict[str, dict]: + """Preserve original bytes; orient, flatten alpha, strip metadata on derivatives.""" + if not 0 < len(data) <= MAX_BYTES or content_type not in {"image/png", "image/jpeg"}: + raise InvalidImage("invalid original") + try: + with Image.open(BytesIO(data)) as source: + if {"PNG": "image/png", "JPEG": "image/jpeg"}.get(source.format) != content_type: + raise InvalidImage("wrong image type") + if source.width * source.height > MAX_PIXELS: + raise InvalidImage("too many pixels") + source.load() + original_size = source.size + oriented = ImageOps.exif_transpose(source) + if oriented.mode in {"RGBA", "LA", "P"}: + rgba = oriented.convert("RGBA") + image = Image.new("RGB", rgba.size, "white") + image.paste(rgba, mask=rgba.getchannel("A")) + else: + image = oriented.convert("RGB") + + thumbnail = _fit(image, 256) + side = min(image.size) + left = (image.width - side) // 2 + top = (image.height - side) // 2 + crop = image.resize( + (128, 128), Image.Resampling.LANCZOS, + box=(left, top, left + side, top + side), + ) + optimized = _fit(image, 1280) + webp = BytesIO() + optimized.save(webp, format="WEBP", quality=82, method=4) + return { + "original": { + "data": data, "content_type": content_type, + "size_bytes": len(data), "width": original_size[0], "height": original_size[1], + "sha256": hashlib.sha256(data).hexdigest(), + }, + "thumbnail": _variant(_jpeg(thumbnail), "image/jpeg", thumbnail), + "crop": _variant(_jpeg(crop), "image/jpeg", crop), + "optimized": _variant(webp.getvalue(), "image/webp", optimized), + } + except (UnidentifiedImageError, OSError, SyntaxError, Image.DecompressionBombError) as exc: + raise InvalidImage("image decoding failed") from exc diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..b0c2f7f3 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,216 @@ +"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects.""" + +import hashlib +import json +import logging +import os +import random +import re +import signal +import time +from contextlib import contextmanager +from datetime import datetime +from io import BytesIO +from pathlib import Path +from uuid import UUID + +import pika +import psycopg +from minio.error import S3Error +from psycopg_pool import PoolTimeout + +from .attachments import BUCKET, storage +from .db import pool +from .image_variants import InvalidImage, MAX_BYTES, render +from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired + + +logger = logging.getLogger(__name__) +QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) +LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) + + +def parse_message(body: bytes) -> dict: + if len(body) > 4096: + raise ValueError("oversized job message") + item = json.loads(body) + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if (not isinstance(item, dict) or not required <= item.keys() or + item.keys() - required - {"traceparent", "tracestate"} or + type(item["schema_version"]) is not int or item["schema_version"] != 1 or + item["type"] != "attachment.process.v1" or + type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or + type(item["attempt"]) is not int or item["attempt"] < 1 or + not isinstance(item["job_id"], str) or + not isinstance(item["attachment_id"], str) or + not isinstance(item["created_at"], str) or + ("traceparent" in item and + (not isinstance(item["traceparent"], str) or + re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or + ("tracestate" in item and + (not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))): + raise ValueError("invalid job message") + UUID(item["job_id"]) + UUID(item["attachment_id"]) + created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00")) + if created_at.utcoffset() is None: + raise ValueError("job timestamp lacks timezone") + return item + + +def _source_bytes(job) -> bytes: + if not 0 < job["size_bytes"] <= MAX_BYTES: + raise InvalidImage("invalid accepted size") + # Reprocessing uses the byte-identical final original, so ready staging + # can be reclaimed without making a future pipeline version impossible. + key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"] + response = storage().get_object(BUCKET, key) + try: + chunks = [] + size = 0 + digest = hashlib.sha256() + for chunk in response.stream(64 * 1024): + size += len(chunk) + if size > job["size_bytes"]: + raise InvalidImage("source exceeds accepted size") + chunks.append(chunk) + digest.update(chunk) + if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]: + raise InvalidImage("source checksum mismatch") + return b"".join(chunks) + finally: + response.close() + response.release_conn() + + +def _write_variants(job, variants: dict) -> dict: + extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"} + stored = {} + for name, variant in variants.items(): + key = (job["object_key"] if name == "original" else + f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}") + data = variant["data"] + storage().put_object( + BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"] + ) + stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"} + stored[name]["object_key"] = key + return stored + + +def _delay(attempt: int) -> float: + base = min(30, 2 ** min(attempt - 1, 5)) + return base + random.uniform(0, base / 2) + + +@contextmanager +def _deadline(seconds: int): + def expired(_signal, _frame): + raise TimeoutError("image job exceeded worker deadline") + + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def process_message(item: dict) -> str: + with pool().connection() as conn: + job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS) + if job is None: + return "duplicate_or_stale" + token = job["lease_token"] + try: + with _deadline(LEASE_SECONDS): + data = _source_bytes(job) + variants = render(data, job["content_type"]) + stored = _write_variants(job, variants) + with pool().connection() as conn: + current = complete_job(conn, job["id"], token, stored) + if current and job["staging_key"]: + try: + storage().remove_object(BUCKET, job["staging_key"]) + except Exception as exc: + # Periodic cleanup recovers this post-COMMIT gap. + logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__) + return "ready" + except psycopg.Error: + raise + except Exception as exc: + if isinstance(exc, InvalidImage): + code, permanent = "invalid_image", True + elif isinstance(exc, TimeoutError): + code, permanent = "worker_timeout", False + elif isinstance(exc, S3Error) or isinstance(exc, OSError): + code, permanent = "storage_unavailable", False + elif isinstance(exc, ValueError): + code, permanent = "processing_failed", True + else: + code, permanent = "processing_failed", False + logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code) + with pool().connection() as conn: + try: + return finish_attempt( + conn, job["id"], token, max_attempts=MAX_ATTEMPTS, + delay_seconds=_delay(job["attempt"]), error_code=code, + permanent=permanent, + ) + except ValueError: + # A lease recovery already owns the next attempt. + return "stale_lease" + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1: + raise SystemExit("invalid worker retry settings") + url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() + params = pika.URLParameters(url) + params.heartbeat = max(120, LEASE_SECONDS * 2) + params.socket_timeout = 5 + params.blocked_connection_timeout = 10 + while True: + broker = None + try: + broker = pika.BlockingConnection(params) + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.basic_qos(prefetch_count=1) + last_recovery = 0.0 + while broker.is_open: + if time.monotonic() - last_recovery >= 5: + with pool().connection() as conn: + recover_expired(conn, max_attempts=MAX_ATTEMPTS) + last_recovery = time.monotonic() + method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False) + if method is None: + broker.process_data_events(time_limit=1) + continue + try: + item = parse_message(body) + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + logger.warning("Rejecting invalid job message") + channel.basic_reject(method.delivery_tag, requeue=False) + continue + try: + result = process_message(item) + channel.basic_ack(method.delivery_tag) + logger.info("Job %s: %s", item["job_id"], result) + except (psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__) + channel.basic_nack(method.delivery_tag, requeue=True) + time.sleep(2) + except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Worker reconnecting after %s", type(exc).__name__) + time.sleep(2) + finally: + if broker is not None and broker.is_open: + broker.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py new file mode 100644 index 00000000..404b8633 --- /dev/null +++ b/student-tests/test_image_pipeline.py @@ -0,0 +1,74 @@ +"""Small checks for the worker's deterministic image and payload boundary.""" + +import json +import unittest +from io import BytesIO + +from PIL import Image + +from src.image_variants import InvalidImage, render +from src.worker import parse_message + + +def encoded(image, format="PNG", **options): + stream = BytesIO() + image.save(stream, format=format, **options) + return stream.getvalue() + + +class ImagePipelineTest(unittest.TestCase): + def test_contract_sizes_original_and_content_types(self): + original = encoded(Image.new("RGB", (320, 200), "red")) + result = render(original, "image/png") + self.assertEqual(result["original"]["data"], original) + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (256, 160)) + self.assertEqual((result["crop"]["width"], result["crop"]["height"]), (128, 128)) + self.assertEqual((result["optimized"]["width"], result["optimized"]["height"]), (320, 200)) + self.assertEqual([result[name]["content_type"] for name in ("thumbnail", "crop", "optimized")], + ["image/jpeg", "image/jpeg", "image/webp"]) + + def test_alpha_is_white_and_small_image_is_not_upscaled(self): + original = encoded(Image.new("RGBA", (24, 12), (255, 0, 0, 0))) + result = render(original, "image/png") + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (24, 12)) + with Image.open(BytesIO(result["thumbnail"]["data"])) as thumb: + self.assertTrue(all(channel >= 245 for channel in thumb.getpixel((0, 0)))) + + def test_exif_orientation_and_center_crop(self): + image = Image.new("RGB", (30, 10), "green") + image.paste("red", (0, 0, 10, 10)) + image.paste("blue", (20, 0, 30, 10)) + exif = Image.Exif() + exif[274] = 6 + original = encoded(image, "JPEG", exif=exif) + result = render(original, "image/jpeg") + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (10, 30)) + with Image.open(BytesIO(result["crop"]["data"])) as crop: + red, green, blue = crop.getpixel((64, 64)) + self.assertGreater(green, red) + self.assertGreater(green, blue) + + def test_rejects_wrong_type_and_corrupt_file(self): + original = encoded(Image.new("RGB", (2, 2), "black")) + with self.assertRaises(InvalidImage): + render(original, "image/jpeg") + with self.assertRaises(InvalidImage): + render(b"broken", "image/png") + + def test_payload_contains_only_versioned_job_identifiers(self): + body = { + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-09-01T12:00:00Z", + } + self.assertEqual(parse_message(json.dumps(body).encode()), body) + with self.assertRaises(ValueError): + parse_message(json.dumps({**body, "staging_key": "private"}).encode()) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_variants_api.py b/student-tests/test_variants_api.py new file mode 100644 index 00000000..fd4aee87 --- /dev/null +++ b/student-tests/test_variants_api.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Check every public variant against committed metadata and original bytes.""" + +import hashlib +import sys +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def main(): + args = arguments("Lab5 ready variants") + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + ready = client.wait_ready(bob, attachment) + variants = {item["name"]: item for item in ready["variants"]} + check(set(variants) == {"thumbnail", "crop", "optimized"}, "incomplete variant set") + expected = { + "thumbnail": ("image/jpeg", 256, 160), + "crop": ("image/jpeg", 128, 128), + "optimized": ("image/webp", 320, 200), + } + for name, (content_type, width, height) in expected.items(): + variant = variants[name] + check((variant["content_type"], variant["width"], variant["height"]) == + (content_type, width, height), f"wrong {name} metadata") + response = client.call( + "GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", actor=bob + ) + check(response["headers"].get_content_type() == content_type, f"wrong {name} MIME") + check(len(response["bytes"]) == variant["size_bytes"] and + hashlib.sha256(response["bytes"]).hexdigest() == variant["sha256"], + f"wrong {name} bytes") + client.call( + "GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", + 404, actor=eve, + ) + downloaded = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob) + check(downloaded["bytes"] == original, "original bytes changed") + print("PASS: all three variants match metadata; original preserved; outsider denied") + + +if __name__ == "__main__": + run(main) -- 2.54.0 From ab88c5ba1b2c76c46aeea32e81fbc8f1c09065e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:44:24 +0300 Subject: [PATCH 5/7] Prove versioned pipeline recovery and audited replay (U4) --- evidence/pipeline-faults-2026-10-02.md | 36 +++++ migrations/007_replay_audit.sql | 16 +++ src/pipeline_jobs.py | 2 +- src/replay.py | 64 +++++++++ src/reprocess.py | 57 ++++++++ src/worker.py | 10 ++ student-tests/test_pipeline_faults.py | 188 +++++++++++++++++++++++++ 7 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 evidence/pipeline-faults-2026-10-02.md create mode 100644 migrations/007_replay_audit.sql create mode 100644 src/replay.py create mode 100644 src/reprocess.py create mode 100644 student-tests/test_pipeline_faults.py diff --git a/evidence/pipeline-faults-2026-10-02.md b/evidence/pipeline-faults-2026-10-02.md new file mode 100644 index 00000000..26743bf5 --- /dev/null +++ b/evidence/pipeline-faults-2026-10-02.md @@ -0,0 +1,36 @@ +# Versioned pipeline fault injection — 2026-10-02 + +## Hypothesis and method + +A process death before or after an S3 side effect, a duplicate delivery, an expired lease, and a reordered older-version delivery should leave one logical variant set per `(attachment_id, pipeline_version, name)`. A failed current job should replay only by an explicit audited operator action. RabbitMQ remains at-least-once: duplicate messages are expected; the durable PostgreSQL state and deterministic S3 keys make their effects idempotent. + +`student-tests/test_pipeline_faults.py` creates synthetic 320×200 PNG uploads in the same PostgreSQL/MinIO stand as the application. It runs the real `src.worker.process_message` in child processes with `LAB5_FAULT_POINT` and `LAB5_FAULT_JOB_ID`; `os._exit(72)` terminates the child at the selected durable boundary. The parent checks database state, expires a simulated dead worker lease, runs recovery, redelivers the same logical job with the next managed attempt, and compares variant rows/checksums. The normal worker and publisher are stopped while injecting faults so their consumers cannot race the test. The test also forces a bounded temporary failure, verifies terminal `failed`, then uses the audited replay operation. No broker message is fabricated by the test; separate handoff/outbox tests exercise publication and confirm. + +Fault points in worker code: `after_claim_before_s3`, `after_s3_before_result_commit`, `after_result_commit_before_ack`. Each is disabled unless both environment variables name a point and exact synthetic job ID. The test runs once on the existing educational volumes and once on a fresh, isolated Compose project. The isolated project had zero `attachment_jobs` and zero `attachment_job_replays` before the run; its volumes were removed afterward. Main lab5 volumes were preserved. + +## Reproduce + +```sh +# Existing local stand, with worker and publisher stopped: +docker compose --env-file .env stop worker publisher +docker compose --env-file .env run --rm --build --no-deps -T worker python - < student-tests/test_pipeline_faults.py + +# Fresh, isolated PostgreSQL and S3 volumes; no API port and no normal consumer: +docker compose -p mtusi2026lab5fresh --env-file .env up -d --build postgres migrate s3-bootstrap +docker compose -p mtusi2026lab5fresh --env-file .env run --rm --build --no-deps -T worker python - < student-tests/test_pipeline_faults.py +docker compose -p mtusi2026lab5fresh --env-file .env down -v +``` + +On this host Docker builds used the ignored `compose.local.yaml` DNS override via `COMPOSE_FILE=compose.yaml:compose.local.yaml`; it does not alter application behavior. The first attempt to use `up --wait` on the isolated project exited 1 because Docker Compose treated the completed one-shot `migrate` service as not running; inspection showed `migrate` exited 0, PostgreSQL/S3 were healthy, and both relevant tables contained zero rows before the fault test. + +## Observed result + +Both runs returned the same anonymized raw summary (one JSON line per run): + +```json +{"after_result_commit_before_ack_duplicate":"pass","after_s3_before_result_commit":"pass","before_side_effect_lease_recovery":"pass","bounded_transient_retry_then_replay":"pass","duplicate_reorder_new_version":"pass","failed_registry_operator_replay":"pass"} +``` + +For each ready version, the test found exactly the four expected variant names (`original`, `thumbnail`, `crop`, `optimized`), a matching original SHA-256, and the expected current `pipeline_version`. Replaying an already ready job returned `duplicate_or_stale`. After a second version became ready, an older delivery left its checksums and current version unchanged; version 1 rows remained available as history. The failed-job replay wrote one audit row and used a new attempt without deleting prior outbox history. The transient retry test exhausted a two-attempt budget, recorded `failed`, then succeeded only after replay reset that budget. + +The controlled process death checks durable boundaries, not the probability of such a crash under load. The local stand has one MinIO and one PostgreSQL instance, so this result does not prove service availability during their outage or exactly-once delivery. The broker outage/confirm boundary is tested separately; this matrix checks the worker effect boundary against real S3 and PostgreSQL. diff --git a/migrations/007_replay_audit.sql b/migrations/007_replay_audit.sql new file mode 100644 index 00000000..cbe4c375 --- /dev/null +++ b/migrations/007_replay_audit.sql @@ -0,0 +1,16 @@ +-- A replay starts a new bounded run without deleting earlier outbox history. +ALTER TABLE attachment_jobs + ADD COLUMN replay_start_attempt integer NOT NULL DEFAULT 1 + CHECK (replay_start_attempt > 0 AND replay_start_attempt <= attempt); + +CREATE TABLE attachment_job_replays ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + previous_attempt integer NOT NULL CHECK (previous_attempt > 0), + new_attempt integer NOT NULL CHECK (new_attempt > previous_attempt), + previous_error_code text NOT NULL, + operator_name text NOT NULL CHECK (length(operator_name) BETWEEN 1 AND 80), + reason text NOT NULL CHECK (length(reason) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (job_id, new_attempt) +); diff --git a/src/pipeline_jobs.py b/src/pipeline_jobs.py index 5fa477f7..0e6f8fee 100644 --- a/src/pipeline_jobs.py +++ b/src/pipeline_jobs.py @@ -149,7 +149,7 @@ def claim_job(conn, job_id, delivery_attempt, lease_seconds=60): def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent): if max_attempts < 1 or delay_seconds < 0: raise ValueError("invalid retry configuration") - failed = permanent or job["attempt"] >= max_attempts + failed = permanent or job["attempt"] - job["replay_start_attempt"] + 1 >= max_attempts if failed: code = error_code if permanent else "retry_exhausted" conn.execute( diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 00000000..96e75965 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,64 @@ +"""Explicit, audited replay of one failed current-version job.""" + +import argparse +from uuid import UUID + +from .db import pool + + +def replay_failed(conn, job_id: UUID, operator: str, reason: str) -> int: + if not 1 <= len(operator) <= 80 or not 1 <= len(reason) <= 200: + raise ValueError("operator and reason are required") + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.status AS attachment_status," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j,a", + (job_id,), + ).fetchone() + if (job is None or job["status"] != "failed" or + job["pipeline_version"] != job["current_pipeline_version"] or + job["attachment_status"] != "failed"): + raise ValueError("only a failed current-version job can be replayed") + attempt = job["attempt"] + 1 + conn.execute( + "UPDATE attachment_jobs SET status='queued',attempt=%s,replay_start_attempt=%s," + "next_attempt_at=clock_timestamp(),error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s", + (attempt, attempt, job_id), + ) + conn.execute( + "UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (job["attachment_id"], job["pipeline_version"]), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,%s)", + (job_id, attempt), + ) + conn.execute( + "INSERT INTO attachment_job_replays " + "(job_id,previous_attempt,new_attempt,previous_error_code,operator_name,reason) " + "VALUES (%s,%s,%s,%s,%s,%s)", + (job_id, job["attempt"], attempt, job["error_code"], operator, reason), + ) + return attempt + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("job_id", type=UUID) + parser.add_argument("--operator", required=True) + parser.add_argument("--reason", required=True) + args = parser.parse_args() + try: + with pool().connection() as conn: + attempt = replay_failed(conn, args.job_id, args.operator, args.reason) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued job {args.job_id} attempt {attempt}") + + +if __name__ == "__main__": + main() diff --git a/src/reprocess.py b/src/reprocess.py new file mode 100644 index 00000000..f7f4354c --- /dev/null +++ b/src/reprocess.py @@ -0,0 +1,57 @@ +"""Start a new pipeline version from a verified, already ready original.""" + +import argparse +from uuid import UUID, uuid4 + +from .db import pool + + +def start_new_version(conn, attachment_id: UUID) -> tuple[int, UUID]: + with conn.transaction(): + row = conn.execute( + "SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,) + ).fetchone() + if row is None or row["status"] != "ready": + raise ValueError("only a ready attachment can start a new version") + original = conn.execute( + "SELECT object_key,size_bytes,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s AND name='original'", + (attachment_id, row["pipeline_version"]), + ).fetchone() + if (original is None or original["object_key"] != row["object_key"] or + original["size_bytes"] != row["size_bytes"] or + original["sha256"] != row["sha256"]): + raise ValueError("verified original is missing") + version = row["pipeline_version"] + 1 + job_id = uuid4() + conn.execute( + "UPDATE attachments SET pipeline_version=%s,status='queued',error_code=NULL," + "updated_at=clock_timestamp() WHERE id=%s", + (version, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) " + "VALUES (%s,%s,%s)", + (job_id, attachment_id, version), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)", + (job_id,), + ) + return version, job_id + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("attachment_id", type=UUID) + args = parser.parse_args() + try: + with pool().connection() as conn: + version, job_id = start_new_version(conn, args.attachment_id) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued attachment {args.attachment_id} version {version} job {job_id}") + + +if __name__ == "__main__": + main() diff --git a/src/worker.py b/src/worker.py index b0c2f7f3..f96f7a59 100644 --- a/src/worker.py +++ b/src/worker.py @@ -104,6 +104,13 @@ def _delay(attempt: int) -> float: return base + random.uniform(0, base / 2) +def _fault(point: str, job_id) -> None: + """Test-only process death at a named durable boundary for one job.""" + if (os.environ.get("LAB5_FAULT_POINT") == point and + os.environ.get("LAB5_FAULT_JOB_ID") == str(job_id)): + os._exit(72) + + @contextmanager def _deadline(seconds: int): def expired(_signal, _frame): @@ -124,13 +131,16 @@ def process_message(item: dict) -> str: if job is None: return "duplicate_or_stale" token = job["lease_token"] + _fault("after_claim_before_s3", job["id"]) try: with _deadline(LEASE_SECONDS): data = _source_bytes(job) variants = render(data, job["content_type"]) stored = _write_variants(job, variants) + _fault("after_s3_before_result_commit", job["id"]) with pool().connection() as conn: current = complete_job(conn, job["id"], token, stored) + _fault("after_result_commit_before_ack", job["id"]) if current and job["staging_key"]: try: storage().remove_object(BUCKET, job["staging_key"]) diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 00000000..63a8e8e0 --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Deterministic fault windows for the versioned attachment pipeline. + +Run in a one-shot worker image while the normal worker and publisher are stopped. +All rows and objects are synthetic and stay in the educational volumes. +""" + +import hashlib +import json +import os +import subprocess +import sys +import time +from io import BytesIO +from uuid import uuid4 + +from PIL import Image + +from src.attachments import BUCKET, storage +from src.db import pool +from src.pipeline_jobs import ( + claim_job, finish_attempt, queue_attachment, recover_expired, reserve_upload, +) +from src.replay import replay_failed +from src.reprocess import start_new_version +from src.worker import process_message + + +image = BytesIO() +Image.new("RGB", (320, 200), "#2765a8").save(image, format="PNG") +DATA = image.getvalue() +SHA = hashlib.sha256(DATA).hexdigest() + + +def create_case(): + user_id, chat_id, attachment_id = uuid4(), uuid4(), uuid4() + staging_key = f"staging/{attachment_id}" + with pool().connection() as conn: + with conn.transaction(): + conn.execute( + "INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)", + (user_id, f"fault_{user_id.hex[:20]}", "Pipeline fault fixture", "synthetic"), + ) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s)", (chat_id, "Pipeline fault fixture")) + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", + (chat_id, user_id), + ) + reserve_upload(conn, attachment_id, staging_key) + storage().put_object(BUCKET, staging_key, BytesIO(DATA), len(DATA), content_type="image/png") + with pool().connection() as conn: + _, job_id = queue_attachment( + conn, attachment_id=attachment_id, chat_id=chat_id, owner_id=user_id, + filename="fixture.png", staging_key=staging_key, + object_key=f"originals/{attachment_id}", content_type="image/png", + size_bytes=len(DATA), sha256=SHA, + ) + return attachment_id, job_id + + +def item(job_id, attempt=1): + return {"job_id": str(job_id), "attempt": attempt} + + +def state(attachment_id, version): + with pool().connection() as conn: + attachment = conn.execute( + "SELECT status,pipeline_version,sha256 FROM attachments WHERE id=%s", (attachment_id,) + ).fetchone() + rows = conn.execute( + "SELECT name,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name", + (attachment_id, version), + ).fetchall() + return attachment, {row["name"]: row["sha256"] for row in rows} + + +def assert_ready(attachment_id, version): + attachment, variants = state(attachment_id, version) + assert attachment["status"] == "ready" and attachment["pipeline_version"] == version + assert attachment["sha256"] == SHA + assert set(variants) == {"original", "thumbnail", "crop", "optimized"} + assert variants["original"] == SHA + return variants + + +def expire_and_recover(job_id): + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_jobs SET lease_expires_at=clock_timestamp()-interval '1 second' " + "WHERE id=%s AND status='processing'", + (job_id,), + ) + recovered = recover_expired(conn, max_attempts=5) + assert (job_id, "queued") in recovered + + +def crash(job_id, point): + child = subprocess.run( + [sys.executable, "-c", "from src.worker import process_message; " + f"process_message({{'job_id': '{job_id}', 'attempt': 1}})"], + env={**os.environ, "LAB5_FAULT_POINT": point, "LAB5_FAULT_JOB_ID": str(job_id)}, + capture_output=True, text=True, + ) + assert child.returncode == 72, (point, child.returncode, child.stderr) + + +def main(): + results = {} + + attachment, job = create_case() + crash(job, "after_result_commit_before_ack") + first = assert_ready(attachment, 1) + assert process_message(item(job)) == "duplicate_or_stale" + assert assert_ready(attachment, 1) == first + results["after_result_commit_before_ack_duplicate"] = "pass" + + attachment, job = create_case() + crash(job, "after_claim_before_s3") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["before_side_effect_lease_recovery"] = "pass" + + attachment, job = create_case() + crash(job, "after_s3_before_result_commit") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["after_s3_before_result_commit"] = "pass" + + attachment, old_job = create_case() + assert process_message(item(old_job)) == "ready" + with pool().connection() as conn: + version, new_job = start_new_version(conn, attachment) + assert version == 2 + assert process_message(item(old_job)) == "duplicate_or_stale" + assert process_message(item(new_job)) == "ready" + newer = assert_ready(attachment, 2) + assert process_message(item(old_job)) == "duplicate_or_stale" + assert assert_ready(attachment, 2) == newer + assert len(state(attachment, 1)[1]) == 4 + results["duplicate_reorder_new_version"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=5, + error_code="invalid_image", permanent=True, + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "verified synthetic replay") + assert attempt == 2 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + with pool().connection() as conn: + count = conn.execute( + "SELECT count(*) AS n FROM attachment_job_replays WHERE job_id=%s", (job,) + ).fetchone()["n"] + assert count == 1 + results["failed_registry_operator_replay"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + delay_seconds=0.01, error_code="storage_unavailable", + ) == "queued" + time.sleep(0.03) + with pool().connection() as conn: + claimed = claim_job(conn, job, 2) + assert claimed is not None + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + error_code="storage_unavailable", + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "retry budget reset after repair") + assert attempt == 3 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + results["bounded_transient_retry_then_replay"] = "pass" + + print(json.dumps(results, sort_keys=True)) + + +if __name__ == "__main__": + main() -- 2.54.0 From 3386e46c3976cacb513578dc93cb96e2da04f5a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:59:32 +0300 Subject: [PATCH 6/7] Document pipeline evidence and load profile (U5) --- REPORT.md | 84 ++++++++++---- evidence/image-load-2026-10-02.json | 40 +++++++ .../image-resource-samples-2026-10-02.txt | 22 ++++ evidence/load-2026-10-02.json | 17 +++ evidence/pipeline-faults-2026-10-02.md | 4 +- scripts/measure-image-pipeline.py | 107 ++++++++++++++++++ student-tests/publish_gap_inner.py | 59 ++++++++++ student-tests/test_publish_confirm_gap.py | 45 ++++++++ 8 files changed, 357 insertions(+), 21 deletions(-) create mode 100644 evidence/image-load-2026-10-02.json create mode 100644 evidence/image-resource-samples-2026-10-02.txt create mode 100644 evidence/load-2026-10-02.json create mode 100644 scripts/measure-image-pipeline.py create mode 100644 student-tests/publish_gap_inner.py create mode 100644 student-tests/test_publish_confirm_gap.py diff --git a/REPORT.md b/REPORT.md index e98f75e9..9cf8f801 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,81 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Самохвалов Андрей Ремович, БПИ-2302. +- Целевая оценка: 5, трек 2 «Доказанная идемпотентность pipeline». Уровни 3 и 4 тоже проверены. +- Ветка: `solution/bpi2302-samohvalov`. Локальные `docs/plans/` и `LAB_NOTES.md` не входят в коммиты. +- Источники: `README.md`, `COURSE.md`, `contracts/README.md`, `contracts/attachment-states.md`, `contracts/jobs.schema.json`, `contracts/openapi.json`, публичные тесты и собственная lab4. Python 3.12.12, FastAPI 0.118.3, Pillow 11.3.0, psycopg 3.2.12, MinIO SDK 7.2.20, pika 1.4.4. Помощь ИИ: Codex; факты ниже проверены на синтетических данных. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose и Python 3.9+. Наружу опубликован только HAProxy на `127.0.0.1:${APP_PORT:-8080}`; паролей в Git нет. -## Архитектура и решения +```sh +cp .env.example .env +python3 scripts/setup-local-secrets.py +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 +docker compose --env-file .env up -d --wait --scale api=2 api lb worker publisher +make check +make test +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Скрипт создаёт только отсутствующие секреты с правами 0600. `migrate`, `s3-volume-init`, `s3-bootstrap`, `broker-init` — одноразовые роли: `Exited (0)` для них штатен. Миграции применяются под PostgreSQL advisory lock. Остановить стенд без потери учебных данных: `docker compose --env-file .env down`. Только для намеренного удаления синтетических данных: `docker compose --env-file .env down -v`. + +Образы: PostgreSQL 17.6, RabbitMQ 4.1.8-management, HAProxy 3.0.11, Python 3.12.12, MinIO `RELEASE.2025-10-15T17-29-55Z` и MinIO Client `2025.7.21-debian-12-r3` (S3-образы закреплены digest). Проверено в Colima/arm64: 2 CPU, 2 GiB RAM. На этом хосте Docker build не разрешал DNS `files.pythonhosted.org`; сборка применяла только локальный ignored `compose.local.yaml` с временным адресом CDN через `COMPOSE_FILE=compose.yaml:compose.local.yaml`. При обычном DNS/PyPI он не нужен. + +## Архитектура и границы надёжности + +Клиент → HAProxy → две API-реплики. PostgreSQL — источник истины для пользователей, сообщений, вложений, jobs, outbox и версий. MinIO хранит байты в закрытом bucket. RabbitMQ переносит только ID и служебные поля `jobs.schema.json`, без изображения, credentials, клиентского пути или произвольного URL. + +| Роль | Health, volume, пределы | Доступ | +| --- | --- | --- | +| `lb` | проверка API backends, без данных | единственный host port | +| `api ×2` | `/health/ready`, 0,5 CPU/256 MiB каждая | DB URL и ограниченные S3 credentials | +| `postgres` | `pg_isready`, `postgres_data` | локальный secret | +| `s3` | `/minio/health/ready`, `s3_data`, 0,5 CPU/384 MiB | root только внутри S3/bootstrap | +| `broker` | `rabbitmq-diagnostics ping`, `broker_data` | AMQP URL через secret, без host port | +| `publisher` | отдельный процесс с publisher confirms | DB и AMQP, без S3 | +| `worker` | отдельный процесс, 0,5 CPU/768 MiB, prefetch=1 | DB, AMQP, объектные S3 права | +| `staging-cleanup` | периодический процесс | DB и отдельные S3 maintenance credentials | + +API потоково принимает до 10 MiB PNG/JPEG, сверяет MIME с содержимым и ограничивает изображение 40 млн пикселей. Имя файла используется только для отображения. Перед S3 PUT создаётся временный upload marker; полный staging-object затем сверяется по размеру и SHA-256. Одна транзакция записывает attachment, job и outbox, и лишь после COMMIT API отвечает `202 queued`. Поэтому остановленный broker/worker не теряет принятую загрузку; другие сообщения и ссылка на queued-вложение того же чата работают. Пока нет `ready`, скачивание отвечает 409. + +Publisher арендует outbox, публикует persistent сообщение в durable quorum queue и отмечает outbox только после broker confirm. Потеря DB-подтверждения может создать вторую доставку. Worker захватывает job под lease, читает доверенные метаданные из PostgreSQL, сверяет исходные байты и пишет объекты с детерминированными ключами. Статус `ready` фиксируется лишь после всех четырёх внутренних эффектов: `original`, `thumbnail`, `crop`, `optimized`. Публичная metadata показывает три производных варианта; доступ к каждому проходит проверку членства в чате через API. + +Original остаётся побайтно неизменным. Thumbnail — JPEG quality=85, `optimize=True`, вписан в 256×256 без увеличения и с округлением вниз: 320×200 → 256×160. Crop — центральный JPEG 128×128; маленький исходник может увеличиваться. Optimized — WebP quality=82, method=4, вписан в 1280×1280 без увеличения. EXIF orientation применяется к производным, alpha кладётся на белый фон, EXIF и прочие метаданные в производные не передаются. Уменьшение размера в байтах каждого файла не обещается. + +Worker имеет 60-секундный deadline/lease, одно текущее сообщение и контейнерный предел памяти. Временная ошибка повторяется до 5 попыток с экспоненциальной базой 1–30 с и jitter до 50%; `invalid_image` постоянна, `storage_unavailable` и `worker_timeout` повторяются. Исчерпанная job остаётся `failed` с безопасным `error_code` в БД: это реестр вместо отдельной DLQ-очереди. Операторский replay требует явной команды и пишет audit, а `ready` job повторно не исполняется: + +```sh +docker compose --env-file .env exec -T worker python -m src.replay JOB_UUID --operator local-operator --reason 'verified repair' +``` + +Вторую версию исследователь запускает через `python -m src.reprocess ATTACHMENT_UUID` в worker-контейнере. Команда принимает только `ready` с проверенным оригиналом. Версия 1 читает staging, следующие версии — неизменный конечный original. Уникальность `(attachment_id, pipeline_version, name)` и lease token защищают текущий результат от старой доставки. После COMMIT worker удаляет staging по возможности; `staging-cleanup` повторяет удаление после сбоя. Orphan без job удаляется только после 3600 с и перепроверки upload marker/БД; queued/processing источник сохраняется. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Команда/сценарий | Наблюдение | Evidence | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Общий стенд и quorum | `docker compose up -d --build --scale api=2`; restart broker; `rabbitmqctl list_queues` | Все роли поднялись; durable quorum queue сохранилась после restart | `evidence/broker-restart.txt`, `compose.yaml` | +| Публичный API | `make test BASE_URL=http://127.0.0.1:8085` после финальной сборки | PASS; original побайтно верен, thumbnail 256×160 | `tests/smoke.py` | +| Durable handoff | `student-tests/test_upload_handoff.py`: broker/worker down → 202/409 → restart API/broker | PASS; сообщение с вложением создано, та же job стала ready | собственный тест | +| Окно publish→confirm | `student-tests/test_publish_confirm_gap.py` | После первой broker confirm outbox остался pending; повтор дал 2 копии в очереди, но один логический ready-результат | `evidence/pipeline-faults-2026-10-02.md` | +| Варианты, доступ и лимиты | `student-tests/test_image_pipeline.py` в worker; `test_variants_api.py` через LB | 5/5 unit; все 3 варианта совпали с metadata/hash, original точен, чужой пользователь 404 | собственные тесты | +| Retry, lease, replay, версия, crash | `student-tests/test_pipeline_faults.py` на существующих и пустых томах | В обеих сериях 6/6 PASS, включая смерть процесса до/после S3 и после result COMMIT | `evidence/pipeline-faults-2026-10-02.md` | +| Очистка staging | `student-tests/test_staging_cleanup.py` в maintenance-контейнере | Старый orphan удалён, active upload сохранён, expired marker/object удалены | собственный тест U2 | +| Persistence/failover | `make test-persistence ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS; сохранены токен/данные после остановки API/LB/publisher/worker/cleanup/broker/PG/S3; 133 успешные записи, 0 ошибок, восстановление 1,327 с | hooks, `tests/resilience.py` | +| Нагрузка | `tests/load.py` 30 с/4 клиента; `scripts/measure-image-pipeline.py` 20+20 файлов/4 клиента | 22 419 сообщений без ошибок, 747,002 RPS; 40/40 изображений ready за 14,150 с | `evidence/load-2026-10-02.json`, `evidence/image-load-2026-10-02.json`, `evidence/image-resource-samples-2026-10-02.txt` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Эксперимент трека 5 и замеры -## Эксперименты и отказы +Гипотеза: подтверждённая job и её эффект останутся логически единичными при дублировании, перестановке версий и смерти worker до/после S3 и COMMIT. `evidence/pipeline-faults-2026-10-02.md` описывает две обезличенные серии: одна на существующих томах, другая в отдельном изначально пустом PostgreSQL/S3 проекте (до теста 0 jobs/replays). В каждой серии 6/6 PASS; у каждой готовой версии ровно четыре внутренние записи, current version и SHA-256 оригинала верны. Тест publish→confirm намеренно создал две копии в RabbitMQ. Это идемпотентность эффекта, не exactly-once доставка. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Текстовая closed-loop нагрузка 30,012 с, concurrency=4: 22 419 успешных POST, 0 ошибок, 747,002 успешных RPS, latency p50/p95/p99=4,109/11,019/36,533 мс. Closed-loop не измеряет предельную пропускную способность image pipeline и имеет coordinated omission/накладные расходы клиента. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Изображения: 20 PNG 320×200 по 111 817 байт и 20 PNG 800×600 с псевдослучайными пикселями по 1 441 103 байт; concurrency=4, 40/40 ready, 0 ошибок, 14,150 с (~2,827 ready/s). Для маленьких upload p50/p95/p99=20,276/29,796/30,022 мс, от начала upload до ready=1273,778/1285,587/1287,102 мс. Для больших upload=29,097/54,469/57,979 мс, до ready=1585,251/2256,920/2571,138 мс. `os.urandom` сохраняет размер/разрешение профиля, но не конкретный hash между запусками. Poll каждые 0,3 с увеличивает наблюдаемое время ready; p95/p99 основаны на 20 результатах каждого размера. Выборочные Docker stats во время серии: API 85–86 MiB на реплику, worker 49–61 MiB, S3 182–184 MiB; это снимки, не пик памяти. -## Ограничения и следующие шаги +После failover вторая API-реплика восстановлена командой `docker compose up -d --wait --scale api=2 api lb`. Основные тома сохраняют синтетические записи; отдельные пустые тома эксперимента удалены только в проекте `mtusi2026lab5fresh`. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Ограничения + +Единственные PostgreSQL, MinIO, RabbitMQ и Docker-host остаются точками отказа; quorum queue на одном узле не означает кластерную доступность. При недоступности зависимости worker повторяет работу или достигает `failed`, а не выдаёт ложный `ready`. Serial worker/prefetch=1 и 60-секундный deadline подходят для учебных лимитов 10 MiB/40 млн пикселей; на другом оборудовании нужны новые замеры. Локальный HTTP используется только с синтетическими данными; удалённой демонстрации нужен TLS. Публичный smoke подтверждает обязательный минимум; отказные сценарии и трек 5 проверены отдельно. diff --git a/evidence/image-load-2026-10-02.json b/evidence/image-load-2026-10-02.json new file mode 100644 index 00000000..d2cf2eab --- /dev/null +++ b/evidence/image-load-2026-10-02.json @@ -0,0 +1,40 @@ +{ + "accepted_ready": 40, + "concurrency": 4, + "elapsed_seconds": 14.15, + "failed": 0, + "failure_types": [], + "lab": 5, + "per_size": 20, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 1585.251, + "p95": 2256.92, + "p99": 2571.138 + }, + "size_bytes": 1441103, + "successful": 20, + "upload_ms": { + "p50": 29.097, + "p95": 54.469, + "p99": 57.979 + } + }, + "small_320x200": { + "ready_ms": { + "p50": 1273.778, + "p95": 1285.587, + "p99": 1287.102 + }, + "size_bytes": 111817, + "successful": 20, + "upload_ms": { + "p50": 20.276, + "p95": 29.796, + "p99": 30.022 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/image-resource-samples-2026-10-02.txt b/evidence/image-resource-samples-2026-10-02.txt new file mode 100644 index 00000000..16973431 --- /dev/null +++ b/evidence/image-resource-samples-2026-10-02.txt @@ -0,0 +1,22 @@ +Lab5 synthetic image run, 2026-10-02, Colima/arm64, 2 CPU / 2 GiB VM. +Command: python3 scripts/measure-image-pipeline.py --base-url http://127.0.0.1:8085 --concurrency 4 --per-size 20 --output evidence/image-load-2026-10-02.json +40 uploads, 20 PNG files per size profile, no failures, elapsed 14.150 s. +Docker stats sampled with --no-stream during that run (CPU is an interval sample, +memory is current usage; these are not CPU/RSS peaks): + +sample 1: +api-1 13.74% 86.09 MiB / 256 MiB +api-2 13.98% 85.24 MiB / 256 MiB +worker 8.38% 60.82 MiB / 768 MiB +postgres 3.26% 70.89 MiB / 1.914 GiB +s3 5.32% 182.10 MiB / 384 MiB + +sample 2: +api-1 14.78% 86.06 MiB / 256 MiB +api-2 0.87% 85.19 MiB / 256 MiB +worker 0.22% 48.93 MiB / 768 MiB +postgres 0.24% 71.05 MiB / 1.914 GiB +s3 0.36% 183.60 MiB / 384 MiB + +The first sample was collected while load was active. The second may include +idle time near the end of the run. No broker/publisher stats were sampled. diff --git a/evidence/load-2026-10-02.json b/evidence/load-2026-10-02.json new file mode 100644 index 00000000..5209e34f --- /dev/null +++ b/evidence/load-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.012, + "concurrency": 4, + "successful_requests": 22419, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 747.002, + "successful_latency_ms": { + "p50": 4.109, + "p95": 11.019, + "p99": 36.533 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/pipeline-faults-2026-10-02.md b/evidence/pipeline-faults-2026-10-02.md index 26743bf5..bbf741e7 100644 --- a/evidence/pipeline-faults-2026-10-02.md +++ b/evidence/pipeline-faults-2026-10-02.md @@ -33,4 +33,6 @@ Both runs returned the same anonymized raw summary (one JSON line per run): For each ready version, the test found exactly the four expected variant names (`original`, `thumbnail`, `crop`, `optimized`), a matching original SHA-256, and the expected current `pipeline_version`. Replaying an already ready job returned `duplicate_or_stale`. After a second version became ready, an older delivery left its checksums and current version unchanged; version 1 rows remained available as history. The failed-job replay wrote one audit row and used a new attempt without deleting prior outbox history. The transient retry test exhausted a two-attempt budget, recorded `failed`, then succeeded only after replay reset that budget. -The controlled process death checks durable boundaries, not the probability of such a crash under load. The local stand has one MinIO and one PostgreSQL instance, so this result does not prove service availability during their outage or exactly-once delivery. The broker outage/confirm boundary is tested separately; this matrix checks the worker effect boundary against real S3 and PostgreSQL. +`student-tests/test_publish_confirm_gap.py` separately stopped the normal publisher and worker, accepted a queued upload, published a persistent broker copy with confirm, intentionally left its outbox row unconfirmed, expired that claim, and ran normal `publish_batch` again. Before starting worker, RabbitMQ reported two queued copies and PostgreSQL reported confirmed outbox. After worker resumed, the attachment reached ready with one public three-variant set. This demonstrates the expected duplicate from publish-before-confirm and safe handling of it. `student-tests/test_upload_handoff.py` also passed with broker and worker down at upload time, then API/broker restart and the original accepted job reaching ready. + +The controlled process death checks durable boundaries, not the probability of such a crash under load. The local stand has one MinIO and one PostgreSQL instance, so this result does not prove service availability during their outage or exactly-once delivery. diff --git a/scripts/measure-image-pipeline.py b/scripts/measure-image-pipeline.py new file mode 100644 index 00000000..55b6e6cc --- /dev/null +++ b/scripts/measure-image-pipeline.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Small synthetic image workload; reports upload and time-to-ready separately.""" + +import argparse +import hashlib +import json +import math +import os +import struct +import sys +import time +import uuid +import zlib +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def png_noise(width, height): + raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height)) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + return (b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")) + + +def percentile(values, percentile_value): + values = sorted(values) + return round(values[max(0, math.ceil(len(values) * percentile_value) - 1)], 3) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--per-size", type=int, default=4) + parser.add_argument("--output", required=True) + args = parser.parse_args() + if not 1 <= args.concurrency <= 16 or not 1 <= args.per_size <= 100: + parser.error("invalid workload size") + + setup = Client(args.base_url) + alice, _bob, _eve, chat = setup.bootstrap() + images = { + "small_320x200": (ROOT / "tests/fixtures/sample.png").read_bytes(), + "noise_800x600": png_noise(800, 600), + } + items = [(name, data) for name, data in images.items() for _ in range(args.per_size)] + + def work(item): + name, data = item + client = Client(args.base_url, timeout=20) + start = time.monotonic() + response = client.call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}.png", + 202, actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + accepted = time.monotonic() + attachment = response["json"] + if attachment["sha256"] != hashlib.sha256(data).hexdigest(): + raise Failure("accepted checksum mismatch") + client.wait_ready(alice, attachment, wait_seconds=180) + ready = time.monotonic() + return {"profile": name, "size_bytes": len(data), + "upload_ms": round((accepted - start) * 1000, 3), + "ready_ms": round((ready - start) * 1000, 3)} + + started = time.monotonic() + completed, failures = [], [] + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + for future in as_completed([executor.submit(work, item) for item in items]): + try: + completed.append(future.result()) + except (Failure, OSError, ValueError) as exc: + failures.append(type(exc).__name__) + elapsed = time.monotonic() - started + result = { + "lab": 5, "workload": "synthetic PNG upload and poll until ready", + "concurrency": args.concurrency, "per_size": args.per_size, + "elapsed_seconds": round(elapsed, 3), "accepted_ready": len(completed), + "failed": len(failures), "failure_types": sorted(set(failures)), + "profiles": {}, + } + for name, data in images.items(): + rows = [item for item in completed if item["profile"] == name] + result["profiles"][name] = { + "size_bytes": len(data), "successful": len(rows), + "upload_ms": {f"p{p}": percentile([r["upload_ms"] for r in rows], p / 100) + for p in (50, 95, 99)} if rows else {}, + "ready_ms": {f"p{p}": percentile([r["ready_ms"] for r in rows], p / 100) + for p in (50, 95, 99)} if rows else {}, + } + Path(args.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, sort_keys=True)) + if failures or len(completed) != len(items): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/publish_gap_inner.py b/student-tests/publish_gap_inner.py new file mode 100644 index 00000000..e1e05596 --- /dev/null +++ b/student-tests/publish_gap_inner.py @@ -0,0 +1,59 @@ +"""Run in a one-shot publisher container to inject publish-before-confirm gap.""" + +import json +import os +from pathlib import Path +from uuid import UUID, uuid4 + +import pika + +from src.db import pool +from src.publisher import QUEUE, job_payload, publish_batch + + +attachment_id = UUID(os.environ["FAULT_ATTACHMENT_ID"]) +url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() +broker = pika.BlockingConnection(pika.URLParameters(url)) +try: + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.confirm_delivery() + token = uuid4() + with pool().connection() as conn: + row = conn.execute( + "SELECT o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at " + "FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id " + "WHERE j.attachment_id=%s AND j.pipeline_version=1 AND o.attempt=1", + (attachment_id,), + ).fetchone() + assert row is not None + claimed = conn.execute( + "UPDATE attachment_outbox SET claim_token=%s," + "claim_expires_at=clock_timestamp()+interval '30 seconds' " + "WHERE id=%s AND claim_token IS NULL AND published_at IS NULL RETURNING id", + (token, row["id"]), + ).fetchone() + assert claimed is not None + body = json.dumps(job_payload(row), separators=(",", ":")).encode() + channel.basic_publish( + exchange="", routing_key=QUEUE, mandatory=True, body=body, + properties=pika.BasicProperties(content_type="application/json", delivery_mode=2), + ) + # Simulate process death after broker confirm, before PostgreSQL confirm. + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_outbox SET claim_expires_at=clock_timestamp()-interval '1 second' " + "WHERE id=%s AND claim_token=%s AND published_at IS NULL", + (row["id"], token), + ) + published = publish_batch(channel) + with pool().connection() as conn: + outbox = conn.execute( + "SELECT published_at IS NOT NULL AS confirmed FROM attachment_outbox " + "WHERE id=%s", (row["id"],), + ).fetchone() + count = channel.queue_declare(queue=QUEUE, passive=True).method.message_count + assert published >= 1 and outbox["confirmed"] and count >= 2 + print(json.dumps({"confirmed_after_republish": True, "queued_copies_before_worker": count})) +finally: + broker.close() diff --git a/student-tests/test_publish_confirm_gap.py b/student-tests/test_publish_confirm_gap.py new file mode 100644 index 00000000..be5b1b65 --- /dev/null +++ b/student-tests/test_publish_confirm_gap.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Show that broker confirm without outbox confirmation safely republishes.""" + +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def compose(*args, input_data=None): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, + input=input_data, check=True, + ) + + +def main(): + args = arguments("Lab5 publish/confirm gap") + client = Client(args.base_url, args.ca_file) + compose("stop", "worker", "publisher") + attachment = None + try: + alice, bob, _eve, chat = client.bootstrap() + attachment, _original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not queued") + source = (ROOT / "student-tests/publish_gap_inner.py").read_bytes() + compose( + "run", "--rm", "--no-deps", "-T", + "-e", f"FAULT_ATTACHMENT_ID={attachment['id']}", "publisher", "python", "-", + input_data=source, + ) + finally: + compose("up", "-d", "publisher", "worker") + ready = client.wait_ready(bob, attachment) + check(ready["status"] == "ready", "republished job did not finish") + check(len(ready["variants"]) == 3, "duplicate delivery changed variant set") + print("PASS: confirmed broker copy republished after lost DB confirm; worker reached ready once") + + +if __name__ == "__main__": + run(main) -- 2.54.0 From 2c46d02205c74dfd963113b780a3d25234c96243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sat, 7 Nov 2026 09:53:49 +0300 Subject: [PATCH 7/7] fix: preserve NUL characters in persisted messages Store message text as UTF-8 bytes and migrate existing rows without loss. Cover HTTP round trips, restart persistence, Unicode limits and idempotent retries. Adapt direct database tools to the binary storage format. (cherry picked from commit 386276532730b8758298080b49db63738ca300a4) --- migrations/009_message_text_utf8.sql | 6 + src/store.py | 4 +- student-tests/test_message_unicode.py | 156 ++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 migrations/009_message_text_utf8.sql create mode 100644 student-tests/test_message_unicode.py diff --git a/migrations/009_message_text_utf8.sql b/migrations/009_message_text_utf8.sql new file mode 100644 index 00000000..86196ee6 --- /dev/null +++ b/migrations/009_message_text_utf8.sql @@ -0,0 +1,6 @@ +-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload. +-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); +ALTER TABLE messages ADD CONSTRAINT messages_text_check + CHECK (octet_length(text) <= 16000); diff --git a/src/store.py b/src/store.py index 298109a7..0ddb32d8 100644 --- a/src/store.py +++ b/src/store.py @@ -78,7 +78,7 @@ def chat_view(row: dict) -> dict: def message_view(row: dict) -> dict: return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), - "sender_id": str(row["sender_id"]), "text": row["text"], + "sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"), "created_at": timestamp(row["created_at"]), "attachment_ids": [str(attachment_id) for attachment_id in row.get("attachment_ids", [])], } @@ -259,7 +259,7 @@ class Store: row = conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) " "RETURNING id,chat_id,sender_id,text,created_at", - (uuid4(), chat_uuid, actor_uuid, text), + (uuid4(), chat_uuid, actor_uuid, text.encode("utf-8")), ).fetchone() for position, attachment_id in enumerate(ids): conn.execute( diff --git a/student-tests/test_message_unicode.py b/student-tests/test_message_unicode.py new file mode 100644 index 00000000..4ff79d81 --- /dev/null +++ b/student-tests/test_message_unicode.py @@ -0,0 +1,156 @@ +"""Run with DATABASE_URL pointing to a disposable PostgreSQL database. + +Checks real HTTP writes, persisted history, retries, and migration of old rows. +""" + +import base64 +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request +from uuid import uuid4 + +import psycopg +from psycopg.conninfo import make_conninfo + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database") +class MessageUnicodeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + cls.port = sock.getsockname()[1] + cls.start_server() + cls.addClassCleanup(cls.stop_server) + + @classmethod + def start_server(cls): + cls.server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"}, + ) + for _ in range(100): + if cls.server.poll() is not None: + raise AssertionError("API exited during startup") + try: + cls.call("GET", "/health/live") + return + except (AssertionError, urllib.error.URLError): + time.sleep(0.05) + cls.stop_server() + raise AssertionError("API did not become ready") + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + + @classmethod + def call(cls, method, path, body=None, headers=None, expected=200): + request = urllib.request.Request( + f"http://127.0.0.1:{cls.port}{path}", method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", **(headers or {})}, + ) + try: + response = urllib.request.urlopen(request, timeout=10) + except urllib.error.HTTPError as exc: + response = exc + with response: + payload = response.read() + assert response.status == expected, (method, path, response.status, expected, payload) + return json.loads(payload) if payload else None + + def test_http_round_trip_after_restart_and_replay(self): + username, password = "u_" + uuid4().hex[:16], "Test-password-123" + user = self.call("POST", "/api/v1/users", { + "username": username, "display_name": "Unicode", "password": password, + }, expected=201) + session = self.call("POST", "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(), + }, expected=201) + auth = {"Authorization": "Bearer " + session["token"]} + chat = self.call("POST", "/api/v1/chats", { + "title": "Unicode", "member_ids": [user["id"]], + }, auth, 201) + path = f"/api/v1/chats/{chat['id']}/messages" + texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000] + messages, keys = [], [] + for text in texts: + key = uuid4().hex + message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(message["text"], text) + messages.append(message) + keys.append(key) + for text in ("", "\0" * 4001, "🙂" * 4001): + self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400) + self.stop_server() + self.start_server() + found, cursor = [], None + while True: + page = self.call("GET", path + "?limit=1" + ( + "&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else "" + ), headers=auth) + found.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(found, messages) + if LAB >= 3: + for text, message, key in zip(texts, messages, keys): + replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(replay, message) + self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409) + self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages)) + + def test_upgrade_preserves_old_text_and_is_repeatable(self): + schema = "unicode_" + uuid4().hex + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'CREATE SCHEMA "{schema}"') + isolated = make_conninfo(URL, options=f"-c search_path={schema}") + try: + old_text = "Старое 🙂 \\ literal \\u0000" + with psycopg.connect(isolated) as conn: + conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)") + for migration in sorted((ROOT / "migrations").glob("*.sql")): + version = int(migration.name.split("_", 1)[0]) + if version >= 9: + continue + conn.execute(migration.read_text()) + conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,)) + user, chat, message = uuid4(), uuid4(), uuid4() + conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,)) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,)) + conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user)) + conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text)) + for _ in range(2): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with psycopg.connect(isolated) as conn: + conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text()) + saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone() + self.assertEqual(saved.decode("utf-8"), old_text) + finally: + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'DROP SCHEMA "{schema}" CASCADE') + + +if __name__ == "__main__": + unittest.main() -- 2.54.0