From 59b2f031bca151211dd00a0c64ef3ec051481c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 17:55:51 +0300 Subject: [PATCH 01/11] feat: add private attachments and safe reconciliation experiment --- .dockerignore | 27 +- .gitignore | 2 + Dockerfile | 19 + Makefile | 25 + compose.yaml | 144 ++++ infra/filer.toml | 3 + infra/haproxy.cfg | 35 + migrations/001_initial.sql | 35 + migrations/002_idempotency.sql | 10 + migrations/003_unicode_storage.sql | 13 + migrations/004_attachments.sql | 35 + migrations/005_attachment_only_message.sql | 3 + pyproject.toml | 29 + research/run.py | 284 ++++++++ scripts/bootstrap.py | 43 ++ scripts/restart.sh | 5 + scripts/stop-one.sh | 4 + scripts/stop_one.py | 19 + src/messenger/__init__.py | 1 + src/messenger/__main__.py | 12 + src/messenger/app.py | 314 ++++++++ src/messenger/attachments.py | 154 ++++ src/messenger/config.py | 10 + src/messenger/domain.py | 54 ++ src/messenger/migrate.py | 32 + src/messenger/models.py | 68 ++ src/messenger/postgres.py | 324 +++++++++ src/messenger/reconcile.py | 176 +++++ src/messenger/storage.py | 96 +++ src/messenger/storage_bootstrap.py | 30 + student-tests/test_api.py | 201 ++++++ student-tests/test_attachments.py | 201 ++++++ student-tests/test_edgecases.py | 99 +++ student-tests/test_sessions.py | 62 ++ uv.lock | 792 +++++++++++++++++++++ 35 files changed, 3341 insertions(+), 20 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/filer.toml create mode 100644 infra/haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_unicode_storage.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_attachment_only_message.sql create mode 100644 pyproject.toml create mode 100644 research/run.py create mode 100644 scripts/bootstrap.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 scripts/stop_one.py create mode 100644 src/messenger/__init__.py create mode 100644 src/messenger/__main__.py create mode 100644 src/messenger/app.py create mode 100644 src/messenger/attachments.py create mode 100644 src/messenger/config.py create mode 100644 src/messenger/domain.py create mode 100644 src/messenger/migrate.py create mode 100644 src/messenger/models.py create mode 100644 src/messenger/postgres.py create mode 100644 src/messenger/reconcile.py create mode 100644 src/messenger/storage.py create mode 100644 src/messenger/storage_bootstrap.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_edgecases.py create mode 100644 student-tests/test_sessions.py create mode 100644 uv.lock diff --git a/.dockerignore b/.dockerignore index bc16e556..438f5083 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,7 @@ -.git -.github -.env -.env.* -!.env.example -secrets -certs -volumes -data -backups -*.key -*.pem -*.p12 -*.pfx -*.jks -.venv -__pycache__ -node_modules -evidence -.DS_Store +* +!pyproject.toml +!uv.lock +!src/ +!src/** +!migrations/ +!migrations/** diff --git a/.gitignore b/.gitignore index b5a71896..94fbfc4c 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +.pytest_cache/ +.ruff_cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..57926cb8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b AS build +COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv +WORKDIR /app +COPY pyproject.toml uv.lock ./ +RUN uv sync --locked --no-dev --no-install-project --python /usr/local/bin/python + +FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b +RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs && rm -rf /var/lib/apt/lists/* +ARG VCS_REF=uncommitted +LABEL org.opencontainers.image.revision=$VCS_REF +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app/src PATH=/app/.venv/bin:$PATH +WORKDIR /app +COPY --from=build /app/.venv /app/.venv +COPY src ./src +COPY migrations ./migrations +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=3s --timeout=2s --start-period=5s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1)"] +CMD ["python", "-m", "messenger"] diff --git a/Makefile b/Makefile index 5d86d171..1dfcdac8 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,28 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# Student implementation commands; original acceptance targets are preserved. +PUBLIC_PORT ?= 8084 +BASE_URL = http://127.0.0.1:$(PUBLIC_PORT) + +bootstrap: + python3 scripts/bootstrap.py + +up: bootstrap + VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait + +down: + docker compose down + +migrate: + docker compose run --rm migrate + +lint: + uv run --locked ruff check src student-tests research + +student-test: + BASE_URL=$(BASE_URL) uv run --locked pytest -q + +reconcile: + docker compose run --rm --no-deps api python -m messenger.reconcile diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..d4baf01c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,144 @@ +name: mtusi-vasilev-lab4 +x-application: &application + image: ${API_IMAGE:-mtusi-vasilev:lab4} + build: + context: . + args: + VCS_REF: ${VCS_REF:-uncommitted} + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + POOL_MAX_SIZE: ${POOL_MAX_SIZE:-8} + secrets: [database_url, cache_url, s3_access_key, s3_secret_key] + cpus: ${API_CPUS:-0.5} + mem_limit: ${API_MEMORY:-512m} + pids_limit: 96 + read_only: true + tmpfs: ["/tmp:size=16m,noexec,nosuid"] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + stop_grace_period: 8s +services: + postgres: + image: postgres:17.10-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 15 + cpus: 1 + mem_limit: 768m + command: [postgres, -c, fsync=on, -c, synchronous_commit=on, -c, full_page_writes=on] + cache: + image: valkey/valkey:8.1.3-alpine + command: [valkey-server, --save, "", --appendonly, "no", --maxmemory, 64mb, --maxmemory-policy, allkeys-lru] + healthcheck: + test: [CMD, valkey-cli, ping] + interval: 2s + timeout: 2s + retries: 15 + cpus: 0.5 + mem_limit: 128m + migrate: + <<: *application + command: [python, -m, messenger.migrate] + healthcheck: + disable: true + depends_on: + postgres: {condition: service_healthy} + api: + <<: *application + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + migrate: {condition: service_completed_successfully} + cache: {condition: service_healthy} + s3-bootstrap: {condition: service_completed_successfully} + storage: + image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e + command: [server, -dir=/data, -ip=storage, -ip.bind=0.0.0.0, -filer, -volume.max=4, -volume.preStopSeconds=0, -master.volumeSizeLimitMB=64] + volumes: ["s3_data:/data", "./infra/filer.toml:/etc/seaweedfs/filer.toml:ro"] + networks: [storage_backend] + cpus: 1 + mem_limit: 512m + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:9333/dir/status"] + interval: 2s + timeout: 2s + retries: 20 + s3: + image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e + command: [s3, -filer=storage:8888, -ip.bind=0.0.0.0, -config=/run/secrets/s3_config, -port=8333] + secrets: [s3_config] + networks: [default, storage_backend] + depends_on: + storage: {condition: service_healthy} + cpus: 0.5 + mem_limit: 256m + s3-bootstrap: + <<: *application + command: [python, -m, messenger.storage_bootstrap] + environment: + S3_ENDPOINT: http://s3:8333 + S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key + S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key + secrets: [s3_admin_access_key, s3_admin_secret_key] + healthcheck: + disable: true + depends_on: + s3: {condition: service_started} + research: + <<: *application + profiles: [tools] + command: [python, /research/run.py] + volumes: ["./research:/research:ro"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: http://s3:8333 + S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key + S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key + secrets: [database_url, cache_url, s3_admin_access_key, s3_admin_secret_key] + lb: + image: haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0 + ports: ["127.0.0.1:${PUBLIC_PORT:-8084}:8080"] + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + depends_on: + api: {condition: service_healthy} + cpus: 0.5 + mem_limit: 128m + read_only: true + tmpfs: ["/tmp:size=8m"] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"] + interval: 2s + timeout: 2s + retries: 10 +volumes: + postgres_data: + s3_data: +networks: + storage_backend: + internal: true +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + s3_access_key: {file: ./secrets/s3_access_key} + s3_secret_key: {file: ./secrets/s3_secret_key} + s3_admin_access_key: {file: ./secrets/s3_admin_access_key} + s3_admin_secret_key: {file: ./secrets/s3_admin_secret_key} + s3_config: {file: ./secrets/s3_config} diff --git a/infra/filer.toml b/infra/filer.toml new file mode 100644 index 00000000..a11e5de2 --- /dev/null +++ b/infra/filer.toml @@ -0,0 +1,3 @@ +[leveldb2] +enabled = true +dir = "/data/filerldb2" diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..ae5f764a --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + maxconn 256 + stats socket /tmp/haproxy.sock mode 600 level admin + +defaults + mode http + log global + timeout connect 1s + timeout client 10s + timeout server 8s + timeout check 1s + timeout queue 2s + retries 0 + option httplog + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + +frontend public + bind :8080 + default_backend apis + +backend apis + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 + server-template api 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..2d05d335 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,35 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL +); +CREATE TABLE members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + position integer NOT NULL CHECK (position >= 0), + PRIMARY KEY(chat_id,user_id), + UNIQUE(chat_id,position) +); +CREATE INDEX members_user_idx ON members(user_id,chat_id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL, + FOREIGN KEY(chat_id,sender_id) REFERENCES members(chat_id,user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id); +CREATE TABLE sessions ( + token_digest text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..3ca43082 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE message_requests ( + actor_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL, + body_digest text NOT NULL, + response jsonb NOT NULL, + expires_at timestamptz NOT NULL, + PRIMARY KEY(actor_id,chat_id,key) +); +CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at); diff --git a/migrations/003_unicode_storage.sql b/migrations/003_unicode_storage.sql new file mode 100644 index 00000000..c79d6b9a --- /dev/null +++ b/migrations/003_unicode_storage.sql @@ -0,0 +1,13 @@ +-- PostgreSQL text rejects U+0000, while the API accepts Unicode code points. +-- Preserve UTF-8 losslessly. API validation enforces code point limits; the DB +-- additionally bounds byte lengths, without stripping or replacing characters. +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name,'UTF8'); +ALTER TABLE users ADD CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title,'UTF8'); +ALTER TABLE chats ADD CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text,'UTF8'); +ALTER TABLE messages ADD CHECK (octet_length(text) BETWEEN 1 AND 16000); +ALTER TABLE message_requests ALTER COLUMN response TYPE bytea USING convert_to(response::text,'UTF8'); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..63273db6 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,35 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL, + status text NOT NULL CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed')), + error_code text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + object_key text NOT NULL UNIQUE, + UNIQUE(id,chat_id) +); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), + attachment_id uuid NOT NULL REFERENCES attachments(id), + position integer NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY(message_id,attachment_id), + UNIQUE(message_id,position) +); +CREATE INDEX message_attachments_attachment_idx ON message_attachments(attachment_id); +CREATE TABLE upload_intents ( + id uuid PRIMARY KEY, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL +); +CREATE INDEX upload_intents_created_idx ON upload_intents(created_at,id); +CREATE TABLE attachment_reconcile_audit ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + event text NOT NULL, + object_key text NOT NULL, + happened_at timestamptz NOT NULL DEFAULT now() +); diff --git a/migrations/005_attachment_only_message.sql b/migrations/005_attachment_only_message.sql new file mode 100644 index 00000000..054d8ce1 --- /dev/null +++ b/migrations/005_attachment_only_message.sql @@ -0,0 +1,3 @@ +-- Empty text is valid only when the API supplies at least one attachment. +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000); diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..5101832c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "mtusi-messenger" +version = "1.0.0" +requires-python = ">=3.12,<3.13" +dependencies = [ + "fastapi", + "uvicorn", + "psycopg[binary,pool]", + "redis", + "argon2-cffi", + "boto3>=1.43,<2", + "pillow>=12.1.1,<13", +] + +[dependency-groups] +dev = ["pytest", "httpx", "ruff", "openapi-spec-validator", "jsonschema"] + +[tool.ruff] +target-version = "py312" +line-length = 100 +include = ["src/**/*.py", "student-tests/**/*.py", "research/**/*.py"] + +[tool.pytest.ini_options] +testpaths = ["student-tests"] +pythonpath = ["src", "tests"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I", "UP", "B"] +ignore = ["B008"] # FastAPI dependencies are declared in endpoint signatures. diff --git a/research/run.py b/research/run.py new file mode 100644 index 00000000..6c272ec3 --- /dev/null +++ b/research/run.py @@ -0,0 +1,284 @@ +"""Real PostgreSQL/S3 fault injection in disposable DB and bucket, never production data.""" + +import hashlib +import json +import os +import time +from datetime import UTC, datetime +from io import BytesIO +from pathlib import Path +from uuid import uuid4 + +import psycopg +from PIL import Image +from psycopg import sql +from psycopg.conninfo import make_conninfo + +from messenger.attachments import AttachmentService +from messenger.config import secret +from messenger.domain import Failure +from messenger.migrate import migrate +from messenger.postgres import PostgresStore +from messenger.reconcile import reconcile +from messenger.storage import ObjectStorage + + +def main(): + original_dsn = secret("DATABASE_URL") + suffix = uuid4().hex[:12] + database, bucket = "research_" + suffix, "research-" + suffix + admin = psycopg.connect(original_dsn, autocommit=True) + admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database))) + os.environ.pop("DATABASE_URL_FILE", None) + os.environ["DATABASE_URL"] = make_conninfo(original_dsn, dbname=database) + os.environ["S3_BUCKET"] = bucket + storage = ObjectStorage(admin=True) + storage.client.create_bucket(Bucket=bucket) + store = None + result = { + "hypothesis": "Durable intent, row locks, grace and live HEAD make reconciliation repeatable and protect fresh/used data", + "started_at": datetime.now(UTC).isoformat(), + "database": database, + "bucket": bucket, + "grace_seconds": 2, + "checks": [], + "scan_cost": [], + "source": {}, + } + for path in sorted(Path("src/messenger").glob("*.py")): + result["source"][str(path)] = hashlib.sha256(path.read_bytes()).hexdigest() + try: + migrate() + store = PostgresStore() + service = AttachmentService(store, storage) + user = store.register( + { + "username": "research_" + suffix, + "display_name": "Research", + "password": "Synthetic-research-password", + } + ) + chat = store.create_chat(user["id"], {"title": "Research", "member_ids": [user["id"]]}) + stream = BytesIO() + Image.new("RGB", (16, 10), "blue").save(stream, "PNG") + data = stream.getvalue() + + def upload(name): + return service.upload( + user["id"], chat["id"], name, "image/png", BytesIO(data), len(data) + ) + + def record(name, condition): + if not condition: + raise AssertionError(name) + result["checks"].append({"name": name, "passed": True}) + + good = upload("used.png") + store.send(user["id"], chat["id"], "", uuid4().hex, [good["id"]]) + missing = upload("missing.png") + store.send(user["id"], chat["id"], "", uuid4().hex, [missing["id"]]) + storage.delete("original/" + missing["id"]) + + # A real PostgreSQL INSERT failure after a successful real S3 PUT. + with store.connection() as conn: + conn.execute( + "CREATE FUNCTION reject_research_upload() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='db-fault.png' THEN RAISE EXCEPTION 'injected DB write failure'; END IF; RETURN NEW; END $$" + ) + conn.execute( + "CREATE TRIGGER research_reject BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION reject_research_upload()" + ) + try: + upload("db-fault.png") + raise AssertionError("DB fault did not fire") + except psycopg.errors.RaiseException: + record("real_db_insert_failure_after_s3_put", True) + finally: + with store.connection() as conn: + conn.execute("DROP TRIGGER research_reject ON attachments") + conn.execute("DROP FUNCTION reject_research_upload()") + + # Real connection failure to a closed S3 endpoint; no final attachment row. + os.environ["S3_ENDPOINT"] = "http://s3:1" + broken_storage = ObjectStorage(admin=True) + try: + AttachmentService(store, broken_storage).upload( + user["id"], chat["id"], "s3-fault.png", "image/png", BytesIO(data), len(data) + ) + raise AssertionError("S3 fault did not fire") + except Failure as exc: + record("real_s3_connect_failure_is_503", exc.status == 503) + finally: + broken_storage.close() + os.environ["S3_ENDPOINT"] = "http://s3:8333" + + # A successful COMMIT followed by an injected loss of its acknowledgement. + os.environ["UPLOAD_FAULT"] = "after_commit" + try: + upload("commit-ack-lost.png") + raise AssertionError("post-COMMIT fault did not fire") + except Failure: + record("commit_ack_loss_injected", True) + finally: + os.environ.pop("UPLOAD_FAULT") + with store.connection() as conn: + committed = conn.execute( + "SELECT * FROM attachments WHERE filename='commit-ack-lost.png'" + ).fetchone() + record( + "uncertain_response_keeps_committed_original", + committed["status"] == "ready" and storage.exists(committed["object_key"]), + ) + + orphan_key = "original/orphan-" + suffix + storage.put(orphan_key, BytesIO(data), len(data), "image/png") + # All initial rows and objects become older than the real grace period. + time.sleep(2.2) + fresh = upload("fresh.png") + fresh_orphan = "original/fresh-orphan-" + suffix + storage.put(fresh_orphan, BytesIO(data), len(data), "image/png") + fresh_intent_id = str(uuid4()) + fresh_intent_key = "original/" + fresh_intent_id + with store.connection() as conn: + conn.execute( + "INSERT INTO upload_intents VALUES (%s,%s,now()-interval '1 day')", + (fresh_intent_id, fresh_intent_key), + ) + storage.put(fresh_intent_key, BytesIO(data), len(data), "image/png") + + # A live writer holds an old intent; neither its row nor object may vanish. + active_id, active_key = str(uuid4()), "original/active-" + suffix + with store.connection() as conn: + conn.execute( + "INSERT INTO upload_intents VALUES (%s,%s,now()-interval '1 day')", + (active_id, active_key), + ) + storage.put(active_key, BytesIO(data), len(data), "image/png") + locked = psycopg.connect(os.environ["DATABASE_URL"], autocommit=False) + locked.execute("SELECT * FROM upload_intents WHERE id=%s FOR UPDATE", (active_id,)) + with store.connection() as conn: + before = { + "attachments": conn.execute("SELECT count(*) AS n FROM attachments").fetchone()[ + "n" + ], + "intents": conn.execute("SELECT count(*) AS n FROM upload_intents").fetchone()["n"], + "audit": conn.execute( + "SELECT count(*) AS n FROM attachment_reconcile_audit" + ).fetchone()["n"], + } + before_keys = sorted(x["Key"] for x in storage.objects()) + dry = reconcile(store, storage, grace_seconds=2, page_size=1) + with store.connection() as conn: + after = { + "attachments": conn.execute("SELECT count(*) AS n FROM attachments").fetchone()[ + "n" + ], + "intents": conn.execute("SELECT count(*) AS n FROM upload_intents").fetchone()["n"], + "audit": conn.execute( + "SELECT count(*) AS n FROM attachment_reconcile_audit" + ).fetchone()["n"], + } + record( + "dry_run_changes_neither_db_nor_s3", + before == after and before_keys == sorted(x["Key"] for x in storage.objects()), + ) + applied = reconcile(store, storage, apply=True, grace_seconds=2, page_size=1) + repeated = reconcile(store, storage, apply=True, grace_seconds=2, page_size=1) + result["dry_run"], result["apply"], result["repeat"] = dry, applied, repeated + record( + "all_reconcile_passes_have_no_errors", + not any(r["errors"] for r in [dry, applied, repeated]), + ) + record("orphan_removed", not storage.exists(orphan_key)) + record( + "missing_marked_failed_metadata_kept", + service.metadata(user["id"], missing["id"])["error_code"] == "storage_missing", + ) + record("used_original_kept", storage.exists("original/" + good["id"])) + record( + "fresh_metadata_and_original_kept", + service.metadata(user["id"], fresh["id"])["status"] == "ready" + and storage.exists("original/" + fresh["id"]), + ) + record("fresh_orphan_kept", storage.exists(fresh_orphan)) + record("old_intent_fresh_object_kept", storage.exists(fresh_intent_key)) + record("active_writer_object_kept", storage.exists(active_key)) + record( + "committed_original_survives_reconciliation", storage.exists(committed["object_key"]) + ) + record( + "repeat_has_no_duplicate_mutations", + repeated["deleted_objects"] + == repeated["deleted_intents"] + == repeated["marked_failed"] + == 0, + ) + locked.rollback() + locked.close() + + # A failed S3 delete leaves the intent retryable. A dry run afterwards must + # not resume deletion or append audit records. + time.sleep(2.2) + + class DeleteFailure: + def __getattr__(self, name): + return getattr(storage, name) + + def delete(self, key): + raise Failure(503, "injected S3 delete failure") + + failed_cleanup = reconcile(store, DeleteFailure(), apply=True, grace_seconds=2) + record("s3_delete_failure_observed", failed_cleanup["errors"] > 0) + with store.connection() as conn: + audit_before = conn.execute( + "SELECT count(*) AS n FROM attachment_reconcile_audit" + ).fetchone()["n"] + recovery_dry = reconcile(store, storage, grace_seconds=2) + with store.connection() as conn: + record( + "dry_run_after_failed_apply_still_read_only", + conn.execute("SELECT count(*) AS n FROM attachment_reconcile_audit").fetchone()["n"] + == audit_before, + ) + recovery_apply = reconcile(store, storage, apply=True, grace_seconds=2) + result["failed_cleanup"], result["recovery_dry"], result["recovery_apply"] = ( + failed_cleanup, + recovery_dry, + recovery_apply, + ) + record("failed_cleanup_recovers", recovery_apply["errors"] == 0) + + # Measure an increasing population of intact, aged metadata/objects. Rows + # share no client names or object paths, and all are in this disposable DB. + current = 0 + for target in (10, 100, 1000): + for _ in range(target - current): + upload("scan.png") + current = target + with store.connection() as conn: + conn.execute("UPDATE attachments SET created_at=now()-interval '1 day'") + for repeat in range(3): + run = reconcile(store, storage, grace_seconds=2, page_size=100) + record("scan_has_no_errors_" + str(target) + "_" + str(repeat), run["errors"] == 0) + result["scan_cost"].append({"seeded_intact": target, "repeat": repeat + 1, **run}) + result["limitations"] = [ + "One local host and warm services; scan measures HEAD/SQL round trips, not production SLA", + "Missing originals are marked failed; reconciliation cannot recreate lost bytes without backup", + "Valid unattached ready attachments are retained; only abandoned intents and untracked old original objects are removed", + "No atomic transaction spans S3 and PostgreSQL; retries plus durable intent recover the selected windows", + "Cost dataset is in disposable isolated DB/bucket, includes a few preserved fault fixtures beyond the seeded intact count", + ] + finally: + if store: + store.close() + for obj in list(storage.objects()): + storage.delete(obj["Key"]) + storage.client.delete_bucket(Bucket=bucket) + storage.close() + admin.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database))) + admin.close() + result["temporary_resources_deleted"] = True + print("RESULT " + json.dumps(result, ensure_ascii=False, default=str)) + + +if __name__ == "__main__": + main() diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..06506986 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Create a private local secret directory once; never rotate existing volumes implicitly.""" + +import os +import json +import secrets +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +private = root / "secrets" +private.mkdir(mode=0o700, exist_ok=True) +os.chmod(private, 0o700) +password_file = private / "postgres_password" +if not password_file.exists(): + password_file.write_text(secrets.token_urlsafe(32)) +password = password_file.read_text().strip() +values = { + "database_url": "postgresql://messenger:" + password + "@postgres:5432/messenger", + "cache_url": "redis://cache:6379/0", + "s3_access_key": "app-" + secrets.token_hex(10), + "s3_secret_key": secrets.token_urlsafe(32), + "s3_admin_access_key": "admin-" + secrets.token_hex(10), + "s3_admin_secret_key": secrets.token_urlsafe(32), +} +for name, value in values.items(): + path = private / name + if not path.exists(): + path.write_text(value) +config = private / "s3_config" +if not config.exists(): + config.write_text(json.dumps({"identities": [ + {"name": "bootstrap", "credentials": [{"accessKey": (private / "s3_admin_access_key").read_text(), + "secretKey": (private / "s3_admin_secret_key").read_text()}], "actions": ["Admin", "Read", "Write", "List"]}, + {"name": "api", "credentials": [{"accessKey": (private / "s3_access_key").read_text(), + "secretKey": (private / "s3_secret_key").read_text()}], "actions": ["Read:messenger", "Write:messenger", "List:messenger"]} + ]})) +# The parent directory is private on the host; files must be readable by non-root API +# through Compose's file-backed secret mount (uid/mode mapping is not portable). +for path in private.iterdir(): + os.chmod(path, 0o644) +if not (root / ".env").exists(): + (root / ".env").write_text("PUBLIC_PORT=8084\nSESSION_TTL_SECONDS=3600\n") +print("Local configuration ready; existing secret values preserved.") diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..6a27217a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres cache storage s3 api lb +docker compose up -d --wait --no-recreate diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..3d9eb6a1 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/stop_one.py diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 00000000..8daf4e0c --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,19 @@ +"""Map a process alias to its real container, then leave exactly it stopped.""" +import json +import os +import subprocess + +wanted = os.environ["TARGET_INSTANCE_ID"] +ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], text=True).split() +for cid in ids: + result = subprocess.run( + ["docker", "exec", cid, "python", "-c", + "import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"], + text=True, capture_output=True, check=False, + ) + if result.returncode == 0 and result.stdout.strip() == wanted: + subprocess.run(["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL) + print(json.dumps({"stopped_container": cid, "instance_id": wanted})) + break +else: + raise SystemExit("Target instance not found among this project's API containers") diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..58ffea15 --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1 @@ +"""Учебный REST-мессенджер.""" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 00000000..5b5cda10 --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,12 @@ +import os + +import uvicorn + +uvicorn.run( + "messenger.app:app", + host=os.getenv("APP_HOST", "0.0.0.0"), + port=int(os.getenv("APP_PORT", "8080")), + workers=1, + access_log=False, + timeout_graceful_shutdown=5, +) diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..fea1a304 --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,314 @@ +import base64 +import binascii +import json +import logging +import os +import re +import time +from contextlib import asynccontextmanager +from threading import BoundedSemaphore +from uuid import uuid4 + +from fastapi import Depends, FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException + +from .attachments import AttachmentService +from .domain import Failure, boundary, identifier, page +from .models import CreateChat, CreateMessage, CreateUser +from .postgres import PostgresStore +from .storage import ObjectStorage + +logger = logging.getLogger("messenger") +logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(message)s") +INSTANCE_ID = str(uuid4()) + +CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + 500: "internal_error", +} + + +@asynccontextmanager +async def lifespan(app): + app.state.store = PostgresStore() + app.state.storage = ObjectStorage() + app.state.attachments = AttachmentService(app.state.store, app.state.storage) + app.state.upload_slots = BoundedSemaphore(1) + logger.info(json.dumps({"event": "started", "storage": "postgres"})) + yield + app.state.storage.close() + app.state.store.close() + logger.info(json.dumps({"event": "stopped"})) + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +def error(request, status, message, headers=None): + return JSONResponse( + { + "error": { + "code": CODES.get(status, "invalid_request"), + "message": message, + "request_id": request.state.request_id, + } + }, + status_code=status, + headers=headers, + ) + + +@app.middleware("http") +async def request_log(request: Request, call_next): + request.state.request_id = str(uuid4()) + start = time.perf_counter() + try: + if ( + request.method == "POST" + and request.url.path == "/api/v1/auth/sessions" + and await request.body() + ): + response = error(request, 400, "Вход не принимает тело запроса") + else: + response = await call_next(request) + except Exception: + logger.error( + json.dumps({"event": "unexpected_error", "request_id": request.state.request_id}) + ) + response = error(request, 500, "Внутренняя ошибка") + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + route = request.scope.get("route") + logger.info( + json.dumps( + { + "event": "request", + "request_id": request.state.request_id, + "method": request.method, + "route": getattr(route, "path", "unknown"), + "status": response.status_code, + "duration_ms": round((time.perf_counter() - start) * 1000, 3), + } + ) + ) + return response + + +@app.exception_handler(Failure) +async def domain_error(request, exc): + return error(request, exc.status, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_error(request, exc): + return error(request, 400, "Неверный JSON или поля запроса") + + +@app.exception_handler(HTTPException) +async def http_error(request, exc): + return error(request, exc.status_code, "Неверный запрос", exc.headers) + + +def bearer_failure(): + return Failure( + 401, "Требуется действующая сессия", {"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + + +def bearer(request): + parts = request.headers.get("Authorization", "").split() + if len(parts) != 2 or parts[0].lower() != "bearer": + raise bearer_failure() + return parts[1] + + +def actor(request: Request): + user = request.app.state.store.authenticate(bearer(request)) + if user is None: + raise bearer_failure() + return user + + +@app.post("/api/v1/auth/sessions", status_code=201) +def login(request: Request): + headers = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + parts = request.headers.get("Authorization", "").split() + try: + if len(parts) != 2 or parts[0].lower() != "basic" or len(parts[1]) > 2048: + raise ValueError + decoded = base64.b64decode(parts[1], validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128: + raise ValueError + except (ValueError, UnicodeError, binascii.Error): + raise Failure(401, "Неверные учётные данные", headers) from None + session = request.app.state.store.login(username, password) + if session is None: + raise Failure(401, "Неверные учётные данные", headers) + return session + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request, user=Depends(actor)): + if not request.app.state.store.logout(bearer(request)): + raise bearer_failure() + return Response(status_code=204) + + +def pagination(request, scope): + allowed = {"limit", "cursor"} + if any(k not in allowed for k in request.query_params): + raise Failure(400, "Неизвестный параметр запроса") + if any(len(request.query_params.getlist(k)) > 1 for k in allowed): + raise Failure(400, "Повторный параметр запроса") + value = request.query_params.get("limit", "50") + if len(value) > 3 or not re.fullmatch(r"[0-9]+", value) or not 1 <= int(value) <= 100: + raise Failure(400, "limit должен быть целым от 1 до 100") + return int(value), boundary(scope, request.query_params.get("cursor")) + + +@app.get("/health/live") +def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def ready(request: Request): + if not request.app.state.store.ready() or not request.app.state.storage.ready(): + raise Failure(503, "Хранилище недоступно") + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +def register(data: CreateUser, request: Request): + return request.app.state.store.register(data.model_dump(mode="json")) + + +@app.get("/api/v1/users/me") +def me(user=Depends(actor)): + return user + + +@app.post("/api/v1/chats", status_code=201) +def create_chat(data: CreateChat, request: Request, user=Depends(actor)): + return request.app.state.store.create_chat(user["id"], data.model_dump(mode="json")) + + +@app.get("/api/v1/chats") +def chats(request: Request, user=Depends(actor)): + scope = "chats:" + user["id"] + limit, after = pagination(request, scope) + return page(request.app.state.store.list_chats(user["id"], after, limit), limit, scope) + + +@app.get("/api/v1/chats/{chat_id}") +def chat(chat_id: str, request: Request, user=Depends(actor)): + return request.app.state.store.chat(user["id"], identifier(chat_id)) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def send(chat_id: str, data: CreateMessage, request: Request, user=Depends(actor)): + key = request.headers.get("Idempotency-Key", "") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise Failure(400, "Требуется корректный Idempotency-Key") + return request.app.state.store.send( + user["id"], identifier(chat_id), data.text, key, [str(a) for a in data.attachment_ids] + ) + + +@app.get("/api/v1/chats/{chat_id}/messages") +def history(chat_id: str, request: Request, user=Depends(actor)): + cid = identifier(chat_id) + scope = "messages:" + cid + limit, after = pagination(request, scope) + return page(request.app.state.store.history(user["id"], cid, after, limit), limit, scope) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment(chat_id: str, request: Request, user=Depends(actor)): + import asyncio + import tempfile + + from starlette.concurrency import run_in_threadpool + + from .attachments import MAX_BYTES, filename_valid + + if ( + set(request.query_params) != {"filename"} + or len(request.query_params.getlist("filename")) != 1 + ): + raise Failure(400, "Требуется одно имя файла") + filename = filename_valid(request.query_params["filename"]) + cid = identifier(chat_id) + await run_in_threadpool(request.app.state.store.chat, user["id"], cid) + content_type = request.headers.get("content-type", "").lower() + if content_type not in {"image/png", "image/jpeg"}: + raise Failure(415, "Ожидается PNG или JPEG") + if not request.app.state.upload_slots.acquire(blocking=False): + raise Failure(429, "Все места загрузки заняты", {"Retry-After": "1"}) + try: + with tempfile.SpooledTemporaryFile(max_size=256 * 1024) as spool: + size = 0 + try: + async with asyncio.timeout(15): + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise Failure(413, "Файл больше 10 MiB") + spool.write(chunk) + except TimeoutError: + raise Failure(400, "Истекло время загрузки") from None + spool.seek(0) + return await run_in_threadpool( + request.app.state.attachments.upload, + user["id"], + cid, + filename, + content_type, + spool, + size, + ) + finally: + request.app.state.upload_slots.release() + + +@app.get("/api/v1/attachments/{attachment_id}") +def attachment_metadata(attachment_id: str, request: Request, user=Depends(actor)): + return request.app.state.attachments.metadata(user["id"], identifier(attachment_id)) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def attachment_content(attachment_id: str, request: Request, user=Depends(actor)): + from fastapi.responses import StreamingResponse + + if ( + any(k != "variant" for k in request.query_params) + or len(request.query_params.getlist("variant")) > 1 + ): + raise Failure(400, "Неверный вариант") + variant = request.query_params.get("variant", "original") + if variant not in {"original", "thumbnail", "crop", "optimized"}: + raise Failure(400, "Неверный вариант") + obj = request.app.state.attachments.content(user["id"], identifier(attachment_id), variant) + + def chunks(): + try: + while data := obj["Body"].read(64 * 1024): + yield data + finally: + obj["Body"].close() + + return StreamingResponse( + chunks(), + media_type=obj["ContentType"], + headers={"Content-Length": str(obj["ContentLength"])}, + ) diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py new file mode 100644 index 00000000..1b037944 --- /dev/null +++ b/src/messenger/attachments.py @@ -0,0 +1,154 @@ +"""Synchronous attachment intake: bounded spool, full image validation, durable intent.""" + +import hashlib +import os +import subprocess +import unicodedata +from uuid import uuid4 + +from PIL import Image, UnidentifiedImageError + +from .domain import Failure, timestamp +from .postgres import public + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +# Check dimensions before decoding. Pillow's own hard bomb threshold remains active. +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +PUBLIC_FIELDS = ( + "id", + "chat_id", + "owner_id", + "filename", + "content_type", + "size_bytes", + "sha256", + "status", + "error_code", + "created_at", + "updated_at", +) + + +def filename_valid(name): + if not isinstance(name, str) or not 1 <= len(name) <= 255 or not name.strip(): + raise Failure(400, "Неверное имя файла") + if any(c in "/\\" or unicodedata.category(c) == "Cc" for c in name): + raise Failure(400, "Неверное имя файла") + return name + + +def validate_image(fileobj, declared_type): + if declared_type not in {"image/png", "image/jpeg"}: + raise Failure(415, "Ожидается PNG или JPEG") + try: + fileobj.seek(0) + with Image.open(fileobj) as image: + width, height = image.size + if width * height > MAX_PIXELS: + raise Failure(413, "Слишком много пикселей") + actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) + if actual != declared_type: + raise Failure(415, "Формат изображения не соответствует MIME") + image.verify() + # verify() alone does not decode IDAT/JPEG scan data. + fileobj.seek(0) + with Image.open(fileobj) as image: + image.load() + if actual == "image/jpeg": + # libjpeg emits a warning but Pillow accepts e.g. a half scan with a + # forged EOI. djpeg's nonzero warning exit rejects that corruption. + fileobj.seek(0) + try: + decoded = subprocess.run( + ["djpeg", "-strict", "-maxmemory", "192m", "-outfile", "/dev/null"], + stdin=fileobj, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + timeout=5, + check=False, + ) + except subprocess.TimeoutExpired: + raise Failure(415, "Изображение не прошло ограниченную проверку") from None + if decoded.returncode: + raise Failure(415, "Повреждённое изображение") + fileobj.seek(0) + return actual, width, height + except Image.DecompressionBombError: + raise Failure(413, "Слишком много пикселей") from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise Failure(415, "Повреждённое изображение") from None + + +def metadata_public(row): + return {**public({k: row[k] for k in PUBLIC_FIELDS}), "variants": []} + + +class AttachmentService: + def __init__(self, store, storage): + self.store = store + self.storage = storage + + def upload(self, actor, cid, filename, content_type, fileobj, size): + filename_valid(filename) + if not 1 <= size <= MAX_BYTES: + raise Failure(413 if size > MAX_BYTES else 415, "Неверный размер изображения") + validate_image(fileobj, content_type) + digest = hashlib.file_digest(fileobj, "sha256").hexdigest() + aid = str(uuid4()) + key = "original/" + aid + now = timestamp() + with self.store.connection() as conn: + self.store._chat(conn, actor, cid) + conn.execute( + "INSERT INTO upload_intents(id,object_key,created_at) VALUES (%s,%s,%s)", + (aid, key, now), + ) + # Keep the intent locked throughout PUT/finalization: reconciler skips live + # uploads even if a slow request crosses the grace boundary. + with self.store.connection() as conn, conn.transaction(): + intent = conn.execute( + "SELECT id FROM upload_intents WHERE id=%s FOR UPDATE", (aid,) + ).fetchone() + if not intent: + raise Failure(503, "Загрузка больше не активна") + self.storage.put(key, fileobj, size, content_type) + if os.getenv("UPLOAD_FAULT") == "after_put": + raise Failure(503, "Учебный сбой после записи файла") + row = conn.execute( + "INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256," + "status,error_code,created_at,updated_at,object_key) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',NULL,%s,%s,%s) RETURNING *", + (aid, cid, actor, filename, content_type, size, digest, now, now, key), + ).fetchone() + conn.execute("DELETE FROM upload_intents WHERE id=%s", (aid,)) + # Never delete S3 on an ambiguous COMMIT: the transaction may have committed. + if os.getenv("UPLOAD_FAULT") == "after_commit": + raise Failure(503, "Учебный сбой после фиксации") + return metadata_public(row) + + def metadata(self, actor, aid): + with self.store.connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s", + (aid, actor), + ).fetchone() + if not row: + raise Failure(404, "Объект не найден") + return metadata_public(row) + + def content(self, actor, aid, variant="original"): + with self.store.connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s", + (aid, actor), + ).fetchone() + if not row: + raise Failure(404, "Объект не найден") + if row["status"] != "ready": + raise Failure(409, "Вложение не готово") + if variant != "original": + raise Failure(404, "Вариант не найден") + return self.storage.get(row["object_key"]) diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 00000000..a4b08821 --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,10 @@ +import os +from pathlib import Path + + +def secret(name: str) -> str: + filename = os.environ.get(name + "_FILE") + value = Path(filename).read_text().strip() if filename else os.environ.get(name, "") + if not value: + raise RuntimeError("Отсутствует настройка " + name) + return value diff --git a/src/messenger/domain.py b/src/messenger/domain.py new file mode 100644 index 00000000..1415db7e --- /dev/null +++ b/src/messenger/domain.py @@ -0,0 +1,54 @@ +import base64 +import json +import re +from datetime import UTC, datetime +from uuid import UUID + + +class Failure(Exception): + def __init__(self, status: int, message: str, headers: dict | None = None): + self.status = status + self.message = message + self.headers = headers or {} + + +def timestamp() -> str: + return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def identifier(value: str) -> str: + try: + return str(UUID(value)) + except (ValueError, TypeError, AttributeError): + raise Failure(400, "Некорректный UUID") from None + + +def boundary(scope: str, cursor: str | None) -> tuple[str, str] | None: + if cursor is None: + return None + try: + if not cursor or len(cursor) > 1024 or not re.fullmatch(r"[A-Za-z0-9_-]+", cursor): + raise ValueError + data = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))) + if not isinstance(data, list) or len(data) != 4 or data[:2] != [1, scope]: + raise ValueError + date, uid = data[2:] + if not isinstance(date, str) or not date.endswith("Z"): + raise ValueError + datetime.fromisoformat(date.replace("Z", "+00:00")) + if not isinstance(uid, str) or str(UUID(uid)) != uid: + raise ValueError + return date, uid + except (ValueError, TypeError, KeyError, UnicodeError): + raise Failure(400, "Некорректный cursor или область запроса") from None + + +def page(items: list[dict], limit: int, scope: str) -> dict: + more = len(items) > limit + selected = items[:limit] + cursor = None + if more: + last = selected[-1] + payload = json.dumps([1, scope, last["created_at"], last["id"]], separators=(",", ":")) + cursor = base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=") + return {"items": selected, "next_cursor": cursor} diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..f4486882 --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,32 @@ +import hashlib +from pathlib import Path + +import psycopg + +from .config import secret + + +def migrate(): + with psycopg.connect(secret("DATABASE_URL")) as conn: + # Serialize independent migration processes without changing application data. + conn.execute("SELECT pg_advisory_xact_lock(2301002)") + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(name text PRIMARY KEY, checksum text NOT NULL)" + ) + for path in sorted(Path("migrations").glob("*.sql")): + checksum = hashlib.sha256(path.read_bytes()).hexdigest() + old = conn.execute( + "SELECT checksum FROM schema_migrations WHERE name=%s", (path.name,) + ).fetchone() + if old: + if old[0] != checksum: + raise RuntimeError("Изменена применённая миграция: " + path.name) + continue + conn.execute(path.read_text()) + conn.execute("INSERT INTO schema_migrations VALUES (%s,%s)", (path.name, checksum)) + print("Applied", path.name) + + +if __name__ == "__main__": + migrate() diff --git a/src/messenger/models.py b/src/messenger/models.py new file mode 100644 index 00000000..af81f3b1 --- /dev/null +++ b/src/messenger/models.py @@ -0,0 +1,68 @@ +import re +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator, model_validator + + +class RequestModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + @field_validator("*", mode="after") + @classmethod + def nonblank(cls, value): + if isinstance(value, str) and not value.strip(): + raise ValueError("Строка не может состоять из пробелов") + return value + + +class CreateUser(RequestModel): + username: StrictStr = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: StrictStr = Field(min_length=1, max_length=100) + password: StrictStr = Field(min_length=12, max_length=128) + + @field_validator("username") + @classmethod + def full_username(cls, value): + if not re.fullmatch(r"[a-z0-9_]{3,32}", value): + raise ValueError("Неверное имя пользователя") + return value + + +class CreateChat(RequestModel): + title: StrictStr = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("member_ids", mode="before") + @classmethod + def string_ids(cls, value): + if not isinstance(value, list) or any(not isinstance(x, str) for x in value): + raise ValueError("Ожидается массив UUID-строк") + return value + + @field_validator("member_ids") + @classmethod + def unique_ids(cls, value): + if len(set(value)) != len(value): + raise ValueError("Участники не должны повторяться") + return value + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: StrictStr = Field(max_length=4000) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids", mode="before") + @classmethod + def string_ids(cls, value): + if not isinstance(value, list) or any(not isinstance(x, str) for x in value): + raise ValueError("Ожидается массив UUID-строк") + return value + + @model_validator(mode="after") + def content(self): + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("Вложения не должны повторяться") + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Пустое сообщение без вложений") + return self diff --git a/src/messenger/postgres.py b/src/messenger/postgres.py new file mode 100644 index 00000000..6c734001 --- /dev/null +++ b/src/messenger/postgres.py @@ -0,0 +1,324 @@ +import hashlib +import json +import logging +import os +import secrets +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from threading import BoundedSemaphore +from uuid import UUID, uuid4 + +import psycopg +import redis +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests + +from .config import secret +from .domain import Failure, timestamp + +logger = logging.getLogger("messenger") +KDF_SLOTS = BoundedSemaphore(2) +KDF = PasswordHasher(time_cost=2, memory_cost=65536, parallelism=1) +DUMMY_HASH = KDF.hash("invalid-user-dummy-password") + + +class DeadlineConnection(psycopg.Connection): + def wait(self, gen, interval=0.1, timeout=None): + try: + return super().wait(gen, interval=interval, timeout=timeout or 3.0) + except psycopg.errors._WaitTimeout: + # A paused server cannot complete cancellation. Discard an uncertain + # connection instead of returning it to the pool or waiting forever. + self.pgconn.finish() + raise + + +def public(row): + return { + k: ( + str(v) + if isinstance(v, UUID) + else v.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z") + if isinstance(v, datetime) + else bytes(v).decode("utf-8") + if isinstance(v, (bytes, memoryview)) + else v + ) + for k, v in row.items() + if k not in {"password_hash", "token_digest"} + } + + +class PostgresStore: + def __init__(self): + self.pool = ConnectionPool( + secret("DATABASE_URL"), + connection_class=DeadlineConnection, + min_size=1, + max_size=int(os.getenv("POOL_MAX_SIZE", "8")), + timeout=2, + max_waiting=32, + kwargs={ + "autocommit": True, + "row_factory": dict_row, + "connect_timeout": 2, + "options": "-c statement_timeout=2500 -c lock_timeout=1500", + }, + check=ConnectionPool.check_connection, + ) + self.pool.wait(timeout=15) + self.cache = redis.Redis.from_url( + secret("CACHE_URL"), + socket_connect_timeout=0.2, + socket_timeout=0.2, + decode_responses=True, + ) + self.ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + if self.ttl < 1: + raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным") + + @contextmanager + def connection(self): + try: + with self.pool.connection() as conn: + yield conn + except ( + psycopg.OperationalError, + psycopg.InterfaceError, + PoolTimeout, + TooManyRequests, + psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable, + ): + raise Failure(503, "Хранилище временно недоступно") from None + + def close(self): + self.cache.close() + self.pool.close() + + def ready(self): + try: + with self.connection() as conn: + conn.execute("SELECT 1") + return True + except Failure: + return False + + def _cached_user(self, uid): + key = "user:v1:" + uid + try: + value = self.cache.get(key) + if value: + cached = json.loads(value) + if ( + isinstance(cached, dict) + and cached.get("id") == uid + and set(cached) == {"id", "username", "display_name", "created_at"} + ): + return cached + except (redis.RedisError, ValueError, TypeError): + logger.warning(json.dumps({"event": "cache_bypass"})) + user = self.user(uid) + if user: + try: + self.cache.setex(key, 60, json.dumps(user)) + except redis.RedisError: + logger.warning(json.dumps({"event": "cache_bypass"})) + return user + + def user(self, uid): + with self.connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at FROM users WHERE id=%s", (uid,) + ).fetchone() + return public(row) if row else None + + def register(self, data): + with KDF_SLOTS: + hashed = KDF.hash(data.pop("password")) + uid = str(uuid4()) + try: + with self.connection() as conn: + row = conn.execute( + "INSERT INTO users VALUES (%s,%s,%s,%s,%s) " + "RETURNING id,username,display_name,created_at", + ( + uid, + data["username"], + data["display_name"].encode("utf-8"), + hashed, + timestamp(), + ), + ).fetchone() + return public(row) + except psycopg.errors.UniqueViolation: + raise Failure(409, "Имя пользователя занято") from None + + def login(self, username, password): + with self.connection() as conn: + row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone() + try: + with KDF_SLOTS: + KDF.verify(row["password_hash"] if row else DUMMY_HASH, password) + except VerificationError: + return None + if row is None: + return None + token = secrets.token_urlsafe(32) + digest = hashlib.sha256(token.encode()).hexdigest() + expires = datetime.now(UTC) + timedelta(seconds=self.ttl) + with self.connection() as conn: + conn.execute("INSERT INTO sessions VALUES (%s,%s,%s)", (digest, row["id"], expires)) + return { + "token": token, + "token_type": "Bearer", + "expires_at": public({"v": expires})["v"], + "user": public(row), + } + + def authenticate(self, token): + if not token or len(token) > 512: + return None + digest = hashlib.sha256(token.encode()).hexdigest() + # Session validity always comes from PostgreSQL, including on a cache hit. + with self.connection() as conn: + row = conn.execute( + "SELECT s.user_id FROM sessions s WHERE s.token_digest=%s AND s.expires_at>now()", + (digest,), + ).fetchone() + return self._cached_user(str(row["user_id"])) if row else None + + def logout(self, token): + with self.connection() as conn: + row = conn.execute( + "DELETE FROM sessions WHERE token_digest=%s AND expires_at>now() RETURNING user_id", + (hashlib.sha256(token.encode()).hexdigest(),), + ).fetchone() + return row is not None + + def create_chat(self, actor, data): + if actor not in data["member_ids"]: + raise Failure(400, "Вызывающий должен входить в чат") + cid = str(uuid4()) + try: + with self.connection() as conn, conn.transaction(): + row = conn.execute( + "INSERT INTO chats VALUES (%s,%s,%s) RETURNING *", + (cid, data["title"].encode("utf-8"), timestamp()), + ).fetchone() + for index, uid in enumerate(data["member_ids"]): + conn.execute("INSERT INTO members VALUES (%s,%s,%s)", (cid, uid, index)) + return {**public(row), "member_ids": data["member_ids"]} + except psycopg.errors.ForeignKeyViolation: + raise Failure(400, "Неизвестный участник") from None + + @staticmethod + def _chat(conn, actor, cid): + row = conn.execute( + "SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id " + "WHERE c.id=%s AND m.user_id=%s", + (cid, actor), + ).fetchone() + if row is None: + raise Failure(404, "Объект не найден") + members = conn.execute( + "SELECT user_id FROM members WHERE chat_id=%s ORDER BY position", (cid,) + ).fetchall() + return {**public(row), "member_ids": [str(m["user_id"]) for m in members]} + + def chat(self, actor, cid): + with self.connection() as conn: + return self._chat(conn, actor, cid) + + def list_chats(self, actor, after, limit): + with self.connection() as conn: + sql = "SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s" + params = [actor] + if after: + sql += " AND (c.created_at,c.id)>(%s,%s)" + params.extend(after) + sql += " ORDER BY c.created_at,c.id LIMIT %s" + params.append(limit + 1) + rows = conn.execute(sql, params).fetchall() + return [self._chat(conn, actor, str(row["id"])) for row in rows] + + def send(self, actor, cid, text, key, attachment_ids=None): + attachment_ids = attachment_ids or [] + body_digest = hashlib.sha256( + json.dumps( + {"text": text, "attachment_ids": attachment_ids}, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + with self.connection() as conn, conn.transaction(): + self._chat(conn, actor, cid) + # Serializes exactly this semantic key across all API processes. Hash + # collisions can only serialize unrelated requests, never mix results. + scope = json.dumps([actor, "POST", cid, key], separators=(",", ":")) + lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True) + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,)) + old = conn.execute( + "SELECT body_digest,response FROM message_requests " + "WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>now()", + (actor, cid, key), + ).fetchone() + if old: + if old["body_digest"] != body_digest: + raise Failure(409, "Ключ уже использован для другого сообщения") + return json.loads(bytes(old["response"])) + conn.execute( + "DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s", + (actor, cid, key), + ) + for aid in attachment_ids: + attachment = conn.execute( + "SELECT id,status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE", + (aid, cid), + ).fetchone() + if not attachment: + raise Failure(404, "Объект не найден") + if attachment["status"] == "failed": + raise Failure(409, "Вложение недоступно") + row = conn.execute( + "INSERT INTO messages VALUES (%s,%s,%s,%s,%s) RETURNING *", + (str(uuid4()), cid, actor, text.encode("utf-8"), timestamp()), + ).fetchone() + for position, aid in enumerate(attachment_ids): + conn.execute( + "INSERT INTO message_attachments VALUES (%s,%s,%s)", (row["id"], aid, position) + ) + response = {**public(row), "attachment_ids": attachment_ids} + conn.execute( + "INSERT INTO message_requests(actor_id,chat_id,key,body_digest,response,expires_at) " + "VALUES (%s,%s,%s,%s,%s,now()+interval '24 hours')", + (actor, cid, key, body_digest, json.dumps(response).encode("utf-8")), + ) + return response + + def history(self, actor, cid, after, limit): + with self.connection() as conn: + self._chat(conn, actor, cid) + sql = "SELECT * FROM messages WHERE chat_id=%s" + params = [cid] + if after: + sql += " AND (created_at,id)>(%s,%s)" + params.extend(after) + sql += " ORDER BY created_at,id LIMIT %s" + params.append(limit + 1) + result = [] + for row in conn.execute(sql, params).fetchall(): + attachments = conn.execute( + "SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", + (row["id"],), + ).fetchall() + result.append( + { + **public(row), + "attachment_ids": [str(a["attachment_id"]) for a in attachments], + } + ) + return result diff --git a/src/messenger/reconcile.py b/src/messenger/reconcile.py new file mode 100644 index 00000000..9cc74dc5 --- /dev/null +++ b/src/messenger/reconcile.py @@ -0,0 +1,176 @@ +"""Explicit, repeatable reconciliation. Default mode is strictly read-only.""" + +import argparse +import json +import time +from datetime import UTC, datetime, timedelta + +from .domain import Failure +from .postgres import PostgresStore +from .storage import ObjectStorage + + +def reconcile(store, storage, *, apply=False, grace_seconds=3600, page_size=100): + if grace_seconds < 1 or not 1 <= page_size <= 1000: + raise ValueError("grace >= 1 second; page size 1..1000") + started = time.perf_counter() + cutoff = datetime.now(UTC) - timedelta(seconds=grace_seconds) + report = { + "mode": "apply" if apply else "dry-run", + "grace_seconds": grace_seconds, + "page_size": page_size, + "intents_scanned": 0, + "objects_scanned": 0, + "metadata_scanned": 0, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + } + + def audit(conn, event, key): + conn.execute( + "INSERT INTO attachment_reconcile_audit(event,object_key) VALUES (%s,%s)", (event, key) + ) + + # Locked live uploads are skipped. A dead upload's committed intent survives + # rollback and makes both its object and metadata absence discoverable. + after = None + while True: + with store.connection() as conn: + rows = conn.execute( + "SELECT id,object_key,created_at FROM upload_intents WHERE created_at<%s " + + ("AND (created_at,id)>(%s,%s) " if after else "") + + "ORDER BY created_at,id LIMIT %s", + [cutoff, *(after or []), page_size], + ).fetchall() + if not rows: + break + for candidate in rows: + report["intents_scanned"] += 1 + try: + with store.connection() as conn, conn.transaction(): + row = conn.execute( + "SELECT * FROM upload_intents WHERE id=%s FOR UPDATE SKIP LOCKED", + (candidate["id"],), + ).fetchone() + if not row: + continue + current_object = storage.head(row["object_key"]) + if current_object and current_object["LastModified"] >= cutoff: + report["fresh_objects_skipped"] += 1 + continue + report["old_intents"] += 1 + known = conn.execute( + "SELECT id FROM attachments WHERE object_key=%s", (row["object_key"],) + ).fetchone() + if apply: + if not known: + storage.delete(row["object_key"]) + report["deleted_objects"] += 1 + conn.execute("DELETE FROM upload_intents WHERE id=%s", (row["id"],)) + audit(conn, "intent_cleaned", row["object_key"]) + report["deleted_intents"] += 1 + except Failure: + report["errors"] += 1 + after = (rows[-1]["created_at"], rows[-1]["id"]) + + # Only server-owned original prefix is eligible. Unknown future prefixes are + # deliberately outside this tool's deletion policy. + try: + for obj in storage.objects(page_size): + report["objects_scanned"] += 1 + if not obj["Key"].startswith("original/"): + continue + if obj["LastModified"] >= cutoff: + report["fresh_objects_skipped"] += 1 + continue + with store.connection() as conn, conn.transaction(): + known = conn.execute( + "SELECT 1 FROM attachments WHERE object_key=%s UNION ALL " + "SELECT 1 FROM upload_intents WHERE object_key=%s LIMIT 1", + (obj["Key"], obj["Key"]), + ).fetchone() + if known: + continue + current_object = storage.head(obj["Key"]) + if current_object and current_object["LastModified"] >= cutoff: + report["fresh_objects_skipped"] += 1 + continue + report["orphan_objects"] += 1 + if apply: + storage.delete(obj["Key"]) + audit(conn, "orphan_deleted", obj["Key"]) + report["deleted_objects"] += 1 + except Failure: + report["errors"] += 1 + + # A listing snapshot cannot prove absence. HEAD is checked live under the row + # lock immediately before changing status. Never delete legitimate metadata, + # including attachments referenced by messages; missing originals are reported. + after_id = None + while True: + with store.connection() as conn: + rows = conn.execute( + "SELECT id FROM attachments WHERE created_at<%s AND status='ready' " + + ("AND id>%s " if after_id else "") + + "ORDER BY id LIMIT %s", + [cutoff, *([after_id] if after_id else []), page_size], + ).fetchall() + if not rows: + break + for candidate in rows: + report["metadata_scanned"] += 1 + try: + with store.connection() as conn, conn.transaction(): + row = conn.execute( + "SELECT * FROM attachments WHERE id=%s AND status='ready' FOR UPDATE SKIP LOCKED", + (candidate["id"],), + ).fetchone() + if not row or storage.exists(row["object_key"]): + continue + report["missing_objects"] += 1 + if apply: + conn.execute( + "UPDATE attachments SET status='failed',error_code='storage_missing'," + "updated_at=now() WHERE id=%s", + (row["id"],), + ) + audit(conn, "missing_marked_failed", row["object_key"]) + report["marked_failed"] += 1 + except Failure: + report["errors"] += 1 + after_id = rows[-1]["id"] + report["duration_seconds"] = round(time.perf_counter() - started, 6) + return report + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true") + parser.add_argument("--grace-seconds", type=int, default=3600) + parser.add_argument("--page-size", type=int, default=100) + args = parser.parse_args() + store, storage = PostgresStore(), ObjectStorage() + try: + result = reconcile( + store, + storage, + apply=args.apply, + grace_seconds=args.grace_seconds, + page_size=args.page_size, + ) + print(json.dumps(result, ensure_ascii=False)) + if result["errors"]: + raise SystemExit(1) + finally: + storage.close() + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/messenger/storage.py b/src/messenger/storage.py new file mode 100644 index 00000000..2719abe3 --- /dev/null +++ b/src/messenger/storage.py @@ -0,0 +1,96 @@ +"""Private S3 access with bounded network waits; never expose storage keys in API.""" + +import os +from contextlib import contextmanager + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import secret +from .domain import Failure + + +class ObjectStorage: + def __init__(self, admin=False): + self.bucket = os.getenv("S3_BUCKET", "messenger") + prefix = "S3_ADMIN" if admin else "S3" + self.client = boto3.client( + "s3", + endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"), + aws_access_key_id=secret(prefix + "_ACCESS_KEY"), + aws_secret_access_key=secret(prefix + "_SECRET_KEY"), + region_name="us-east-1", + config=Config( + connect_timeout=1, + read_timeout=2, + retries={"total_max_attempts": 1}, + s3={"addressing_style": "path"}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + + @contextmanager + def boundary(self): + try: + yield + except (BotoCoreError, ClientError): + raise Failure(503, "Хранилище файлов временно недоступно") from None + + def put(self, key, fileobj, size, content_type): + fileobj.seek(0) + with self.boundary(): + self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=fileobj, + ContentLength=size, + ContentType=content_type, + ) + + def get(self, key): + try: + return self.client.get_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}: + raise Failure(404, "Объект не найден") from None + raise Failure(503, "Хранилище файлов временно недоступно") from None + except BotoCoreError: + raise Failure(503, "Хранилище файлов временно недоступно") from None + + def head(self, key): + try: + return self.client.head_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}: + return None + raise Failure(503, "Хранилище файлов временно недоступно") from None + except BotoCoreError: + raise Failure(503, "Хранилище файлов временно недоступно") from None + + def exists(self, key): + return self.head(key) is not None + + def delete(self, key): + with self.boundary(): + self.client.delete_object(Bucket=self.bucket, Key=key) + + def objects(self, page_size=100): + with self.boundary(): + paginator = self.client.get_paginator("list_objects_v2") + for page in paginator.paginate( + Bucket=self.bucket, PaginationConfig={"PageSize": page_size} + ): + yield from page.get("Contents", []) + + def ready(self): + try: + with self.boundary(): + self.client.head_bucket(Bucket=self.bucket) + return True + except Failure: + return False + + def close(self): + self.client.close() diff --git a/src/messenger/storage_bootstrap.py b/src/messenger/storage_bootstrap.py new file mode 100644 index 00000000..34d10e63 --- /dev/null +++ b/src/messenger/storage_bootstrap.py @@ -0,0 +1,30 @@ +"""Idempotent private bucket bootstrap; this process alone receives admin keys.""" + +import time + +from botocore.exceptions import BotoCoreError, ClientError + +from .storage import ObjectStorage + + +def main(): + storage = ObjectStorage(admin=True) + for attempt in range(30): + try: + try: + storage.client.head_bucket(Bucket=storage.bucket) + except ClientError as exc: + if exc.response["Error"]["Code"] not in {"404", "NoSuchBucket", "NotFound"}: + raise + storage.client.create_bucket(Bucket=storage.bucket) + print("Private bucket ready; anonymous identity is not configured.") + storage.close() + return + except (ClientError, BotoCoreError): + if attempt == 29: + raise RuntimeError("S3 bootstrap timeout") from None + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..c7f5caf5 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,201 @@ +import base64 +import os +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import quote + +import pytest +from client import LAB, Client + + +@pytest.fixture +def scenario(): + c = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8081")) + return c, *c.bootstrap() + + +@pytest.mark.parametrize( + "body", + [ + {"text": ""}, + {"text": " \n\t"}, + {"text": 1}, + {"text": True}, + {"text": None}, + {"text": "x", "sender_id": str(uuid.uuid4())}, + {"text": "x", "unknown": 1}, + {"text": "x" * 4001}, + {}, + [], + ], +) +def test_invalid_message(scenario, body): + c, alice, _, _, chat = scenario + c.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + actor=alice, + body=body, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + + +def test_malformed_json(scenario): + c, alice, _, _, chat = scenario + c.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + actor=alice, + raw=b'{"text":', + headers={"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}, + ) + + +def test_acl_and_unknown_chat(scenario): + c, alice, _, eve, chat = scenario + for cid in [chat["id"], str(uuid.uuid4())]: + for suffix in ["", "/messages"]: + c.call("GET", f"/api/v1/chats/{cid}{suffix}", 404, actor=eve) + c.call( + "POST", + f"/api/v1/chats/{cid}/messages", + 404, + actor=eve, + body={"text": "x"}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + c.call("GET", "/api/v1/chats/not-a-uuid", 400, actor=alice) + + +@pytest.mark.parametrize("name", ["ab", "UPPER", "a-b", "a" * 33, "имя", "abc\n"]) +def test_username_validation(scenario, name): + c, alice, *_ = scenario + body = {**alice["registration"], "username": name} + c.call("POST", "/api/v1/users", 400, body=body) + + +def test_unicode_and_limits(scenario): + c, alice, _, _, chat = scenario + text = "Ё👩‍💻漢字" * 500 + assert c.message(alice, chat, text)["json"]["text"] == text + c.call( + "POST", + "/api/v1/users", + 400, + body={ + **alice["registration"], + "username": "u_" + uuid.uuid4().hex[:12], + "display_name": " ", + }, + ) + + +@pytest.mark.parametrize("value", ["0", "101", "1.5", "true", "-1", "", "1&limit=2"]) +def test_invalid_limit(scenario, value): + c, alice, _, _, chat = scenario + c.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit={value}", 400, actor=alice) + + +def test_pagination_and_cursor_scope(scenario): + c, alice, bob, _, chat = scenario + expected = [c.message(alice, chat, str(i))["json"] for i in range(13)] + path = f"/api/v1/chats/{chat['id']}/messages" + result, cursor, first = [], None, None + for i in range(20): + url = path + f"?limit={2 if i == 0 else 3}" + if cursor: + url += "&cursor=" + quote(cursor, safe="") + data = c.call("GET", url, actor=bob, schema="MessagePage")["json"] + result += data["items"] + cursor = data["next_cursor"] + first = first or cursor + if cursor is None: + break + assert result == sorted(expected, key=lambda m: (m["created_at"], m["id"])) + other = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "другой", "member_ids": [alice["id"]]}, + schema="Chat", + )["json"] + c.call("GET", f"/api/v1/chats/{other['id']}/messages?cursor={first}", 400, actor=alice) + c.call("GET", path + "?cursor=broken", 400, actor=alice) + + +def test_chat_validation_and_atomicity(scenario): + c, alice, bob, _, _ = scenario + before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"] + for members in [ + [bob["id"]], + [alice["id"], alice["id"]], + [alice["id"], str(uuid.uuid4())], + [], + [123], + ]: + c.call( + "POST", + "/api/v1/chats", + 400, + actor=alice, + body={"title": "не должен появиться", "member_ids": members}, + ) + after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"] + assert before == after + + +def test_concurrent_messages(scenario): + c, alice, bob, _, chat = scenario + barrier = threading.Barrier(10) + + def sender(i): + local = Client(c.base) + barrier.wait() + return [local.message(alice, chat, f"{i}:{j}")["json"] for j in range(10)] + + with ThreadPoolExecutor(max_workers=10) as pool: + sent = [m for batch in pool.map(sender, range(10)) for m in batch] + data = c.call( + "GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", actor=bob, schema="MessagePage" + )["json"] + assert len(sent) == len({m["id"] for m in sent}) == len(data["items"]) == 100 + assert {m["id"] for m in sent} == {m["id"] for m in data["items"]} + assert data["next_cursor"] is None + + +def test_registration_race(scenario): + c, alice, *_ = scenario + body = {**alice["registration"], "username": "race_" + uuid.uuid4().hex[:12]} + barrier = threading.Barrier(10) + + def register(_): + local = Client(c.base) + barrier.wait() + return local.call("POST", "/api/v1/users", [201, 409], body=body)["status"] + + with ThreadPoolExecutor(max_workers=10) as pool: + statuses = list(pool.map(register, range(10))) + assert statuses.count(201) == 1 + assert statuses.count(409) == 9 + + +@pytest.mark.skipif(LAB < 2, reason="Сессии появляются в ЛР2") +def test_sessions(scenario): + c, alice, _, eve, _ = scenario + credential = base64.b64encode( + (alice["registration"]["username"] + ":wrong-password").encode() + ).decode() + c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + credential}) + c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]}) + assert ( + c.call( + "GET", "/api/v1/users/me", actor=alice, headers={"X-User-Id": eve["id"]}, schema="User" + )["json"] + == alice["user"] + ) + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + c.call("GET", "/api/v1/users/me", 401, actor=alice) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..fbdf7a90 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,201 @@ +"""Black-box regressions derived from the contract and teacher review comments.""" + +import hashlib +import os +import struct +import uuid +import zlib +from io import BytesIO +from urllib.parse import quote + +import pytest +from client import Client +from PIL import Image + + +@pytest.fixture(scope="module") +def scenario(): + c = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8084")) + return c, *c.bootstrap() + + +def image_bytes(fmt="PNG", size=(32, 20)): + stream = BytesIO() + Image.new("RGB", size, (100, 130, 200)).save(stream, fmt) + return stream.getvalue() + + +def upload(scenario, data=None, name="sample.png", mime="image/png", actor=None, status=201): + c, alice, _, _, chat = scenario + return c.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename={quote(name, safe='')}", + status, + actor=actor or alice, + raw=data if data is not None else image_bytes(), + headers={"Content-Type": mime}, + schema="Attachment" if status == 201 else None, + ) + + +def test_roundtrip_acl_and_variant(scenario): + c, alice, bob, eve, chat = scenario + data = image_bytes() + a = upload(scenario, data)["json"] + assert a["sha256"] == hashlib.sha256(data).hexdigest() + assert a["status"] == "ready" and a["variants"] == [] + path = f"/api/v1/attachments/{a['id']}" + assert c.call("GET", path + "/content", 200, actor=bob)["bytes"] == data + c.call("GET", path, 404, actor=eve) + c.call("GET", path + "/content", 404, actor=eve) + c.call("GET", path + "/content?variant=thumbnail", 404, actor=alice) + c.call("GET", path + "/content?variant=unknown", 400, actor=alice) + message = c.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + actor=alice, + body={"text": "", "attachment_ids": [a["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + schema="Message", + )["json"] + assert message["text"] == "" and message["attachment_ids"] == [a["id"]] + + +@pytest.mark.parametrize( + "name", [" ", "../x.png", "a\\b.png", "a\0.png", "a\x7f.png", "a\x85.png", "a\n.png"] +) +def test_bad_filename(scenario, name): + upload(scenario, name=name, status=400) + + +def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + +def png_with_data(data, width=1, height=1, color=2, palette=b""): + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, color, 0, 0, 0)) + + (chunk(b"PLTE", palette) if palette else b"") + + chunk(b"IDAT", data) + + chunk(b"IEND", b"") + ) + + +@pytest.mark.parametrize( + "data,mime", + [ + (image_bytes()[:33], "image/png"), + (image_bytes()[:-15], "image/png"), + (png_with_data(b"broken-zlib"), "image/png"), + (png_with_data(zlib.compress(b"\x05\0\0\0")), "image/png"), + (png_with_data(zlib.compress(b"\0\0"), color=3, palette=bytes(771)), "image/png"), + (bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "image/jpeg"), + (bytes.fromhex("ffd8ffc00008080001000101ffda0002ffd9"), "image/jpeg"), + (image_bytes(), "image/jpeg"), + (b"not-an-image", "image/png"), + ], +) +def test_corrupt_or_spoofed_image(scenario, data, mime): + upload(scenario, data=data, mime=mime, status=415) + + +@pytest.mark.parametrize("size", [(8000, 5001), (20000, 10000)]) +def test_pixel_limit_before_decode(scenario, size): + data = png_with_data(b"", width=size[0], height=size[1]) + upload(scenario, data=data, status=413) + + +def test_byte_limit_and_exact_boundary(scenario): + data = image_bytes() + exact = data + bytes(10 * 1024 * 1024 - len(data)) + a = upload(scenario, data=exact)["json"] + assert a["size_bytes"] == len(exact) + upload(scenario, data=exact + b"x", status=413) + + +@pytest.mark.parametrize( + "body", + [ + {"attachment_ids": []}, + {"text": None}, + {"text": "x", "attachment_ids": None}, + {"text": "x", "attachment_ids": [123]}, + ], +) +def test_strict_message_fields(scenario, body): + c, alice, _, _, chat = scenario + c.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + actor=alice, + body=body, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + + +def test_foreign_unknown_and_ordered_attachments(scenario): + c, alice, _, _, chat = scenario + first, second = upload(scenario)["json"], upload(scenario)["json"] + other = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "other", "member_ids": [alice["id"]]}, + schema="Chat", + )["json"] + for aid in [first["id"], str(uuid.uuid4())]: + c.call( + "POST", + f"/api/v1/chats/{other['id']}/messages", + 404, + actor=alice, + body={"text": "", "attachment_ids": [aid]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + key = uuid.uuid4().hex + body = {"text": "with files", "attachment_ids": [second["id"], first["id"]]} + p = f"/api/v1/chats/{chat['id']}/messages" + sent = c.call( + "POST", p, 201, actor=alice, body=body, headers={"Idempotency-Key": key}, schema="Message" + )["json"] + repeat = c.call( + "POST", p, 201, actor=alice, body=body, headers={"Idempotency-Key": key}, schema="Message" + )["json"] + assert repeat == sent + c.call( + "POST", + p, + 409, + actor=alice, + body={**body, "attachment_ids": list(reversed(body["attachment_ids"]))}, + headers={"Idempotency-Key": key}, + ) + history = c.call("GET", p, 200, actor=alice, schema="MessagePage")["json"]["items"] + assert next(m for m in history if m["id"] == sent["id"]) == sent + + +def test_nul_message_roundtrip(scenario): + c, alice, _, _, chat = scenario + assert c.message(alice, chat, "abc\0def")["json"]["text"] == "abc\0def" + + +def test_truncated_jpeg_with_forged_eoi(scenario): + import random + + stream = BytesIO() + Image.frombytes("RGB", (512, 512), random.Random(42).randbytes(512 * 512 * 3)).save( + stream, "JPEG", quality=95 + ) + original = stream.getvalue() + upload(scenario, data=original, name="real.jpg", mime="image/jpeg") + upload( + scenario, + data=original[: len(original) // 2] + b"\xff\xd9", + name="broken.jpg", + mime="image/jpeg", + status=415, + ) diff --git a/student-tests/test_edgecases.py b/student-tests/test_edgecases.py new file mode 100644 index 00000000..88ead657 --- /dev/null +++ b/student-tests/test_edgecases.py @@ -0,0 +1,99 @@ +import base64 +import json +import os +import uuid +from urllib.parse import quote + +import pytest +from client import LAB, Client + + +@pytest.fixture +def edge_scenario(): + client = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8081")) + return client, *client.bootstrap() + + +def encoded_cursor(value): + return base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip("=") + + +@pytest.mark.parametrize("identity", ["missing", "malformed", "unknown"]) +def test_invalid_identity(edge_scenario, identity): + client, _, _, _, chat = edge_scenario + headers = {} + if identity == "malformed": + headers["X-User-Id"] = "not-a-uuid" + elif identity == "unknown": + headers["X-User-Id"] = str(uuid.uuid4()) + for path in ["/api/v1/users/me", "/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"]: + client.call("GET", path, 401, headers=headers) + + +@pytest.mark.parametrize("uid", [None, [], {}, 123, True, "not-a-uuid"]) +def test_cursor_rejects_non_uuid_boundary(edge_scenario, uid): + client, alice, _, _, chat = edge_scenario + payload = [1, "messages:" + chat["id"], "2026-10-30T12:00:00.000000Z", uid] + cursor = quote(encoded_cursor(payload), safe="") + client.call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, actor=alice) + + +@pytest.mark.parametrize( + "payload", [None, {}, [], [1], [1, "wrong-scope", "2026-10-30T12:00:00Z", str(uuid.uuid4())]] +) +def test_cursor_rejects_wrong_shape(edge_scenario, payload): + client, alice, _, _, chat = edge_scenario + cursor = quote(encoded_cursor(payload), safe="") + client.call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, actor=alice) + + +def test_request_id_is_present_and_not_reused(edge_scenario): + client, alice, _, _, _ = edge_scenario + first = client.call("GET", "/health/live", schema="Health") + second = client.call("GET", "/health/ready", schema="Health") + error = client.call("GET", "/api/v1/chats/malformed", 400, actor=alice) + values = [r["headers"]["X-Request-Id"] for r in [first, second, error]] + assert len(set(values)) == 3 + assert error["json"]["error"]["request_id"] == values[-1] + + +def test_unicode_codepoint_boundaries(edge_scenario): + client, alice, _, _, chat = edge_scenario + username = "unicode_" + uuid.uuid4().hex[:12] + registration = {"username": username, "display_name": "😀" * 100} + if LAB >= 2: + registration["password"] = "unicode-test-" + uuid.uuid4().hex + user = client.call( + "POST", + "/api/v1/users", + 201, + body=registration, + schema="User", + )["json"] + assert user["display_name"] == "😀" * 100 + client.call( + "POST", "/api/v1/users", 400, body={"username": username + "x", "display_name": "😀" * 101} + ) + message = client.message(alice, chat, "😀" * 4000)["json"] + assert message["text"] == "😀" * 4000 + client.call( + "POST", f"/api/v1/chats/{chat['id']}/messages", 400, actor=alice, body={"text": "😀" * 4001} + ) + + +def test_empty_history_and_cursor_exhaustion(edge_scenario): + client, alice, _, _, chat = edge_scenario + path = f"/api/v1/chats/{chat['id']}/messages" + empty = client.call("GET", path, actor=alice, schema="MessagePage")["json"] + assert empty == {"items": [], "next_cursor": None} + message = client.message(alice, chat, "одна запись")["json"] + data = client.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"] + assert data == {"items": [message], "next_cursor": None} + + +def test_missing_chat_is_hidden_like_foreign_chat(edge_scenario): + client, _, _, eve, chat = edge_scenario + foreign = client.call("GET", f"/api/v1/chats/{chat['id']}", 404, actor=eve)["json"] + missing = client.call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, actor=eve)["json"] + assert foreign["error"]["code"] == missing["error"]["code"] == "not_found" + assert foreign["error"]["message"] == missing["error"]["message"] diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 00000000..0519f812 --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,62 @@ +import base64 +import uuid + +import pytest +from client import Client + + +@pytest.mark.parametrize("password", ["x" * 11, "x" * 129, "😀" * 129, 123, None, True]) +def test_password_contract(password): + import os + + c = Client(os.environ["BASE_URL"]) + c.call( + "POST", + "/api/v1/users", + 400, + body={ + "username": "pwd_" + uuid.uuid4().hex[:12], + "display_name": "Тест", + "password": password, + }, + ) + + +def test_full_unicode_password_and_colon(): + import os + + c = Client(os.environ["BASE_URL"]) + username = "pwd_" + uuid.uuid4().hex[:12] + password = "😀" * 126 + ":Я" + user = c.call( + "POST", + "/api/v1/users", + 201, + body={"username": username, "display_name": "Пароль128", "password": password}, + schema="User", + )["json"] + good = base64.b64encode((username + ":" + password).encode()).decode() + session = c.call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "bAsIc " + good}, + schema="Session", + )["json"] + actor = {"id": user["id"], "token": session["token"]} + assert session["user"] == user + bad = base64.b64encode((username + ":" + password[:-1] + "Б").encode()).decode() + c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + bad}) + c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good}) + c.call("GET", "/api/v1/users/me", actor=actor, schema="User") + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor) + c.call("GET", "/api/v1/users/me", 401, actor=actor) + + +@pytest.mark.parametrize("credential", ["Basic !invalid", "Basic", "Bearer abc", "", "Basic YTpi"]) +def test_invalid_basic(credential): + import os + + Client(os.environ["BASE_URL"]).call( + "POST", "/api/v1/auth/sessions", 401, headers={"Authorization": credential} + ) diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..410a13af --- /dev/null +++ b/uv.lock @@ -0,0 +1,792 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/d2/0ae991f1b2181e5be49007c574710a800ad36c2978683addb3e67c474e55/argon2_cffi_bindings-26.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2", size = 25521, upload-time = "2026-08-20T07:32:43.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e4/ad91d8297638aa2258aad4501c306aca99480dfe76ccd638173fa3702db9/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69", size = 27177, upload-time = "2026-08-20T07:32:44.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/86/5363df11b86d02cf3662208e7406496327649cc90eb365bf6f4e8a54a41f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29", size = 26597, upload-time = "2026-08-20T07:32:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b5/a14dcc592652347dad23ee93b278a4da5d2a25c9ed3ebd10d68eea823a4f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d", size = 27403, upload-time = "2026-08-20T07:32:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/b4a20d4902af7f796390bf9245ff83c5217dfa7367efa1d14986956c482b/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728", size = 27132, upload-time = "2026-08-20T07:32:47.13Z" }, + { url = "https://files.pythonhosted.org/packages/7e/1b/c8de358af07b1c490e0fcb863ef98e46ddb486e45567aca5a60bd68d9daa/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81", size = 27588, upload-time = "2026-08-20T07:32:48.087Z" }, + { url = "https://files.pythonhosted.org/packages/48/2f/7ee62a6e79f9309f9d9982d301b22a00010adb580c05c8109b94d7b33de0/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4", size = 26785, upload-time = "2026-08-20T07:32:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/960d0ee93d4897741bcaf4799c697dae2d81499f66fd1ed042a7dd54c1f4/argon2_cffi_bindings-26.1.0-cp310-abi3-win32.whl", hash = "sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb", size = 23898, upload-time = "2026-08-20T07:32:50.114Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3a/0cc14a05810e6add9bce5e87693334baa2222de5f647fa31781885b6573f/argon2_cffi_bindings-26.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e", size = 25730, upload-time = "2026-08-20T07:32:51.091Z" }, + { url = "https://files.pythonhosted.org/packages/4e/db/d83cf2af140547f0b9cdaece05b2dc2dcbf991be4667331d073eff771435/argon2_cffi_bindings-26.1.0-cp310-abi3-win_arm64.whl", hash = "sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638", size = 24478, upload-time = "2026-08-20T07:32:52.111Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.125" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/c2/9285c749ca49c213ad90aa239884d0c5cbbdc58f02c4c44bcc4a3bf59023/boto3-1.43.125.tar.gz", hash = "sha256:5a635a0fbbd51f319e465b809cf043668705defbb9dddaf26ae66747f70b2a49", size = 112651, upload-time = "2026-11-05T20:24:13.843Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/12/1e295c564a8319e6e27353c4c849ec31b0e3857b2614c36c7cc050ca1b9c/boto3-1.43.125-py3-none-any.whl", hash = "sha256:101fe3ee99664e671efb9af65c8e98ec0bceddf6de922ea89685da4adc8c6907", size = 140043, upload-time = "2026-11-05T20:24:12.048Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.125" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/57/6d638614c940c73a750b8a4203989b3edbde5ff7f83cd9031e65cb97d69b/botocore-1.43.125.tar.gz", hash = "sha256:e3ae5ebbfa8033015958a9cb16cef67a0592902ca8afb46ef06e8ae77475183f", size = 16520959, upload-time = "2026-11-05T20:24:08.59Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/c1/0dca9a7ea362219b1e98f7fa8aebca1c19f6516eb91792b82595d7b79dc0/botocore-1.43.125-py3-none-any.whl", hash = "sha256:e3969c8191d3435440037310187d324f37c00da53987ea31df2eceff19a2795a", size = 16223768, upload-time = "2026-11-05T20:24:04.594Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, +] + +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "fastapi" +version = "0.162.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/eb/2cf8c80abf09b2800aef0a83b41066fbfda8ee5495b7a08b9d1af40f7046/fastapi-0.162.0.tar.gz", hash = "sha256:6a22c1e2e6e32e0633ee7e170f00009a29b8228b16feb219ee42f32fd8d88a8f", size = 527867, upload-time = "2026-10-23T18:47:33.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/4f/3f2c25c9b43970b5e9a08beb5e04776247edea684ddc0930f4f02f0bb6cb/fastapi-0.162.0-py3-none-any.whl", hash = "sha256:3785963de305dd09912426cb81f2d230fce98ab3521cdfe97265c8d14983128c", size = 148912, upload-time = "2026-10-23T18:47:31.734Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-path" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "pathable" }, + { name = "pyyaml" }, + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/79/cd02a4df6d9270efdc7d3feefe6edd730b0820c39eeaa107a2faee8322d5/jsonschema_path-0.5.0.tar.gz", hash = "sha256:493b156ba895c97602655b620a8456caa2ce08c1aa389f5a7addec065e6e855c", size = 19597, upload-time = "2026-05-19T20:45:00.971Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/2c/9e69d73c4297508be9e3b64a970ea3971b3eb8db64ffc5802d40bd25981f/jsonschema_path-0.5.0-py3-none-any.whl", hash = "sha256:2790a070bc7abb08ea3dbe4d340ece4efadf639223001f020c7503229ba068e2", size = 24077, upload-time = "2026-05-19T20:44:59.225Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "lazy-object-proxy" +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/8e/f26b09778fcb26261fdeb4cc2b011975292316529bff9b541a5a02acb976/lazy_object_proxy-1.13.0.tar.gz", hash = "sha256:ce34046cb021d69c72d05ddb83821ecfd4d2d0a5d43a93c5cd01c9e34779cf14", size = 45649, upload-time = "2026-10-17T16:38:01.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/a5/98c03827458988def73622e075486fe1ebbb85ae53ac3f21479fbfd9988c/lazy_object_proxy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:087eba19ea5ce02ea9eb1749d8fdcfb229eeeaf13abaebecf00d579d306d1fd9", size = 26808, upload-time = "2026-10-17T16:37:42.616Z" }, + { url = "https://files.pythonhosted.org/packages/c3/88/30aa37866ff9122c7b00341396781d30e27ae9d863c2698d1f42bb25a8ab/lazy_object_proxy-1.13.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3df6b7db013fbbd00001a43af6b188b545442b4d125b5fddda79366108e02286", size = 71403, upload-time = "2026-10-20T01:05:33.169Z" }, + { url = "https://files.pythonhosted.org/packages/65/fa/099b2931430f87231fa31cc39893d047a1971a57fc7f90df70b9c82961a6/lazy_object_proxy-1.13.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bdc617c7b6f58aa5e59f514c7d619d7f60b52b343b77c9149d2fbfefc412f3a", size = 70989, upload-time = "2026-10-20T01:05:34.591Z" }, + { url = "https://files.pythonhosted.org/packages/8b/17/87891d3fee9c03764691239745ee3f6b793062b88cf5d029bf515aa0836a/lazy_object_proxy-1.13.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47d26b4c5a5ea2d3a1993a1c9509110beb030a2739ca69ebbe516fb04b6842b7", size = 65742, upload-time = "2026-10-20T01:05:36.015Z" }, + { url = "https://files.pythonhosted.org/packages/a9/9a/02acae69e1d0930f9f430d6e4ea77235c72a735672cd8d91164909815fb6/lazy_object_proxy-1.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e6261a9c38f629c3566ab5caec3909f5c6f3333a711c7e39f6883715aa38c566", size = 69280, upload-time = "2026-10-20T01:05:37.217Z" }, + { url = "https://files.pythonhosted.org/packages/75/0e/e697f92c1a5410bddf0571077f38684b16fcf3925bb9ba5430355028e0ba/lazy_object_proxy-1.13.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:337b25ac5b0f5103c72343f74eea438ad3216b0a3cc77a3f6e4ec54c80d0256b", size = 65028, upload-time = "2026-10-20T01:05:38.378Z" }, + { url = "https://files.pythonhosted.org/packages/f2/4f/ec5c7f4907305086a55e9372d3deccec369b30e731b92c14d34df455cf4c/lazy_object_proxy-1.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:faa395d97edfafd3cd0cbd8855473c4673c7e3080fccf31f97a9bcfa7218d7ec", size = 70619, upload-time = "2026-10-20T01:05:39.551Z" }, + { url = "https://files.pythonhosted.org/packages/e9/23/67edce3954b280049b87dfe914fa16400c3ce2f3fa236388cf64a1bba600/lazy_object_proxy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:81437ad7883fe637620bc63a3df3e9974c9a63562e76d7e8a444fe724adccb5c", size = 25815, upload-time = "2026-10-17T16:37:43.889Z" }, + { url = "https://files.pythonhosted.org/packages/21/21/d70d4ca285bb6ed3629ac432ea0910b8f923e50580888e37990b31f3fac9/lazy_object_proxy-1.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:da66d53e0c350e6e6876f2dc060b240dc06cc159ca7f279d59fe4ed48aec2ece", size = 24467, upload-time = "2026-10-17T16:37:44.82Z" }, +] + +[[package]] +name = "mtusi-messenger" +version = "1.0.0" +source = { virtual = "." } +dependencies = [ + { name = "argon2-cffi" }, + { name = "boto3" }, + { name = "fastapi" }, + { name = "pillow" }, + { name = "psycopg", extra = ["binary", "pool"] }, + { name = "redis" }, + { name = "uvicorn" }, +] + +[package.dev-dependencies] +dev = [ + { name = "httpx" }, + { name = "jsonschema" }, + { name = "openapi-spec-validator" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "argon2-cffi" }, + { name = "boto3", specifier = ">=1.43,<2" }, + { name = "fastapi" }, + { name = "pillow", specifier = ">=12.1.1,<13" }, + { name = "psycopg", extras = ["binary", "pool"] }, + { name = "redis" }, + { name = "uvicorn" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "httpx" }, + { name = "jsonschema" }, + { name = "openapi-spec-validator" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[[package]] +name = "openapi-schema-validator" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "jsonschema-specifications" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "referencing" }, + { name = "rfc3339-validator" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/e8/ab3f27dbca54ec645f7fab714b640907d5d36c2ebb07e87eebd30bd5c81b/openapi_schema_validator-0.9.0.tar.gz", hash = "sha256:b72db64315b89d21834cd3ffef37e3e6893bc876327be2d366e8424b1029afd3", size = 24686, upload-time = "2026-04-27T17:31:27.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/c0/5467967d95378b2cfce312e09cbd0c9ab64354a0922379b734f793edd04f/openapi_schema_validator-0.9.0-py3-none-any.whl", hash = "sha256:faa3bbe7c3aa8ca2087ad83f709dc3b7d920283153a570c03e24ea182558aa25", size = 19980, upload-time = "2026-04-27T17:31:25.965Z" }, +] + +[[package]] +name = "openapi-spec-validator" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "jsonschema-path" }, + { name = "lazy-object-proxy" }, + { name = "openapi-schema-validator" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/d2/640b5149cd5688bc0ad1fdbb4df6a2f7b84a093c8d787c27d566132f8b8b/openapi_spec_validator-0.9.0.tar.gz", hash = "sha256:6d648cff6490ebb799dcfe273792f2941c050158854c721f086599d845da78b8", size = 1756839, upload-time = "2026-05-20T09:23:18.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/d8/321ff889330acca2e3097f3d4f80a40bcc41b6d34d302978ab32c449520b/openapi_spec_validator-0.9.0-py3-none-any.whl", hash = "sha256:222fecffc7714f6d0a6ad62c0e4b66cc2b7dbfafb7b93acfc6c308abbdb51af8", size = 50328, upload-time = "2026-05-20T09:23:17.017Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.45.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/cf3a6b6448849656e1e5f4e2101f3988dc6789724050096aba1bc5ccf592/opentelemetry_api-1.45.2.tar.gz", hash = "sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4", size = 73277, upload-time = "2026-10-29T17:29:37.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/2d/610db864d7513158cebb8f36cc88e67e73bd6c7aa1905ef18d89e1281c81/opentelemetry_api-1.45.2-py3-none-any.whl", hash = "sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34", size = 60335, upload-time = "2026-10-29T17:29:06.955Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pathable" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/f3/5a20387de9bcd0607871bfc2198ee0e15836da7baa4592ccd7f24c27c986/pathable-0.6.0.tar.gz", hash = "sha256:6404b8b82aef5ff0fd478934137128b99b12212ba35afdde5525ca4f8388ea58", size = 18970, upload-time = "2026-05-19T18:15:11.911Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/e8/6d75ffd9784bce2e93d1ae4415649427e39a53bb172d4672b2b59c6f0a7b/pathable-0.6.0-py3-none-any.whl", hash = "sha256:82c4ca6c98c502ad12e0d4e9779b6210afee93c38990988c8c5d1b49bdcdf566", size = 18983, upload-time = "2026-05-19T18:15:10.728Z" }, +] + +[[package]] +name = "pillow" +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] +pool = [ + { name = "psycopg-pool" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/01/2cdd1824e58b4467ee0b9498664cd28c42d8794db6b1e35b6bcb834f0044/psycopg_binary-3.3.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3f84dab25e0385692ee13274c68678377e0b1a70ab9d14e56264cbf61f60c62d", size = 4707086, upload-time = "2026-09-18T13:18:05.138Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/de9948ac06895261c84d5b9fbe283d8f3c5bc9f070691b8d9eaa1b51e322/psycopg_binary-3.3.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:612382ac3ed13651c7fa44b5fee9fbf7baaa2ddbc6f500391672682c5f1df9e0", size = 4769607, upload-time = "2026-09-18T13:18:12.83Z" }, + { url = "https://files.pythonhosted.org/packages/76/a9/72436c9915ee4905964689e7f0e182ce7767cc0a0390b3ce703be8177625/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:366db6e97e66b37211475f20c4c1324a2dc0dd825e46d4e87f9d599304d276f9", size = 5554134, upload-time = "2026-09-18T13:18:21.175Z" }, + { url = "https://files.pythonhosted.org/packages/0a/42/948bb3d2617795093512613fd96ba380e922992c7908fbc073858147d196/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1679a1cb93fbe5a6d1fd58d82cbddcc6fcb8c61446ba7cae6eb2a7b19bc585de", size = 5235723, upload-time = "2026-09-18T13:18:27.071Z" }, + { url = "https://files.pythonhosted.org/packages/99/47/93e823ff1b0088400703410939c9bda3e63ed9c850b3ee088e8769f4c10b/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37d40450659401600e6d043ff586c89a71a69f33cbb8bcdba6cdb2569beecdbe", size = 6833587, upload-time = "2026-09-18T13:18:33.794Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2d/ecc69c847795aa704041a9f5667a6b0938a088cf1853636d762a6938e493/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a5165300324efd5a772c48a88ab3a928513ab3979fca76553e62ee815f7b2b9c", size = 5070013, upload-time = "2026-09-18T13:18:39.628Z" }, + { url = "https://files.pythonhosted.org/packages/92/36/6126f0dac21713dcae91404f2a76da18598a6252339a8c669c46370d43b2/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d636338c8f21b0df2f84657b00bc34f9313f826ef93f1155bc743607e4a0c5eb", size = 4597367, upload-time = "2026-09-18T13:18:45.023Z" }, + { url = "https://files.pythonhosted.org/packages/4d/29/7ecfc04243b46c89ffd49924e9c5634ea904ef96c7d0f37e4073623584c1/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a4ee3bdd5468a725f2a4d9aab8a74b6d0279f768c8b5d3aeb102c5307ff3d59c", size = 4275419, upload-time = "2026-09-18T13:18:49.299Z" }, + { url = "https://files.pythonhosted.org/packages/6e/90/2f46d2e0de79706ac170df0a3637fe63c4498fc04f131f6049520b78b806/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:289aadd6a00e151203c081f708348ec89f1e483c9b510ef4ac3981f847f01f79", size = 4007358, upload-time = "2026-09-18T13:18:53.944Z" }, + { url = "https://files.pythonhosted.org/packages/03/48/6744e91291b751a8cf12d63d719977974bb94c84ceba913e7ddb2e478e51/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f21d057f3e5f5491067e5b292498073b73847d48799b099803fef100775fcc52", size = 4320156, upload-time = "2026-09-18T13:18:59.258Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9b/94ff7fce53a64d5b286e2ec454e0a025cf3d6e6b4a9189bef16aa5de98b2/psycopg_binary-3.3.6-cp312-cp312-win_amd64.whl", hash = "sha256:e23a66a763fbe83fcc210bc77c27e5a5ea380ebf091c06f34d8561b695e5a40f", size = 3658864, upload-time = "2026-09-18T13:19:06.503Z" }, +] + +[[package]] +name = "psycopg-pool" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pydantic" +version = "2.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/d5/b4867fa7c7f04c34997d7d8aad744f64d210bb99d67cf18bf3fb1eea1aed/pydantic-2.14.2.tar.gz", hash = "sha256:fb7b2db6cefe8414dd612edeede49c4c433acb0432b0f92b95024e45f66c505d", size = 911596, upload-time = "2026-10-29T16:37:12.476Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/01/c71a71258b58659c72504e5fd65b451f6e36e34dc931d0b237ded213c814/pydantic-2.14.2-py3-none-any.whl", hash = "sha256:7efbf7507b937931cd32a5755acb00ea1cf5ccb1d4e8067a5850d5069e382118", size = 490403, upload-time = "2026-10-29T16:37:10.181Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.50.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/8e/23ec1021834e35041fbb2f10946141e6acdfd39379c45cc4d65a4e8a1d98/pydantic_core-2.50.2.tar.gz", hash = "sha256:1b0e8dc3932cf4e9ed5727570325cacc716b9409ffc794a221c0ccc72e8eb775", size = 518711, upload-time = "2026-10-29T16:34:57.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/65/6609a66cda909d414c58438f9bd70b1e947fe12c25f6bd40c93121afd7ef/pydantic_core-2.50.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e3d824b6152e1c0fd58dc6266f2de470eb7b2286135294f5f7cd26a0f0693738", size = 2220341, upload-time = "2026-10-29T16:32:10.193Z" }, + { url = "https://files.pythonhosted.org/packages/f9/0b/92463ae7be78d73885487f72c4599902c2cc190378bccaad5718b7495250/pydantic_core-2.50.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:596921489390a002d6a5e303384a0f1f98384f8bb550fbf38a1aafa6093da0f9", size = 2093701, upload-time = "2026-10-29T16:32:11.502Z" }, + { url = "https://files.pythonhosted.org/packages/62/d3/24147a1aa22f7eade546172750e1110fd7b2793998057809800d3ebeae4e/pydantic_core-2.50.2-cp312-cp312-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:36d8b9725684ca3bf658b90a101d335fc8cae405bb22196b7b5ef41344913756", size = 2253622, upload-time = "2026-10-29T16:32:13.051Z" }, + { url = "https://files.pythonhosted.org/packages/f6/50/2628ca3c83a0d9a6fda3305fe59b1b28e40c0221e8f234cd1ef86fb64745/pydantic_core-2.50.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:43a8fd56f3c681f3cda2d5c842ad28e88241df742dfcbe8fd1dd437a3056e1f6", size = 2114101, upload-time = "2026-10-29T16:32:14.702Z" }, + { url = "https://files.pythonhosted.org/packages/21/c8/cc6d7304a93f91f5e06260ccad71cc7d882489663a37dec009532379fc46/pydantic_core-2.50.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:6f5a432c6c1ef18a1a02a04b3616b8d3f2f88b13faa46caae1ac6559ab202234", size = 2199541, upload-time = "2026-10-29T16:32:16.153Z" }, + { url = "https://files.pythonhosted.org/packages/65/d9/18f44ac5696b3e2b2384cb89c9fbe6ed69860f528e232b9fb46cc5b45a17/pydantic_core-2.50.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5d3cd8be71c4da3c9b42cc92f17e5e3036bdca98bf18555a0e072c3fe87abbb4", size = 2368597, upload-time = "2026-10-29T16:32:17.544Z" }, + { url = "https://files.pythonhosted.org/packages/1d/22/5f2a48ef819a7e1b74fb003a85d27576ad2f9ddf3901b88998ac63ce1e00/pydantic_core-2.50.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a438cb8a3f87f2c49e82e43fb19511d4d5043f53e0787dc4df937ccb904373a", size = 2451701, upload-time = "2026-10-29T16:32:18.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/93/2a6556bbdb8e918175af9d849a5d0b367b5644e41939d040f3a7215b1339/pydantic_core-2.50.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9667637665acba166a7dbbc786f14c50512d3a01f7199e0e1a13b929e92d7c7a", size = 2242782, upload-time = "2026-10-29T16:32:20.381Z" }, + { url = "https://files.pythonhosted.org/packages/90/ed/dd9f29a92257f32eb5b99d5120826186ee0632ecc3aaf6c23e2fab89ed6d/pydantic_core-2.50.2-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:b3e681208e2d39fde13fbea449ef2736a85ec1ea2afa41e9193a8c536e71dafe", size = 2253706, upload-time = "2026-10-29T16:32:22.293Z" }, + { url = "https://files.pythonhosted.org/packages/11/19/1ccf3bf38eb4ed97d8aeefc558458834ae0ad53fdbb1edf61af7228ea500/pydantic_core-2.50.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6eb6ff07964f39e33c6f57fefe939514e76b8c0d4a694e5663e2e04e80cc1798", size = 2288603, upload-time = "2026-10-29T16:32:23.465Z" }, + { url = "https://files.pythonhosted.org/packages/ea/cd/bae2320fc89a65bcd87bf66ec83aac51b3b43b970a2edece15feb8af64dd/pydantic_core-2.50.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:4eb54c25e3463d73e1e4ed439ffc45d1d7e590df40bc7bb06f11a906afd996c3", size = 2476597, upload-time = "2026-10-29T16:32:24.824Z" }, + { url = "https://files.pythonhosted.org/packages/db/35/1bdb8d52c8e562e95c5e6ebb5784792af568ac6a81470dd546d95f392b60/pydantic_core-2.50.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:91238117cd296352f54f24117ff5a227bdea428865dba1244aa0cdd1424c0bc2", size = 2478664, upload-time = "2026-10-29T16:32:26.197Z" }, + { url = "https://files.pythonhosted.org/packages/12/72/0179b40ac6a6b0899c61826f2e3e80f88063f6d821e0dd9641927de46700/pydantic_core-2.50.2-cp312-cp312-win32.whl", hash = "sha256:98722314e12a8e4fa346e0afd3fff2098b5732875ff45592b450a7b589b43c4f", size = 2108820, upload-time = "2026-10-29T16:32:27.736Z" }, + { url = "https://files.pythonhosted.org/packages/b4/fb/679144362d7e5162519d9f6631ec982521417de3196351c6c56485f5b29f/pydantic_core-2.50.2-cp312-cp312-win_amd64.whl", hash = "sha256:b381574614245f18fdb2d6d1aa8f149e225869c47b5ee7f81812de8977817314", size = 2252238, upload-time = "2026-10-29T16:32:29.572Z" }, + { url = "https://files.pythonhosted.org/packages/0f/68/e3ccc885c3e2086ff84d7ca496f45b974a9f0a49ce2da8973aaa3a369a4b/pydantic_core-2.50.2-cp312-cp312-win_arm64.whl", hash = "sha256:f07c965a8aa6d62145f122641f292129984f39a3e3697585b19298a1488287db", size = 2162565, upload-time = "2026-10-29T16:32:30.853Z" }, + { url = "https://files.pythonhosted.org/packages/64/4d/23560b11366b547e3e68a3b8f9c02557cee707effa54fd39dbb3c4fff1fa/pydantic_core-2.50.2-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:d18d56c1ec63b304f4bb3c7db7e0d311e61cca1b1c944590a4b13ac39b2ca92b", size = 2204868, upload-time = "2026-10-29T16:34:34.745Z" }, + { url = "https://files.pythonhosted.org/packages/06/58/b9ddfaad189143d6568627166a730b6624ca90bd9a9a4af4854df88d444b/pydantic_core-2.50.2-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:41ed8abeffda6a880207fdc0cd1343a448cec3cc6b38b53f730c0745f745292d", size = 2049162, upload-time = "2026-10-29T16:34:36.186Z" }, + { url = "https://files.pythonhosted.org/packages/8b/20/99d259bf7835b2f13c6d0b3eb7bcf669fb6349986736fa7a8308a9e620b0/pydantic_core-2.50.2-graalpy312-graalpy250_312_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b7abcfaf66c1dc14c5ed928c349614f223c972a8b021b4be9548dbea6c817371", size = 2112135, upload-time = "2026-10-29T16:34:37.777Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/652c8fe599abacafd852479059d209e608596ad3de82f1bff22e9f0f02fe/pydantic_core-2.50.2-graalpy312-graalpy250_312_native-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e7460abce637f90e37f8c443559fbd687c38884d05cb69ef265e634945f4324", size = 2270339, upload-time = "2026-10-29T16:34:39.039Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.16.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/31/21d21e0e38a8f5f7a7f0cde22820117b6d17d6bc36f348129527f915d7f4/pydantic_settings-2.16.2.tar.gz", hash = "sha256:3dc08ffabed2bbd5dc2dd55ead7b78b4e4c5cd776aecd04310db5f1224caaf46", size = 309225, upload-time = "2026-10-28T11:14:28.066Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/88/4131c51ff87b7cd7f82a1194d0c0a2b7ca68c72d9da04c3b7e69ab91a79f/pydantic_settings-2.16.2-py3-none-any.whl", hash = "sha256:8d523e5c129b097686bde019bd6109bff53f4c8c3f63f1ddf9933ffa42e5ab4f", size = 81994, upload-time = "2026-10-28T11:14:26.716Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/42/bb4641652e58dedeac2ec51abc1f8cae4fee4ec63899a0999016229a0f2a/pytest-9.2.0.tar.gz", hash = "sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b", size = 1797555, upload-time = "2026-10-27T21:30:20.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/26/2fbeedb218a787a5eea551c7532cac4e009f83d689dd2faa0d0353473f86/python_dotenv-1.2.4.tar.gz", hash = "sha256:f0d53e69935a851c0dcc78f3ab7aaccd8cabef0b92382b576b824212902873c0", size = 60824, upload-time = "2026-10-01T05:36:10Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/d1/38f3a3405989a89ac18390803e70c6ad7c7760da4f9b83cbeca0c44a0c72/python_dotenv-1.2.4-py3-none-any.whl", hash = "sha256:42269a8a5b3fd54ffa6f3d84b18abed50064717576b4ecf03dc4a55d8aa04fdc", size = 23266, upload-time = "2026-10-01T05:36:08.633Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "redis" +version = "8.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa", size = 3490, upload-time = "2021-05-12T16:37:52.536Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz", hash = "sha256:4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12", size = 63948, upload-time = "2026-10-04T16:32:36.469Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/34/a828586ea3329fbb50895b50e9cf98ca3d924a9f41a0b26d443bff1b3794/rpds_py-2026.9.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:50906f5aea24b5a865cbd0a589698288631d9f3a54c3a937c83aefa95a0d14af", size = 346822, upload-time = "2026-10-04T16:29:16.258Z" }, + { url = "https://files.pythonhosted.org/packages/90/81/ac6a0d064982251856ce009c9e1dd51a34110b3c055aa7d1aad18b4899a3/rpds_py-2026.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e21c1429e205828ea886a2293a4a2c8e01f4c25d9893ca330e97a6cf73f52e7b", size = 341380, upload-time = "2026-10-04T16:29:17.706Z" }, + { url = "https://files.pythonhosted.org/packages/42/ff/bf7d54f362748fd6a49277b9d6531c394074110fedf50ad791ec59133fbb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693b2728bbcc48d09a981a356954b0c47c53ff25b545856f28a889ea619f69a", size = 373053, upload-time = "2026-10-04T16:29:19.063Z" }, + { url = "https://files.pythonhosted.org/packages/60/de/74b0ccdbbd28687b9b5fdb34c1cabed88352182facaced9d6f5486b8b9ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8601470267d938bcb7f3ab1a336100af51a4fd5b6ed030ef52461bb3ef5e7e07", size = 377742, upload-time = "2026-10-04T16:29:20.543Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6d/b979775a3057b2a5c26d75ecbff60a20a24ef081db6dd76836fca3f20247/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3890a6aa36e6baa53d5258a2a25d3ef8b37ad165a6ab27a892d7c3e3a432cd69", size = 487289, upload-time = "2026-10-04T16:29:21.958Z" }, + { url = "https://files.pythonhosted.org/packages/1d/5d/7c34734ce3ece943d9d6ee0122a74a21bbc75b47acb5b7053c83f6efb1ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b5b393eda5ea42cca1c1a6665f2a4882b4fd5d1777e41ce0545a107fb008c9d", size = 390710, upload-time = "2026-10-04T16:29:23.378Z" }, + { url = "https://files.pythonhosted.org/packages/1d/6d/b26eb1e75395925b3a142ed351cbed2ec8212f5c9d937aee3df32c701baa/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab", size = 371261, upload-time = "2026-10-04T16:29:25.1Z" }, + { url = "https://files.pythonhosted.org/packages/7c/98/b2fdfe10301a9e27af21e634337fbba7baac0bc17fe44619a17c26bba2cb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d9edf30457d74eebfd76b045535e36f1cd89062566a128a0db2145ca042d787e", size = 376075, upload-time = "2026-10-04T16:29:26.748Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b1/c4b8d954e49c3c69cf8063c0c9e0919c99f3d2cecc46311606a1cbe835dc/rpds_py-2026.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815d26356930846a40c7bc1366e7b1b0320ab8a063e66c11298a208bed0fd237", size = 399358, upload-time = "2026-10-04T16:29:28.282Z" }, + { url = "https://files.pythonhosted.org/packages/b3/59/9559a7293c97dff0cbb293efffbd36644103883ecc27741af8ef90c84ca8/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b5a6f40f0a1486b4b36c888123afc67acdbd9f33235927acf5ff295429a0ba3", size = 550267, upload-time = "2026-10-04T16:29:29.851Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9d/6dc60e49511de4c8b00e74f9c1d43aed4d27f712cdb1ade92f2c199cbf64/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b5b8b0753718d258fd454283fbd57e14545d3b40583fa672e27cb4f987626bcc", size = 614018, upload-time = "2026-10-04T16:29:31.54Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bc/92a4ecf27301b886232a413360f6b348a81d55ffef654f2ebda5970ebc3d/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:46d80bc76b51a6c24f9944368c28d38b8bcbcea1da4f2f8d3ebc31a67e8c6ec6", size = 578509, upload-time = "2026-10-04T16:29:33.01Z" }, + { url = "https://files.pythonhosted.org/packages/08/53/efb97f2e6589b7ab8394385a73870e0b2ab2a195281c117b21fd6d7d4855/rpds_py-2026.9.1-cp312-cp312-win32.whl", hash = "sha256:befc2d6a953e563f8a7bfd87a42c22ebf8a3e980dcb7b6a4d17b70b0e914e8a3", size = 205401, upload-time = "2026-10-04T16:29:34.451Z" }, + { url = "https://files.pythonhosted.org/packages/1c/84/1700cc748d0eaa747486e4e82882d2575224e6e9a3148df583058850c629/rpds_py-2026.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:5ce8943f79c2210f7abcc28e86367b03b28d95027fd01c46d2472373ae70c86f", size = 222923, upload-time = "2026-10-04T16:29:35.709Z" }, + { url = "https://files.pythonhosted.org/packages/c3/89/0302215373c2f8b4408b0fdfee955368cc378a98ff59508d47a7e0dc2dbf/rpds_py-2026.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:501909f2e4a1e2dee528ef766fe3c469060ebc17e54a8383d404ba07a81a6f02", size = 218415, upload-time = "2026-10-04T16:29:37.263Z" }, + { url = "https://files.pythonhosted.org/packages/a3/90/a9ba81408369d34c1aca73319c15adc91ed45c1d21e9e8ee4832ffb7fe0b/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdc537c8633d7fd92a82e2e0d2ab74320a3f63d5e59fb9cf08711e08fe151c4", size = 381109, upload-time = "2026-10-04T16:32:08.994Z" }, + { url = "https://files.pythonhosted.org/packages/b3/44/5192a0bed94cec86bd2a5e1cc5a1bb8f7eec3aec907eacd2deb13e87e326/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:10e208f2425d973938afcd56e28a7c4be32e27b6a60b5d381f49fb9d8acf9759", size = 388222, upload-time = "2026-10-04T16:32:11.321Z" }, + { url = "https://files.pythonhosted.org/packages/92/cd/5356549711448f18b52f7a11ffbe90f1228774996fbf3c6cd1b18d22abaf/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c0dbbcc19735fe5f8b0a54c07659d154a9e69f47e15d0a6ab7299215daf62cb", size = 500384, upload-time = "2026-10-04T16:32:13.478Z" }, + { url = "https://files.pythonhosted.org/packages/7a/88/ddda9d28adfe33119c75b2e733d4ba7e326f41d7e1b1093951771768ca48/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684fd492fff4fead00587544e059be2bbcb6f93454f21fa2a91b66fc7508be82", size = 401661, upload-time = "2026-10-04T16:32:15.653Z" }, + { url = "https://files.pythonhosted.org/packages/32/c3/bb59ba16a6b4a57995d15b8c04c00f28df3b938c4dcdde48fbe0f73edb0c/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1028417bb44037eb3069c1009bd7b7277212876cda22fbe565b0bca9fab6d2c", size = 380748, upload-time = "2026-10-04T16:32:17.727Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b8/0580faa1c5a32ddc160130dd7ae54d2d007272a89a40a6b5db3e50f9f585/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:492e5e428cbe126221611f47e068f01660352feec4ad18bc0f5ea9b2ae88fb14", size = 385731, upload-time = "2026-10-04T16:32:19.855Z" }, + { url = "https://files.pythonhosted.org/packages/bf/70/f73564642bbe3322c7eeef2c2f258cf040b71c41a430b532e59d04a1b838/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88b5268892fde430d5531f95bc560b6efbbd67c929662c586afd729a96e7461c", size = 412131, upload-time = "2026-10-04T16:32:22.285Z" }, + { url = "https://files.pythonhosted.org/packages/8c/5d/17fff2e1f8f68721a2afb5cb48f7e442c751bb6b4883157c5abaf618cd08/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:01445c8d194aa032a08e944f16567672da1c62dbdbefd8b6d0693032e290cf68", size = 558193, upload-time = "2026-10-04T16:32:24.4Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d0/869ab6fb08531f97aae4a780b4d61b4190d8aef48fe054ee28bcf7114466/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_i686.whl", hash = "sha256:eef6a03b0b6d08d0835ccfa8ec8d1bc70525e3801387567137b50c557695e6da", size = 622448, upload-time = "2026-10-04T16:32:26.844Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a5/9672e532fe02cd3b92c78cfd8743abca939d96853006c63835009a1ecb6f/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6b9bf3135b4ad5981df9a73d71a35272d650a2985ae9c2746357b24d59de2448", size = 591126, upload-time = "2026-10-04T16:32:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/80/ef/3a9f8e4279c7920af561deed4cb7ebebed2794a8f608cf404d5eef14a105/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_10_12_x86_64.whl", hash = "sha256:56c6952a9b15047466d0c2347c446a761d4527f89976156341e68f0ce5cc08b0", size = 357108, upload-time = "2026-10-04T16:32:31.641Z" }, + { url = "https://files.pythonhosted.org/packages/b7/55/4b2fa381a583760aea5c92841e4e928a358e1c6511b129219e9c2826a226/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:b242c27c8f836305a4a72df9cdd564386ac57b807bd252a063223331c9316b37", size = 346877, upload-time = "2026-10-04T16:32:34.061Z" }, +] + +[[package]] +name = "ruff" +version = "0.17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/73/81f140c4edd9cc323de5478e6bbd3bbcfae9a82692df1a258de8cec6a6d2/ruff-0.17.3.tar.gz", hash = "sha256:a57049c8addb3662073859bb55a7ad35aa54f8a7c9ed268911d1333c5aadaaa9", size = 5339444, upload-time = "2026-10-27T00:34:05.914Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/8c/083365310c4e79c842f25bf5a842facbd9b0c0ec336f96dffa5074fe723d/ruff-0.17.3-py3-none-linux_armv6l.whl", hash = "sha256:ab86b84cd9d05e56d329742c0cabdcbe2d1d56c98f4a04aab9b5b66196febfb4", size = 10216522, upload-time = "2026-10-27T00:33:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/72/26/4b2edf31cfebe9605d8f6ed8f1d6f376d3a53d4c9a123e62661c64c440f0/ruff-0.17.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e407b4a6852853a9a2cdce96963e94ad9775578bb44e7561a0f2829f3cc04b11", size = 10217786, upload-time = "2026-10-27T00:33:33.949Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7e/537aba4b7f54397a374d2f6a6fc93e9c8015e18553f31a4a7b7dbced48b5/ruff-0.17.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b75cc113b3aca388933d8296b667b1ac401a80317ea3074c9b0a66541199da29", size = 9440362, upload-time = "2026-10-27T00:33:35.784Z" }, + { url = "https://files.pythonhosted.org/packages/3d/66/283e39734f66c8a5ebaa93c22eb7b7515779a0b1d5a5c0ad20e05c0edb41/ruff-0.17.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab5fa7ed5ce18554ea69d850ad162cbaa0c85fc47c08d83a569f806681c9a6ad", size = 10094281, upload-time = "2026-10-27T00:33:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/21/4c/1ffc768497bd025a32c72152771e40c0b99bddc5cc6e4f50c7949a395346/ruff-0.17.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c673c170e27834d1262e3a9c1e97bdc9b9c3b180c63d89173ce69747232474c1", size = 10022543, upload-time = "2026-10-27T00:33:40.181Z" }, + { url = "https://files.pythonhosted.org/packages/73/c3/f90db547301080fcb82c92abf914c8bd57b89911d9b65a6c4cc6b52d0635/ruff-0.17.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a754c0341bc9ca151b7b335988b3263921c08859f84ca219b70cee825571b6f", size = 10683921, upload-time = "2026-10-27T00:33:42.182Z" }, + { url = "https://files.pythonhosted.org/packages/e5/98/92d4a8a26c6c96cf1a5b34dd59036d8a71e52d650aab065c58867e964e63/ruff-0.17.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4590b9d86c0b1543ebb09094254e191cd43fe8edaf330baa035aab03da251532", size = 11373408, upload-time = "2026-10-27T00:33:44.19Z" }, + { url = "https://files.pythonhosted.org/packages/43/02/1f24d2e3064c9753926deeb901afa57777b37ee46799b09ddd7aee6e5145/ruff-0.17.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bbde18beccd98c29a227c63c99022e3443b99258e08326df6425dfe0443705ef", size = 10977469, upload-time = "2026-10-27T00:33:46.179Z" }, + { url = "https://files.pythonhosted.org/packages/f0/20/fecedea2209f574ab92033b344e01a7f0c2dc76c6d6d19a14c9f8141173f/ruff-0.17.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fa4b0f931cdbcb8a52044f370a3eb429fb42eb88807b0e800fcaa98dd42b24e", size = 10577491, upload-time = "2026-10-27T00:33:48.233Z" }, + { url = "https://files.pythonhosted.org/packages/33/36/b93fdf0bc6f84f0cd487aadfb0354deb40013669ea35ac3d26855de1e959/ruff-0.17.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:39405f341deec934b4289634324ad695f72df2970f9121582872eaf5b722f0f5", size = 10608748, upload-time = "2026-10-27T00:33:50.284Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b7/7d3f5f14145daead1efb0b9a8289ed1daa100f419d60774375951723ce00/ruff-0.17.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:68074094f55ca3d29ab19491f640282f7a84061c4c7977aa5094b968ca457e9a", size = 10075057, upload-time = "2026-10-27T00:33:52.074Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ed/93ff7a5909ee56e2bd4b982deae9e0efb2724417874a90c9e41ad66c6dd7/ruff-0.17.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:27bb571ac78fef178fe5118d2827df95c2aae74a48fff2b15f6acd03af5ff5bb", size = 10022431, upload-time = "2026-10-27T00:33:54.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/07/a83b9eeec8c1c3c0e4eda15f06c3a91001bfd2687ea2f07b750719bf9712/ruff-0.17.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:029c37cbd3de48ca37aa12117bead2d065f15da17d66bf84e0d4274013705db2", size = 10394903, upload-time = "2026-10-27T00:33:56.158Z" }, + { url = "https://files.pythonhosted.org/packages/af/b2/05c39857c2526fcbebe46ad93a8dc97424656dfef14d3ef933abe1ffd331/ruff-0.17.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:692fdb46d0fd9c71e380947dc5fc72f2c3f881073bc1f739e4579376231affbe", size = 10799780, upload-time = "2026-10-27T00:33:58.204Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d9/47171a16739eb7a353936b31c20cf54340531e68c9af7d72fd7a784e235d/ruff-0.17.3-py3-none-win32.whl", hash = "sha256:46539cb031b01fd29ae0a02142483c46640b233ee12bb1c1b690bffa1555d5ca", size = 9908481, upload-time = "2026-10-27T00:34:00.087Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a0/f08c1631bfd81faa6a90eaf70c37c786c11bed2046302b56bb506c25e1a0/ruff-0.17.3-py3-none-win_amd64.whl", hash = "sha256:56b8ffb3f3b3cbccb8b76b71b38261b7ed982ef5d20f5b447b1b61a7f4949b24", size = 10341333, upload-time = "2026-10-27T00:34:02.099Z" }, + { url = "https://files.pythonhosted.org/packages/99/d5/4ad4ed31413c301430918e70d3d38ae6f097af0766cb9ab8c9d502b2a4d6/ruff-0.17.3-py3-none-win_arm64.whl", hash = "sha256:81fce74bcd6be0e9da7c67c7553fd0f1f12964a2052cd728ef917b937a5ca1d9", size = 9854332, upload-time = "2026-10-27T00:34:03.847Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "starlette" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/bb/3d8757f21e9a53d9035c9eb039a777047139a16bfdbedfc7a58241c3d914/starlette-1.10.0.tar.gz", hash = "sha256:b8399098d4c1e3522734f7235dc79b1fc967f96cf139c81e0fa30479e0685f9b", size = 2734356, upload-time = "2026-10-24T15:04:12.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/2e/947cfe02def9371c07cd5bf5b6b61c829e9aecb9904db02a66cfafbc1eec/starlette-1.10.0-py3-none-any.whl", hash = "sha256:478c0ab3b4ced6e82719849c118fa406f0278a3499c1a898cd15cd31bdc38ce4", size = 80571, upload-time = "2026-10-24T15:04:10.371Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/29/6a1391d31cd00ff9e927215a224a880e1e84e826b64ba80a499e51c074d8/tzdata-2026.6.tar.gz", hash = "sha256:85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d", size = 201017, upload-time = "2026-10-28T09:57:09.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/66/9a06c2227fedfd4919f0eaaa8b94377d9730882a7e4f95823c9a331ab5e1/tzdata-2026.6-py2.py3-none-any.whl", hash = "sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec", size = 348210, upload-time = "2026-10-28T09:57:07.276Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283, upload-time = "2026-09-25T06:52:37.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427, upload-time = "2026-09-25T06:52:35.829Z" }, +] -- 2.54.0 From ca9570f49524a95d2af846a0e6ae3edb201ae71a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 17:59:42 +0300 Subject: [PATCH 02/11] test: cover reconciliation failure windows and bounded upload memory --- REVIEW_NOTES.md | 23 ++++++ compose.yaml | 2 +- research/README.md | 53 +++++++++++++ research/run.py | 38 +++++++++ research/upload_memory.py | 161 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 REVIEW_NOTES.md create mode 100644 research/README.md create mode 100644 research/upload_memory.py diff --git a/REVIEW_NOTES.md b/REVIEW_NOTES.md new file mode 100644 index 00000000..3ee48280 --- /dev/null +++ b/REVIEW_NOTES.md @@ -0,0 +1,23 @@ +# Учтённые замечания преподавателей + +Чужие PR использованы для изучения требований и типовых ошибок до реализации, не как источник кода или результатов. Отсутствие комментария не трактуется как доказательство правильности решения. Исходные комментарии доступны по ссылкам ниже; обзор собран 2026-11-06. + +| Источник | Риск | Наше решение / проверка | +|---|---|---| +| [PR1](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/1#issuecomment-238) | Dry-run продолжает частичное удаление; missing/null в JSON принимаются | Каждая мутация только внутри apply; real DB DELETE fault → dry-run → repeat. Strict модели отличают обязательный text и массив attachment_ids | +| [PR4](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/4#issuecomment-506), [PR7](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/7#issuecomment-571) | Повреждённые изображения, неверные byte/pixel лимиты, root credentials, S3 500/timeouts | Полное verify+decode, проверка размеров до decode, 10 MiB включительно через LB, bounded S3 calls, отдельные app/admin keys | +| [PR8](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/8#issuecomment-572) | PNG palette 257 и недопустимый filter | Проверяет Pillow 12.3; отдельные воспроизведения | +| [PR12](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/12#issuecomment-630) | Удаление S3 при неопределённом COMMIT | Никакого немедленного delete в catch; durable upload_intent, сверка после grace | +| [PR13](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/13#issuecomment-735) | Пробельный filename, тест ACL без Idempotency-Key | Filename проверяется до I/O; тесты авторизации предъявляют действительный ключ | +| [PR24](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/24#issuecomment-990), [PR25](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/25#issuecomment-1007), [PR49](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/49#issuecomment-1507) | JPEG без scan/половина JPEG с поддельным EOI; OOM LB | Pillow дополняется строгим djpeg с timeout; 10 конкурентных upload через HAProxy, memory.events/peak API и LB | +| [PR27](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/27#issuecomment-1442), [PR15](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/15#issuecomment-1444) | Анонимный filer GET обходит приватный S3 | Storage backend в отдельной internal network; API видит только S3 gateway | +| [PR22](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/22#issuecomment-1446) | Filename C0/C1/DEL, неверные 400/404/415, потеря порядка вложений | Unicode category Cc; контрактные коды; position в связях и сохранённый JSON ответа | +| [PR21](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/21#issuecomment-1448) | Pending никогда не очищается | Старые intent удаляются с их остаточными объектами после grace; активная row lock защищает живой PUT | +| [PR47](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/47#issuecomment-1508) | Pixel bomb неверный код; отсутствующий filename 422 | Dimension check до декодирования, DecompressionBombError→413; собственная query validation→400 | +| [PR56](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/56#issuecomment-1550) | JSON null и повреждённые PNG/JPEG | Strict model + generated corrupt fixtures | +| [PR60](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/60#issuecomment-1551) | Проверка только заголовка крупных файлов, NUL text, paused PostgreSQL | Полный decode во всём допустимом диапазоне; bytea для lossless UTF-8; client-side deadline connection | +| [PR59](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/59#issuecomment-1552), [PR64](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/64#issuecomment-1576), [PR63](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/63#issuecomment-1625) | Серверный SQL timeout не срабатывает при pause БД | Клиентский I/O deadline закрывает неопределённое соединение; реальные pause/resume проверки | +| [PR28](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/28#issuecomment-1555) | Username с завершающим LF | Полный re.fullmatch до INSERT | +| [PR58](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/58#issuecomment-1573) | Игнорирование variant/status и воскресающая сессия из кэша | Проверки ready/variant перед GET; каждая сессия проверяется в PostgreSQL | + +Дополнительно при сверке учитываются возраст объекта независимо от возраста intent, повторный HEAD перед мутацией вместо старого снимка LIST, сохранение используемых и свежих вложений и запрет сортировать attachment_ids при сравнении идемпотентного запроса. Детальные результаты находятся в REPORT.md и evidence. diff --git a/compose.yaml b/compose.yaml index d4baf01c..cd46a693 100644 --- a/compose.yaml +++ b/compose.yaml @@ -67,7 +67,7 @@ services: s3-bootstrap: {condition: service_completed_successfully} storage: image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e - command: [server, -dir=/data, -ip=storage, -ip.bind=0.0.0.0, -filer, -volume.max=4, -volume.preStopSeconds=0, -master.volumeSizeLimitMB=64] + command: [server, -dir=/data, -ip=storage, -ip.bind=0.0.0.0, -filer, -volume.max=32, -volume.preStopSeconds=0, -master.volumeSizeLimitMB=64] volumes: ["s3_data:/data", "./infra/filer.toml:/etc/seaweedfs/filer.toml:ro"] networks: [storage_backend] cpus: 1 diff --git a/research/README.md b/research/README.md new file mode 100644 index 00000000..05c9a4fa --- /dev/null +++ b/research/README.md @@ -0,0 +1,53 @@ +# Трек 2: согласованность PostgreSQL и S3 + +## Гипотеза + +Durable intent до S3 PUT, row lock на время PUT/финализации и консервативный reconciler с grace + актуальным HEAD позволяют обнаружить и безопасно убрать остатки незавершённых upload, сохранив подтверждённые и свежие данные. Один COMMIT не делает распределённую транзакцию с S3, поэтому результат проверяется fault-injection. + +## Протокол upload + +1. Поток HTTP складывается в ограниченный spool: 256 KiB в RAM, затем временный файл на ограниченном tmpfs; максимум 10 MiB. +2. После полного приёма выполняется полная проверка PNG/JPEG. Пиксельный предел применяется до decode. Для JPEG строгий djpeg отвергает предупреждения о повреждении, которые Pillow иногда допускает. +3. Создаётся и фиксируется upload_intent с уникальным server key. +4. Новая транзакция блокирует intent, выполняет S3 PUT и создаёт ready attachment, удаляет intent, COMMIT. +5. Только после успешной фиксации возвращается 201. При неопределённом исходе COMMIT объект немедленно не удаляется. + +## Reconciler + +По умолчанию dry-run. `--apply` явно разрешает изменения; default grace 3600 секунд. Обход SQL keyset, S3 paginator, page-size 1..1000. Метрики выводятся без credentials и пользовательских текстов. + +- Старые intent: `FOR UPDATE SKIP LOCKED` пропускает живой upload. Недавний LastModified объекта защищает его даже при старом intent. При отсутствии ready metadata apply удаляет остаточный объект, затем intent. +- Старые S3 objects в server prefix original/: повторно проверяются PostgreSQL и HEAD. Свежие, учтённые и live intent объекты не удаляются. Чужие prefix вне политики удаления. +- Ready metadata без оригинала: актуальный HEAD под row lock; после grace apply меняет status на failed/storage_missing. Метаданные и связи сообщений сохраняются. Reconciler не может восстановить потерянные bytes без backup. +- Полноценные ready вложения, даже пока не прикреплённые к сообщению, не удаляются. +- Audit пишется только в apply. После частично завершённого удаления повторный dry-run не продолжает очистку. + +## Воспроизведение + +Из корня репозитория после `make up`: + +```sh +# Исследование получает admin keys только в отдельном profile tools. +# Создаёт отдельные БД и bucket, очищает только их в finally. +docker compose run --rm --no-deps research > evidence/reconcile-repeat-raw.txt 2>&1 +``` + +В строке `RESULT ` находится JSON с исходными счётчиками, проверками, хешами кода и результатами времени. Основная messenger БД и bucket не очищаются. + +Ошибки: настоящий trigger PostgreSQL после PUT; реальная ошибка подключения S3; успешный COMMIT с искусственно потерянным подтверждением; ошибка DELETE PostgreSQL после удаления bytes; ошибка S3 delete. Проверяются повтор, dry-run после сбоя, старый intent со свежим объектом, live row lock, missing used metadata и сохранность подтверждённого оригинала. + +Стоимость обхода: 10/100/1000 специально добавленных intact attachment, по три dry-run на точку, page_size=100. Дополнительно остаются несколько fault fixtures; точное число просмотренных metadata/objects записывается в каждой строке. Сервисы прогреты, однопоточный обход; временные БД/bucket используются на том же PostgreSQL/S3 без внешней нагрузки. Это цена выбранной сверки, не HTTP throughput и не производственный SLA. + +## Память десяти конкурентных upload + +```sh +uv run --locked python research/upload_memory.py --restart-api --output evidence/upload-memory-repeat.json +``` + +Генератор использует настоящие PNG 512×512 и 1700×1700. На каждой точке десять клиентов стартуют через barrier и идут через LB. По одному upload slot на API-реплику, занятые места возвращают контролируемый 429 с Retry-After. Сохраняются status/latency и cgroup current/peak/memory.events, VmHWM API и LB. Проверяются readiness и отсутствие OOM. + +Kernel peaks накопительные с момента старта; before/after показаны честно, без приписывания всей памяти image buffer. Регистрация и импорт Argon2 тоже влияют на high-water. Это проверка ограниченной памяти/backpressure, а не доказательство роста максимальной пропускной способности. + +## Ограничения + +Нет атомарной транзакции через обе системы. Консервативная очистка допускает временные остатки до grace и явного apply. Сбой самого S3/БД не интерпретируется как отсутствие объекта. Ручной администратор с правом перезаписи может изменить объект после HEAD; инструмент рассчитан на уникальные immutable server keys штатного API. Один host/LB/PostgreSQL/Seaweed backend остаются SPOF. Нет репликации и внешнего backup в этом эксперименте. diff --git a/research/run.py b/research/run.py index 6c272ec3..b8081ae4 100644 --- a/research/run.py +++ b/research/run.py @@ -247,6 +247,44 @@ def main(): ) record("failed_cleanup_recovers", recovery_apply["errors"] == 0) + # Failure after deleting bytes but before deleting the durable intent. + delete_id, delete_key = str(uuid4()), "original/delete-ack-fault-" + suffix + with store.connection() as conn: + conn.execute( + "INSERT INTO upload_intents VALUES (%s,%s,now()-interval '1 day')", + (delete_id, delete_key), + ) + storage.put(delete_key, BytesIO(data), len(data), "image/png") + time.sleep(2.2) + with store.connection() as conn: + conn.execute( + "CREATE FUNCTION reject_intent_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected intent DELETE failure'; END $$" + ) + conn.execute( + "CREATE TRIGGER research_reject_delete BEFORE DELETE ON upload_intents FOR EACH ROW EXECUTE FUNCTION reject_intent_delete()" + ) + try: + reconcile(store, storage, apply=True, grace_seconds=2) + raise AssertionError("DELETE fault did not fire") + except psycopg.errors.RaiseException: + record("real_db_delete_failure_after_s3_delete", not storage.exists(delete_key)) + finally: + with store.connection() as conn: + conn.execute("DROP TRIGGER research_reject_delete ON upload_intents") + conn.execute("DROP FUNCTION reject_intent_delete()") + with store.connection() as conn: + before_count = conn.execute("SELECT count(*) AS n FROM upload_intents").fetchone()["n"] + dry_after_delete = reconcile(store, storage, grace_seconds=2) + with store.connection() as conn: + after_count = conn.execute("SELECT count(*) AS n FROM upload_intents").fetchone()["n"] + record( + "dry_run_does_not_resume_partial_delete", + before_count == after_count and dry_after_delete["deleted_intents"] == 0, + ) + resumed = reconcile(store, storage, apply=True, grace_seconds=2) + record("partial_delete_resumes_only_on_apply", resumed["deleted_intents"] == 1) + result["partial_delete_dry"], result["partial_delete_apply"] = dry_after_delete, resumed + # Measure an increasing population of intact, aged metadata/objects. Rows # share no client names or object paths, and all are in this disposable DB. current = 0 diff --git a/research/upload_memory.py b/research/upload_memory.py new file mode 100644 index 00000000..a1824eff --- /dev/null +++ b/research/upload_memory.py @@ -0,0 +1,161 @@ +"""Ten simultaneous HTTP uploads through LB, with kernel memory high-water readings.""" + +import argparse +import json +import random +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from io import BytesIO +from pathlib import Path + +import httpx +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/upload-memory.json") + parser.add_argument("--base-url", default="http://127.0.0.1:8084") + parser.add_argument("--restart-api", action="store_true") + args = parser.parse_args() + c = Client(args.base_url) + alice, _, _, chat = c.bootstrap() + if args.restart_api: + subprocess.run( + ["docker", "compose", "restart", "api"], check=True, stdout=subprocess.DEVNULL + ) + for _ in range(40): + try: + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + break + except AssertionError: + time.sleep(0.5) + else: + raise AssertionError("API did not recover with old token") + ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], text=True).split() + + api_count = len(ids) + ids += subprocess.check_output(["docker", "compose", "ps", "-q", "lb"], text=True).split() + + def memory(): + result = {} + for container in ids: + lines = subprocess.check_output( + [ + "docker", + "exec", + container, + "cat", + "/sys/fs/cgroup/memory.current", + "/sys/fs/cgroup/memory.peak", + "/sys/fs/cgroup/memory.events", + "/proc/1/status", + ], + text=True, + ).splitlines() + result[container] = { + "current": int(lines[0]), + "peak": int(lines[1]), + "events": { + line.split()[0]: int(line.split()[1]) + for line in lines[2:] + if line.split() + and line.split()[0] + in {"low", "high", "max", "oom", "oom_kill", "oom_group_kill"} + }, + "process_hwm": next(line for line in lines if line.startswith("VmHWM:")), + } + return result + + result = { + "concurrency": 10, + "api_replicas": api_count, + "sampled_api_and_lb_containers": ids, + "api_restarted_before_trial": args.restart_api, + "runs": [], + "limitations": [ + "Kernel cgroup peak and process VmHWM are lifetime peaks; before/after reported, not attributed solely to upload buffers", + "Fast rejection 429 is intentional backpressure, not a successful upload", + "Local two-replica stand; not a throughput benchmark", + ], + } + for side in (512, 1700): + output = BytesIO() + Image.frombytes("RGB", (side, side), random.Random(42).randbytes(side * side * 3)).save( + output, "PNG" + ) + data = output.getvalue() + barrier = threading.Barrier(10) + before = memory() + + def send(_, barrier=barrier, data=data): + barrier.wait() + started = time.perf_counter() + try: + with httpx.Client(timeout=20, trust_env=False) as client: + response = client.post( + args.base_url + + f"/api/v1/chats/{chat['id']}/attachments?filename=memory.png", + content=data, + headers={ + "Authorization": "Bearer " + alice["token"], + "Content-Type": "image/png", + }, + ) + record = { + "status": response.status_code, + "elapsed": time.perf_counter() - started, + } + if response.status_code == 201: + record["attachment_id"] = response.json()["id"] + return record + except httpx.HTTPError as exc: + return {"error": type(exc).__name__, "elapsed": time.perf_counter() - started} + + with ThreadPoolExecutor(max_workers=10) as pool: + responses = list(pool.map(send, range(10))) + after = memory() + c.call("GET", "/health/ready", schema="Health") + for record in responses: + if record.get("status") not in {201, 429}: + raise AssertionError(record) + for container in ids: + state = json.loads( + subprocess.check_output(["docker", "inspect", container], text=True) + )[0]["State"] + if state["OOMKilled"] or not state["Running"]: + raise AssertionError("API OOM or not running") + result["runs"].append( + { + "image_dimensions": [side, side], + "size_bytes": len(data), + "before": before, + "after": after, + "responses": responses, + "ready_after": True, + } + ) + Path(args.output).write_text(json.dumps(result, indent=2) + "\n") + print( + json.dumps( + { + "runs": [ + { + "size_bytes": r["size_bytes"], + "statuses": [x.get("status", x.get("error")) for x in r["responses"]], + } + for r in result["runs"] + ] + } + ) + ) + + +if __name__ == "__main__": + main() -- 2.54.0 From 6ac558ee0895b445cfd3ef180269c1224a5f3476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 18:04:38 +0300 Subject: [PATCH 03/11] docs: explain attachment lifecycle, research and reproducible setup --- .env.example | 38 +++++----- Dockerfile | 2 +- REPORT.md | 149 ++++++++++++++++++++++++++++++++------ SCHEMA.md | 37 ++++++++++ compose.yaml | 2 +- scripts/verify_storage.py | 143 ++++++++++++++++++++++++++++++++++++ 6 files changed, 328 insertions(+), 43 deletions(-) create mode 100644 SCHEMA.md create mode 100644 scripts/verify_storage.py diff --git a/.env.example b/.env.example index d99d26a2..eb30c66b 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,20 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# Copying is optional: make up runs scripts/bootstrap.py and creates .env once. +# Never put credentials in this file. Bootstrap writes ignored ./secrets/ files. +PUBLIC_PORT=8084 API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +API_CPUS=0.5 +API_MEMORY=512m +POOL_MAX_SIZE=8 +SESSION_TTL_SECONDS=3600 +# Optional custom tag; default is mtusi-vasilev:lab4. +API_IMAGE=mtusi-vasilev:lab4 + +# Fixed internal topology in compose.yaml (comments, not .env overrides): +# API binds 0.0.0.0:8080; only LB publishes 127.0.0.1:${PUBLIC_PORT}:8080. +# PostgreSQL host=postgres, database/user=messenger; DATABASE_URL_FILE is mounted. +# CACHE_URL_FILE points to redis://cache:6379/0, public user cards only. +# S3_ENDPOINT=http://s3:8333, S3_BUCKET=messenger. +# S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key +# S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key +# Bootstrap alone receives S3_ADMIN_ACCESS_KEY_FILE/S3_ADMIN_SECRET_KEY_FILE. +# Storage backend/filer/master are on the isolated storage_backend network. diff --git a/Dockerfile b/Dockerfile index 57926cb8..6b171bc6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ COPY pyproject.toml uv.lock ./ RUN uv sync --locked --no-dev --no-install-project --python /usr/local/bin/python FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b -RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-4 && rm -rf /var/lib/apt/lists/* ARG VCS_REF=uncommitted LABEL org.opencontainers.image.revision=$VCS_REF ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app/src PATH=/app/.venv/bin:$PATH diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..0ca71e35 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,142 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Васильев Никита, БВТ2301. Целевая оценка: 5. Трек 2 — согласованность двух хранилищ.** + +Локальное выполнение без публикации PR. Оригинальные README, COURSE, CONTRIBUTING, contracts и публичные tests сохранены. Собственный код ЛР3 перенесён в отдельный skeleton ЛР4; добавлены вложения и исследование согласованности. Использована помощь ИИ при реализации, тестировании и подготовке отчёта. Замечания преподавателей изучены заранее, ссылки и регрессии — в [REVIEW_NOTES.md](REVIEW_NOTES.md). Код и чужие измерения не заимствованы. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose v2, Python 3.12 и uv, make. Для публичных тестов достаточно системного Python 3.9+. API и все хранилища запускаются из этого репозитория; ручная установка PostgreSQL/S3 не нужна. -## Архитектура и решения +```sh +uv sync --locked --python 3.12 +make up +make check validate PYTHON=.venv/bin/python +make test +make student-test +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`make up` создаёт локальные секреты один раз, сохраняет существующие значения, собирает образ, ждёт PostgreSQL, применяет миграции и идемпотентно создаёт приватный bucket. Затем запускаются две API-реплики и HAProxy. На хост опубликован только `127.0.0.1:8084` балансировщика; API и storage напрямую не публикуются. -## Доказательства критериев +Обычная остановка с сохранением данных: -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +```sh +make down +``` -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Намеренное удаление учебных данных выполняется отдельно командой `docker compose down --volumes`; она не нужна для restart и не применяется к основному стенду при проверке сохранности. -## Эксперименты и отказы +Полная проверка рестарта и отказа одной API: -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +```sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +docker compose up -d --wait --no-recreate +``` -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +После восстановления первой реплики failover повторяется для другой. Hook использует TARGET_INSTANCE_ID, читает действительный alias процесса и останавливает соответствующий контейнер. Сопоставление alias/контейнера и timestamps до/после фиксируются в evidence. -## Ограничения и следующие шаги +Переменные: `PUBLIC_PORT=8084`, `API_REPLICAS=2`, `API_CPUS=0.5`, `API_MEMORY=512m`, `POOL_MAX_SIZE=8`, `SESSION_TTL_SECONDS=3600`. DSN, S3 access/secret keys читаются из файлов Compose secrets; app и admin keys разделены. `.env`, secrets и volumes не находятся в Git. `.env.example` содержит поддерживаемые переменные и комментарии с фиксированными внутренними адресами; реальные секреты генерирует scripts/bootstrap.py. Изменение S3 bucket/endpoint требует согласованного изменения Compose и bucket-scoped identity, а не только строки в .env. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Версии и ограничения ресурсов + +- Python 3.12, Python-зависимости закреплены в uv.lock (FastAPI 0.162.0, Pillow 12.3.0, boto3/botocore 1.43.125 и прочие). +- Python base закреплён digest, uv в Docker 0.11.7. Native decoder libjpeg-turbo-progs 1:2.1.5-4 закреплён явно. +- PostgreSQL 17.10-alpine: 1 CPU, 768 MiB, durable WAL settings включены. +- Valkey 8.1.3-alpine: 0.5 CPU, 128 MiB; кэш публичных пользователей, не сессий. +- HAProxy 3.2-alpine закреплён digest: 0.5 CPU, 128 MiB; stream forwarding без полного буферирования тела. +- SeaweedFS 4.15 закреплён digest: backend 1 CPU/512 MiB, S3 gateway 0.5 CPU/256 MiB. +- Каждая API: 0.5 CPU/512 MiB, один worker, non-root UID10001, read-only FS, tmpfs16MiB, drop capabilities, no-new-privileges. + +При первоначальном переносе bytea-миграция несовместима со старым runtime ЛР2: такой переход требует maintenance window. Поочерёдная замена одинаковой версии с уже применённой схемой не требует повторной миграции. + +## Архитектура и источник истины + +```text +Клиент → HAProxy → API × 2 → PostgreSQL (сессии/данные/метаданные/intent) + ↘ Valkey (кэш публичных карточек пользователей) + ↘ S3 gateway → Seaweed backend (bytes) +``` + +Seaweed backend подключён только к отдельной `storage_backend` internal network. API к ней не подключается: filer/master/volume интерфейсы недоступны из общей API-сети. S3 gateway подключён к обеим сетям и предоставляет только S3-интерфейс на общей сети; публичных host ports у него нет. Admin identity присутствует только у gateway и bootstrap/research, API получает bucket-scoped Read/Write/List identity. + +Ни случайный UUID, ни приватный bucket не заменяют ACL: метаданные и content выдаются только после проверки PostgreSQL-сессии и членства в чате. Сессии не кэшируются. Валидный токен не открывает чужой чат. Оригинал скачивается через авторизованный API без redirect. + +Сообщения сохраняют порядок attachment_ids в message_attachments.position. Идемпотентность из ЛР3 атомарно сохраняет сообщение, связи и точный ответ; перестановка вложений при том же ключе считается другим body. Пропущенный attachment_ids и [] эквивалентны. UTF-8 поля сообщений/имён хранятся lossless в bytea, включая NUL, который PostgreSQL text не принимает. + +## Upload и download + +1. Проверяются сессия, чат, filename, MIME. +2. Тело читается частями в spool: первые 256 KiB в RAM, далее временный файл в ограниченном tmpfs. Максимум 10 MiB, таймаут чтения 15 секунд. +3. По одному upload slot на API, лишняя работа получает 429/Retry-After. Размеры изображения проверяются до decode: максимум 40 млн пикселей. +4. Pillow verify и повторный полный load проверяют PNG/JPEG. Для JPEG djpeg -strict дополнительно отклоняет недодекодируемые изображения, которые Pillow может принять с предупреждением; timeout5s, maxmemory192m. +5. После полной валидации фиксируется durable upload_intent. Под его row lock выполняются S3 PUT и транзакция ready metadata + удаление intent. +6. Только успешное завершение COMMIT даёт 201 ready с variants=[] и error_code=null. SHA-256/size/MIME относятся к исходным bytes. + +Filename не используется в object key. Отклоняются slash, backslash, все Unicode Cc (включая C0, C1 и DEL), пустые/пробельные имена и длина вне1..255. MIME сравнивается с фактическим PNG/JPEG. Download получает S3 stream заранее и отдаёт части по64KiB; отсутствие variant у ready возвращает404, failed content409. Ошибка S3 до начала ответа переводится в503. Если сеть оборвётся после отправки HTTP headers, клиент увидит оборванный поток и должен проверить размер/checksum. + +Прерванный HTTP приём закрывает spool и освобождает slot до создания intent/объекта. Неопределённый COMMIT не приводит к немедленному удалению S3: возможно, ready уже зафиксирован. Остатки очищаются сверкой после grace. + +## Согласованность и исследование на 5 + +Подробная гипотеза, окна отказа и команды находятся в [research/README.md](research/README.md). + +Reconciler по умолчанию dry-run. Явный apply может удалять старые orphan objects и завершать очистку abandoned intent; live intent защищён row lock, свежие bytes — независимой проверкой LastModified. Перед мутацией выполняется актуальный HEAD, не используется только старый LIST snapshot. Пропавший оригинал старой ready metadata помечается failed/storage_missing, но запись и связи сообщений сохраняются. Валидные ready вложения не удаляются даже без сообщения. + +```sh +# Обычная сверка основного стенда — без изменений: +docker compose run --rm --no-deps api python -m messenger.reconcile +# Явное исправление после просмотра dry-run: +docker compose run --rm --no-deps api python -m messenger.reconcile --apply --grace-seconds 3600 +# Изолированное исследование в новых временных БД и bucket: +docker compose run --rm --no-deps research +``` + +Проверяются реальные PG INSERT failure после PUT, ошибка подключения S3, потеря подтверждения после успешного COMMIT, PG DELETE failure после удаления S3, повторный dry-run после частичного удаления, S3 delete failure/recovery, fresh orphan, old intent с fresh bytes, live writer lock, missing used metadata. Временные исследовательские ресурсы удаляются в finally, рабочая messenger не очищается. + +Стоимость обхода измеряется на10/100/1000 добавленных intact attachments, по3 dry-run, page_size100. Точные размеры с учётом сохранённых fault fixtures находятся в JSON. Измеряются время и количество просмотренных объектов/метаданных; это SQL+S3 обход, не HTTP throughput. + +### Результат исследовательского прогона + +[reconcile-study.json](evidence/reconcile-study.json) содержит **30 успешных проверок**, девять замеров обхода и hashes runtime. Первый apply удалил остатки двух старых intent и orphan, пометил один missing original failed; повтор не сделал новых изменений. Проверены также ошибка DELETE после удаления bytes и последующий read-only dry-run. Временные БД и bucket удалены. + +| Добавлено intact | Фактически просмотрено metadata | Медиана трёх обходов | +|---:|---:|---:| +| 10 | 13 | 0,018528 с | +| 100 | 103 | 0,126891 с | +| 1000 | 1003 | 1,684725 с | + +Рост близок к росту числа HEAD/SQL round trips. Это локальная стоимость консервативного обхода; нет утверждения о линейности для любой БД/сети. Помимо добавленных строк остаются три intact fault fixtures; missing failed исключён из ready scan. + +### Результат десяти конкурентных upload + +После рестарта API с сохранением старого токена проверены PNG 788100 и 8683594 байт. На каждом размере десять клиентов начали одновременно; **2 получили 201, 8 — контролируемый 429**, затем readiness200. У API и LB `oom=0`, `oom_kill=0`, процессы остались живы. Cgroup lifetime peaks: API 130342912 и130285568 байт, LB18243584 байт. Лимиты соответственно512/512/128MiB. + +Before/after peaks не выросли: начальная загрузка приложения/Argon2 уже создала больший high-water. Поэтому опыт доказывает ограниченное поведение выбранной нагрузки без OOM, но не выделяет точную память одной загрузки и не доказывает экономию RAM самим spool. + +## Проверки и доказательства + +| Критерий | Сценарий | Доказательство | +|---|---|---| +| Контракт и схема | make check validate | evidence/contract-checks.txt | +| Обязательный API, PNG/ACL/checksum/idempotency | make test | evidence/smoke-strict.txt | +| Негативные/параллельные/Unicode/вложения | make student-test | evidence/tests-strict.txt: 88 passed | +| Повреждённый JPEG с поддельным EOI | regression до исправления и полный тест после djpeg | evidence/attachments-before-strict-jpeg.txt; evidence/tests-strict.txt | +| Независимая проверка реального HTTP | root external regression | Отдельная итоговая проверка общего пакета | +| Десять одновременных upload | research/upload_memory.py | evidence/upload-memory-final.json | +| Reconciler/fault-injection/стоимость | tools profile research | evidence/reconcile-study.json | +| Сохранность/переключение/отказы | публичные resilience + собственные реальные stop/pause | Итоговые evidence persistence/failover/storage-checks | + +## Ограничения + +- Один host, LB, PostgreSQL и Seaweed backend остаются точками отказа; две API-реплики не создают HA всех зависимостей. +- Сверка может пометить потерянные bytes failed, но не восстановить их без backup. Здесь не настраивается внешнее резервное копирование S3. +- Нет распределённого COMMIT через PostgreSQL и S3: остатки допустимы до grace/apply. Уникальные immutable server keys и intent делают выбранную очистку безопасной для штатного API; конкурентное ручное изменение администратором после HEAD находится вне этой гарантии. +- Очереди обработки изображений нет: ЛР4 синхронная, thumbnail/crop/optimized относятся к ЛР5. +- HTTP допустим для локального изолированного стенда с синтетическими данными. Удалённая передача секретов требует TLS. +- Native decoder/libjpeg ограничен memory/time/cgroup; полная корректность каждого экзотического изображения зависит от библиотечного декодера. Специально проверены конкретные повреждения из замечаний преподавателя. +- Client-side deadline ограничивает ожидание одного обращения к PostgreSQL, а не весь HTTP-путь; при неопределённом результате записи ключ идемпотентности защищает повтор сообщения. Upload не имеет такого ключа и не должен автоматически повторяться как будто это тот же объект. + +## Источники + +Оригинальные contracts/README и OpenAPI текущего skeleton, Docker Compose reference, документация PostgreSQL/psycopg, boto3 S3 client и SeaweedFS S3 configuration, Pillow Image API и libjpeg-turbo djpeg. Проверки кода опираются на сохранённые версии зависимостей и реальные ответы стенда. diff --git a/SCHEMA.md b/SCHEMA.md new file mode 100644 index 00000000..b9ee9804 --- /dev/null +++ b/SCHEMA.md @@ -0,0 +1,37 @@ +# Схема PostgreSQL и связь с S3 + +Основной источник истины — PostgreSQL. В S3 находятся только исходные bytes вложения. Valkey содержит временные копии публичных пользователей, TTL60, без сессий и истории. + +| Таблица | Ключи и назначение | +|---|---| +| users | UUID PK, username UNIQUE, Argon2id password_hash, UTF-8 display_name в bytea | +| chats | UUID PK, UTF-8 title в bytea, server UTC created_at | +| members | PK(chat_id,user_id), FK к chats/users, UNIQUE(chat_id,position) | +| messages | UUID PK, FK(chat_id,sender_id)→members, UTF-8 text в bytea, UTC created_at | +| message_requests | Уникальный scope(actor_id,chat_id,key), body_digest, точный JSON response в bytea, expires_at24h | +| sessions | SHA-256 token_digest PK, user FK, абсолютный expires_at | +| attachments | UUID PK, chat/owner FK, filename/MIME/size/SHA-256, status/error, created_at/updated_at, object_key UNIQUE | +| message_attachments | PK(message_id,attachment_id), FK к message/attachment, UNIQUE(message_id,position), position0..9 | +| upload_intents | UUID PK, object_key UNIQUE, created_at; durable запись до PUT | +| attachment_reconcile_audit | Генерируемый PK, безопасный event, внутренний object_key и время apply | +| schema_migrations | Имя SQL-файла PK и SHA-256 применённого содержимого | + +## Миграции + +1. 001_initial: базовые таблицы/ограничения/индексы ЛР2. +2. 002_idempotency: общий результат POST message для всех API-реплик. +3. 003_unicode_storage: lossless UTF-8 в bytea для полей, допускающих NUL; старому runtime нужна остановка на переход. +4. 004_attachments: attachments, links, durable intents и audit. +5. 005_attachment_only_message: допускает нулевую длину bytea text; API требует хотя бы одно вложение для пустого/пробельного текста. + +Мигратор применяет файлы последовательно в транзакции под advisory lock. Повторный запуск пропускает неизменившиеся миграции; изменённый применённый файл вызывает ошибку. + +## Индексы и транзакции + +История читается keyset по `(chat_id,created_at,id)` с соответствующим B-tree. ID разрешает совпадения времени. Cursor scoped к чату и валидируется до SQL; он не создаёт snapshot между страницами. + +`message_attachments_attachment_idx` помогает находить ссылки на вложение. `upload_intents_created_idx` поддерживает обход старых intent через `(created_at,id)` без OFFSET. Публичный порядок вложений берётся из position, не сортируется по UUID. + +Создание чата с участниками атомарно. Отправка сообщения, проверка вложений того же чата под row lock, связи и сохранённый ответ идемпотентности выполняются одной транзакцией. Read-after-write обеспечивается чтением общего PostgreSQL, история не кэшируется. + +Upload сначала фиксирует intent, затем держит его `FOR UPDATE` во время S3 PUT и финализации ready metadata. PostgreSQL и S3 не имеют общей транзакции. Неопределённый COMMIT не запускает delete; только консервативная сверка после grace может убрать abandoned остаток. Публичный API не раскрывает object_key, внутренние адреса или credentials. diff --git a/compose.yaml b/compose.yaml index cd46a693..ddf84003 100644 --- a/compose.yaml +++ b/compose.yaml @@ -79,7 +79,7 @@ services: retries: 20 s3: image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e - command: [s3, -filer=storage:8888, -ip.bind=0.0.0.0, -config=/run/secrets/s3_config, -port=8333] + command: [s3, -filer=storage:8888, -ip.bind=0.0.0.0, -config=/run/secrets/s3_config, -port=8333, -port.iceberg=0] secrets: [s3_config] networks: [default, storage_backend] depends_on: diff --git a/scripts/verify_storage.py b/scripts/verify_storage.py new file mode 100644 index 00000000..2921400b --- /dev/null +++ b/scripts/verify_storage.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Real storage ACL and dependency failure checks; tokens never leave memory/stdin.""" + +import hashlib +import json +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args): + return subprocess.run( + ["docker", "compose", *args], cwd=ROOT, check=True, capture_output=True, text=True + ) + + +def inside(code, payload=None): + result = subprocess.run( + ["docker", "compose", "exec", "-T", "--index", "1", "api", "python", "-c", code], + cwd=ROOT, + input=json.dumps(payload or {}), + capture_output=True, + text=True, + check=True, + ) + return json.loads(result.stdout) + + +def request(path, token=None): + return inside( + """import json,sys,time,urllib.request,urllib.error +p=json.load(sys.stdin);h={"Authorization":"Bearer "+p["token"]} if p.get("token") else {} +start=time.monotonic() +try:r=urllib.request.urlopen(urllib.request.Request("http://127.0.0.1:8080"+p["path"],headers=h),timeout=8) +except urllib.error.HTTPError as e:r=e +data=r.read() +print(json.dumps({"status":r.code,"elapsed":time.monotonic()-start,"json":json.loads(data) if r.headers.get_content_type()=="application/json" else None,"sha256":__import__("hashlib").sha256(data).hexdigest()})) +""", + {"path": path, "token": token}, + ) + + +def main(): + c = Client("http://127.0.0.1:8084") + alice, _, _, chat = c.bootstrap() + attachment, content = c.attachment(alice, chat) + report = {"checks": [], "faults": []} + + def check(name, condition): + if not condition: + raise AssertionError(name) + report["checks"].append({"name": name, "passed": True}) + + security = inside( + """import json,sys,os,socket,urllib.request,urllib.error,uuid +from pathlib import Path +from messenger.storage import ObjectStorage +from messenger.postgres import PostgresStore +from botocore.exceptions import ClientError +p=json.load(sys.stdin);s=ObjectStorage();db=PostgresStore() +with db.connection() as conn:key=conn.execute("SELECT object_key FROM attachments WHERE id=%s",(p["id"],)).fetchone()["object_key"] +try:r=urllib.request.urlopen(os.getenv("S3_ENDPOINT")+"/"+s.bucket+"/"+key,timeout=3);anon=r.code;r.close() +except urllib.error.HTTPError as e:anon=e.code +try:s.client.create_bucket(Bucket="forbidden-"+uuid.uuid4().hex);denied=False +except ClientError as e:denied=e.response["ResponseMetadata"]["HTTPStatusCode"]==403 +try:socket.create_connection(("storage",8888),timeout=.5).close();backend=False +except OSError:backend=True +try:socket.create_connection(("s3",8888),timeout=.5).close();gateway=False +except OSError:gateway=True +print(json.dumps({"anonymous_status":anon,"unrelated_bucket_denied":denied,"backend_unreachable":backend,"gateway_filer_unreachable":gateway,"no_admin_secrets":not Path("/run/secrets/s3_admin_access_key").exists() and not any(k.startswith("S3_ADMIN") for k in os.environ)}));db.close();s.close() +""", + {"id": attachment["id"]}, + ) + report["security"] = security + for key in [ + "unrelated_bucket_denied", + "backend_unreachable", + "gateway_filer_unreachable", + "no_admin_secrets", + ]: + check(key, security[key]) + check("anonymous_original_denied", security["anonymous_status"] in {401, 403}) + compose("run", "--rm", "--no-deps", "s3-bootstrap") + check("bucket_bootstrap_repeatable", True) + try: + compose("stop", "s3") + ready = request("/health/ready") + download = request(f"/api/v1/attachments/{attachment['id']}/content", alice["token"]) + check("s3_down_ready_503", ready["status"] == 503 and ready["elapsed"] < 8) + check("s3_down_content_503", download["status"] == 503 and download["elapsed"] < 8) + report["faults"].append({"service": "s3", "ready": ready, "download": download}) + finally: + compose("start", "s3") + for _ in range(30): + try: + downloaded = request(f"/api/v1/attachments/{attachment['id']}/content", alice["token"]) + if downloaded["status"] == 200: + break + except (AssertionError, subprocess.CalledProcessError): + pass + time.sleep(0.5) + else: + raise AssertionError("S3 recovery timed out") + check( + "s3_restart_original_unchanged", downloaded["sha256"] == hashlib.sha256(content).hexdigest() + ) + try: + compose("pause", "postgres") + ready = request("/health/ready") + me = request("/api/v1/users/me", alice["token"]) + check( + "paused_pg_ready_503_with_client_deadline", + ready["status"] == 503 and ready["elapsed"] < 8, + ) + check("paused_pg_auth_503_with_client_deadline", me["status"] == 503 and me["elapsed"] < 8) + report["faults"].append({"service": "paused_postgres", "ready": ready, "me": me}) + finally: + compose("unpause", "postgres") + for _ in range(30): + try: + me = request("/api/v1/users/me", alice["token"]) + if me["status"] == 200: + break + except subprocess.CalledProcessError: + pass + time.sleep(0.5) + else: + raise AssertionError("PostgreSQL recovery timed out") + check("old_session_recovers_after_unpause", me["json"]["id"] == alice["id"]) + Path(ROOT / "evidence/storage-checks.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n" + ) + print(json.dumps({"passed": len(report["checks"])})) + + +if __name__ == "__main__": + main() -- 2.54.0 From 1779b5f03688d5eac8be8049b3201940e81a1426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 18:05:59 +0300 Subject: [PATCH 04/11] ci: add isolated application build and API checks --- .github/workflows/application.yml | 24 ++++++++++++++++++++++++ REPORT.md | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/application.yml diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..e5e8678a --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,24 @@ +name: Application checks +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + # Intended for an isolated hosted VM, never a runner sharing production Docker. + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + COMPOSE_PROJECT_NAME: ci-lab4-${{ github.run_id }}-${{ github.run_attempt }} + PUBLIC_PORT: '8084' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install uv==0.11.7 + - run: uv sync --locked + - run: make up + - run: make check validate PYTHON=.venv/bin/python + - run: make test student-test + - if: always() + run: docker compose down --volumes --remove-orphans diff --git a/REPORT.md b/REPORT.md index 0ca71e35..86e16c90 100644 --- a/REPORT.md +++ b/REPORT.md @@ -2,7 +2,7 @@ **Васильев Никита, БВТ2301. Целевая оценка: 5. Трек 2 — согласованность двух хранилищ.** -Локальное выполнение без публикации PR. Оригинальные README, COURSE, CONTRIBUTING, contracts и публичные tests сохранены. Собственный код ЛР3 перенесён в отдельный skeleton ЛР4; добавлены вложения и исследование согласованности. Использована помощь ИИ при реализации, тестировании и подготовке отчёта. Замечания преподавателей изучены заранее, ссылки и регрессии — в [REVIEW_NOTES.md](REVIEW_NOTES.md). Код и чужие измерения не заимствованы. +Локальное выполнение без публикации PR. Отдельный application workflow описывает build/readiness/smoke/own tests на изолированной hosted VM; удалённый CI в рамках этой работы не запускался. Оригинальные README, COURSE, CONTRIBUTING, contracts и публичные tests сохранены. Собственный код ЛР3 перенесён в отдельный skeleton ЛР4; добавлены вложения и исследование согласованности. Использована помощь ИИ при реализации, тестировании и подготовке отчёта. Замечания преподавателей изучены заранее, ссылки и регрессии — в [REVIEW_NOTES.md](REVIEW_NOTES.md). Код и чужие измерения не заимствованы. ## Запуск из чистого clone -- 2.54.0 From 14c659fda4c9c4b923134aaf1f9d56173aa9a0b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 18:09:32 +0300 Subject: [PATCH 05/11] fix: validate required schema and support bounded JPEG file objects --- REPORT.md | 2 ++ scripts/verify_storage.py | 16 ++++++++++++++++ src/messenger/attachments.py | 21 ++++++++++++++++++++- src/messenger/postgres.py | 8 ++++++-- student-tests/test_attachments.py | 14 ++++++++++++++ 5 files changed, 58 insertions(+), 3 deletions(-) diff --git a/REPORT.md b/REPORT.md index 86e16c90..bb25e15d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -60,6 +60,8 @@ docker compose up -d --wait --no-recreate Seaweed backend подключён только к отдельной `storage_backend` internal network. API к ней не подключается: filer/master/volume интерфейсы недоступны из общей API-сети. S3 gateway подключён к обеим сетям и предоставляет только S3-интерфейс на общей сети; публичных host ports у него нет. Admin identity присутствует только у gateway и bootstrap/research, API получает bucket-scoped Read/Write/List identity. +Readiness проверяет обязательные таблицы и столбцы PostgreSQL запросом WHERE false, затем доступность S3 bucket. Простого SELECT 1 недостаточно для обнаружения отсутствующей схемы. + Ни случайный UUID, ни приватный bucket не заменяют ACL: метаданные и content выдаются только после проверки PostgreSQL-сессии и членства в чате. Сессии не кэшируются. Валидный токен не открывает чужой чат. Оригинал скачивается через авторизованный API без redirect. Сообщения сохраняют порядок attachment_ids в message_attachments.position. Идемпотентность из ЛР3 атомарно сохраняет сообщение, связи и точный ответ; перестановка вложений при том же ключе считается другим body. Пропущенный attachment_ids и [] эквивалентны. UTF-8 поля сообщений/имён хранятся lossless в bytea, включая NUL, который PostgreSQL text не принимает. diff --git a/scripts/verify_storage.py b/scripts/verify_storage.py index 2921400b..b225a39d 100644 --- a/scripts/verify_storage.py +++ b/scripts/verify_storage.py @@ -133,6 +133,22 @@ print(json.dumps({"anonymous_status":anon,"unrelated_bucket_denied":denied,"back else: raise AssertionError("PostgreSQL recovery timed out") check("old_session_recovers_after_unpause", me["json"]["id"] == alice["id"]) + inside( + "from messenger.postgres import PostgresStore; s=PostgresStore(); " + "c=s.pool.getconn(); c.execute('ALTER TABLE message_requests RENAME TO message_requests_readiness_test'); " + "s.pool.putconn(c); s.close(); print('{}')" + ) + try: + absent_table = request("/health/ready") + check("missing_required_table_ready_503", absent_table["status"] == 503) + report["faults"].append({"service": "missing_message_requests", "ready": absent_table}) + finally: + inside( + "from messenger.postgres import PostgresStore; s=PostgresStore(); " + "c=s.pool.getconn(); c.execute('ALTER TABLE message_requests_readiness_test RENAME TO message_requests'); " + "s.pool.putconn(c); s.close(); print('{}')" + ) + check("schema_restore_ready_200", request("/health/ready")["status"] == 200) Path(ROOT / "evidence/storage-checks.json").write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n" ) diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py index 1b037944..ecae250c 100644 --- a/src/messenger/attachments.py +++ b/src/messenger/attachments.py @@ -3,6 +3,7 @@ import hashlib import os import subprocess +import tempfile import unicodedata from uuid import uuid4 @@ -59,10 +60,25 @@ def validate_image(fileobj, declared_type): # libjpeg emits a warning but Pillow accepts e.g. a half scan with a # forged EOI. djpeg's nonzero warning exit rejects that corruption. fileobj.seek(0) + fallback = None try: + decoder_input = fileobj + try: + fileobj.fileno() + except (AttributeError, OSError): + # Internal callers may supply BytesIO; bound the fallback too. + fallback = tempfile.TemporaryFile() + copied = 0 + while chunk := fileobj.read(64 * 1024): + copied += len(chunk) + if copied > MAX_BYTES: + raise Failure(413, "Файл больше 10 MiB") from None + fallback.write(chunk) + fallback.seek(0) + decoder_input = fallback decoded = subprocess.run( ["djpeg", "-strict", "-maxmemory", "192m", "-outfile", "/dev/null"], - stdin=fileobj, + stdin=decoder_input, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=5, @@ -70,6 +86,9 @@ def validate_image(fileobj, declared_type): ) except subprocess.TimeoutExpired: raise Failure(415, "Изображение не прошло ограниченную проверку") from None + finally: + if fallback is not None: + fallback.close() if decoded.returncode: raise Failure(415, "Повреждённое изображение") fileobj.seek(0) diff --git a/src/messenger/postgres.py b/src/messenger/postgres.py index 6c734001..3fa8791a 100644 --- a/src/messenger/postgres.py +++ b/src/messenger/postgres.py @@ -101,9 +101,13 @@ class PostgresStore: def ready(self): try: with self.connection() as conn: - conn.execute("SELECT 1") + conn.execute( + "SELECT u.id,c.id,m.chat_id,msg.id,s.token_digest,r.key,a.id,i.id,ma.message_id " + "FROM users u,chats c,members m,messages msg,sessions s,message_requests r," + "attachments a,upload_intents i,message_attachments ma WHERE false" + ) return True - except Failure: + except (Failure, psycopg.Error): return False def _cached_user(self, uid): diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index fbdf7a90..1c105076 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -199,3 +199,17 @@ def test_truncated_jpeg_with_forged_eoi(scenario): mime="image/jpeg", status=415, ) + + +def test_internal_jpeg_bytesio_fallback(): + import shutil + + from messenger.attachments import validate_image + + if shutil.which("djpeg") is None: + pytest.skip( + "Internal BytesIO helper requires host djpeg; HTTP JPEG validation runs in Docker regardless" + ) + stream = BytesIO(image_bytes("JPEG")) + assert validate_image(stream, "image/jpeg") == ("image/jpeg", 32, 20) + assert stream.tell() == 0 -- 2.54.0 From 9d9d425205da6898cde95697d5dfb5abd7ea25a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 18:10:55 +0300 Subject: [PATCH 06/11] test: measure aged reconciliation scans and cross-replica retries --- REPORT.md | 2 +- research/README.md | 2 +- research/analyze.py | 53 +++++++++++++++++++++++++++++++ research/run.py | 2 ++ student-tests/test_attachments.py | 30 +++++++++++++++++ 5 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 research/analyze.py diff --git a/REPORT.md b/REPORT.md index bb25e15d..e1415258 100644 --- a/REPORT.md +++ b/REPORT.md @@ -96,7 +96,7 @@ docker compose run --rm --no-deps research Проверяются реальные PG INSERT failure после PUT, ошибка подключения S3, потеря подтверждения после успешного COMMIT, PG DELETE failure после удаления S3, повторный dry-run после частичного удаления, S3 delete failure/recovery, fresh orphan, old intent с fresh bytes, live writer lock, missing used metadata. Временные исследовательские ресурсы удаляются в finally, рабочая messenger не очищается. -Стоимость обхода измеряется на10/100/1000 добавленных intact attachments, по3 dry-run, page_size100. Точные размеры с учётом сохранённых fault fixtures находятся в JSON. Измеряются время и количество просмотренных объектов/метаданных; это SQL+S3 обход, не HTTP throughput. +Стоимость обхода измеряется на10/100/1000 добавленных intact attachments, по3 dry-run, page_size100. Перед измерением все добавленные объекты выдерживаются за границу grace, чтобы fresh-skip не менял профиль работы. Точные размеры с учётом сохранённых fault fixtures находятся в JSON. Измеряются время и количество просмотренных объектов/метаданных; это SQL+S3 обход, не HTTP throughput. ### Результат исследовательского прогона diff --git a/research/README.md b/research/README.md index 05c9a4fa..15e5118e 100644 --- a/research/README.md +++ b/research/README.md @@ -36,7 +36,7 @@ docker compose run --rm --no-deps research > evidence/reconcile-repeat-raw.txt 2 Ошибки: настоящий trigger PostgreSQL после PUT; реальная ошибка подключения S3; успешный COMMIT с искусственно потерянным подтверждением; ошибка DELETE PostgreSQL после удаления bytes; ошибка S3 delete. Проверяются повтор, dry-run после сбоя, старый intent со свежим объектом, live row lock, missing used metadata и сохранность подтверждённого оригинала. -Стоимость обхода: 10/100/1000 специально добавленных intact attachment, по три dry-run на точку, page_size=100. Дополнительно остаются несколько fault fixtures; точное число просмотренных metadata/objects записывается в каждой строке. Сервисы прогреты, однопоточный обход; временные БД/bucket используются на том же PostgreSQL/S3 без внешней нагрузки. Это цена выбранной сверки, не HTTP throughput и не производственный SLA. +Стоимость обхода: 10/100/1000 специально добавленных intact attachment, по три dry-run на точку, page_size=100. Перед каждым размером объекты выдерживаются за границу grace, чтобы fresh-skip не менял профиль работы. Дополнительно остаются несколько fault fixtures; точное число просмотренных metadata/objects записывается в каждой строке. Сервисы прогреты, однопоточный обход; временные БД/bucket используются на том же PostgreSQL/S3 без внешней нагрузки. Это цена выбранной сверки, не HTTP throughput и не производственный SLA. ## Память десяти конкурентных upload diff --git a/research/analyze.py b/research/analyze.py new file mode 100644 index 00000000..19c02fc5 --- /dev/null +++ b/research/analyze.py @@ -0,0 +1,53 @@ +"""Summarize the saved raw reconciliation measurements without re-running them.""" + +import argparse +import json +import statistics +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("input", nargs="?", default="evidence/reconcile-study.json") + parser.add_argument("--output", default="evidence/reconcile-summary.md") + args = parser.parse_args() + result = json.loads(Path(args.input).read_text()) + checks = result["checks"] + lines = [ + "# Исследование согласованности PostgreSQL/S3", + "", + f"Дата начала (UTC): {result['started_at']}.", + "", + f"Успешных проверок: {sum(c['passed'] for c in checks)}/{len(checks)}.", + "", + "| Добавлено intact | Просмотрено metadata | Медиана обхода, с |", + "|---:|---:|---:|", + ] + for size in sorted({r["seeded_intact"] for r in result["scan_cost"]}): + runs = [r for r in result["scan_cost"] if r["seeded_intact"] == size] + lines.append( + f"| {size} | {runs[0]['metadata_scanned']} | {statistics.median(r['duration_seconds'] for r in runs):.6f} |" + ) + lines += [ + "", + "По три dry-run на размер, page_size100, прогретый локальный стенд.", + "Дополнительные три intact fault fixtures входят в фактический scan.", + "", + f"Первый apply: удалено intent {result['apply']['deleted_intents']}, " + f"операций удаления объекта {result['apply']['deleted_objects']}, " + f"missing помечено failed {result['apply']['marked_failed']}.", + "Счётчик удаления объекта — число успешных идемпотентных DELETE, включая отсутствие bytes у intent.", + "", + f"Временные ресурсы удалены: {result['temporary_resources_deleted']}.", + "", + "## Проверенные инварианты", + "", + ] + lines += [f"- {'PASS' if c['passed'] else 'FAIL'}: {c['name']}" for c in checks] + lines += ["", "## Ограничения", ""] + ["- " + s for s in result["limitations"]] + Path(args.output).write_text("\n".join(lines) + "\n") + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/research/run.py b/research/run.py index b8081ae4..7ae6c0c1 100644 --- a/research/run.py +++ b/research/run.py @@ -292,6 +292,8 @@ def main(): for _ in range(target - current): upload("scan.png") current = target + # Equalize age policy: no point may skip freshly seeded object checks. + time.sleep(2.2) with store.connection() as conn: conn.execute("UPDATE attachments SET created_at=now()-interval '1 day'") for repeat in range(3): diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index 1c105076..09864f40 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -213,3 +213,33 @@ def test_internal_jpeg_bytesio_fallback(): stream = BytesIO(image_bytes("JPEG")) assert validate_image(stream, "image/jpeg") == ("image/jpeg", 32, 20) assert stream.tell() == 0 + + +def test_parallel_same_key_across_replicas(scenario): + import threading + from concurrent.futures import ThreadPoolExecutor + + c, alice, _, _, chat = scenario + key = uuid.uuid4().hex + barrier = threading.Barrier(10) + path = f"/api/v1/chats/{chat['id']}/messages" + body = {"text": "concurrent idempotent message"} + + def send(_): + barrier.wait() + return Client(c.base).call( + "POST", + path, + 201, + actor=alice, + body=body, + headers={"Idempotency-Key": key}, + schema="Message", + ) + + with ThreadPoolExecutor(max_workers=10) as pool: + responses = list(pool.map(send, range(10))) + assert len({r["json"]["id"] for r in responses}) == 1 + assert len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2 + history = c.call("GET", path, actor=alice, schema="MessagePage")["json"]["items"] + assert sum(m["id"] == responses[0]["json"]["id"] for m in history) == 1 -- 2.54.0 From 619155f1e01457ee02da561200a571cbd5674055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 18:12:56 +0300 Subject: [PATCH 07/11] research: save repeatable experiment and sanitized environment manifest --- research/README.md | 4 +- research/run_local.py | 85 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 research/run_local.py diff --git a/research/README.md b/research/README.md index 15e5118e..2806242a 100644 --- a/research/README.md +++ b/research/README.md @@ -29,10 +29,10 @@ Durable intent до S3 PUT, row lock на время PUT/финализации ```sh # Исследование получает admin keys только в отдельном profile tools. # Создаёт отдельные БД и bucket, очищает только их в finally. -docker compose run --rm --no-deps research > evidence/reconcile-repeat-raw.txt 2>&1 +uv run --locked python research/run_local.py --output evidence/reconcile-repeat.json ``` -В строке `RESULT ` находится JSON с исходными счётчиками, проверками, хешами кода и результатами времени. Основная messenger БД и bucket не очищаются. +Контроллер сохраняет raw output, JSON со счётчиками, проверками, хешами кода, результатами времени и кратким host manifest; рядом создаётся summary.md. В исходном выводе контейнера JSON начинается со строки `RESULT `. Основная messenger БД и bucket не очищаются. Ошибки: настоящий trigger PostgreSQL после PUT; реальная ошибка подключения S3; успешный COMMIT с искусственно потерянным подтверждением; ошибка DELETE PostgreSQL после удаления bytes; ошибка S3 delete. Проверяются повтор, dry-run после сбоя, старый intent со свежим объектом, live row lock, missing used metadata и сохранность подтверждённого оригинала. diff --git a/research/run_local.py b/research/run_local.py new file mode 100644 index 00000000..edde11ba --- /dev/null +++ b/research/run_local.py @@ -0,0 +1,85 @@ +"""Run and save the isolated experiment, plus a small sanitized host manifest.""" + +import argparse +import json +import platform +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/reconcile-study.json") + args = parser.parse_args() + output = ROOT / args.output + output.parent.mkdir(parents=True, exist_ok=True) + run = subprocess.run( + ["docker", "compose", "run", "--rm", "--no-deps", "research"], + cwd=ROOT, + capture_output=True, + text=True, + ) + output.with_suffix(".raw.txt").write_text(run.stdout + run.stderr) + if run.returncode: + raise SystemExit(f"Experiment failed; inspect {output.with_suffix('.raw.txt')}") + marker = next( + (line[7:] for line in run.stdout.splitlines() if line.startswith("RESULT ")), None + ) + if marker is None: + raise SystemExit("Experiment did not emit RESULT") + result = json.loads(marker) + result["host_manifest"] = { + "system": platform.system(), + "machine": platform.machine(), + "release": platform.release(), + "git_revision": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip(), + } + result["host_manifest"]["docker"] = json.loads( + subprocess.check_output( + [ + "docker", + "info", + "--format", + '{"version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}', + ], + text=True, + ) + ) + containers = subprocess.check_output( + ["docker", "compose", "ps", "-q"], cwd=ROOT, text=True + ).split() + result["host_manifest"]["containers"] = [] + for container in containers: + record = json.loads(subprocess.check_output(["docker", "inspect", container], text=True))[0] + result["host_manifest"]["containers"].append( + { + "service": record["Config"]["Labels"].get("com.docker.compose.service"), + "image": record["Image"], + "cpu_nano": record["HostConfig"]["NanoCpus"], + "memory_limit": record["HostConfig"]["Memory"], + } + ) + output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + subprocess.run( + [ + sys.executable, + "research/analyze.py", + str(output), + "--output", + str(output.with_suffix(".summary.md")), + ], + cwd=ROOT, + check=True, + ) + print( + f"Saved {len(result['checks'])} checks and {len(result['scan_cost'])} scan runs to {output}" + ) + + +if __name__ == "__main__": + main() -- 2.54.0 From ff17362bcd8617928e0e6f2aa83b0c409ee48e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 18:20:32 +0300 Subject: [PATCH 08/11] fix: reject noncanonical cursors and verify storage failover --- REPORT.md | 28 +++++-- infra/haproxy.cfg | 2 + scripts/stop_one.py | 31 ++++++-- scripts/verify_attachment_failover.py | 101 ++++++++++++++++++++++++++ scripts/verify_storage.py | 12 ++- src/messenger/domain.py | 13 +++- student-tests/test_attachments.py | 36 +++++++++ student-tests/test_cursor_format.py | 39 ++++++++++ 8 files changed, 246 insertions(+), 16 deletions(-) create mode 100644 scripts/verify_attachment_failover.py create mode 100644 student-tests/test_cursor_format.py diff --git a/REPORT.md b/REPORT.md index e1415258..43fbc541 100644 --- a/REPORT.md +++ b/REPORT.md @@ -98,21 +98,23 @@ docker compose run --rm --no-deps research Стоимость обхода измеряется на10/100/1000 добавленных intact attachments, по3 dry-run, page_size100. Перед измерением все добавленные объекты выдерживаются за границу grace, чтобы fresh-skip не менял профиль работы. Точные размеры с учётом сохранённых fault fixtures находятся в JSON. Измеряются время и количество просмотренных объектов/метаданных; это SQL+S3 обход, не HTTP throughput. +После сохранённых измерений внесена отдельная правка `domain.boundary`: строгая RFC3339Z-дата с календарной проверкой и целочисленная версия cursor вместо расширений Python ISO parser. Этот путь не используется исследованием сверки или image upload. Исходные hashes в raw JSON оставлены как снимок измеренного runtime, не переписаны под новый файл. Финальные HTTP/unit регрессии проверяют эту правку отдельно. + ### Результат исследовательского прогона [reconcile-study.json](evidence/reconcile-study.json) содержит **30 успешных проверок**, девять замеров обхода и hashes runtime. Первый apply удалил остатки двух старых intent и orphan, пометил один missing original failed; повтор не сделал новых изменений. Проверены также ошибка DELETE после удаления bytes и последующий read-only dry-run. Временные БД и bucket удалены. | Добавлено intact | Фактически просмотрено metadata | Медиана трёх обходов | |---:|---:|---:| -| 10 | 13 | 0,018528 с | -| 100 | 103 | 0,126891 с | -| 1000 | 1003 | 1,684725 с | +| 10 | 13 | 0,023085 с | +| 100 | 103 | 0,170581 с | +| 1000 | 1003 | 1,581705 с | Рост близок к росту числа HEAD/SQL round trips. Это локальная стоимость консервативного обхода; нет утверждения о линейности для любой БД/сети. Помимо добавленных строк остаются три intact fault fixtures; missing failed исключён из ready scan. ### Результат десяти конкурентных upload -После рестарта API с сохранением старого токена проверены PNG 788100 и 8683594 байт. На каждом размере десять клиентов начали одновременно; **2 получили 201, 8 — контролируемый 429**, затем readiness200. У API и LB `oom=0`, `oom_kill=0`, процессы остались живы. Cgroup lifetime peaks: API 130342912 и130285568 байт, LB18243584 байт. Лимиты соответственно512/512/128MiB. +После рестарта API с сохранением старого токена проверены PNG 788100 и 8683594 байт. На каждом размере десять клиентов начали одновременно; **2 получили 201, 8 — контролируемый 429**, затем readiness200. У API и LB `oom=0`, `oom_kill=0`, процессы остались живы. Cgroup lifetime peaks: API 122773504 и122679296 байт, LB19656704 байт. Лимиты соответственно512/512/128MiB. Before/after peaks не выросли: начальная загрузка приложения/Argon2 уже создала больший high-water. Поэтому опыт доказывает ограниченное поведение выбранной нагрузки без OOM, но не выделяет точную память одной загрузки и не доказывает экономию RAM самим spool. @@ -121,13 +123,23 @@ Before/after peaks не выросли: начальная загрузка пр | Критерий | Сценарий | Доказательство | |---|---|---| | Контракт и схема | make check validate | evidence/contract-checks.txt | -| Обязательный API, PNG/ACL/checksum/idempotency | make test | evidence/smoke-strict.txt | -| Негативные/параллельные/Unicode/вложения | make student-test | evidence/tests-strict.txt: 88 passed | -| Повреждённый JPEG с поддельным EOI | regression до исправления и полный тест после djpeg | evidence/attachments-before-strict-jpeg.txt; evidence/tests-strict.txt | +| Обязательный API, PNG/ACL/checksum/idempotency | make test | evidence/smoke-final.txt | +| Негативные/параллельные/Unicode/вложения | make student-test | evidence/tests-final.txt | +| Повреждённый JPEG с поддельным EOI | regression до исправления и полный тест после djpeg | evidence/attachments-before-strict-jpeg.txt; evidence/tests-final.txt | | Независимая проверка реального HTTP | root external regression | Отдельная итоговая проверка общего пакета | | Десять одновременных upload | research/upload_memory.py | evidence/upload-memory-final.json | | Reconciler/fault-injection/стоимость | tools profile research | evidence/reconcile-study.json | -| Сохранность/переключение/отказы | публичные resilience + собственные реальные stop/pause | Итоговые evidence persistence/failover/storage-checks | +| Сохранность данных/прежней сессии/файла | make test-persistence + реальные timestamps | evidence/persistence-final.txt; persistence-before-containers.json; persistence-after-containers.json | +| Отказ каждой физической API | 4 публичных failover до покрытия двух разных контейнеров | evidence/failover-1.txt .. failover-4.txt; failover-hooks.jsonl | +| Отказ именно загрузившей API и download другой | scripts/verify_attachment_failover.py | evidence/attachment-failover.json | +| Приватный S3/ограниченные credentials/backend isolation/отказы/schema | scripts/verify_storage.py: 15 проверок | evidence/storage-checks.json | +| JSON503 без выдуманного instance при отсутствии всех API | explicit stop all APIs | evidence/attachment-failover.json | + +### Наблюдения отказов + +Действительный upload обслужила одна API, затем она была остановлена по её X-Instance-Id. Другая API вернула те же исходные bytes через 2,318 секунды; сохранены оба alias и checksum. Публичные failover-прогоны допускают краткие ошибки обнаружения: точные успешные записи, failed_attempts и first_recovery_seconds находятся в четырёх raw outputs. После каждого отказа реплика восстановлена; audit доказывает остановку двух разных контейнеров, а не только смену заголовка. + +При выключенном S3 direct API readiness и content возвращают503 в пределах сетевых таймаутов. После запуска оригинал совпадает. Paused PostgreSQL даёт503 с client-side deadline; после unpause прежняя сессия работает. Переименование message_requests делает readiness503, возврат схемы восстанавливает200. Когда LB не видит ни одной готовой API, он возвращает JSON503 с динамическим согласованным X-Request-Id и без фиктивного X-Instance-Id. ## Ограничения diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg index ae5f764a..62b0a227 100644 --- a/infra/haproxy.cfg +++ b/infra/haproxy.cfg @@ -25,6 +25,8 @@ resolvers docker frontend public bind :8080 + http-request set-var(txn.lb_request_id) uuid() + http-request return status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"No ready API","request_id":"%[var(txn.lb_request_id)]"}}' hdr X-Request-Id '%[var(txn.lb_request_id)]' if { nbsrv(apis) eq 0 } default_backend apis backend apis diff --git a/scripts/stop_one.py b/scripts/stop_one.py index 8daf4e0c..164f0238 100644 --- a/scripts/stop_one.py +++ b/scripts/stop_one.py @@ -1,19 +1,40 @@ """Map a process alias to its real container, then leave exactly it stopped.""" + import json import os import subprocess +from datetime import UTC, datetime +from pathlib import Path wanted = os.environ["TARGET_INSTANCE_ID"] ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], text=True).split() for cid in ids: result = subprocess.run( - ["docker", "exec", cid, "python", "-c", - "import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"], - text=True, capture_output=True, check=False, + [ + "docker", + "exec", + cid, + "python", + "-c", + "import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])", + ], + text=True, + capture_output=True, + check=False, ) if result.returncode == 0 and result.stdout.strip() == wanted: - subprocess.run(["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL) - print(json.dumps({"stopped_container": cid, "instance_id": wanted})) + subprocess.run( + ["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL + ) + record = { + "stopped_container": cid, + "instance_id": wanted, + "stopped_at": datetime.now(UTC).isoformat(), + } + Path("evidence").mkdir(exist_ok=True) + with Path("evidence/failover-hooks.jsonl").open("a") as output: + output.write(json.dumps(record) + "\n") + print(json.dumps(record)) break else: raise SystemExit("Target instance not found among this project's API containers") diff --git a/scripts/verify_attachment_failover.py b/scripts/verify_attachment_failover.py new file mode 100644 index 00000000..9e07fec1 --- /dev/null +++ b/scripts/verify_attachment_failover.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Stop the actual uploader, read via another process, and test LB's no-backend JSON.""" + +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 - load the unchanged public helper after adding tests path. + + +def main(): + c = Client(os.getenv("BASE_URL", "http://127.0.0.1:8084")) + alice, bob, _, chat = c.bootstrap() + data = (ROOT / "tests/fixtures/sample.png").read_bytes() + uploaded = c.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=failover.png", + 201, + actor=alice, + raw=data, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + target = uploaded["headers"]["X-Instance-Id"] + started = time.monotonic() + try: + subprocess.run( + [str(ROOT / "scripts/stop-one.sh")], + cwd=ROOT, + env={**os.environ, "TARGET_INSTANCE_ID": target}, + check=True, + capture_output=True, + ) + for _ in range(40): + try: + downloaded = c.call( + "GET", f"/api/v1/attachments/{uploaded['json']['id']}/content", actor=bob + ) + if downloaded["headers"]["X-Instance-Id"] != target: + break + except AssertionError: + pass + time.sleep(0.2) + else: + raise AssertionError("Uploader failover timed out") + assert downloaded["bytes"] == data + result = { + "uploader_instance": target, + "download_instance": downloaded["headers"]["X-Instance-Id"], + "same_original_bytes": True, + "sha256": uploaded["json"]["sha256"], + "elapsed_seconds": time.monotonic() - started, + } + finally: + subprocess.run( + ["docker", "compose", "up", "-d", "--wait", "--no-recreate"], + cwd=ROOT, + check=True, + capture_output=True, + ) + subprocess.run(["docker", "compose", "stop", "api"], cwd=ROOT, check=True, capture_output=True) + try: + time.sleep(3) + try: + response = urllib.request.urlopen(c.base + "/api/v1/users/me", timeout=3) + except urllib.error.HTTPError as exc: + response = exc + with response: + body = json.loads(response.read()) + assert response.code == 503 and body["error"]["code"] == "unavailable" + assert response.headers["X-Request-Id"] == body["error"]["request_id"] + assert not response.headers.get("X-Instance-Id") + result["lb_no_backend"] = { + "status": response.code, + "request_id": response.headers["X-Request-Id"], + "no_fake_instance": True, + } + finally: + subprocess.run( + ["docker", "compose", "up", "-d", "--wait", "--no-recreate"], + cwd=ROOT, + check=True, + capture_output=True, + ) + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + result["existing_session_recovers"] = True + (ROOT / "evidence/attachment-failover.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n" + ) + print(json.dumps(result, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_storage.py b/scripts/verify_storage.py index b225a39d..5ebcde71 100644 --- a/scripts/verify_storage.py +++ b/scripts/verify_storage.py @@ -3,6 +3,7 @@ import hashlib import json +import os import subprocess import sys import time @@ -47,7 +48,7 @@ print(json.dumps({"status":r.code,"elapsed":time.monotonic()-start,"json":json.l def main(): - c = Client("http://127.0.0.1:8084") + c = Client(os.getenv("BASE_URL", "http://127.0.0.1:8084")) alice, _, _, chat = c.bootstrap() attachment, content = c.attachment(alice, chat) report = {"checks": [], "faults": []} @@ -149,6 +150,15 @@ print(json.dumps({"anonymous_status":anon,"unrelated_bucket_denied":denied,"back "s.pool.putconn(c); s.close(); print('{}')" ) check("schema_restore_ready_200", request("/health/ready")["status"] == 200) + for _ in range(30): + try: + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + break + except AssertionError: + time.sleep(0.5) + else: + raise AssertionError("LB did not readmit recovered API") + check("lb_readmits_recovered_api", True) Path(ROOT / "evidence/storage-checks.json").write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n" ) diff --git a/src/messenger/domain.py b/src/messenger/domain.py index 1415db7e..7d1a462a 100644 --- a/src/messenger/domain.py +++ b/src/messenger/domain.py @@ -30,10 +30,19 @@ def boundary(scope: str, cursor: str | None) -> tuple[str, str] | None: if not cursor or len(cursor) > 1024 or not re.fullmatch(r"[A-Za-z0-9_-]+", cursor): raise ValueError data = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))) - if not isinstance(data, list) or len(data) != 4 or data[:2] != [1, scope]: + if ( + not isinstance(data, list) + or len(data) != 4 + or type(data[0]) is not int + or data[0] != 1 + or data[1] != scope + ): raise ValueError date, uid = data[2:] - if not isinstance(date, str) or not date.endswith("Z"): + if not isinstance(date, str) or not re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z", + date, + ): raise ValueError datetime.fromisoformat(date.replace("Z", "+00:00")) if not isinstance(uid, str) or str(UUID(uid)) != uid: diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index 09864f40..e69d2097 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -1,6 +1,8 @@ """Black-box regressions derived from the contract and teacher review comments.""" +import base64 import hashlib +import json import os import struct import uuid @@ -243,3 +245,37 @@ def test_parallel_same_key_across_replicas(scenario): assert len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2 history = c.call("GET", path, actor=alice, schema="MessagePage")["json"]["items"] assert sum(m["id"] == responses[0]["json"]["id"] for m in history) == 1 + + +def test_exact_pixel_boundary_is_accepted(scenario): + uploaded = upload(scenario, data=image_bytes(size=(8000, 5000)))["json"] + assert uploaded["status"] == "ready" + + +@pytest.mark.parametrize( + "version,date", + [ + (True, "2026-01-01T00:00:00Z"), + (1.0, "2026-01-01T00:00:00Z"), + (1, "2026-W01-1T00:00:00Z"), + (1, "20260101T000000Z"), + (1, "2026-01-01 00:00:00Z"), + (1, "2026-01-01X00:00:00Z"), + (1, "2026-02-30T00:00:00Z"), + (1, "2026-01-01T00:00:00.1234567Z"), + (1, "0000-01-01T00:00:00Z"), + (1, "2026-01-01T24:00:00Z"), + ], +) +def test_noncanonical_cursor_over_http(scenario, version, date): + c, alice, _, _, chat = scenario + for path, scope in [ + ("/api/v1/chats", "chats:" + alice["id"]), + (f"/api/v1/chats/{chat['id']}/messages", "messages:" + chat["id"]), + ]: + cursor = ( + base64.urlsafe_b64encode(json.dumps([version, scope, date, str(uuid.uuid4())]).encode()) + .decode() + .rstrip("=") + ) + c.call("GET", path + "?cursor=" + cursor, 400, actor=alice) diff --git a/student-tests/test_cursor_format.py b/student-tests/test_cursor_format.py new file mode 100644 index 00000000..ceb15c54 --- /dev/null +++ b/student-tests/test_cursor_format.py @@ -0,0 +1,39 @@ +import base64 +import json +from uuid import uuid4 + +import pytest + +from messenger.domain import Failure, boundary + + +@pytest.mark.parametrize( + "version,date", + [ + (True, "2026-01-01T00:00:00Z"), + (1.0, "2026-01-01T00:00:00Z"), + (1, "2026-W01-1T00:00:00Z"), + (1, "20260101T000000Z"), + (1, "2026-01-01 00:00:00Z"), + (1, "2026-01-01X00:00:00Z"), + (1, "2026-02-30T00:00:00Z"), + (1, "2026-01-01T00:00:00.1234567Z"), + ], +) +def test_boundary_rejects_python_iso_extensions(version, date): + payload = [version, "scope", date, str(uuid4())] + cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=") + with pytest.raises(Failure) as error: + boundary("scope", cursor) + assert error.value.status == 400 + + +@pytest.mark.parametrize( + "date", ["2026-01-01T00:00:00Z", "2026-01-01T00:00:00.1Z", "2026-01-01T00:00:00.123456Z"] +) +def test_boundary_preserves_generated_format(date): + uid = str(uuid4()) + cursor = ( + base64.urlsafe_b64encode(json.dumps([1, "scope", date, uid]).encode()).decode().rstrip("=") + ) + assert boundary("scope", cursor) == (date, uid) -- 2.54.0 From 8de05d38998a59e0a3fd7487386c162a92731309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 18:21:51 +0300 Subject: [PATCH 09/11] docs: record verified attachment, failover and reconciliation evidence --- REPORT.md | 4 +- evidence/attachment-failover.json | 13 + evidence/attachment-failover.txt | 1 + evidence/attachments-before-strict-jpeg.txt | 43 ++ evidence/build-final-cursor.txt | 155 ++++++ evidence/build-final.txt | 214 ++++++++ evidence/contract-checks.txt | 4 + evidence/exact-pixel-boundary.txt | 2 + evidence/failover-1.txt | 4 + evidence/failover-2.txt | 4 + evidence/failover-3.txt | 4 + evidence/failover-4.txt | 4 + evidence/failover-hooks.jsonl | 5 + evidence/lint-final.txt | 1 + evidence/persistence-after-containers.json | 56 +++ evidence/persistence-before-containers.json | 56 +++ evidence/persistence-final.txt | 3 + evidence/reconcile-study.json | 515 ++++++++++++++++++++ evidence/reconcile-study.raw.txt | 8 + evidence/reconcile-study.summary.md | 60 +++ evidence/smoke-final.txt | 2 + evidence/storage-checks.json | 142 ++++++ evidence/storage-checks.txt | 1 + evidence/tests-final.txt | 3 + evidence/upload-memory-final.json | 285 +++++++++++ evidence/upload-memory-final.txt | 5 + 26 files changed, 1592 insertions(+), 2 deletions(-) create mode 100644 evidence/attachment-failover.json create mode 100644 evidence/attachment-failover.txt create mode 100644 evidence/attachments-before-strict-jpeg.txt create mode 100644 evidence/build-final-cursor.txt create mode 100644 evidence/build-final.txt create mode 100644 evidence/contract-checks.txt create mode 100644 evidence/exact-pixel-boundary.txt create mode 100644 evidence/failover-1.txt create mode 100644 evidence/failover-2.txt create mode 100644 evidence/failover-3.txt create mode 100644 evidence/failover-4.txt create mode 100644 evidence/failover-hooks.jsonl create mode 100644 evidence/lint-final.txt create mode 100644 evidence/persistence-after-containers.json create mode 100644 evidence/persistence-before-containers.json create mode 100644 evidence/persistence-final.txt create mode 100644 evidence/reconcile-study.json create mode 100644 evidence/reconcile-study.raw.txt create mode 100644 evidence/reconcile-study.summary.md create mode 100644 evidence/smoke-final.txt create mode 100644 evidence/storage-checks.json create mode 100644 evidence/storage-checks.txt create mode 100644 evidence/tests-final.txt create mode 100644 evidence/upload-memory-final.json create mode 100644 evidence/upload-memory-final.txt diff --git a/REPORT.md b/REPORT.md index 43fbc541..d970a0c3 100644 --- a/REPORT.md +++ b/REPORT.md @@ -6,7 +6,7 @@ ## Запуск из чистого clone -Нужны Docker Engine с Compose v2, Python 3.12 и uv, make. Для публичных тестов достаточно системного Python 3.9+. API и все хранилища запускаются из этого репозитория; ручная установка PostgreSQL/S3 не нужна. +Нужны Docker Engine с Compose v2, Python 3.12 и uv, make. Для публичных тестов достаточно системного Python 3.9+. Host djpeg необязателен: один дополнительный unit-тест внутреннего BytesIO helper пропускается, если этого бинарника нет на хосте; HTTP-регрессии JPEG всегда выполняются против Docker, где decoder установлен и закреплён. API и все хранилища запускаются из этого репозитория; ручная установка PostgreSQL/S3 не нужна. ```sh uv sync --locked --python 3.12 @@ -124,7 +124,7 @@ Before/after peaks не выросли: начальная загрузка пр |---|---|---| | Контракт и схема | make check validate | evidence/contract-checks.txt | | Обязательный API, PNG/ACL/checksum/idempotency | make test | evidence/smoke-final.txt | -| Негативные/параллельные/Unicode/вложения | make student-test | evidence/tests-final.txt | +| Негативные/параллельные/Unicode/вложения/cursor | make student-test | evidence/tests-final.txt | | Повреждённый JPEG с поддельным EOI | regression до исправления и полный тест после djpeg | evidence/attachments-before-strict-jpeg.txt; evidence/tests-final.txt | | Независимая проверка реального HTTP | root external regression | Отдельная итоговая проверка общего пакета | | Десять одновременных upload | research/upload_memory.py | evidence/upload-memory-final.json | diff --git a/evidence/attachment-failover.json b/evidence/attachment-failover.json new file mode 100644 index 00000000..80eca297 --- /dev/null +++ b/evidence/attachment-failover.json @@ -0,0 +1,13 @@ +{ + "uploader_instance": "2ce0bbc4-1cd3-4f73-95a0-e4ca5b2a752c", + "download_instance": "52106d2a-9828-4995-8a50-02f0607c18da", + "same_original_bytes": true, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "elapsed_seconds": 2.3175257910043, + "lb_no_backend": { + "status": 503, + "request_id": "0a7be913-caf6-490f-88a6-9407cda9391c", + "no_fake_instance": true + }, + "existing_session_recovers": true +} diff --git a/evidence/attachment-failover.txt b/evidence/attachment-failover.txt new file mode 100644 index 00000000..fba0b373 --- /dev/null +++ b/evidence/attachment-failover.txt @@ -0,0 +1 @@ +{"uploader_instance": "2ce0bbc4-1cd3-4f73-95a0-e4ca5b2a752c", "download_instance": "52106d2a-9828-4995-8a50-02f0607c18da", "same_original_bytes": true, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", "elapsed_seconds": 2.3175257910043, "lb_no_backend": {"status": 503, "request_id": "0a7be913-caf6-490f-88a6-9407cda9391c", "no_fake_instance": true}, "existing_session_recovers": true} diff --git a/evidence/attachments-before-strict-jpeg.txt b/evidence/attachments-before-strict-jpeg.txt new file mode 100644 index 00000000..b22b6237 --- /dev/null +++ b/evidence/attachments-before-strict-jpeg.txt @@ -0,0 +1,43 @@ +..........................F [100%] +=================================== FAILURES =================================== +_____________________ test_truncated_jpeg_with_forged_eoi ______________________ + +scenario = (, {'id': '881b708c-1c42-4d33-a7ff-0a24d24180ed', 'user': {'id': '881b708c-1c42-4...-06T14:52:44.187538Z', 'member_ids': ['881b708c-1c42-4d33-a7ff-0a24d24180ed', 'e9f642b2-d052-420a-9f41-113bc79347fe']}) + + def test_truncated_jpeg_with_forged_eoi(scenario): + import random + + stream = BytesIO() + Image.frombytes("RGB", (512, 512), random.Random(42).randbytes(512 * 512 * 3)).save( + stream, "JPEG", quality=95 + ) + original = stream.getvalue() + upload(scenario, data=original, name="real.jpg", mime="image/jpeg") +> upload( + scenario, + data=original[: len(original) // 2] + b"\xff\xd9", + name="broken.jpg", + mime="image/jpeg", + status=415, + ) + +student-tests/test_attachments.py:195: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +student-tests/test_attachments.py:30: in upload + return c.call( +tests/client.py:132: in call + check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +condition = False +message = 'POST /api/v1/chats/ed1d5486-a609-41e0-8066-e2abfcd078c0/attachments: expected [415], received 201' + + def check(condition, message): + if not condition: +> raise Failure(message) +E client.Failure: POST /api/v1/chats/ed1d5486-a609-41e0-8066-e2abfcd078c0/attachments: expected [415], received 201 + +tests/client.py:24: Failure +=========================== short test summary info ============================ +FAILED student-tests/test_attachments.py::test_truncated_jpeg_with_forged_eoi +1 failed, 26 passed in 0.93s diff --git a/evidence/build-final-cursor.txt b/evidence/build-final-cursor.txt new file mode 100644 index 00000000..92d4e73f --- /dev/null +++ b/evidence/build-final-cursor.txt @@ -0,0 +1,155 @@ + Image mtusi-vasilev:lab4 Building + Image mtusi-vasilev:lab4 Building + Image mtusi-vasilev:lab4 Building +#1 [internal] load local bake definitions +#1 reading from stdin 1.91kB done +#1 DONE 0.0s + +#2 [s3-bootstrap internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.07kB done +#2 DONE 0.0s + +#3 [migrate internal] load metadata for ghcr.io/astral-sh/uv:0.11.7 +#3 DONE 0.0s + +#4 [s3-bootstrap internal] load metadata for docker.io/library/python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b +#4 DONE 0.0s + +#5 [api internal] load .dockerignore +#5 transferring context: 109B done +#5 DONE 0.0s + +#6 [migrate internal] load build context +#6 transferring context: 32.99kB done +#6 DONE 0.0s + +#7 [s3-bootstrap build 1/5] FROM docker.io/library/python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b +#7 resolve docker.io/library/python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b done +#7 DONE 0.0s + +#8 [migrate] FROM ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a +#8 resolve ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a 0.0s done +#8 DONE 0.0s + +#9 [api stage-1 2/6] RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-4 && rm -rf /var/lib/apt/lists/* +#9 CACHED + +#10 [api build 2/5] COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv +#10 CACHED + +#11 [api build 5/5] RUN uv sync --locked --no-dev --no-install-project --python /usr/local/bin/python +#11 CACHED + +#12 [api build 4/5] COPY pyproject.toml uv.lock ./ +#12 CACHED + +#13 [api stage-1 3/6] WORKDIR /app +#13 CACHED + +#14 [api build 3/5] WORKDIR /app +#14 CACHED + +#15 [s3-bootstrap stage-1 4/6] COPY --from=build /app/.venv /app/.venv +#15 CACHED + +#16 [s3-bootstrap stage-1 5/6] COPY src ./src +#16 DONE 0.0s + +#17 [api stage-1 6/6] COPY migrations ./migrations +#17 DONE 0.0s + +#18 [api] exporting to image +#18 exporting layers 0.0s done +#18 exporting manifest sha256:76e4717644b7216802f263cecdc39b1b8909111738b9595eb11d2a69fded5cf5 done +#18 exporting config sha256:a5eb2b1b6617dc8eeb1b37ab4a36f375b33482ea9e0372bc6eafaa32b69354b6 +#18 exporting config sha256:a5eb2b1b6617dc8eeb1b37ab4a36f375b33482ea9e0372bc6eafaa32b69354b6 0.0s done +#18 exporting attestation manifest sha256:052d9bea4bb919ea1aedd937166fb57f95b69f8b7afd94ff08013d8c3be404e3 0.0s done +#18 exporting manifest list sha256:cfcd162da81cd5a5f1e6729a245b7c1fc814b5cbc1a543b2a50d0131dae5120b done +#18 naming to docker.io/library/mtusi-vasilev:lab4 done +#18 unpacking to docker.io/library/mtusi-vasilev:lab4 0.0s done +#18 DONE 0.1s + +#19 [s3-bootstrap] exporting to image +#19 exporting layers 0.0s done +#19 exporting manifest sha256:23e60390e8d2edca4c8664bda72b63b7fbfaab5ef91c09c9dc3dd09ccbf889dc done +#19 exporting config sha256:808eb376512c2f8bd0148c5a1ac9a3ddbe2a34184cb7c3cb1e9a081955bae2a6 0.0s done +#19 exporting attestation manifest sha256:e8977fb690c0b5986719e2e960dd939b8f9ccadc2b5a009370959a3d85e9390f 0.0s done +#19 exporting manifest list sha256:593cdabf59cf1bf0ff446446a6138270a064bce3da5b08095b95096700da485d done +#19 naming to docker.io/library/mtusi-vasilev:lab4 done +#19 unpacking to docker.io/library/mtusi-vasilev:lab4 0.0s done +#19 DONE 0.1s + +#20 [migrate] exporting to image +#20 exporting layers 0.0s done +#20 exporting manifest sha256:85518f8cd6ca3f1af9a60e4b594d4dc02717b9ca2b402823eecebec387f67afe 0.0s done +#20 exporting config sha256:e6cbef2d7963fb852a2fdd3d2012b69a4c9f0ab81e6dd65bc3d9a17cbb37561e 0.0s done +#20 exporting attestation manifest sha256:70ce3b90e76d010cbd587cc1a2c3fa66e31a427c996cb86be9975765493529e5 0.0s done +#20 exporting manifest list sha256:d8e01d722065f1b4b46706f834e2727124409d8c90dd9a707a2b97bd91600980 done +#20 naming to docker.io/library/mtusi-vasilev:lab4 done +#20 unpacking to docker.io/library/mtusi-vasilev:lab4 0.0s done +#20 DONE 0.1s + +#21 [api] resolving provenance for metadata file +#21 DONE 0.0s + +#22 [s3-bootstrap] resolving provenance for metadata file +#22 DONE 0.0s + +#23 [migrate] resolving provenance for metadata file +#23 DONE 0.0s + Image mtusi-vasilev:lab4 Built + Image mtusi-vasilev:lab4 Built + Image mtusi-vasilev:lab4 Built + Container mtusi-vasilev-lab4-cache-1 Running + Container mtusi-vasilev-lab4-postgres-1 Running + Container mtusi-vasilev-lab4-s3-1 Running + Container mtusi-vasilev-lab4-storage-1 Running + Container mtusi-vasilev-lab4-lb-1 Running + Container mtusi-vasilev-lab4-migrate-1 Recreate + Container mtusi-vasilev-lab4-s3-bootstrap-1 Recreate + Container mtusi-vasilev-lab4-s3-bootstrap-1 Recreated + Container mtusi-vasilev-lab4-migrate-1 Recreated + Container mtusi-vasilev-lab4-api-1 Recreate + Container mtusi-vasilev-lab4-api-2 Recreate + Container mtusi-vasilev-lab4-api-2 Recreated + Container mtusi-vasilev-lab4-api-1 Recreated + Container mtusi-vasilev-lab4-storage-1 Waiting + Container mtusi-vasilev-lab4-postgres-1 Waiting + Container mtusi-vasilev-lab4-postgres-1 Healthy + Container mtusi-vasilev-lab4-migrate-1 Starting + Container mtusi-vasilev-lab4-storage-1 Healthy + Container mtusi-vasilev-lab4-s3-bootstrap-1 Starting + Container mtusi-vasilev-lab4-s3-bootstrap-1 Started + Container mtusi-vasilev-lab4-migrate-1 Started + Container mtusi-vasilev-lab4-s3-bootstrap-1 Waiting + Container mtusi-vasilev-lab4-migrate-1 Waiting + Container mtusi-vasilev-lab4-cache-1 Waiting + Container mtusi-vasilev-lab4-cache-1 Healthy + Container mtusi-vasilev-lab4-migrate-1 Exited + Container mtusi-vasilev-lab4-s3-bootstrap-1 Exited + Container mtusi-vasilev-lab4-api-1 Starting + Container mtusi-vasilev-lab4-api-1 Started + Container mtusi-vasilev-lab4-api-2 Starting + Container mtusi-vasilev-lab4-api-2 Started + Container mtusi-vasilev-lab4-api-1 Waiting + Container mtusi-vasilev-lab4-api-2 Waiting + Container mtusi-vasilev-lab4-api-1 Healthy + Container mtusi-vasilev-lab4-api-2 Healthy + Container mtusi-vasilev-lab4-api-1 Waiting + Container mtusi-vasilev-lab4-api-2 Waiting + Container mtusi-vasilev-lab4-lb-1 Waiting + Container mtusi-vasilev-lab4-s3-bootstrap-1 Waiting + Container mtusi-vasilev-lab4-migrate-1 Waiting + Container mtusi-vasilev-lab4-cache-1 Waiting + Container mtusi-vasilev-lab4-postgres-1 Waiting + Container mtusi-vasilev-lab4-s3-1 Waiting + Container mtusi-vasilev-lab4-storage-1 Waiting + Container mtusi-vasilev-lab4-lb-1 Healthy + Container mtusi-vasilev-lab4-migrate-1 Exited + Container mtusi-vasilev-lab4-s3-1 Healthy + Container mtusi-vasilev-lab4-s3-bootstrap-1 Exited + Container mtusi-vasilev-lab4-storage-1 Healthy + Container mtusi-vasilev-lab4-postgres-1 Healthy + Container mtusi-vasilev-lab4-cache-1 Healthy + Container mtusi-vasilev-lab4-api-1 Healthy + Container mtusi-vasilev-lab4-api-2 Healthy diff --git a/evidence/build-final.txt b/evidence/build-final.txt new file mode 100644 index 00000000..8e26020f --- /dev/null +++ b/evidence/build-final.txt @@ -0,0 +1,214 @@ + Image mtusi-vasilev:lab4 Building + Image mtusi-vasilev:lab4 Building + Image mtusi-vasilev:lab4 Building +#1 [internal] load local bake definitions +#1 reading from stdin 1.91kB done +#1 DONE 0.0s + +#2 [api] FROM ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a +#2 resolve ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a done +#2 DONE 0.0s + +#3 [api build 1/5] FROM docker.io/library/python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b +#3 resolve docker.io/library/python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b 0.0s done +#3 CACHED + +#4 [s3-bootstrap build 5/5] RUN uv sync --locked --no-dev --no-install-project --python /usr/local/bin/python +#4 CACHED + +#5 [migrate internal] load build definition from Dockerfile +#5 transferring dockerfile: 1.07kB done +#5 DONE 0.0s + +#6 [s3-bootstrap internal] load metadata for ghcr.io/astral-sh/uv:0.11.7 +#6 DONE 0.0s + +#7 [s3-bootstrap internal] load metadata for docker.io/library/python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b +#7 DONE 0.0s + +#8 [api internal] load .dockerignore +#8 transferring context: 109B done +#8 DONE 0.0s + +#9 [api internal] load build context +#9 transferring context: 56.14kB done +#9 DONE 0.0s + +#10 [api stage-1 2/6] RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-4 && rm -rf /var/lib/apt/lists/* +#10 0.316 Get:1 http://deb.debian.org/debian trixie InRelease [140 kB] +#10 0.381 Get:2 http://deb.debian.org/debian trixie-updates InRelease [47.3 kB] +#10 0.463 Get:3 http://deb.debian.org/debian-security trixie-security InRelease [43.4 kB] +#10 0.591 Get:4 http://deb.debian.org/debian trixie/main arm64 Packages [9614 kB] +#10 0.965 Get:5 http://deb.debian.org/debian trixie-updates/main arm64 Packages [1108 B] +#10 1.030 Get:6 http://deb.debian.org/debian-security trixie-security/main arm64 Packages [299 kB] +#10 1.575 Fetched 10.1 MB in 1s (6874 kB/s) +#10 1.575 Reading package lists... +#10 1.918 Reading package lists... +#10 2.236 Building dependency tree... +#10 2.328 Reading state information... +#10 2.436 The following additional packages will be installed: +#10 2.436 libjpeg62-turbo libturbojpeg0 +#10 2.461 The following NEW packages will be installed: +#10 2.461 libjpeg-turbo-progs libjpeg62-turbo libturbojpeg0 +#10 2.543 0 upgraded, 3 newly installed, 0 to remove and 28 not upgraded. +#10 2.543 Need to get 523 kB of archives. +#10 2.543 After this operation, 1650 kB of additional disk space will be used. +#10 2.543 Get:1 http://deb.debian.org/debian trixie/main arm64 libjpeg62-turbo arm64 1:2.1.5-4 [173 kB] +#10 2.818 Get:2 http://deb.debian.org/debian trixie/main arm64 libturbojpeg0 arm64 1:2.1.5-4 [211 kB] +#10 2.854 Get:3 http://deb.debian.org/debian trixie/main arm64 libjpeg-turbo-progs arm64 1:2.1.5-4 [140 kB] +#10 2.985 debconf: unable to initialize frontend: Dialog +#10 2.985 debconf: (TERM is not set, so the dialog frontend is not usable.) +#10 2.985 debconf: falling back to frontend: Readline +#10 2.985 debconf: unable to initialize frontend: Readline +#10 2.985 debconf: (Can't locate Term/ReadLine.pm in @INC (you may need to install the Term::ReadLine module) (@INC entries checked: /etc/perl /usr/local/lib/aarch64-linux-gnu/perl/5.40.1 /usr/local/share/perl/5.40.1 /usr/lib/aarch64-linux-gnu/perl5/5.40 /usr/share/perl5 /usr/lib/aarch64-linux-gnu/perl-base /usr/lib/aarch64-linux-gnu/perl/5.40 /usr/share/perl/5.40 /usr/local/lib/site_perl) at /usr/share/perl5/Debconf/FrontEnd/Readline.pm line 8, line 3.) +#10 2.985 debconf: falling back to frontend: Teletype +#10 2.987 debconf: unable to initialize frontend: Teletype +#10 2.987 debconf: (This frontend requires a controlling tty.) +#10 2.987 debconf: falling back to frontend: Noninteractive +#10 3.306 Fetched 523 kB in 0s (1191 kB/s) +#10 3.316 Selecting previously unselected package libjpeg62-turbo:arm64. +#10 3.316 (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 5642 files and directories currently installed.) +#10 3.318 Preparing to unpack .../libjpeg62-turbo_1%3a2.1.5-4_arm64.deb ... +#10 3.320 Unpacking libjpeg62-turbo:arm64 (1:2.1.5-4) ... +#10 3.337 Selecting previously unselected package libturbojpeg0:arm64. +#10 3.337 Preparing to unpack .../libturbojpeg0_1%3a2.1.5-4_arm64.deb ... +#10 3.338 Unpacking libturbojpeg0:arm64 (1:2.1.5-4) ... +#10 3.354 Selecting previously unselected package libjpeg-turbo-progs. +#10 3.354 Preparing to unpack .../libjpeg-turbo-progs_1%3a2.1.5-4_arm64.deb ... +#10 3.355 Unpacking libjpeg-turbo-progs (1:2.1.5-4) ... +#10 3.371 Setting up libturbojpeg0:arm64 (1:2.1.5-4) ... +#10 3.373 Setting up libjpeg62-turbo:arm64 (1:2.1.5-4) ... +#10 3.375 Setting up libjpeg-turbo-progs (1:2.1.5-4) ... +#10 3.379 Processing triggers for libc-bin (2.41-12+deb13u3) ... +#10 DONE 3.4s + +#11 [migrate stage-1 3/6] WORKDIR /app +#11 DONE 0.0s + +#12 [s3-bootstrap stage-1 4/6] COPY --from=build /app/.venv /app/.venv +#12 DONE 0.3s + +#13 [migrate stage-1 4/6] COPY --from=build /app/.venv /app/.venv +#13 CACHED + +#14 [migrate build 4/5] COPY pyproject.toml uv.lock ./ +#14 CACHED + +#15 [migrate build 5/5] RUN uv sync --locked --no-dev --no-install-project --python /usr/local/bin/python +#15 CACHED + +#16 [migrate build 2/5] COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv +#16 CACHED + +#17 [migrate build 3/5] WORKDIR /app +#17 CACHED + +#18 [s3-bootstrap stage-1 5/6] COPY src ./src +#18 DONE 0.0s + +#19 [migrate stage-1 6/6] COPY migrations ./migrations +#19 DONE 0.0s + +#20 [s3-bootstrap] exporting to image +#20 exporting layers +#20 exporting layers 1.4s done +#20 exporting manifest sha256:9c7beb537d46f62e0a0125cca10a389b079d69ff236f6ebf103c48a668fe48a9 0.0s done +#20 exporting config sha256:4b06563cc9df4800046c57198947d33b3de7ce8ab5e489172a3b79fc1e32a206 0.0s done +#20 exporting attestation manifest sha256:63b281e33599928fc2e05e1f503d3a248bd05c3891f609c6183515222d97762c 0.0s done +#20 exporting manifest list sha256:02ba01cac8063d6d9d92a423e07a799cb887605f5c2d4c45b0cb2be678555f62 done +#20 naming to docker.io/library/mtusi-vasilev:lab4 done +#20 unpacking to docker.io/library/mtusi-vasilev:lab4 +#20 unpacking to docker.io/library/mtusi-vasilev:lab4 0.8s done +#20 DONE 2.4s + +#21 [api] exporting to image +#21 exporting layers 1.4s done +#21 exporting manifest sha256:e099f891bbc8556c8d165432df0607fb19fe94c118a41f07100c83a2070ed33a done +#21 exporting config sha256:206eac1e185e7d2e0c51de40eefacd3451e957a541592b347742643f26d513f0 0.0s done +#21 exporting attestation manifest sha256:048e39632ccadafe299f7ee0e9b14c6e510cc0cdadb3ad30646cb18620cccc7c 0.0s done +#21 exporting manifest list sha256:fb1fa012b813c17f834b01a58e3dff5c48a7376b11942226426e0ed7637ef719 done +#21 naming to docker.io/library/mtusi-vasilev:lab4 done +#21 unpacking to docker.io/library/mtusi-vasilev:lab4 0.9s done +#21 DONE 2.4s + +#22 [migrate] exporting to image +#22 exporting layers 1.4s done +#22 exporting manifest sha256:b07d34138667c73e38e0a14aeaaab517a72d3f83e52fa9c5abf1b437802396f8 0.0s done +#22 exporting config sha256:5c1fc0ea8ee20d8ba557b492b0700e78b506ac52aedf466d1258456061177fd8 0.0s done +#22 exporting attestation manifest sha256:56327f9b7aaaa4d121696463d86e51cf52b698f7c9f7efcb989344541c33ac27 0.0s done +#22 exporting manifest list sha256:1beedea111bd3f8f5de5a31b7a67b2013ffc9c0271b51313bb6968bcaea96f6e 0.0s done +#22 naming to docker.io/library/mtusi-vasilev:lab4 done +#22 unpacking to docker.io/library/mtusi-vasilev:lab4 0.9s done +#22 DONE 2.5s + +#23 [s3-bootstrap] resolving provenance for metadata file +#23 DONE 0.0s + +#24 [api] resolving provenance for metadata file +#24 DONE 0.0s + +#25 [migrate] resolving provenance for metadata file +#25 DONE 0.0s + Image mtusi-vasilev:lab4 Built + Image mtusi-vasilev:lab4 Built + Image mtusi-vasilev:lab4 Built + Container mtusi-vasilev-lab4-migrate-1 Recreate + Container mtusi-vasilev-lab4-s3-1 Recreate + Container mtusi-vasilev-lab4-s3-1 Recreated + Container mtusi-vasilev-lab4-s3-bootstrap-1 Recreate + Container mtusi-vasilev-lab4-migrate-1 Recreated + Container mtusi-vasilev-lab4-s3-bootstrap-1 Recreated + Container mtusi-vasilev-lab4-api-1 Recreate + Container mtusi-vasilev-lab4-api-2 Recreate + Container mtusi-vasilev-lab4-api-1 Recreated + Container mtusi-vasilev-lab4-api-2 Recreated + Container mtusi-vasilev-lab4-storage-1 Starting + Container mtusi-vasilev-lab4-cache-1 Starting + Container mtusi-vasilev-lab4-postgres-1 Starting + Container mtusi-vasilev-lab4-storage-1 Started + Container mtusi-vasilev-lab4-storage-1 Waiting + Container mtusi-vasilev-lab4-postgres-1 Started + Container mtusi-vasilev-lab4-postgres-1 Waiting + Container mtusi-vasilev-lab4-cache-1 Started + Container mtusi-vasilev-lab4-storage-1 Healthy + Container mtusi-vasilev-lab4-s3-1 Starting + Container mtusi-vasilev-lab4-postgres-1 Healthy + Container mtusi-vasilev-lab4-migrate-1 Starting + Container mtusi-vasilev-lab4-s3-1 Started + Container mtusi-vasilev-lab4-s3-bootstrap-1 Starting + Container mtusi-vasilev-lab4-migrate-1 Started + Container mtusi-vasilev-lab4-s3-bootstrap-1 Started + Container mtusi-vasilev-lab4-cache-1 Waiting + Container mtusi-vasilev-lab4-s3-bootstrap-1 Waiting + Container mtusi-vasilev-lab4-migrate-1 Waiting + Container mtusi-vasilev-lab4-cache-1 Healthy + Container mtusi-vasilev-lab4-migrate-1 Exited + Container mtusi-vasilev-lab4-s3-bootstrap-1 Exited + Container mtusi-vasilev-lab4-api-1 Starting + Container mtusi-vasilev-lab4-api-1 Started + Container mtusi-vasilev-lab4-api-2 Starting + Container mtusi-vasilev-lab4-api-2 Started + Container mtusi-vasilev-lab4-api-1 Waiting + Container mtusi-vasilev-lab4-api-2 Waiting + Container mtusi-vasilev-lab4-api-1 Healthy + Container mtusi-vasilev-lab4-api-2 Healthy + Container mtusi-vasilev-lab4-lb-1 Starting + Container mtusi-vasilev-lab4-lb-1 Started + Container mtusi-vasilev-lab4-api-1 Waiting + Container mtusi-vasilev-lab4-api-2 Waiting + Container mtusi-vasilev-lab4-s3-1 Waiting + Container mtusi-vasilev-lab4-postgres-1 Waiting + Container mtusi-vasilev-lab4-migrate-1 Waiting + Container mtusi-vasilev-lab4-cache-1 Waiting + Container mtusi-vasilev-lab4-lb-1 Waiting + Container mtusi-vasilev-lab4-s3-bootstrap-1 Waiting + Container mtusi-vasilev-lab4-storage-1 Waiting + Container mtusi-vasilev-lab4-cache-1 Healthy + Container mtusi-vasilev-lab4-migrate-1 Exited + Container mtusi-vasilev-lab4-s3-1 Healthy + Container mtusi-vasilev-lab4-s3-bootstrap-1 Exited + Container mtusi-vasilev-lab4-storage-1 Healthy + Container mtusi-vasilev-lab4-api-1 Healthy + Container mtusi-vasilev-lab4-api-2 Healthy + Container mtusi-vasilev-lab4-postgres-1 Healthy + Container mtusi-vasilev-lab4-lb-1 Healthy diff --git a/evidence/contract-checks.txt b/evidence/contract-checks.txt new file mode 100644 index 00000000..ed9cd475 --- /dev/null +++ b/evidence/contract-checks.txt @@ -0,0 +1,4 @@ +.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/exact-pixel-boundary.txt b/evidence/exact-pixel-boundary.txt new file mode 100644 index 00000000..c6f3b556 --- /dev/null +++ b/evidence/exact-pixel-boundary.txt @@ -0,0 +1,2 @@ +. [100%] +1 passed in 1.66s diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 00000000..0310a745 --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8084" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 136, "failed_attempts": 2, "first_recovery_seconds": 1.543} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 00000000..f23c9f1e --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8084" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 3, "first_recovery_seconds": 2.105} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-3.txt b/evidence/failover-3.txt new file mode 100644 index 00000000..83ac07ed --- /dev/null +++ b/evidence/failover-3.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8084" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 3, "first_recovery_seconds": 2.213} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-4.txt b/evidence/failover-4.txt new file mode 100644 index 00000000..448efcdd --- /dev/null +++ b/evidence/failover-4.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8084" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 3, "first_recovery_seconds": 0.868} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-hooks.jsonl b/evidence/failover-hooks.jsonl new file mode 100644 index 00000000..6cc9c415 --- /dev/null +++ b/evidence/failover-hooks.jsonl @@ -0,0 +1,5 @@ +{"stopped_container": "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9", "instance_id": "41205fc8-2249-4028-8387-2a9bb6f40b74", "stopped_at": "2026-11-06T15:16:55.346421+00:00"} +{"stopped_container": "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9", "instance_id": "730d80b7-0f1d-443f-b817-69b96c42257a", "stopped_at": "2026-11-06T15:17:18.680493+00:00"} +{"stopped_container": "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9", "instance_id": "6e460a1c-fe88-4108-bc03-0f5f668e8a65", "stopped_at": "2026-11-06T15:17:42.685676+00:00"} +{"stopped_container": "fb116950e109b2c4f68378be994e4fa685b1e32d5df70e4f4ba88b9aaac11734", "instance_id": "6a5a80d4-582a-47a1-9cef-0179b9a423b3", "stopped_at": "2026-11-06T15:18:05.937471+00:00"} +{"stopped_container": "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9", "instance_id": "2ce0bbc4-1cd3-4f73-95a0-e4ca5b2a752c", "stopped_at": "2026-11-06T15:18:34.416070+00:00"} diff --git a/evidence/lint-final.txt b/evidence/lint-final.txt new file mode 100644 index 00000000..1f5f344d --- /dev/null +++ b/evidence/lint-final.txt @@ -0,0 +1 @@ +All checks passed! diff --git a/evidence/persistence-after-containers.json b/evidence/persistence-after-containers.json new file mode 100644 index 00000000..5a753883 --- /dev/null +++ b/evidence/persistence-after-containers.json @@ -0,0 +1,56 @@ +[ + { + "service": "api", + "container": "fb116950e109b2c4f68378be994e4fa685b1e32d5df70e4f4ba88b9aaac11734", + "started_at": "2026-11-06T15:15:49.279244611Z", + "running": true + }, + { + "service": "api", + "container": "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9", + "started_at": "2026-11-06T15:15:49.329476128Z", + "running": true + }, + { + "service": "cache", + "container": "5b1f796409334488ab6738161733ab9bf5a18168f130a97adb3e26115a37ca58", + "started_at": "2026-11-06T15:15:48.865336893Z", + "running": true + }, + { + "service": "lb", + "container": "52284c66754124d6476f53fdb606b5c60548d12faa9ead291572e25b054c7998", + "started_at": "2026-11-06T15:15:48.806688697Z", + "running": true + }, + { + "service": "migrate", + "container": "2b0e607724660e7759eb19441e55f876fd8f7a5ed70786abff1b598f5a8f6902", + "started_at": "2026-11-06T15:15:51.577483807Z", + "running": false + }, + { + "service": "postgres", + "container": "f7114ab3fbe35b16192783e8a6482e6498af0add82239395a3855c672ff04973", + "started_at": "2026-11-06T15:15:48.896278712Z", + "running": true + }, + { + "service": "s3", + "container": "2d81a2bfeaf92d8fee8221e7a1747c30d1da158fe987ce977940dae91307d9c0", + "started_at": "2026-11-06T15:15:48.917467586Z", + "running": true + }, + { + "service": "s3-bootstrap", + "container": "83ea8d79ab0e55db166db204f0e2c297a0773e1718552b6c4723fea097b3e758", + "started_at": "2026-11-06T15:15:51.576960641Z", + "running": false + }, + { + "service": "storage", + "container": "d680a5826a35dcaf058c08801e477ffa23ae9114ff1deabccbadcded7cc76ecb", + "started_at": "2026-11-06T15:15:48.934995346Z", + "running": true + } +] diff --git a/evidence/persistence-before-containers.json b/evidence/persistence-before-containers.json new file mode 100644 index 00000000..c1164f06 --- /dev/null +++ b/evidence/persistence-before-containers.json @@ -0,0 +1,56 @@ +[ + { + "service": "api", + "container": "fb116950e109b2c4f68378be994e4fa685b1e32d5df70e4f4ba88b9aaac11734", + "started_at": "2026-11-06T15:13:47.712944078Z", + "running": true + }, + { + "service": "api", + "container": "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9", + "started_at": "2026-11-06T15:13:47.729250356Z", + "running": true + }, + { + "service": "cache", + "container": "5b1f796409334488ab6738161733ab9bf5a18168f130a97adb3e26115a37ca58", + "started_at": "2026-11-06T15:13:09.057038043Z", + "running": true + }, + { + "service": "lb", + "container": "52284c66754124d6476f53fdb606b5c60548d12faa9ead291572e25b054c7998", + "started_at": "2026-11-06T15:13:29.211107429Z", + "running": true + }, + { + "service": "migrate", + "container": "2b0e607724660e7759eb19441e55f876fd8f7a5ed70786abff1b598f5a8f6902", + "started_at": "2026-11-06T15:13:11.679089233Z", + "running": false + }, + { + "service": "postgres", + "container": "f7114ab3fbe35b16192783e8a6482e6498af0add82239395a3855c672ff04973", + "started_at": "2026-11-06T15:13:09.057550292Z", + "running": true + }, + { + "service": "s3", + "container": "2d81a2bfeaf92d8fee8221e7a1747c30d1da158fe987ce977940dae91307d9c0", + "started_at": "2026-11-06T15:14:23.761816807Z", + "running": true + }, + { + "service": "s3-bootstrap", + "container": "83ea8d79ab0e55db166db204f0e2c297a0773e1718552b6c4723fea097b3e758", + "started_at": "2026-11-06T15:13:11.756618786Z", + "running": false + }, + { + "service": "storage", + "container": "d680a5826a35dcaf058c08801e477ffa23ae9114ff1deabccbadcded7cc76ecb", + "started_at": "2026-11-06T15:13:09.05733731Z", + "running": true + } +] diff --git a/evidence/persistence-final.txt b/evidence/persistence-final.txt new file mode 100644 index 00000000..67bad796 --- /dev/null +++ b/evidence/persistence-final.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8084" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/reconcile-study.json b/evidence/reconcile-study.json new file mode 100644 index 00000000..eb6e0ab3 --- /dev/null +++ b/evidence/reconcile-study.json @@ -0,0 +1,515 @@ +{ + "hypothesis": "Durable intent, row locks, grace and live HEAD make reconciliation repeatable and protect fresh/used data", + "started_at": "2026-11-06T15:13:23.135411+00:00", + "database": "research_fc070dd14e38", + "bucket": "research-fc070dd14e38", + "grace_seconds": 2, + "checks": [ + { + "name": "real_db_insert_failure_after_s3_put", + "passed": true + }, + { + "name": "real_s3_connect_failure_is_503", + "passed": true + }, + { + "name": "commit_ack_loss_injected", + "passed": true + }, + { + "name": "uncertain_response_keeps_committed_original", + "passed": true + }, + { + "name": "dry_run_changes_neither_db_nor_s3", + "passed": true + }, + { + "name": "all_reconcile_passes_have_no_errors", + "passed": true + }, + { + "name": "orphan_removed", + "passed": true + }, + { + "name": "missing_marked_failed_metadata_kept", + "passed": true + }, + { + "name": "used_original_kept", + "passed": true + }, + { + "name": "fresh_metadata_and_original_kept", + "passed": true + }, + { + "name": "fresh_orphan_kept", + "passed": true + }, + { + "name": "old_intent_fresh_object_kept", + "passed": true + }, + { + "name": "active_writer_object_kept", + "passed": true + }, + { + "name": "committed_original_survives_reconciliation", + "passed": true + }, + { + "name": "repeat_has_no_duplicate_mutations", + "passed": true + }, + { + "name": "s3_delete_failure_observed", + "passed": true + }, + { + "name": "dry_run_after_failed_apply_still_read_only", + "passed": true + }, + { + "name": "failed_cleanup_recovers", + "passed": true + }, + { + "name": "real_db_delete_failure_after_s3_delete", + "passed": true + }, + { + "name": "dry_run_does_not_resume_partial_delete", + "passed": true + }, + { + "name": "partial_delete_resumes_only_on_apply", + "passed": true + }, + { + "name": "scan_has_no_errors_10_0", + "passed": true + }, + { + "name": "scan_has_no_errors_10_1", + "passed": true + }, + { + "name": "scan_has_no_errors_10_2", + "passed": true + }, + { + "name": "scan_has_no_errors_100_0", + "passed": true + }, + { + "name": "scan_has_no_errors_100_1", + "passed": true + }, + { + "name": "scan_has_no_errors_100_2", + "passed": true + }, + { + "name": "scan_has_no_errors_1000_0", + "passed": true + }, + { + "name": "scan_has_no_errors_1000_1", + "passed": true + }, + { + "name": "scan_has_no_errors_1000_2", + "passed": true + } + ], + "scan_cost": [ + { + "seeded_intact": 10, + "repeat": 1, + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 0, + "objects_scanned": 13, + "metadata_scanned": 13, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.023085 + }, + { + "seeded_intact": 10, + "repeat": 2, + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 0, + "objects_scanned": 13, + "metadata_scanned": 13, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.018874 + }, + { + "seeded_intact": 10, + "repeat": 3, + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 0, + "objects_scanned": 13, + "metadata_scanned": 13, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.025435 + }, + { + "seeded_intact": 100, + "repeat": 1, + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 0, + "objects_scanned": 103, + "metadata_scanned": 103, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.189366 + }, + { + "seeded_intact": 100, + "repeat": 2, + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 0, + "objects_scanned": 103, + "metadata_scanned": 103, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.146632 + }, + { + "seeded_intact": 100, + "repeat": 3, + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 0, + "objects_scanned": 103, + "metadata_scanned": 103, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.170581 + }, + { + "seeded_intact": 1000, + "repeat": 1, + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 0, + "objects_scanned": 1003, + "metadata_scanned": 1003, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 1.559015 + }, + { + "seeded_intact": 1000, + "repeat": 2, + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 0, + "objects_scanned": 1003, + "metadata_scanned": 1003, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 1.868045 + }, + { + "seeded_intact": 1000, + "repeat": 3, + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 0, + "objects_scanned": 1003, + "metadata_scanned": 1003, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 1.581705 + } + ], + "source": { + "src/messenger/__init__.py": "9a72a49106367ca514cc6147d0e727e62c4d33fefd61572d209a5c9494551a2c", + "src/messenger/__main__.py": "ba9a59ad44ee8884cd65cf88d5ea15cbd7051b10f7c729043ddd44ca2f34defe", + "src/messenger/app.py": "cb708a0f2ccd99ff421883ec318ccba5cecc0c228ec2155ee00a64aa8ebfac4d", + "src/messenger/attachments.py": "0d14697afb4064a530f1ca99f59460879ef81af5b7d49af9e5d31661fa11ba28", + "src/messenger/config.py": "e4d4aaf674ede8b5abdc361f77c7793ee94d8d7856030c28834eb7c95291b193", + "src/messenger/domain.py": "33086cb229083ecee3bcee09f143b4bc09e48896a0c497bd9902e738e2e6b1af", + "src/messenger/migrate.py": "eb5cad7ac6abe38dd40b2db307d298b297abe9543b3928c36bab8ef06283664e", + "src/messenger/models.py": "c43769d2d4ba0369129a86bf80364a626ab91442f9b9020e6bc904e363a39665", + "src/messenger/postgres.py": "8a9c0cd275bbb04604d94c1a0155176e9f0348d1e05abe2120c74b0b83f36f6c", + "src/messenger/reconcile.py": "792e21f50e777458df367069694e5ff751817fff4dbec4276e4d73e9c64a2fd7", + "src/messenger/storage.py": "70b0fb157b285b674a7388a2bfe5c5a7feb8149a9266ac7ad035961d769008a5", + "src/messenger/storage_bootstrap.py": "b29a2a1f8864ec3425a7499b1752688f69d8f402f3881957e790ed4522d35589" + }, + "dry_run": { + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 1, + "intents_scanned": 4, + "objects_scanned": 8, + "metadata_scanned": 3, + "old_intents": 2, + "orphan_objects": 1, + "missing_objects": 1, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 5, + "errors": 0, + "duration_seconds": 0.019806 + }, + "apply": { + "mode": "apply", + "grace_seconds": 2, + "page_size": 1, + "intents_scanned": 4, + "objects_scanned": 7, + "metadata_scanned": 3, + "old_intents": 2, + "orphan_objects": 1, + "missing_objects": 1, + "deleted_objects": 3, + "deleted_intents": 2, + "marked_failed": 1, + "fresh_objects_skipped": 5, + "errors": 0, + "duration_seconds": 0.016948 + }, + "repeat": { + "mode": "apply", + "grace_seconds": 2, + "page_size": 1, + "intents_scanned": 2, + "objects_scanned": 6, + "metadata_scanned": 2, + "old_intents": 0, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 5, + "errors": 0, + "duration_seconds": 0.008812 + }, + "failed_cleanup": { + "mode": "apply", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 2, + "objects_scanned": 6, + "metadata_scanned": 3, + "old_intents": 2, + "orphan_objects": 1, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 3, + "duration_seconds": 0.013409 + }, + "recovery_dry": { + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 2, + "objects_scanned": 6, + "metadata_scanned": 3, + "old_intents": 2, + "orphan_objects": 1, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.011906 + }, + "recovery_apply": { + "mode": "apply", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 2, + "objects_scanned": 4, + "metadata_scanned": 3, + "old_intents": 2, + "orphan_objects": 1, + "missing_objects": 0, + "deleted_objects": 3, + "deleted_intents": 2, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.013727 + }, + "partial_delete_dry": { + "mode": "dry-run", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 1, + "objects_scanned": 3, + "metadata_scanned": 3, + "old_intents": 1, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.018603 + }, + "partial_delete_apply": { + "mode": "apply", + "grace_seconds": 2, + "page_size": 100, + "intents_scanned": 1, + "objects_scanned": 3, + "metadata_scanned": 3, + "old_intents": 1, + "orphan_objects": 0, + "missing_objects": 0, + "deleted_objects": 1, + "deleted_intents": 1, + "marked_failed": 0, + "fresh_objects_skipped": 0, + "errors": 0, + "duration_seconds": 0.012837 + }, + "limitations": [ + "One local host and warm services; scan measures HEAD/SQL round trips, not production SLA", + "Missing originals are marked failed; reconciliation cannot recreate lost bytes without backup", + "Valid unattached ready attachments are retained; only abandoned intents and untracked old original objects are removed", + "No atomic transaction spans S3 and PostgreSQL; retries plus durable intent recover the selected windows", + "Cost dataset is in disposable isolated DB/bucket, includes a few preserved fault fixtures beyond the seeded intact count" + ], + "temporary_resources_deleted": true, + "host_manifest": { + "system": "Darwin", + "machine": "arm64", + "release": "24.3.0", + "git_revision": "619155f1e01457ee02da561200a571cbd5674055", + "docker": { + "version": "29.5.2", + "cpus": 4, + "memory_bytes": 8307167232 + }, + "containers": [ + { + "service": "api", + "image": "sha256:1beedea111bd3f8f5de5a31b7a67b2013ffc9c0271b51313bb6968bcaea96f6e", + "cpu_nano": 500000000, + "memory_limit": 536870912 + }, + { + "service": "api", + "image": "sha256:1beedea111bd3f8f5de5a31b7a67b2013ffc9c0271b51313bb6968bcaea96f6e", + "cpu_nano": 500000000, + "memory_limit": 536870912 + }, + { + "service": "cache", + "image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773", + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "service": "lb", + "image": "sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0", + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "service": "postgres", + "image": "sha256:742f40ea20b9ff2ff31db5458d127452988a2164df9e17441e191f3b72252193", + "cpu_nano": 1000000000, + "memory_limit": 805306368 + }, + { + "service": "s3", + "image": "sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e", + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "service": "storage", + "image": "sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e", + "cpu_nano": 1000000000, + "memory_limit": 536870912 + } + ] + } +} diff --git a/evidence/reconcile-study.raw.txt b/evidence/reconcile-study.raw.txt new file mode 100644 index 00000000..24e0f547 --- /dev/null +++ b/evidence/reconcile-study.raw.txt @@ -0,0 +1,8 @@ +Applied 001_initial.sql +Applied 002_idempotency.sql +Applied 003_unicode_storage.sql +Applied 004_attachments.sql +Applied 005_attachment_only_message.sql +RESULT {"hypothesis": "Durable intent, row locks, grace and live HEAD make reconciliation repeatable and protect fresh/used data", "started_at": "2026-11-06T15:13:23.135411+00:00", "database": "research_fc070dd14e38", "bucket": "research-fc070dd14e38", "grace_seconds": 2, "checks": [{"name": "real_db_insert_failure_after_s3_put", "passed": true}, {"name": "real_s3_connect_failure_is_503", "passed": true}, {"name": "commit_ack_loss_injected", "passed": true}, {"name": "uncertain_response_keeps_committed_original", "passed": true}, {"name": "dry_run_changes_neither_db_nor_s3", "passed": true}, {"name": "all_reconcile_passes_have_no_errors", "passed": true}, {"name": "orphan_removed", "passed": true}, {"name": "missing_marked_failed_metadata_kept", "passed": true}, {"name": "used_original_kept", "passed": true}, {"name": "fresh_metadata_and_original_kept", "passed": true}, {"name": "fresh_orphan_kept", "passed": true}, {"name": "old_intent_fresh_object_kept", "passed": true}, {"name": "active_writer_object_kept", "passed": true}, {"name": "committed_original_survives_reconciliation", "passed": true}, {"name": "repeat_has_no_duplicate_mutations", "passed": true}, {"name": "s3_delete_failure_observed", "passed": true}, {"name": "dry_run_after_failed_apply_still_read_only", "passed": true}, {"name": "failed_cleanup_recovers", "passed": true}, {"name": "real_db_delete_failure_after_s3_delete", "passed": true}, {"name": "dry_run_does_not_resume_partial_delete", "passed": true}, {"name": "partial_delete_resumes_only_on_apply", "passed": true}, {"name": "scan_has_no_errors_10_0", "passed": true}, {"name": "scan_has_no_errors_10_1", "passed": true}, {"name": "scan_has_no_errors_10_2", "passed": true}, {"name": "scan_has_no_errors_100_0", "passed": true}, {"name": "scan_has_no_errors_100_1", "passed": true}, {"name": "scan_has_no_errors_100_2", "passed": true}, {"name": "scan_has_no_errors_1000_0", "passed": true}, {"name": "scan_has_no_errors_1000_1", "passed": true}, {"name": "scan_has_no_errors_1000_2", "passed": true}], "scan_cost": [{"seeded_intact": 10, "repeat": 1, "mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 0, "objects_scanned": 13, "metadata_scanned": 13, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.023085}, {"seeded_intact": 10, "repeat": 2, "mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 0, "objects_scanned": 13, "metadata_scanned": 13, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.018874}, {"seeded_intact": 10, "repeat": 3, "mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 0, "objects_scanned": 13, "metadata_scanned": 13, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.025435}, {"seeded_intact": 100, "repeat": 1, "mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 0, "objects_scanned": 103, "metadata_scanned": 103, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.189366}, {"seeded_intact": 100, "repeat": 2, "mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 0, "objects_scanned": 103, "metadata_scanned": 103, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.146632}, {"seeded_intact": 100, "repeat": 3, "mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 0, "objects_scanned": 103, "metadata_scanned": 103, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.170581}, {"seeded_intact": 1000, "repeat": 1, "mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 0, "objects_scanned": 1003, "metadata_scanned": 1003, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 1.559015}, {"seeded_intact": 1000, "repeat": 2, "mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 0, "objects_scanned": 1003, "metadata_scanned": 1003, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 1.868045}, {"seeded_intact": 1000, "repeat": 3, "mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 0, "objects_scanned": 1003, "metadata_scanned": 1003, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 1.581705}], "source": {"src/messenger/__init__.py": "9a72a49106367ca514cc6147d0e727e62c4d33fefd61572d209a5c9494551a2c", "src/messenger/__main__.py": "ba9a59ad44ee8884cd65cf88d5ea15cbd7051b10f7c729043ddd44ca2f34defe", "src/messenger/app.py": "cb708a0f2ccd99ff421883ec318ccba5cecc0c228ec2155ee00a64aa8ebfac4d", "src/messenger/attachments.py": "0d14697afb4064a530f1ca99f59460879ef81af5b7d49af9e5d31661fa11ba28", "src/messenger/config.py": "e4d4aaf674ede8b5abdc361f77c7793ee94d8d7856030c28834eb7c95291b193", "src/messenger/domain.py": "33086cb229083ecee3bcee09f143b4bc09e48896a0c497bd9902e738e2e6b1af", "src/messenger/migrate.py": "eb5cad7ac6abe38dd40b2db307d298b297abe9543b3928c36bab8ef06283664e", "src/messenger/models.py": "c43769d2d4ba0369129a86bf80364a626ab91442f9b9020e6bc904e363a39665", "src/messenger/postgres.py": "8a9c0cd275bbb04604d94c1a0155176e9f0348d1e05abe2120c74b0b83f36f6c", "src/messenger/reconcile.py": "792e21f50e777458df367069694e5ff751817fff4dbec4276e4d73e9c64a2fd7", "src/messenger/storage.py": "70b0fb157b285b674a7388a2bfe5c5a7feb8149a9266ac7ad035961d769008a5", "src/messenger/storage_bootstrap.py": "b29a2a1f8864ec3425a7499b1752688f69d8f402f3881957e790ed4522d35589"}, "dry_run": {"mode": "dry-run", "grace_seconds": 2, "page_size": 1, "intents_scanned": 4, "objects_scanned": 8, "metadata_scanned": 3, "old_intents": 2, "orphan_objects": 1, "missing_objects": 1, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 5, "errors": 0, "duration_seconds": 0.019806}, "apply": {"mode": "apply", "grace_seconds": 2, "page_size": 1, "intents_scanned": 4, "objects_scanned": 7, "metadata_scanned": 3, "old_intents": 2, "orphan_objects": 1, "missing_objects": 1, "deleted_objects": 3, "deleted_intents": 2, "marked_failed": 1, "fresh_objects_skipped": 5, "errors": 0, "duration_seconds": 0.016948}, "repeat": {"mode": "apply", "grace_seconds": 2, "page_size": 1, "intents_scanned": 2, "objects_scanned": 6, "metadata_scanned": 2, "old_intents": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 5, "errors": 0, "duration_seconds": 0.008812}, "failed_cleanup": {"mode": "apply", "grace_seconds": 2, "page_size": 100, "intents_scanned": 2, "objects_scanned": 6, "metadata_scanned": 3, "old_intents": 2, "orphan_objects": 1, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 3, "duration_seconds": 0.013409}, "recovery_dry": {"mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 2, "objects_scanned": 6, "metadata_scanned": 3, "old_intents": 2, "orphan_objects": 1, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.011906}, "recovery_apply": {"mode": "apply", "grace_seconds": 2, "page_size": 100, "intents_scanned": 2, "objects_scanned": 4, "metadata_scanned": 3, "old_intents": 2, "orphan_objects": 1, "missing_objects": 0, "deleted_objects": 3, "deleted_intents": 2, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.013727}, "partial_delete_dry": {"mode": "dry-run", "grace_seconds": 2, "page_size": 100, "intents_scanned": 1, "objects_scanned": 3, "metadata_scanned": 3, "old_intents": 1, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_intents": 0, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.018603}, "partial_delete_apply": {"mode": "apply", "grace_seconds": 2, "page_size": 100, "intents_scanned": 1, "objects_scanned": 3, "metadata_scanned": 3, "old_intents": 1, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 1, "deleted_intents": 1, "marked_failed": 0, "fresh_objects_skipped": 0, "errors": 0, "duration_seconds": 0.012837}, "limitations": ["One local host and warm services; scan measures HEAD/SQL round trips, not production SLA", "Missing originals are marked failed; reconciliation cannot recreate lost bytes without backup", "Valid unattached ready attachments are retained; only abandoned intents and untracked old original objects are removed", "No atomic transaction spans S3 and PostgreSQL; retries plus durable intent recover the selected windows", "Cost dataset is in disposable isolated DB/bucket, includes a few preserved fault fixtures beyond the seeded intact count"], "temporary_resources_deleted": true} + Container mtusi-vasilev-lab4-research-run-cd5af1b396b4 Creating + Container mtusi-vasilev-lab4-research-run-cd5af1b396b4 Created diff --git a/evidence/reconcile-study.summary.md b/evidence/reconcile-study.summary.md new file mode 100644 index 00000000..c9e166ed --- /dev/null +++ b/evidence/reconcile-study.summary.md @@ -0,0 +1,60 @@ +# Исследование согласованности PostgreSQL/S3 + +Дата начала (UTC): 2026-11-06T15:13:23.135411+00:00. + +Успешных проверок: 30/30. + +| Добавлено intact | Просмотрено metadata | Медиана обхода, с | +|---:|---:|---:| +| 10 | 13 | 0.023085 | +| 100 | 103 | 0.170581 | +| 1000 | 1003 | 1.581705 | + +По три dry-run на размер, page_size100, прогретый локальный стенд. +Дополнительные три intact fault fixtures входят в фактический scan. + +Первый apply: удалено intent 2, операций удаления объекта 3, missing помечено failed 1. +Счётчик удаления объекта — число успешных идемпотентных DELETE, включая отсутствие bytes у intent. + +Временные ресурсы удалены: True. + +## Проверенные инварианты + +- PASS: real_db_insert_failure_after_s3_put +- PASS: real_s3_connect_failure_is_503 +- PASS: commit_ack_loss_injected +- PASS: uncertain_response_keeps_committed_original +- PASS: dry_run_changes_neither_db_nor_s3 +- PASS: all_reconcile_passes_have_no_errors +- PASS: orphan_removed +- PASS: missing_marked_failed_metadata_kept +- PASS: used_original_kept +- PASS: fresh_metadata_and_original_kept +- PASS: fresh_orphan_kept +- PASS: old_intent_fresh_object_kept +- PASS: active_writer_object_kept +- PASS: committed_original_survives_reconciliation +- PASS: repeat_has_no_duplicate_mutations +- PASS: s3_delete_failure_observed +- PASS: dry_run_after_failed_apply_still_read_only +- PASS: failed_cleanup_recovers +- PASS: real_db_delete_failure_after_s3_delete +- PASS: dry_run_does_not_resume_partial_delete +- PASS: partial_delete_resumes_only_on_apply +- PASS: scan_has_no_errors_10_0 +- PASS: scan_has_no_errors_10_1 +- PASS: scan_has_no_errors_10_2 +- PASS: scan_has_no_errors_100_0 +- PASS: scan_has_no_errors_100_1 +- PASS: scan_has_no_errors_100_2 +- PASS: scan_has_no_errors_1000_0 +- PASS: scan_has_no_errors_1000_1 +- PASS: scan_has_no_errors_1000_2 + +## Ограничения + +- One local host and warm services; scan measures HEAD/SQL round trips, not production SLA +- Missing originals are marked failed; reconciliation cannot recreate lost bytes without backup +- Valid unattached ready attachments are retained; only abandoned intents and untracked old original objects are removed +- No atomic transaction spans S3 and PostgreSQL; retries plus durable intent recover the selected windows +- Cost dataset is in disposable isolated DB/bucket, includes a few preserved fault fixtures beyond the seeded intact count diff --git a/evidence/smoke-final.txt b/evidence/smoke-final.txt new file mode 100644 index 00000000..75f85629 --- /dev/null +++ b/evidence/smoke-final.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:8084" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/storage-checks.json b/evidence/storage-checks.json new file mode 100644 index 00000000..44b51f36 --- /dev/null +++ b/evidence/storage-checks.json @@ -0,0 +1,142 @@ +{ + "checks": [ + { + "name": "unrelated_bucket_denied", + "passed": true + }, + { + "name": "backend_unreachable", + "passed": true + }, + { + "name": "gateway_filer_unreachable", + "passed": true + }, + { + "name": "no_admin_secrets", + "passed": true + }, + { + "name": "anonymous_original_denied", + "passed": true + }, + { + "name": "bucket_bootstrap_repeatable", + "passed": true + }, + { + "name": "s3_down_ready_503", + "passed": true + }, + { + "name": "s3_down_content_503", + "passed": true + }, + { + "name": "s3_restart_original_unchanged", + "passed": true + }, + { + "name": "paused_pg_ready_503_with_client_deadline", + "passed": true + }, + { + "name": "paused_pg_auth_503_with_client_deadline", + "passed": true + }, + { + "name": "old_session_recovers_after_unpause", + "passed": true + }, + { + "name": "missing_required_table_ready_503", + "passed": true + }, + { + "name": "schema_restore_ready_200", + "passed": true + }, + { + "name": "lb_readmits_recovered_api", + "passed": true + } + ], + "faults": [ + { + "service": "s3", + "ready": { + "status": 503, + "elapsed": 0.010417848010547459, + "json": { + "error": { + "code": "unavailable", + "message": "Хранилище недоступно", + "request_id": "eacb2683-a734-450a-ab88-873976aaf1a2" + } + }, + "sha256": "6c93ed3dd714ac1d6e6ea952a436cb1ed67fe5be9e1401ded99c66ef9b50120e" + }, + "download": { + "status": 503, + "elapsed": 0.012548128957860172, + "json": { + "error": { + "code": "unavailable", + "message": "Хранилище файлов временно недоступно", + "request_id": "19f68dc9-a1c5-41ba-b8c1-7764f3c715f1" + } + }, + "sha256": "be4a4560dfe7e74fb4c243b760e103c31cd16be4195c9a5c852e2ff1ed2b1ef5" + } + }, + { + "service": "paused_postgres", + "ready": { + "status": 503, + "elapsed": 3.0085260499618016, + "json": { + "error": { + "code": "unavailable", + "message": "Хранилище недоступно", + "request_id": "400bdaa7-e2e5-43e3-873e-fdfd8a5da545" + } + }, + "sha256": "82ff75670fbb92a7d5852887154f6baf684f2782a6aa145aa406fae8fc16e5a0" + }, + "me": { + "status": 503, + "elapsed": 2.0085200910107233, + "json": { + "error": { + "code": "unavailable", + "message": "Хранилище временно недоступно", + "request_id": "bfd8f577-3cd7-4d8e-981a-04537fb47c7d" + } + }, + "sha256": "38aa5fc7df41d74f4c3cde71a84e96805568e9a469d2be9ff6340fb578af0268" + } + }, + { + "service": "missing_message_requests", + "ready": { + "status": 503, + "elapsed": 0.08961376297520474, + "json": { + "error": { + "code": "unavailable", + "message": "Хранилище недоступно", + "request_id": "3e778ff4-c1fd-485d-973a-7fa367556469" + } + }, + "sha256": "6f1f6ba3ac28b6ac691d4dc7da3e4102cc5169bc01040701210a4200b25dcd43" + } + } + ], + "security": { + "anonymous_status": 403, + "unrelated_bucket_denied": true, + "backend_unreachable": true, + "gateway_filer_unreachable": true, + "no_admin_secrets": true + } +} diff --git a/evidence/storage-checks.txt b/evidence/storage-checks.txt new file mode 100644 index 00000000..24d80ee5 --- /dev/null +++ b/evidence/storage-checks.txt @@ -0,0 +1 @@ +{"passed": 15} diff --git a/evidence/tests-final.txt b/evidence/tests-final.txt new file mode 100644 index 00000000..a4609016 --- /dev/null +++ b/evidence/tests-final.txt @@ -0,0 +1,3 @@ +........................................................................ [ 64%] +........................................ [100%] +112 passed in 34.16s diff --git a/evidence/upload-memory-final.json b/evidence/upload-memory-final.json new file mode 100644 index 00000000..2b192519 --- /dev/null +++ b/evidence/upload-memory-final.json @@ -0,0 +1,285 @@ +{ + "concurrency": 10, + "api_replicas": 2, + "sampled_api_and_lb_containers": [ + "fb116950e109b2c4f68378be994e4fa685b1e32d5df70e4f4ba88b9aaac11734", + "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9", + "52284c66754124d6476f53fdb606b5c60548d12faa9ead291572e25b054c7998" + ], + "api_restarted_before_trial": true, + "runs": [ + { + "image_dimensions": [ + 512, + 512 + ], + "size_bytes": 788100, + "before": { + "fb116950e109b2c4f68378be994e4fa685b1e32d5df70e4f4ba88b9aaac11734": { + "current": 79519744, + "peak": 122773504, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 139516 kB" + }, + "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9": { + "current": 79613952, + "peak": 122679296, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 139728 kB" + }, + "52284c66754124d6476f53fdb606b5c60548d12faa9ead291572e25b054c7998": { + "current": 11587584, + "peak": 13217792, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 13704 kB" + } + }, + "after": { + "fb116950e109b2c4f68378be994e4fa685b1e32d5df70e4f4ba88b9aaac11734": { + "current": 83845120, + "peak": 122773504, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 139516 kB" + }, + "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9": { + "current": 84389888, + "peak": 122679296, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 139728 kB" + }, + "52284c66754124d6476f53fdb606b5c60548d12faa9ead291572e25b054c7998": { + "current": 11927552, + "peak": 14299136, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 13704 kB" + } + }, + "responses": [ + { + "status": 201, + "elapsed": 0.10770674998639151, + "attachment_id": "4e85ee5d-1b9b-49de-b78a-df0a2b025927" + }, + { + "status": 201, + "elapsed": 0.09620504203485325, + "attachment_id": "848f0e91-208e-4272-a80c-6cdc3badfba2" + }, + { + "status": 429, + "elapsed": 0.049307000008411705 + }, + { + "status": 429, + "elapsed": 0.0555269590113312 + }, + { + "status": 429, + "elapsed": 0.056070291029755026 + }, + { + "status": 429, + "elapsed": 0.04627674998482689 + }, + { + "status": 429, + "elapsed": 0.059664790984243155 + }, + { + "status": 429, + "elapsed": 0.05963275005342439 + }, + { + "status": 429, + "elapsed": 0.05560016701929271 + }, + { + "status": 429, + "elapsed": 0.0469216660130769 + } + ], + "ready_after": true + }, + { + "image_dimensions": [ + 1700, + 1700 + ], + "size_bytes": 8683594, + "before": { + "fb116950e109b2c4f68378be994e4fa685b1e32d5df70e4f4ba88b9aaac11734": { + "current": 83865600, + "peak": 122773504, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 139516 kB" + }, + "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9": { + "current": 84439040, + "peak": 122679296, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 139728 kB" + }, + "52284c66754124d6476f53fdb606b5c60548d12faa9ead291572e25b054c7998": { + "current": 11866112, + "peak": 14299136, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 13704 kB" + } + }, + "after": { + "fb116950e109b2c4f68378be994e4fa685b1e32d5df70e4f4ba88b9aaac11734": { + "current": 86425600, + "peak": 122773504, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 139516 kB" + }, + "255f6fa790fd2b2f5f308846391634c9ff7d308e1d6064085fda31f364b7a6d9": { + "current": 86462464, + "peak": 122679296, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 139728 kB" + }, + "52284c66754124d6476f53fdb606b5c60548d12faa9ead291572e25b054c7998": { + "current": 11878400, + "peak": 19656704, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "process_hwm": "VmHWM:\t 13704 kB" + } + }, + "responses": [ + { + "status": 429, + "elapsed": 0.31627862504683435 + }, + { + "status": 429, + "elapsed": 0.32798779202857986 + }, + { + "status": 201, + "elapsed": 0.8054331250023097, + "attachment_id": "f0a451ee-d667-4f29-9409-4304ec59a156" + }, + { + "status": 429, + "elapsed": 0.32699191698338836 + }, + { + "status": 429, + "elapsed": 0.32789891597349197 + }, + { + "status": 429, + "elapsed": 0.3272826250176877 + }, + { + "status": 201, + "elapsed": 0.8380368750076741, + "attachment_id": "e35b8424-19b3-4ef5-8e37-23bdc23f1ccf" + }, + { + "status": 429, + "elapsed": 0.32766412501223385 + }, + { + "status": 429, + "elapsed": 0.3256588340154849 + }, + { + "status": 429, + "elapsed": 0.19960300001548603 + } + ], + "ready_after": true + } + ], + "limitations": [ + "Kernel cgroup peak and process VmHWM are lifetime peaks; before/after reported, not attributed solely to upload buffers", + "Fast rejection 429 is intentional backpressure, not a successful upload", + "Local two-replica stand; not a throughput benchmark" + ] +} diff --git a/evidence/upload-memory-final.txt b/evidence/upload-memory-final.txt new file mode 100644 index 00000000..a686aff4 --- /dev/null +++ b/evidence/upload-memory-final.txt @@ -0,0 +1,5 @@ + Container mtusi-vasilev-lab4-api-2 Restarting + Container mtusi-vasilev-lab4-api-1 Restarting + Container mtusi-vasilev-lab4-api-1 Started + Container mtusi-vasilev-lab4-api-2 Started +{"runs": [{"size_bytes": 788100, "statuses": [201, 201, 429, 429, 429, 429, 429, 429, 429, 429]}, {"size_bytes": 8683594, "statuses": [429, 429, 201, 429, 429, 429, 201, 429, 429, 429]}]} -- 2.54.0 From de36c967cc73199e1e3a9a2d5831aa2274e89a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 19:08:18 +0300 Subject: [PATCH 10/11] fix: bound full image validation and cover progressive JPEG and APNG --- REPORT.md | 8 +- research/README.md | 2 +- src/messenger/app.py | 19 +++ src/messenger/attachments.py | 108 ++++++------ src/messenger/image_validation.py | 146 +++++++++++++++++ student-tests/test_image_edges.py | 199 +++++++++++++++++++++++ student-tests/test_json_encoding.py | 140 ++++++++++++++++ student-tests/test_validation_process.py | 57 +++++++ 8 files changed, 617 insertions(+), 62 deletions(-) create mode 100644 src/messenger/image_validation.py create mode 100644 student-tests/test_image_edges.py create mode 100644 student-tests/test_json_encoding.py create mode 100644 student-tests/test_validation_process.py diff --git a/REPORT.md b/REPORT.md index d970a0c3..5d8319e4 100644 --- a/REPORT.md +++ b/REPORT.md @@ -70,8 +70,8 @@ Readiness проверяет обязательные таблицы и стол 1. Проверяются сессия, чат, filename, MIME. 2. Тело читается частями в spool: первые 256 KiB в RAM, далее временный файл в ограниченном tmpfs. Максимум 10 MiB, таймаут чтения 15 секунд. -3. По одному upload slot на API, лишняя работа получает 429/Retry-After. Размеры изображения проверяются до decode: максимум 40 млн пикселей. -4. Pillow verify и повторный полный load проверяют PNG/JPEG. Для JPEG djpeg -strict дополнительно отклоняет недодекодируемые изображения, которые Pillow может принять с предупреждением; timeout5s, maxmemory192m. +3. По одному upload slot на API, лишняя работа получает 429/Retry-After. Сжатое содержимое передаётся короткоживущему процессу ограниченной копией до10MiB; второй10MiB-файл в tmpfs не создаётся. API не сохраняет декодированные пиксели между запросами. +4. Child проверяет MIME и размеры до decode. Для PNG/APNG Pillow проверяет CRC и декодирует каждый кадр по очереди; общий бюджет всех canvas, включая default image, не более40млн пикселей. Raw acTL/fcTL сверяются, чтобы нулевое/поддельное число кадров не скрывало повреждение. Для JPEG после проверки метаданных строгий djpeg полностью декодирует пиксели построчно в/dev/null, с nativebudget384MiB и timeout4s. Второй полный Pillow.load JPEG не выполняется: он избыточен и у progressiveCMYK мог одновременно держать коэффициенты320MB и canvas160MB. Child Linux address-space limit512MiB, CPU4s; parent walltimeout5s завершает всю process group, включая native grandchildren. Ошибка/падение/таймаут декодера даёт415, превышение bytes/pixel budget413; недоступность запуска процесса503. 5. После полной валидации фиксируется durable upload_intent. Под его row lock выполняются S3 PUT и транзакция ready metadata + удаление intent. 6. Только успешное завершение COMMIT даёт 201 ready с variants=[] и error_code=null. SHA-256/size/MIME относятся к исходным bytes. @@ -98,7 +98,7 @@ docker compose run --rm --no-deps research Стоимость обхода измеряется на10/100/1000 добавленных intact attachments, по3 dry-run, page_size100. Перед измерением все добавленные объекты выдерживаются за границу grace, чтобы fresh-skip не менял профиль работы. Точные размеры с учётом сохранённых fault fixtures находятся в JSON. Измеряются время и количество просмотренных объектов/метаданных; это SQL+S3 обход, не HTTP throughput. -После сохранённых измерений внесена отдельная правка `domain.boundary`: строгая RFC3339Z-дата с календарной проверкой и целочисленная версия cursor вместо расширений Python ISO parser. Этот путь не используется исследованием сверки или image upload. Исходные hashes в raw JSON оставлены как снимок измеренного runtime, не переписаны под новый файл. Финальные HTTP/unit регрессии проверяют эту правку отдельно. +После сохранённых измерений отдельно исправлены строгий RFC3339Z cursor, UTF-8 JSON guard и validation edge cases progressive JPEG/APNG. Сам reconciler, S3 adapter и схема этого эксперимента не изменились. Исходные hashes в raw JSON оставлены как снимок измеренного runtime, не переписаны под новые файлы. Финальные HTTP/unit регрессии и отдельные image-validation receipts проверяют последние изменения; они не выдаются за повторный замер исследования сверки. ### Результат исследовательского прогона @@ -148,7 +148,7 @@ Before/after peaks не выросли: начальная загрузка пр - Нет распределённого COMMIT через PostgreSQL и S3: остатки допустимы до grace/apply. Уникальные immutable server keys и intent делают выбранную очистку безопасной для штатного API; конкурентное ручное изменение администратором после HEAD находится вне этой гарантии. - Очереди обработки изображений нет: ЛР4 синхронная, thumbnail/crop/optimized относятся к ЛР5. - HTTP допустим для локального изолированного стенда с синтетическими данными. Удалённая передача секретов требует TLS. -- Native decoder/libjpeg ограничен memory/time/cgroup; полная корректность каждого экзотического изображения зависит от библиотечного декодера. Специально проверены конкретные повреждения из замечаний преподавателя. +- Native decoder/libjpeg и Pillow ограничены memory/time/cgroup; корректность декодирования зависит от закреплённых библиотек. Проверены конкретные повреждения из замечаний,36M RGB progressive,40M CMYK progressive и последовательность APNG40M→CMYK40M→RGB36M→обычныйPNG при512MiB API безOOM. Кадры APNG валидируются все, а thumbnail следующей ЛР5 строится по первому кадру; такая политика не изменяет original bytes. - Client-side deadline ограничивает ожидание одного обращения к PostgreSQL, а не весь HTTP-путь; при неопределённом результате записи ключ идемпотентности защищает повтор сообщения. Upload не имеет такого ключа и не должен автоматически повторяться как будто это тот же объект. ## Источники diff --git a/research/README.md b/research/README.md index 2806242a..6956e2ad 100644 --- a/research/README.md +++ b/research/README.md @@ -7,7 +7,7 @@ Durable intent до S3 PUT, row lock на время PUT/финализации ## Протокол upload 1. Поток HTTP складывается в ограниченный spool: 256 KiB в RAM, затем временный файл на ограниченном tmpfs; максимум 10 MiB. -2. После полного приёма выполняется полная проверка PNG/JPEG. Пиксельный предел применяется до decode. Для JPEG строгий djpeg отвергает предупреждения о повреждении, которые Pillow иногда допускает. +2. После полного приёма короткоживущий ограниченный child проверяет PNG/JPEG. Пиксельный предел применяется до decode: для APNG это сумма canvas всех кадров≤40M. JPEG полностью декодирует строгий djpeg384MiB построчно, PNG/APNG декодирует Pillow; decoded buffers не остаются в API. Последние edge исправлены после исходного измерения сверки, его raw snapshot не переписывался. 3. Создаётся и фиксируется upload_intent с уникальным server key. 4. Новая транзакция блокирует intent, выполняет S3 PUT и создаёт ready attachment, удаляет intent, COMMIT. 5. Только после успешной фиксации возвращается 201. При неопределённом исходе COMMIT объект немедленно не удаляется. diff --git a/src/messenger/app.py b/src/messenger/app.py index fea1a304..89d34912 100644 --- a/src/messenger/app.py +++ b/src/messenger/app.py @@ -67,6 +67,23 @@ def error(request, status, message, headers=None): ) +async def invalid_json_encoding(request: Request): + path = request.url.path + if request.method != "POST" or not ( + path in {"/api/v1/users", "/api/v1/chats"} + or re.fullmatch(r"/api/v1/chats/[^/]+/messages", path) + ): + return False + raw = await request.body() + try: + raw.decode("utf-8") + except UnicodeDecodeError: + return True + # Literal NUL is invalid JSON; it also prevents json.loads(bytes) from + # autodetecting BOM-less UTF-16/32. Escaped JSON \u0000 remains valid. + return b"\x00" in raw + + @app.middleware("http") async def request_log(request: Request, call_next): request.state.request_id = str(uuid4()) @@ -78,6 +95,8 @@ async def request_log(request: Request, call_next): and await request.body() ): response = error(request, 400, "Вход не принимает тело запроса") + elif await invalid_json_encoding(request): + response = error(request, 400, "JSON должен быть закодирован в UTF-8") else: response = await call_next(request) except Exception: diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py index ecae250c..cb6fcff4 100644 --- a/src/messenger/attachments.py +++ b/src/messenger/attachments.py @@ -1,21 +1,19 @@ """Synchronous attachment intake: bounded spool, full image validation, durable intent.""" import hashlib +import json import os +import signal import subprocess -import tempfile +import sys import unicodedata +from pathlib import Path from uuid import uuid4 -from PIL import Image, UnidentifiedImageError - from .domain import Failure, timestamp +from .image_validation import MAX_BYTES from .postgres import public -MAX_BYTES = 10 * 1024 * 1024 -MAX_PIXELS = 40_000_000 -# Check dimensions before decoding. Pillow's own hard bomb threshold remains active. -Image.MAX_IMAGE_PIXELS = MAX_PIXELS PUBLIC_FIELDS = ( "id", "chat_id", @@ -42,61 +40,57 @@ def filename_valid(name): def validate_image(fileobj, declared_type): if declared_type not in {"image/png", "image/jpeg"}: raise Failure(415, "Ожидается PNG или JPEG") + fileobj.seek(0) + encoded = fileobj.read(MAX_BYTES + 1) + if len(encoded) > MAX_BYTES: + raise Failure(413, "Файл больше 10 MiB") try: - fileobj.seek(0) - with Image.open(fileobj) as image: - width, height = image.size - if width * height > MAX_PIXELS: - raise Failure(413, "Слишком много пикселей") - actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) - if actual != declared_type: - raise Failure(415, "Формат изображения не соответствует MIME") - image.verify() - # verify() alone does not decode IDAT/JPEG scan data. - fileobj.seek(0) - with Image.open(fileobj) as image: - image.load() - if actual == "image/jpeg": - # libjpeg emits a warning but Pillow accepts e.g. a half scan with a - # forged EOI. djpeg's nonzero warning exit rejects that corruption. - fileobj.seek(0) - fallback = None + with subprocess.Popen( + [sys.executable, "-m", "messenger.image_validation", declared_type], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + env={ + "PATH": os.environ.get("PATH", os.defpath), + "PYTHONPATH": str(Path(__file__).resolve().parents[1]), + "PYTHONDONTWRITEBYTECODE": "1", + "LANG": "C.UTF-8", + }, + ) as process: try: - decoder_input = fileobj - try: - fileobj.fileno() - except (AttributeError, OSError): - # Internal callers may supply BytesIO; bound the fallback too. - fallback = tempfile.TemporaryFile() - copied = 0 - while chunk := fileobj.read(64 * 1024): - copied += len(chunk) - if copied > MAX_BYTES: - raise Failure(413, "Файл больше 10 MiB") from None - fallback.write(chunk) - fallback.seek(0) - decoder_input = fallback - decoded = subprocess.run( - ["djpeg", "-strict", "-maxmemory", "192m", "-outfile", "/dev/null"], - stdin=decoder_input, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - timeout=5, - check=False, - ) + output, _ = process.communicate(input=encoded, timeout=5) except subprocess.TimeoutExpired: + # The validator can itself run djpeg: kill its whole process group + # before releasing the upload slot, not only the Python parent. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass # The child may have exited at the timeout boundary. + process.communicate() raise Failure(415, "Изображение не прошло ограниченную проверку") from None - finally: - if fallback is not None: - fallback.close() - if decoded.returncode: - raise Failure(415, "Повреждённое изображение") - fileobj.seek(0) - return actual, width, height - except Image.DecompressionBombError: - raise Failure(413, "Слишком много пикселей") from None - except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + if process.returncode: + raise Failure(415, "Изображение не прошло ограниченную проверку") + except OSError: + raise Failure(503, "Проверка изображения временно недоступна") from None + finally: + del encoded + try: + outcome = json.loads(output) + except (ValueError, UnicodeError): raise Failure(415, "Повреждённое изображение") from None + if not isinstance(outcome, dict): + raise Failure(415, "Изображение не прошло ограниченную проверку") + if "error" in outcome: + status = outcome["error"] + if type(status) is not int or status not in {413, 415}: + status = 415 + raise Failure(status, "Изображение не прошло ограниченную проверку") + values = outcome.get("ok") + if not isinstance(values, list) or len(values) != 3: + raise Failure(415, "Изображение не прошло ограниченную проверку") + fileobj.seek(0) + return tuple(values) def metadata_public(row): diff --git a/src/messenger/image_validation.py b/src/messenger/image_validation.py new file mode 100644 index 00000000..13c69880 --- /dev/null +++ b/src/messenger/image_validation.py @@ -0,0 +1,146 @@ +"""Short-lived bounded pixel validation; no DB, credentials or application imports.""" + +import json +import resource +import subprocess +import sys +from io import BytesIO + +from PIL import Image, UnidentifiedImageError + +from .domain import Failure + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +def inspect_image(fileobj, declared_type): + if declared_type not in {"image/png", "image/jpeg"}: + raise Failure(415, "Ожидается PNG или JPEG") + try: + fileobj.seek(0) + with Image.open(fileobj) as header: + width, height = header.size + if width * height > MAX_PIXELS: + raise Failure(413, "Слишком много пикселей") + actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(header.format) + if actual != declared_type: + raise Failure(415, "Формат изображения не соответствует MIME") + frames = getattr(header, "n_frames", 1) + default_image = bool(getattr(header, "default_image", False)) + if width * height * frames > MAX_PIXELS: + raise Failure(413, "Слишком много пикселей во всех кадрах") + header.verify() + # Drop parsed metadata before the separate strict decoder process starts. + del header + if actual == "image/png": + # verify() checks CRC, but does not compare acTL's frame count with + # every fcTL. Do not let a forged count hide a corrupt later frame. + fileobj.seek(8) + controls = frame_controls = 0 + declared_frames = None + animation_data = False + while True: + chunk_header = fileobj.read(8) + if len(chunk_header) != 8: + raise ValueError("Truncated PNG chunk") + length = int.from_bytes(chunk_header[:4], "big") + kind = chunk_header[4:] + controls += kind == b"acTL" + frame_controls += kind == b"fcTL" + animation_data |= kind == b"fdAT" + if kind == b"acTL": + if length != 8: + raise ValueError("Invalid acTL length") + declared_frames = int.from_bytes(fileobj.read(4), "big") + fileobj.seek(length, 1) # remaining payload plus CRC + else: + fileobj.seek(length + 4, 1) + if kind == b"IEND": + break + if controls: + if ( + controls != 1 + or not declared_frames + or frame_controls != declared_frames + or declared_frames + int(default_image) != frames + ): + raise ValueError("Invalid APNG frame count") + elif frame_controls or animation_data: + raise ValueError("Animation chunks without acTL") + if actual == "image/jpeg": + # Validate scan warnings before allocating Pillow's decoded canvas. + # 384 MiB covers coefficient arrays for <=40M pixels and <=4 channels; + # keeping both full decoded buffers alive could exceed the API cgroup. + fileobj.seek(0) + # A bounded compressed copy avoids sharing buffered file-descriptor + # offsets with the child, and supports BytesIO callers as well. + encoded = fileobj.read(MAX_BYTES + 1) + if len(encoded) > MAX_BYTES: + raise Failure(413, "Файл больше 10 MiB") + try: + decoded = subprocess.run( + ["djpeg", "-strict", "-maxmemory", "384m", "-outfile", "/dev/null"], + input=encoded, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + timeout=4, + check=False, + ) + except subprocess.TimeoutExpired: + raise Failure(415, "Изображение не прошло ограниченную проверку") from None + finally: + del encoded + if decoded.returncode: + raise Failure(415, "Повреждённое изображение") + # djpeg has already decoded every JPEG pixel, streaming output rows. + # A second Pillow load would overlap full progressive coefficient + # arrays with a 160 MiB CMYK canvas and violate the API's 512 MiB cap. + fileobj.seek(0) + return actual, width, height + # Decode one frame at a time. Pillow may retain a previous APNG canvas + # for compositing, but no list/copies of every decoded frame is collected. + fileobj.seek(0) + with Image.open(fileobj) as image: + decoded_pixels = 0 + for frame in range(frames): + if decoded_pixels + width * height > MAX_PIXELS: + raise Failure(413, "Слишком много пикселей во всех кадрах") + image.seek(frame) + decoded_pixels += image.size[0] * image.size[1] + if decoded_pixels > MAX_PIXELS: + raise Failure(413, "Слишком много пикселей во всех кадрах") + image.load() + # Image's context manager closes its input handle, not the decoded core. + # Releasing the object frees native canvases before hashing/S3 publication. + del image + fileobj.seek(0) + return actual, width, height + except Image.DecompressionBombError: + raise Failure(413, "Слишком много пикселей") from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError): + raise Failure(415, "Повреждённое изображение") from None + + +def main(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + resource.setrlimit(resource.RLIMIT_CPU, (4, 5)) + if sys.platform == "linux": + resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024)) + try: + data = sys.stdin.buffer.read(MAX_BYTES + 1) + if len(data) > MAX_BYTES: + raise Failure(413, "Файл больше 10 MiB") + result = inspect_image(BytesIO(data), sys.argv[1]) + print(json.dumps({"ok": result})) + except Failure as exc: + print(json.dumps({"error": exc.status})) + except MemoryError: + print(json.dumps({"error": 415})) + except Exception: + print(json.dumps({"error": 415})) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_image_edges.py b/student-tests/test_image_edges.py new file mode 100644 index 00000000..ca2eb3ba --- /dev/null +++ b/student-tests/test_image_edges.py @@ -0,0 +1,199 @@ +"""Full-range JPEG and bounded multi-frame PNG regressions.""" + +import os +import struct +import zlib +from io import BytesIO + +import pytest +from client import Client +from PIL import Image + + +@pytest.fixture(scope="module") +def scenario(): + c = Client(os.getenv("BASE_URL", "http://127.0.0.1:8084")) + return c, *c.bootstrap() + + +def upload(scenario, data, mime, expected): + c, alice, _, _, chat = scenario + return c.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=edge.png", + expected, + actor=alice, + raw=data, + headers={"Content-Type": mime}, + schema="Attachment" if expected == 201 else None, + ) + + +def progressive_jpeg(): + output = BytesIO() + image = Image.new("RGB", (6000, 6000), "blue") + try: + image.save(output, "JPEG", quality=90, progressive=True, subsampling=0) + finally: + image.close() + return output.getvalue() + + +def apng(size=(16, 16), default_image=False): + output = BytesIO() + first, second = Image.new("RGB", size, "red"), Image.new("RGB", size, "blue") + try: + first.save( + output, + "PNG", + save_all=True, + append_images=[second], + duration=100, + loop=0, + default_image=default_image, + ) + finally: + first.close() + second.close() + return output.getvalue() + + +def broken_later_frame(data): + result, position = bytearray(data[:8]), 8 + while position < len(data): + length = struct.unpack(">I", data[position : position + 4])[0] + kind = data[position + 4 : position + 8] + payload = data[position + 8 : position + 8 + length] + if kind == b"fdAT": + payload = payload[:4] + b"not-a-valid-deflate-stream" + result += ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload)) + ) + position += length + 12 + return bytes(result) + + +def test_36m_progressive_jpeg_is_valid_and_unchanged(scenario): + data = progressive_jpeg() + attachment = upload(scenario, data, "image/jpeg", 201)["json"] + c, _, bob, _, _ = scenario + assert ( + c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data + ) + + +def test_apng_corrupt_later_frame_is_rejected(scenario): + upload(scenario, broken_later_frame(apng()), "image/png", 415) + + +def test_valid_small_apng_is_accepted_and_unchanged(scenario): + data = apng() + attachment = upload(scenario, data, "image/png", 201)["json"] + c, _, bob, _, _ = scenario + assert ( + c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data + ) + + +def test_apng_cumulative_canvas_limit(scenario): + # Two 25M-pixel canvases exceed the total 40M decode budget. + upload(scenario, apng((5000, 5000)), "image/png", 413) + + +def test_apng_forged_frame_count_cannot_hide_bad_frame(scenario): + data = bytearray(broken_later_frame(apng())) + position = 8 + while position < len(data): + length = struct.unpack(">I", data[position : position + 4])[0] + kind = bytes(data[position + 4 : position + 8]) + if kind == b"acTL": + data[position + 8 : position + 12] = struct.pack(">I", 1) + payload = bytes(data[position + 8 : position + 8 + length]) + data[position + 8 + length : position + 12 + length] = struct.pack( + ">I", zlib.crc32(kind + payload) + ) + break + position += length + 12 + upload(scenario, bytes(data), "image/png", 415) + + +def test_apng_exact_cumulative_limit_is_accepted(scenario): + upload(scenario, apng((4000, 5000)), "image/png", 201) + + +def test_40m_cmyk_progressive_jpeg_is_accepted(scenario): + output = BytesIO() + image = Image.new("CMYK", (8000, 5000), (0, 128, 128, 0)) + try: + image.save(output, "JPEG", quality=90, progressive=True, subsampling=0) + finally: + image.close() + data = output.getvalue() + attachment = upload(scenario, data, "image/jpeg", 201)["json"] + c, _, bob, _, _ = scenario + assert ( + c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data + ) + + +def test_repeated_large_image_sequence_stays_available(scenario): + # Formerly retained APNG canvases could overlap the next CMYK decoder. + c, _, bob, _, _ = scenario + data = apng((4000, 5000)) + attachment = upload(scenario, data, "image/png", 201)["json"] + assert ( + c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data + ) + del data + test_40m_cmyk_progressive_jpeg_is_accepted(scenario) + test_36m_progressive_jpeg_is_valid_and_unchanged(scenario) + data = BytesIO() + image = Image.new("RGB", (16, 16), "green") + image.save(data, "PNG") + image.close() + attachment = upload(scenario, data.getvalue(), "image/png", 201)["json"] + assert ( + c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] + == data.getvalue() + ) + + +@pytest.mark.parametrize("progressive", [False, True]) +def test_small_cmyk_jpeg(progressive, scenario): + output = BytesIO() + image = Image.new("CMYK", (24, 18), (0, 128, 128, 0)) + image.save(output, "JPEG", progressive=progressive) + image.close() + upload(scenario, output.getvalue(), "image/jpeg", 201) + + +def test_invalid_zero_frame_count_is_rejected(scenario): + data = apng() + result, position, controls = bytearray(data[:8]), 8, 0 + while position < len(data): + length = struct.unpack(">I", data[position : position + 4])[0] + kind = data[position + 4 : position + 8] + payload = data[position + 8 : position + 8 + length] + position += length + 12 + if kind == b"fcTL": + controls += 1 + if controls > 1: + continue + if kind == b"fdAT": + continue + if kind == b"acTL": + payload = bytes(4) + payload[4:] + result += ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload)) + ) + upload(scenario, bytes(result), "image/png", 415) + + +def test_valid_apng_default_image(scenario): + upload(scenario, apng(default_image=True), "image/png", 201) diff --git a/student-tests/test_json_encoding.py b/student-tests/test_json_encoding.py new file mode 100644 index 00000000..b614d873 --- /dev/null +++ b/student-tests/test_json_encoding.py @@ -0,0 +1,140 @@ +import base64 +import codecs +import json +import os +import uuid + +import pytest +from client import Client + +ENCODINGS = [ + "utf-16", + "utf-16-le", + "utf-16-be", + "utf-16-be-bom", + "utf-32", + "utf-32-le", + "utf-32-be", + "utf-32-be-bom", +] + + +def encoded(body, encoding): + text = json.dumps(body, ensure_ascii=False) + if encoding == "utf-16-be-bom": + return codecs.BOM_UTF16_BE + text.encode("utf-16-be") + if encoding == "utf-32-be-bom": + return codecs.BOM_UTF32_BE + text.encode("utf-32-be") + return text.encode(encoding) + + +@pytest.fixture(scope="module") +def encoding_scenario(): + c = Client(os.environ["BASE_URL"]) + return c, *c.bootstrap() + + +@pytest.mark.parametrize("encoding", ENCODINGS) +def test_non_utf8_registration_has_no_write(encoding_scenario, encoding): + c, *_ = encoding_scenario + body = { + "username": "enc_" + uuid.uuid4().hex[:14], + "display_name": "Никита\u0000имя", + "password": "Пароль:тестовый-12345", + } + c.call( + "POST", + "/api/v1/users", + 400, + raw=encoded(body, encoding), + headers={"Content-Type": "application/json"}, + ) + # UNIQUE username proves the rejected request did not insert a user. + created = c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + assert created["display_name"] == body["display_name"] + + +@pytest.mark.parametrize("encoding", ENCODINGS) +def test_non_utf8_chat_has_no_write(encoding_scenario, encoding): + c, alice, *_ = encoding_scenario + before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"] + body = {"title": "Чат\u0000" + uuid.uuid4().hex, "member_ids": [alice["id"]]} + c.call( + "POST", + "/api/v1/chats", + 400, + actor=alice, + raw=encoded(body, encoding), + headers={"Content-Type": "application/json"}, + ) + after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"] + assert before == after + assert ( + c.call("POST", "/api/v1/chats", 201, actor=alice, body=body, schema="Chat")["json"]["title"] + == body["title"] + ) + + +@pytest.mark.parametrize("encoding", ENCODINGS) +def test_non_utf8_message_has_no_write(encoding_scenario, encoding): + c, alice, _, _, chat = encoding_scenario + path = f"/api/v1/chats/{chat['id']}/messages" + key = uuid.uuid4().hex + body = {"text": "Сообщение\u0000" + key} + c.call( + "POST", + path, + 400, + actor=alice, + raw=encoded(body, encoding), + headers={"Content-Type": "application/json", "Idempotency-Key": key}, + ) + history = c.call("GET", path + "?limit=100", actor=alice, schema="MessagePage")["json"]["items"] + assert all(item["text"] != body["text"] for item in history) + good = c.call( + "POST", + path, + 201, + actor=alice, + body=body, + headers={"Idempotency-Key": key}, + schema="Message", + )["json"] + repeat = c.call( + "POST", + path, + 201, + actor=alice, + body=body, + headers={"Idempotency-Key": key}, + schema="Message", + )["json"] + assert good == repeat and good["text"] == body["text"] + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +def test_valid_utf8_registration_and_login(encoding_scenario, encoding): + c, *_ = encoding_scenario + body = { + "username": "utf8_" + uuid.uuid4().hex[:14], + "display_name": "Юникод\u0000😀", + "password": "Пароль:тестовый-12345", + } + user = c.call( + "POST", + "/api/v1/users", + 201, + raw=encoded(body, encoding), + headers={"Content-Type": "application/json"}, + schema="User", + )["json"] + assert user["display_name"] == body["display_name"] + credential = base64.b64encode((body["username"] + ":" + body["password"]).encode()).decode() + session = c.call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "Basic " + credential}, + schema="Session", + )["json"] + assert session["user"] == user diff --git a/student-tests/test_validation_process.py b/student-tests/test_validation_process.py new file mode 100644 index 00000000..f9f53f54 --- /dev/null +++ b/student-tests/test_validation_process.py @@ -0,0 +1,57 @@ +"""Process boundary failures must stay bounded and map to contract errors.""" + +import signal +import subprocess +from io import BytesIO + +import pytest + +from messenger.attachments import validate_image +from messenger.domain import Failure + + +class FakeProcess: + pid = 999999999 + returncode = 0 + + def __init__(self, output=b'{"ok":["image/png",1,1]}', crash=False, timeout=False): + self.output, self.timeout, self.calls = output, timeout, 0 + self.returncode = -11 if crash else 0 + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def communicate(self, input=None, timeout=None): + self.calls += 1 + if self.timeout and self.calls == 1: + raise subprocess.TimeoutExpired("validator", 5) + return self.output, b"" + + +def test_timeout_kills_group_and_returns_415(monkeypatch): + process = FakeProcess(timeout=True) + killed = [] + monkeypatch.setattr("messenger.attachments.subprocess.Popen", lambda *a, **k: process) + monkeypatch.setattr( + "messenger.attachments.os.killpg", lambda pid, sig: killed.append((pid, sig)) + ) + with pytest.raises(Failure) as failure: + validate_image(BytesIO(b"synthetic"), "image/png") + assert failure.value.status == 415 + assert killed == [(process.pid, signal.SIGKILL)] + assert process.calls == 2 + + +@pytest.mark.parametrize( + "output,crash", [(b"", True), (b"not-json", False), (b"null", False), (b"[]", False)] +) +def test_crash_or_malformed_child_result_returns_415(monkeypatch, output, crash): + monkeypatch.setattr( + "messenger.attachments.subprocess.Popen", lambda *a, **k: FakeProcess(output, crash=crash) + ) + with pytest.raises(Failure) as failure: + validate_image(BytesIO(b"synthetic"), "image/png") + assert failure.value.status == 415 -- 2.54.0 From f32e7e18a62594f93ae4c1c557c9ff0f344b8f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Fri, 6 Nov 2026 19:12:03 +0300 Subject: [PATCH 11/11] docs: verify full-range image validation and record bounded-memory receipts --- REPORT.md | 15 +++- evidence/contract-image-fix-final.txt | 4 + evidence/image-validation-after.txt | 2 + evidence/image-validation-before.txt | 82 +++++++++++++++++++ .../image-validation-memory-single-api.json | 23 ++++++ evidence/image-validation-memory.json | 63 ++++++++++++++ evidence/lint-image-fix-final.txt | 1 + evidence/smoke-image-fix-final.txt | 1 + evidence/tests-image-fix-final.txt | 4 + evidence/validation-process-final.txt | 2 + src/messenger/attachments.py | 4 + 11 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 evidence/contract-image-fix-final.txt create mode 100644 evidence/image-validation-after.txt create mode 100644 evidence/image-validation-before.txt create mode 100644 evidence/image-validation-memory-single-api.json create mode 100644 evidence/image-validation-memory.json create mode 100644 evidence/lint-image-fix-final.txt create mode 100644 evidence/smoke-image-fix-final.txt create mode 100644 evidence/tests-image-fix-final.txt create mode 100644 evidence/validation-process-final.txt diff --git a/REPORT.md b/REPORT.md index 5d8319e4..33be65d5 100644 --- a/REPORT.md +++ b/REPORT.md @@ -114,18 +114,27 @@ docker compose run --rm --no-deps research ### Результат десяти конкурентных upload +`upload-memory-final.json` — исходный снимок до поздних исправлений валидатора. Его данные и hashes не переписаны. Отдельная новая image-validation проверка ниже проверяет текущий short-lived child на больших файлах; это другой сценарий, не подмена старого эксперимента. + + После рестарта API с сохранением старого токена проверены PNG 788100 и 8683594 байт. На каждом размере десять клиентов начали одновременно; **2 получили 201, 8 — контролируемый 429**, затем readiness200. У API и LB `oom=0`, `oom_kill=0`, процессы остались живы. Cgroup lifetime peaks: API 122773504 и122679296 байт, LB19656704 байт. Лимиты соответственно512/512/128MiB. Before/after peaks не выросли: начальная загрузка приложения/Argon2 уже создала больший high-water. Поэтому опыт доказывает ограниченное поведение выбранной нагрузки без OOM, но не выделяет точную память одной загрузки и не доказывает экономию RAM самим spool. +### Поздняя проверка полного диапазона изображений + +Повторное изучение замечаний обнаружило допустимый36M progressiveRGBJPEG, который прежний nativebudget192MiB отклонял, и APNG с повреждённым вторым кадром. Регрессия на отдельном Docker project до правки:3 failed/1 passed. После правки проверены большие RGB/CMYK progressiveJPEG, маленькиеCMYK baseline/progressive, полный APNG decode, повреждённый later frame, rawacTL0/forgedcount, default image,40M cumulative boundary и50M rejection. Последовательность APNG40M→CMYK40M→RGB36M→обычныйPNG возвращает201 с неизменными original bytes при API512MiB и безOOM. Пик/счётчики и текущие source hashes сохранены в отдельном receipt. + +Полный собственный набор после основной правки:155 passed; child timeout/crash boundary дополнительно проверена пятью unit-тестами. Финальные smoke/schema/lint и целевые image-регрессии находятся в новых evidence с суффиксом image-fix. + ## Проверки и доказательства | Критерий | Сценарий | Доказательство | |---|---|---| | Контракт и схема | make check validate | evidence/contract-checks.txt | -| Обязательный API, PNG/ACL/checksum/idempotency | make test | evidence/smoke-final.txt | -| Негативные/параллельные/Unicode/вложения/cursor | make student-test | evidence/tests-final.txt | -| Повреждённый JPEG с поддельным EOI | regression до исправления и полный тест после djpeg | evidence/attachments-before-strict-jpeg.txt; evidence/tests-final.txt | +| Обязательный API, PNG/ACL/checksum/idempotency | make test | evidence/smoke-image-fix-final.txt | +| Негативные/параллельные/Unicode/вложения/cursor | make student-test | evidence/tests-image-fix-final.txt | +| Повреждённый JPEG с поддельным EOI | regression до исправления и полный тест после djpeg | evidence/attachments-before-strict-jpeg.txt; evidence/tests-image-fix-final.txt | | Независимая проверка реального HTTP | root external regression | Отдельная итоговая проверка общего пакета | | Десять одновременных upload | research/upload_memory.py | evidence/upload-memory-final.json | | Reconciler/fault-injection/стоимость | tools profile research | evidence/reconcile-study.json | diff --git a/evidence/contract-image-fix-final.txt b/evidence/contract-image-fix-final.txt new file mode 100644 index 00000000..ed9cd475 --- /dev/null +++ b/evidence/contract-image-fix-final.txt @@ -0,0 +1,4 @@ +.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/image-validation-after.txt b/evidence/image-validation-after.txt new file mode 100644 index 00000000..eca467c7 --- /dev/null +++ b/evidence/image-validation-after.txt @@ -0,0 +1,2 @@ +...................... [100%] +22 passed in 13.29s diff --git a/evidence/image-validation-before.txt b/evidence/image-validation-before.txt new file mode 100644 index 00000000..7f6b893d --- /dev/null +++ b/evidence/image-validation-before.txt @@ -0,0 +1,82 @@ +FF.F [100%] +=================================== FAILURES =================================== +_______________ test_36m_progressive_jpeg_is_valid_and_unchanged _______________ + +scenario = (, {'id': 'ef1e6ee9-739a-4dfe-8b05-78bef274c542', 'user': {'id': 'ef1e6ee9-739a-4...-06T15:51:51.913231Z', 'member_ids': ['ef1e6ee9-739a-4dfe-8b05-78bef274c542', '830c471d-1354-43d3-9241-ef98ffc87c7c']}) + + def test_36m_progressive_jpeg_is_valid_and_unchanged(scenario): + data = progressive_jpeg() +> attachment = upload(scenario, data, "image/jpeg", 201)["json"] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +student-tests/test_image_edges.py:73: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +student-tests/test_image_edges.py:21: in upload + return c.call( +tests/client.py:132: in call + check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +condition = False +message = 'POST /api/v1/chats/e9b3f246-cba5-470b-b88f-35da5340b635/attachments: expected [201], received 415' + + def check(condition, message): + if not condition: +> raise Failure(message) +E client.Failure: POST /api/v1/chats/e9b3f246-cba5-470b-b88f-35da5340b635/attachments: expected [201], received 415 + +tests/client.py:24: Failure +__________________ test_apng_corrupt_later_frame_is_rejected ___________________ + +scenario = (, {'id': 'ef1e6ee9-739a-4dfe-8b05-78bef274c542', 'user': {'id': 'ef1e6ee9-739a-4...-06T15:51:51.913231Z', 'member_ids': ['ef1e6ee9-739a-4dfe-8b05-78bef274c542', '830c471d-1354-43d3-9241-ef98ffc87c7c']}) + + def test_apng_corrupt_later_frame_is_rejected(scenario): +> upload(scenario, broken_later_frame(apng()), "image/png", 415) + +student-tests/test_image_edges.py:81: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +student-tests/test_image_edges.py:21: in upload + return c.call( +tests/client.py:132: in call + check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +condition = False +message = 'POST /api/v1/chats/e9b3f246-cba5-470b-b88f-35da5340b635/attachments: expected [415], received 201' + + def check(condition, message): + if not condition: +> raise Failure(message) +E client.Failure: POST /api/v1/chats/e9b3f246-cba5-470b-b88f-35da5340b635/attachments: expected [415], received 201 + +tests/client.py:24: Failure +______________________ test_apng_cumulative_canvas_limit _______________________ + +scenario = (, {'id': 'ef1e6ee9-739a-4dfe-8b05-78bef274c542', 'user': {'id': 'ef1e6ee9-739a-4...-06T15:51:51.913231Z', 'member_ids': ['ef1e6ee9-739a-4dfe-8b05-78bef274c542', '830c471d-1354-43d3-9241-ef98ffc87c7c']}) + + def test_apng_cumulative_canvas_limit(scenario): + # Two 25M-pixel canvases exceed the total 40M decode budget. +> upload(scenario, apng((5000, 5000)), "image/png", 413) + +student-tests/test_image_edges.py:95: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +student-tests/test_image_edges.py:21: in upload + return c.call( +tests/client.py:132: in call + check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +condition = False +message = 'POST /api/v1/chats/e9b3f246-cba5-470b-b88f-35da5340b635/attachments: expected [413], received 201' + + def check(condition, message): + if not condition: +> raise Failure(message) +E client.Failure: POST /api/v1/chats/e9b3f246-cba5-470b-b88f-35da5340b635/attachments: expected [413], received 201 + +tests/client.py:24: Failure +=========================== short test summary info ============================ +FAILED student-tests/test_image_edges.py::test_36m_progressive_jpeg_is_valid_and_unchanged +FAILED student-tests/test_image_edges.py::test_apng_corrupt_later_frame_is_rejected +FAILED student-tests/test_image_edges.py::test_apng_cumulative_canvas_limit +3 failed, 1 passed in 2.41s diff --git a/evidence/image-validation-memory-single-api.json b/evidence/image-validation-memory-single-api.json new file mode 100644 index 00000000..153b2e40 --- /dev/null +++ b/evidence/image-validation-memory-single-api.json @@ -0,0 +1,23 @@ +{ + "scenario": "APNG40M\u2192CMYK40M\u2192RGB36M\u2192ordinaryPNG, plus malformed frames/JPEG and repeated cases", + "api_limit_bytes": 536870912, + "containers": [ + { + "id": "7b1331ef2f48894e180f55aa43a7fc926ebdb8119b70c6b12e6154ddd7718c41", + "image": "sha256:0f1975a55bf57b7a413815f678e29d500924f18a1485e2e3000aa9b09c2e23ae", + "current_bytes": 88059904, + "peak_bytes": 426946560, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "running": true, + "oom_killed": false, + "limit_bytes": 536870912 + } + ] +} diff --git a/evidence/image-validation-memory.json b/evidence/image-validation-memory.json new file mode 100644 index 00000000..2e804be9 --- /dev/null +++ b/evidence/image-validation-memory.json @@ -0,0 +1,63 @@ +{ + "api_limit_bytes": 536870912, + "native_budget_mib": 384, + "child_linux_as_mib": 512, + "child_cpu_seconds": 4, + "parent_wall_seconds": 5, + "scenario": "APNG40M\u2192CMYK40M\u2192RGB36M\u2192ordinaryPNG plus malformed/full-range cases", + "targeted_result": "22 passed in 13.29s", + "full_suite_result": "155 passed in 50.44s", + "final_source_hashes": { + "src/messenger/__init__.py": "9a72a49106367ca514cc6147d0e727e62c4d33fefd61572d209a5c9494551a2c", + "src/messenger/__main__.py": "ba9a59ad44ee8884cd65cf88d5ea15cbd7051b10f7c729043ddd44ca2f34defe", + "src/messenger/app.py": "bea8bc70db73e99cb26a794db9549ff6d5ffc51951e587c44c36ea2656410e54", + "src/messenger/attachments.py": "fb8735c19140c96b16ecdc7b643c175c8e418bfa9683a986d5c9c92617dc8082", + "src/messenger/config.py": "e4d4aaf674ede8b5abdc361f77c7793ee94d8d7856030c28834eb7c95291b193", + "src/messenger/domain.py": "7bf6f177ab5bdeea09b303dc886160335d348f64f47912b9bba45355db378bef", + "src/messenger/image_validation.py": "d9aa8db65edaab88c9a76cf2e5459ad3028ba808579311aeb0d8c4f01f875720", + "src/messenger/migrate.py": "eb5cad7ac6abe38dd40b2db307d298b297abe9543b3928c36bab8ef06283664e", + "src/messenger/models.py": "c43769d2d4ba0369129a86bf80364a626ab91442f9b9020e6bc904e363a39665", + "src/messenger/postgres.py": "8a9c0cd275bbb04604d94c1a0155176e9f0348d1e05abe2120c74b0b83f36f6c", + "src/messenger/reconcile.py": "792e21f50e777458df367069694e5ff751817fff4dbec4276e4d73e9c64a2fd7", + "src/messenger/storage.py": "70b0fb157b285b674a7388a2bfe5c5a7feb8149a9266ac7ad035961d769008a5", + "src/messenger/storage_bootstrap.py": "b29a2a1f8864ec3425a7499b1752688f69d8f402f3881957e790ed4522d35589" + }, + "containers": [ + { + "id": "beb1633f31867054f0f1d251df15769de8c8a7a8ee955fb1c5a4df75b0a7fb88", + "image": "sha256:362e7406d132beae97a8824381832939f98de42b02ac914367b0a6edf8549ce6", + "current_bytes": 83705856, + "peak_bytes": 422154240, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "running": true, + "oom_killed": false, + "limit_bytes": 536870912, + "runtime_source_matches": true + }, + { + "id": "8368fe5482e93bba9a337f21aca3e7e72c6a3844bb2ecd451878c66bbb07aa33", + "image": "sha256:362e7406d132beae97a8824381832939f98de42b02ac914367b0a6edf8549ce6", + "current_bytes": 83824640, + "peak_bytes": 422637568, + "events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "running": true, + "oom_killed": false, + "limit_bytes": 536870912, + "runtime_source_matches": true + } + ] +} diff --git a/evidence/lint-image-fix-final.txt b/evidence/lint-image-fix-final.txt new file mode 100644 index 00000000..1f5f344d --- /dev/null +++ b/evidence/lint-image-fix-final.txt @@ -0,0 +1 @@ +All checks passed! diff --git a/evidence/smoke-image-fix-final.txt b/evidence/smoke-image-fix-final.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke-image-fix-final.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/tests-image-fix-final.txt b/evidence/tests-image-fix-final.txt new file mode 100644 index 00000000..bdd6655d --- /dev/null +++ b/evidence/tests-image-fix-final.txt @@ -0,0 +1,4 @@ +........................................................................ [ 46%] +........................................................................ [ 92%] +........... [100%] +155 passed in 50.44s diff --git a/evidence/validation-process-final.txt b/evidence/validation-process-final.txt new file mode 100644 index 00000000..d506421d --- /dev/null +++ b/evidence/validation-process-final.txt @@ -0,0 +1,2 @@ +..... [100%] +5 passed in 0.14s diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py index cb6fcff4..3036db07 100644 --- a/src/messenger/attachments.py +++ b/src/messenger/attachments.py @@ -70,6 +70,10 @@ def validate_image(fileobj, declared_type): process.communicate() raise Failure(415, "Изображение не прошло ограниченную проверку") from None if process.returncode: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass raise Failure(415, "Изображение не прошло ограниченную проверку") except OSError: raise Failure(503, "Проверка изображения временно недоступна") from None -- 2.54.0