From 064a6c1ce50159342c4503f989ca955506535fda Mon Sep 17 00:00:00 2001 From: Pavel Grigorev Date: Fri, 13 Nov 2026 16:22:54 +0300 Subject: [PATCH 1/3] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F:=20S3?= =?UTF-8?q?-=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B5=20Seawe?= =?UTF-8?q?edFS,=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D1=83=20?= =?UTF-8?q?=D0=B8=20=D1=81=D0=BA=D0=B0=D1=87=D0=B8=D0=B2=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D0=B5=20=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20API,=20=D1=81=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D1=89=D0=B8=D0=BA=20PostgreSQL=20=D0=B8=20S3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 12 ++ .env.example | 30 ++- .gitattributes | 2 + Dockerfile | 25 +++ Makefile | 5 +- compose.yaml | 257 ++++++++++++++++++++++++ infra/haproxy/haproxy.cfg | 53 +++++ migrations/0001_init.sql | 46 +++++ migrations/0002_unicode_text.sql | 14 ++ migrations/0003_session_revocations.sql | 6 + migrations/0004_idempotency_keys.sql | 11 + migrations/0005_attachments.sql | 30 +++ requirements.txt | 27 +++ scripts/init_secrets.py | 38 ++++ scripts/restart.cmd | 6 + scripts/restart.sh | 7 + scripts/rolling_update.py | 77 +++++++ scripts/stop-one.cmd | 11 + scripts/stop-one.sh | 14 ++ src/app.py | 252 +++++++++++++++++++++++ src/auth.py | 120 +++++++++++ src/cache.py | 89 ++++++++ src/config.py | 54 +++++ src/db.py | 133 ++++++++++++ src/errors.py | 37 ++++ src/images.py | 139 +++++++++++++ src/main.py | 34 ++++ src/migrate.py | 47 +++++ src/models.py | 62 ++++++ src/pagination.py | 31 +++ src/reconcile.py | 165 +++++++++++++++ src/s3.py | 65 ++++++ src/s3_bootstrap.py | 55 +++++ src/store.py | 246 +++++++++++++++++++++++ 34 files changed, 2194 insertions(+), 6 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_unicode_text.sql create mode 100644 migrations/0003_session_revocations.sql create mode 100644 migrations/0004_idempotency_keys.sql create mode 100644 migrations/0005_attachments.sql create mode 100644 requirements.txt create mode 100644 scripts/init_secrets.py create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 scripts/rolling_update.py create mode 100644 scripts/stop-one.cmd create mode 100755 scripts/stop-one.sh create mode 100644 src/app.py create mode 100644 src/auth.py create mode 100644 src/cache.py create mode 100644 src/config.py create mode 100644 src/db.py create mode 100644 src/errors.py create mode 100644 src/images.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/models.py create mode 100644 src/pagination.py create mode 100644 src/reconcile.py create mode 100644 src/s3.py create mode 100644 src/s3_bootstrap.py create mode 100644 src/store.py diff --git a/.dockerignore b/.dockerignore index bc16e556..93f3500f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,15 @@ __pycache__ node_modules evidence .DS_Store +tests +student-tests +contracts +scripts +examples +*.md +Makefile +lab.json +requirements-dev.txt +Dockerfile* +.dockerignore +.gitignore diff --git a/.env.example b/.env.example index d99d26a2..d8f951b3 100644 --- a/.env.example +++ b/.env.example @@ -3,18 +3,38 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +APP_SHUTDOWN_TIMEOUT=5 +APP_DRAIN_SECONDS=3 +# Host ports of the load balancer (change to 8090/8405 to run next to another lab stack). +LB_PORT=8080 +STATS_PORT=8404 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. +# Credentials are generated locally by: python scripts/init_secrets.py +# File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# Sessions: PostgreSQL is the source of truth, Valkey is a read-through cache. +CACHE_URL=redis://valkey:6379/0 +SESSION_CACHE_TTL_SECONDS=60 +REVOCATION_SYNC_SECONDS=1 API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +PASSWORD_HASH_CONCURRENCY=4 +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_TIMEOUT_SECONDS=2 +DB_STATEMENT_TIMEOUT_MS=3000 +DB_REQUEST_TIMEOUT_SECONDS=4 +# S3 (SeaweedFS). API reads its own limited keys from secret files. +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_TIMEOUT_SECONDS=15 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +UPLOAD_MEMORY_BYTES=1048576 +UPLOAD_CONCURRENCY=4 +# Periodic reconciler service: cleans orphans and stale uploads older than the grace period. +RECONCILE_INTERVAL_SECONDS=600 +RECONCILE_GRACE_SECONDS=900 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f4609ace --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.sh text eol=lf +*.cmd text eol=crlf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7df36ae4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.12.15-slim-trixie@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_ROOT_USER_ACTION=ignore + +WORKDIR /app + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY migrations/ ./migrations/ +COPY src/ ./src/ + +RUN useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=60s --start-interval=1s --retries=3 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"] + +CMD ["python", "src/main.py"] diff --git a/Makefile b/Makefile index 5d86d171..ee7fcb85 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover student-test help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -18,6 +18,9 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +student-test: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -v + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..e609c131 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,257 @@ +name: grigoriev-lab4 + +x-app-env: &app-env + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + APP_SHUTDOWN_TIMEOUT: ${APP_SHUTDOWN_TIMEOUT:-5} + APP_DRAIN_SECONDS: ${APP_DRAIN_SECONDS:-3} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + REVOCATION_SYNC_SECONDS: ${REVOCATION_SYNC_SECONDS:-1} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-2} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-3000} + DB_REQUEST_TIMEOUT_SECONDS: ${DB_REQUEST_TIMEOUT_SECONDS:-4} + MALLOC_ARENA_MAX: "2" + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: ${CACHE_URL:-redis://valkey:6379/0} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-15} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_MEMORY_BYTES: ${UPLOAD_MEMORY_BYTES:-1048576} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4} + FAULTS: ${FAULTS:-} + +x-dns-opt: &dns-opt + - timeout:1 + - attempts:1 + +services: + lb: + image: haproxy:3.0.30-alpine@sha256:d2b3567d836f3b7d174edb6d6f611ed5308e7966e53a9cd9a597e0415a298efd + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${LB_PORT:-8080}:8080" + - "127.0.0.1:${STATS_PORT:-8404}:8404" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/lb/health"] + interval: 5s + timeout: 2s + retries: 6 + depends_on: + api: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + networks: [frontend, backend] + restart: unless-stopped + + api: + build: . + image: mtusi-messenger:lab4 + environment: + <<: *app-env + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - database_url + - {source: s3_api_access_key_id, target: s3_access_key_id} + - {source: s3_api_secret_access_key, target: s3_secret_access_key} + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + valkey: + condition: service_started + dns_opt: *dns-opt + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "0.50", memory: 320M} + networks: [backend] + restart: unless-stopped + + migrate: + build: . + image: mtusi-messenger:lab4 + command: ["python", "src/migrate.py"] + environment: *app-env + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + networks: [backend] + restart: "no" + + s3: + image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d + command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1", + "-s3", "-s3.port=8333", "-s3.ip.bind=0.0.0.0", "-s3.iam=false", + "-s3.config=/run/secrets/s3_config.json", + "-s3.concurrentUploadLimitMB=64", "-filer.concurrentUploadLimitMB=64", + "-s3.cacheCapacityMB=16", "-s3.readerCacheSizeMB=32", + "-master.volumeSizeLimitMB=256", "-volume.max=40", "-master.telemetry=false"] + environment: + GOMEMLIMIT: 300MiB + secrets: [s3_config.json] + volumes: + - s3data:/data + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + networks: [backend] + restart: unless-stopped + + s3-init: + build: . + image: mtusi-messenger:lab4 + command: ["python", "src/s3_bootstrap.py"] + environment: + <<: *app-env + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: [database_url, s3_admin_access_key_id, s3_admin_secret_access_key] + depends_on: + s3: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + networks: [backend] + restart: "no" + + reconcile: + build: . + image: mtusi-messenger:lab4 + profiles: [tools] + entrypoint: ["python", "src/reconcile.py"] + environment: + <<: *app-env + S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id + S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key + secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key] + deploy: + resources: + limits: {cpus: "0.50", memory: 256M} + networks: [backend] + + reconciler: + build: . + image: mtusi-messenger:lab4 + command: ["python", "src/reconcile.py", "--apply", + "--interval", "${RECONCILE_INTERVAL_SECONDS:-600}", "--grace-seconds", "${RECONCILE_GRACE_SECONDS:-900}"] + environment: + <<: *app-env + S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id + S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key + RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-600} + secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + # жив цикл: файл-пульс обновлялся не позже интервала + 2 мин назад + test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.path.getmtime('/tmp/reconciler.alive') > float(os.environ['RECONCILE_INTERVAL_SECONDS']) + 120)"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 60s + start_interval: 1s + stop_grace_period: 5s + deploy: + resources: + limits: {cpus: "0.25", memory: 192M} + networks: [backend] + restart: unless-stopped + + postgres: + image: postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: ["postgres", "-c", "track_io_timing=on"] + secrets: [postgres_password] + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 384M} + networks: [backend] + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a + command: ["valkey-server", "--save", "", "--appendonly", "no", + "--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl"] + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + deploy: + resources: + limits: {cpus: "0.25", memory: 96M} + networks: [backend] + restart: unless-stopped + +volumes: + pgdata: + s3data: + +networks: + frontend: + backend: + internal: true + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + s3_config.json: + file: ./secrets/s3_config.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_api_access_key_id: + file: ./secrets/s3_api_access_key_id + s3_api_secret_access_key: + file: ./secrets/s3_api_secret_access_key + s3_reconciler_access_key_id: + file: ./secrets/s3_reconciler_access_key_id + s3_reconciler_secret_access_key: + file: ./secrets/s3_reconciler_secret_access_key diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..49894f84 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,53 @@ +global + log stdout format raw local0 info + maxconn 2000 + +defaults + mode http + log global + option httplog + option http-server-close + option forwardfor + option redispatch + retries 2 + retry-on conn-failure + timeout connect 1s + timeout client 30s + timeout server 60s + timeout check 1s + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 2s + hold nx 2s + accepted_payload_size 8192 + +frontend messenger + bind :8080 + monitor-uri /lb/health + use_backend api_live if { nbsrv(api) lt 1 } + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none + server-template api 1-4 api:8080 + +backend api_live + balance roundrobin + option httpchk GET /health/live + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none + server-template live 1-4 api:8080 + +listen stats + bind :8404 + stats enable + stats uri / + stats refresh 2s diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..44c4ed38 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,46 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_by uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL CHECK (position >= 1), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32), + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_user_idx ON sessions (user_id); +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/migrations/0002_unicode_text.sql b/migrations/0002_unicode_text.sql new file mode 100644 index 00000000..bf25a000 --- /dev/null +++ b/migrations/0002_unicode_text.sql @@ -0,0 +1,14 @@ +ALTER TABLE users + DROP CONSTRAINT users_display_name_check, + ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'), + ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400); + +ALTER TABLE chats + DROP CONSTRAINT chats_title_check, + ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'), + ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400); + +ALTER TABLE messages + DROP CONSTRAINT messages_text_check, + ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'), + ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/migrations/0003_session_revocations.sql b/migrations/0003_session_revocations.sql new file mode 100644 index 00000000..c2bcb84c --- /dev/null +++ b/migrations/0003_session_revocations.sql @@ -0,0 +1,6 @@ +CREATE TABLE session_revocations ( + token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32), + expires_at timestamptz NOT NULL +); + +CREATE INDEX session_revocations_expires_idx ON session_revocations (expires_at); diff --git a/migrations/0004_idempotency_keys.sql b/migrations/0004_idempotency_keys.sql new file mode 100644 index 00000000..e4db9257 --- /dev/null +++ b/migrations/0004_idempotency_keys.sql @@ -0,0 +1,11 @@ +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + scope text NOT NULL, + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash bytea NOT NULL CHECK (octet_length(request_hash) = 32), + message_id uuid REFERENCES messages (id), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, scope, key) +); + +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/0005_attachments.sql b/migrations/0005_attachments.sql new file mode 100644 index 00000000..fdca7ba9 --- /dev/null +++ b/migrations/0005_attachments.sql @@ -0,0 +1,30 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 bytea NOT NULL CHECK (octet_length(sha256) = 32), + object_key text NOT NULL UNIQUE, + status text NOT NULL CHECK (status IN ('uploading', 'ready')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id); +CREATE INDEX attachments_status_created_idx ON attachments (status, created_at); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + position smallint NOT NULL CHECK (position >= 1), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); + +ALTER TABLE messages + DROP CONSTRAINT messages_text_check, + ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..7b66ae00 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,27 @@ +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +boto3==1.44.1 +botocore==1.44.1 +cffi==2.1.1 +click==8.5.0 +fastapi==0.115.6 +h11==0.16.0 +idna==3.20 +jmespath==1.1.0 +pillow==13.0.0 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +pycparser==3.12 +pydantic==2.10.4 +pydantic_core==2.27.2 +python-dateutil==2.9.0.post0 +redis==8.1.0 +s3transfer==0.19.3 +six==1.17.0 +starlette==0.41.3 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.0 diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..0288f7e3 --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,38 @@ +import json +import os +from pathlib import Path +import secrets + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / 'secrets' + + +def write(name, value): + path = SECRETS / name + if path.exists(): + print('keep existing secrets/%s' % name) + return path.read_text(encoding='utf-8').strip() + path.write_text(value + '\n', encoding='utf-8') + os.chmod(path, 0o644) + print('created secrets/%s' % name) + return value + + +def main(): + SECRETS.mkdir(exist_ok=True) + password = write('postgres_password', secrets.token_hex(24)) + write('database_url', 'postgresql://messenger:%s@postgres:5432/messenger' % password) + bucket = 'messenger-attachments' + rights = {'admin': ['Admin', 'Read', 'List', 'Tagging', 'Write'], + 'api': ['Read:' + bucket, 'Write:' + bucket], + 'reconciler': ['Read:' + bucket, 'List:' + bucket, 'Write:' + bucket]} + identities = [] + for role, actions in rights.items(): + key = write('s3_%s_access_key_id' % role, role + '-' + secrets.token_hex(8)) + secret = write('s3_%s_secret_access_key' % role, secrets.token_hex(24)) + identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions}) + write('s3_config.json', json.dumps({'identities': identities}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..1c08e20c --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,6 @@ +@echo off +cd /d "%~dp0.." +docker compose stop api || exit /b 1 +docker compose restart postgres valkey s3 || exit /b 1 +docker compose up -d --no-build --no-recreate --wait postgres valkey s3 || exit /b 1 +docker compose start api || exit /b 1 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..957806bb --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose stop api +docker compose restart postgres valkey s3 +docker compose up -d --no-build --no-recreate --wait postgres valkey s3 +docker compose start api diff --git a/scripts/rolling_update.py b/scripts/rolling_update.py new file mode 100644 index 00000000..bf79856e --- /dev/null +++ b/scripts/rolling_update.py @@ -0,0 +1,77 @@ +"""Поочерёдная замена API-реплик без остановки сервиса. + + python scripts/rolling_update.py # пересобрать образ и заменить реплики по одной + python scripts/rolling_update.py --no-build # заменить на текущий образ + +1. Один раз применяет миграции (отдельный контейнер migrate; совместимые, только добавляющие изменения). +2. Для каждой старой реплики: поднимает одну новую (--scale +1, --no-recreate), ждёт, пока LB начнёт + отправлять на неё трафик, затем штатно останавливает старую (SIGTERM → вывод из ротации → завершение) + и удаляет её контейнер. Число реплик в ротации не опускается ниже исходного. +""" +import argparse +import os +from pathlib import Path +import subprocess +import sys +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/') + + +def compose(*args): + return subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout + + +def running_api(): + return compose('ps', '-q', 'api').split() + + +def instance_of_next_response(): + try: + with urllib.request.urlopen(BASE_URL + '/health/ready', timeout=2) as response: + return response.headers.get('X-Instance-Id') + except OSError: + return None + + +def wait_in_rotation(container, seconds=90): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if instance_of_next_response() == container[:12]: + return + time.sleep(0.1) + raise SystemExit('new replica %s did not enter LB rotation' % container[:12]) + + +def image_of(container): + return subprocess.run(['docker', 'inspect', '-f', '{{.Image}}', container], capture_output=True, text=True).stdout.strip()[7:19] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--no-build', action='store_true') + args = parser.parse_args() + old = running_api() + print('old replicas:', ', '.join('%s(image %s)' % (c[:12], image_of(c)) for c in old)) + if not args.no_build: + compose('build', 'api') + compose('run', '--rm', '--no-deps', 'migrate') + print('migrations applied once') + for container in old: + before = set(running_api()) + compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (len(before) + 1), 'api') + new = (set(running_api()) - before).pop() + started = time.monotonic() + wait_in_rotation(new) + print('+ %s (image %s) in rotation after %.1fs' % (new[:12], image_of(new), time.monotonic() - started)) + started = time.monotonic() + subprocess.run(['docker', 'stop', '-t', '15', container], check=True, capture_output=True) + subprocess.run(['docker', 'rm', container], check=True, capture_output=True) + print('- %s stopped and removed after %.1fs' % (container[:12], time.monotonic() - started)) + print('new replicas:', ', '.join('%s(image %s)' % (c[:12], image_of(c)) for c in running_api())) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..ea9641bb --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,11 @@ +@echo off +if "%TARGET_INSTANCE_ID%"=="" exit /b 2 +cd /d "%~dp0.." +for /f %%i in ('docker compose ps -q api') do ( + echo %%i| findstr /b /i "%TARGET_INSTANCE_ID%" >nul && ( + docker stop -t 10 %%i >nul + exit /b 0 + ) +) +echo no running api replica matches TARGET_INSTANCE_ID 1>&2 +exit /b 1 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..20a5f14c --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +for id in $(docker compose ps -q api); do + case "$id" in + "$TARGET_INSTANCE_ID"*) + docker stop -t 10 "$id" >/dev/null + exit 0 + ;; + esac +done +echo "no running api replica matches TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..cce03c92 --- /dev/null +++ b/src/app.py @@ -0,0 +1,252 @@ +from contextlib import asynccontextmanager +import hashlib +import json +import logging +import re +import socket +import time +import tempfile +import threading +from typing import Annotated, Literal, Optional +import uuid + +from fastapi import Depends, FastAPI, Header, Query, Request, Response +from fastapi.responses import StreamingResponse +from fastapi.exceptions import RequestValidationError +import psycopg +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +import auth +from cache import cache +from config import settings +from db import connection, pool, request_deadline, restart_budget +from errors import ApiError, error_response +import images +from models import CreateChat, CreateMessage, CreateUser +import s3 +import store + +log = logging.getLogger('messenger') + +Limit = Annotated[int, Query(ge=1, le=100)] +Cursor = Annotated[Optional[str], Query(min_length=1)] + +INSTANCE_ID = socket.gethostname() +IDEMPOTENCY_KEY = re.compile(r'^[A-Za-z0-9._:-]{1,128}$') + +PUBLIC = {('POST', '/api/v1/users'), ('POST', '/api/v1/auth/sessions')} + +state = {'ready': False} + + +@asynccontextmanager +async def lifespan(app): + pool.open(wait=False) + log.info('instance %s, session cache: %s', INSTANCE_ID, 'valkey' if cache.client else 'disabled') + state['ready'] = True + log.info('application started') + yield + state['ready'] = False + log.info('application stopping') + pool.close(timeout=2) + + +app = FastAPI(title='MTUSI Messenger lab4', lifespan=lifespan) + + +@app.middleware('http') +async def request_context(request: Request, call_next): + request.state.request_id = uuid.uuid4().hex + request_deadline.set({'deadline': None}) + started = time.perf_counter() + try: + response = await call_next(request) + except Exception: + log.exception('unhandled error request_id=%s', request.state.request_id) + response = error_response(request, 500, 'Внутренняя ошибка сервера') + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = INSTANCE_ID + log.info('%s %s -> %d %.1fms request_id=%s', request.method, request.url.path, + response.status_code, (time.perf_counter() - started) * 1000, request.state.request_id) + return response + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, exc: ApiError): + return error_response(request, exc.status, exc.message, exc.headers) + + +@app.exception_handler(StarletteHTTPException) +async def handle_http_error(request: Request, exc: StarletteHTTPException): + messages = {404: 'Ресурс не найден', 405: 'Метод не поддерживается'} + return error_response(request, exc.status_code, messages.get(exc.status_code, 'Ошибка запроса')) + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error(request: Request, exc: RequestValidationError): + if request.url.path.startswith('/api/v1/') and (request.method, request.url.path) not in PUBLIC: + try: + await run_in_threadpool(auth.authenticate, request.headers.get('authorization')) + except ApiError as error: + return error_response(request, error.status, error.message, error.headers) + except (psycopg.OperationalError, psycopg.InterfaceError): + return error_response(request, 503, 'Хранилище временно недоступно') + errors = exc.errors() + where = '.'.join(str(part) for part in errors[0]['loc']) if errors else 'request' + return error_response(request, 400, 'Некорректный запрос: ' + where) + + +@app.exception_handler(psycopg.OperationalError) +@app.exception_handler(psycopg.InterfaceError) +async def handle_database_unavailable(request: Request, exc: psycopg.Error): + log.warning('database unavailable: %s request_id=%s', type(exc).__name__, request.state.request_id) + return error_response(request, 503, 'Хранилище временно недоступно') + + +@app.get('/health/live') +async def liveness(): + return {'status': 'ok'} + + +@app.get('/health/ready') +def readiness(): + if not state['ready']: + raise ApiError(503, 'Сервис не готов') + with connection(limit=1) as conn: + conn.execute('SELECT 1') + return {'status': 'ok'} + + +def current_session(authorization: Annotated[Optional[str], Header()] = None): + return auth.authenticate(authorization) + + +def current_user(session: Annotated[dict, Depends(current_session)]): + return session['user'] + + +@app.post('/api/v1/users', status_code=201) +def register_user(body: CreateUser): + return store.create_user(body.username, body.display_name, auth.hash_password(body.password)) + + +@app.post('/api/v1/auth/sessions', status_code=201) +def create_session(request: Request, authorization: Annotated[Optional[str], Header()] = None): + if request.headers.get('content-length', '0') != '0' or 'transfer-encoding' in request.headers: + raise ApiError(400, 'Вход принимает только заголовок Authorization: Basic, без тела') + return auth.login(authorization) + + +@app.delete('/api/v1/auth/sessions/current', status_code=204) +def delete_session(session: Annotated[dict, Depends(current_session)]): + auth.logout(session) + return Response(status_code=204) + + +@app.get('/api/v1/users/me') +def get_me(user: Annotated[dict, Depends(current_user)]): + return user + + +@app.post('/api/v1/chats', status_code=201) +def create_chat(body: CreateChat, user: Annotated[dict, Depends(current_user)]): + return store.create_chat(user['id'], body.title, body.member_ids) + + +@app.get('/api/v1/chats') +def list_chats(user: Annotated[dict, Depends(current_user)], limit: Limit = 50, cursor: Cursor = None): + return store.list_chats(user['id'], cursor, limit) + + +@app.get('/api/v1/chats/{chat_id}') +def get_chat(chat_id: uuid.UUID, user: Annotated[dict, Depends(current_user)]): + return store.get_chat(str(chat_id), user['id']) + + +@app.post('/api/v1/chats/{chat_id}/messages', status_code=201) +def create_message(chat_id: uuid.UUID, body: CreateMessage, user: Annotated[dict, Depends(current_user)], + idempotency_key: Annotated[Optional[str], Header()] = None): + if idempotency_key is None or not IDEMPOTENCY_KEY.match(idempotency_key): + raise ApiError(400, 'Нужен заголовок Idempotency-Key: [A-Za-z0-9._:-]{1,128}') + canonical = json.dumps({'text': body.text, 'attachment_ids': body.attachment_ids}, ensure_ascii=False, sort_keys=True) + return store.create_message(str(chat_id), user['id'], body.text, body.attachment_ids, idempotency_key, + hashlib.sha256(canonical.encode('utf-8')).digest(), settings.idempotency_ttl) + + +@app.get('/api/v1/chats/{chat_id}/messages') +def list_messages(chat_id: uuid.UUID, user: Annotated[dict, Depends(current_user)], + limit: Limit = 50, cursor: Cursor = None): + return store.list_messages(str(chat_id), user['id'], cursor, limit) + + +uploads = threading.BoundedSemaphore(settings.upload_concurrency) + + +def save_attachment(chat_id, owner_id, filename, content_type, spool, size, sha256): + if not uploads.acquire(timeout=settings.upload_queue_seconds): + raise ApiError(503, 'Слишком много одновременных загрузок, повторите позже') + try: + return store_upload(chat_id, owner_id, filename, content_type, spool, size, sha256) + finally: + uploads.release() + + +def store_upload(chat_id, owner_id, filename, content_type, spool, size, sha256): + images.inspect(spool, content_type) + attachment_id = str(uuid.uuid4()) + object_key = 'attachments/' + attachment_id + restart_budget() + store.start_attachment(attachment_id, chat_id, owner_id, filename, content_type, size, sha256, object_key) + restart_budget() + try: + s3.put(object_key, spool, size, content_type) + except ApiError: + try: + store.forget_attachment(attachment_id) + except psycopg.Error: + pass + raise + if 'after_s3_put' in settings.faults and filename.startswith('fault-'): + raise RuntimeError('fault injection: process failed after S3 PUT, before COMMIT of ready') + restart_budget() + return store.finish_attachment(attachment_id) + + +@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201) +async def upload_attachment(chat_id: uuid.UUID, request: Request, user: Annotated[dict, Depends(current_user)], + filename: Optional[str] = None): + images.check_filename(filename) + content_type = images.check_content_type(request.headers.get('content-type')) + await run_in_threadpool(store.check_member, str(chat_id), user['id']) + declared = request.headers.get('content-length', '') + if declared.isdigit() and int(declared) > settings.max_upload_bytes: + raise ApiError(413, 'Файл больше %d байт' % settings.max_upload_bytes) + spool = tempfile.SpooledTemporaryFile(max_size=settings.upload_memory_bytes) + try: + size, digest = 0, hashlib.sha256() + async for chunk in request.stream(): + size += len(chunk) + if size > settings.max_upload_bytes: + raise ApiError(413, 'Файл больше %d байт' % settings.max_upload_bytes) + digest.update(chunk) + spool.write(chunk) + if size == 0: + raise ApiError(415, 'Пустое тело вместо изображения') + return await run_in_threadpool(save_attachment, str(chat_id), user['id'], filename, content_type, + spool, size, digest.digest()) + finally: + spool.close() + + +@app.get('/api/v1/attachments/{attachment_id}') +def get_attachment(attachment_id: uuid.UUID, user: Annotated[dict, Depends(current_user)]): + return store.attachment_out(store.get_attachment(str(attachment_id), user['id'])) + + +@app.get('/api/v1/attachments/{attachment_id}/content') +def download_attachment(attachment_id: uuid.UUID, user: Annotated[dict, Depends(current_user)], + variant: Literal['original'] = 'original'): + row = store.get_attachment(str(attachment_id), user['id']) + return StreamingResponse(s3.open_stream(row['object_key']), media_type=row['content_type'], + headers={'Content-Length': str(row['size_bytes'])}) diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..9af4a6d6 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,120 @@ +import base64 +import binascii +import hashlib +import re +import secrets +import threading + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +import psycopg + +from cache import REVOKED, cache +from config import settings +from db import iso +from errors import BASIC_CHALLENGE, BEARER_CHALLENGE, unauthorized +import store + +REVOCATION_MARGIN_SECONDS = 5 + +USERNAME = re.compile(r'^[a-z0-9_]{3,32}$') + +hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +hashing = threading.BoundedSemaphore(settings.password_hash_concurrency) +DUMMY_HASH = hasher.hash('dummy-password-for-equal-timing') + + +def hash_password(password): + with hashing: + return hasher.hash(password) + + +def verify_password(password_hash, password): + try: + with hashing: + return hasher.verify(password_hash, password) + except (VerificationError, InvalidHashError): + return False + + +def token_hash(token): + return hashlib.sha256(token.encode('ascii')).digest() + + +def parse_credentials(header, scheme): + name, _, value = (header or '').strip().partition(' ') + value = value.strip() + if name.lower() != scheme or not value or ' ' in value: + return None + return value + + +def parse_basic(header): + value = parse_credentials(header, 'basic') + if value is None: + return None + try: + decoded = base64.b64decode(value, validate=True).decode('utf-8') + except (binascii.Error, UnicodeDecodeError): + return None + username, separator, password = decoded.partition(':') + if not separator or not USERNAME.match(username) or not 12 <= len(password) <= 128: + return None + return username, password + + +def login(header): + credentials = parse_basic(header) + row = store.find_credentials(credentials[0]) if credentials else None + if row is None: + verify_password(DUMMY_HASH, credentials[1] if credentials else 'x' * 12) + raise unauthorized(BASIC_CHALLENGE, 'Неверное имя пользователя или пароль') + if not verify_password(row['password_hash'], credentials[1]): + raise unauthorized(BASIC_CHALLENGE, 'Неверное имя пользователя или пароль') + token = secrets.token_urlsafe(32) + digest = token_hash(token) + expires_at = store.create_session(row['id'], digest, settings.session_ttl) + user = store.user_out(row) + cache.put(digest, user, expires_at) + return {'token': token, 'token_type': 'Bearer', 'expires_at': iso(expires_at), 'user': user} + + +def authenticate(header): + token = parse_credentials(header, 'bearer') + if token is None or not token.isascii(): + raise unauthorized(BEARER_CHALLENGE) + digest = token_hash(token) + if cache.client is not None and not cache.trusted(): + sync_revocations() + user = cache.get(digest) + if user == REVOKED: + raise unauthorized(BEARER_CHALLENGE) + if user is None: + row = store.session_user(digest) + if row is None: + raise unauthorized(BEARER_CHALLENGE) + user = store.user_out(row) + cache.put(digest, user, row['expires_at']) + return {'user': user, 'token_hash': digest} + + +def sync_revocations(): + errors_before = cache.errors + pending = store.pending_revocations() + if not cache.revoke_all(pending): + return + if pending: + try: + store.clear_revocations(pending) + except psycopg.Error: + pass + cache.mark_synced(errors_before) + + +def logout(session): + store.revoke_session(session['token_hash'], cache.max_ttl + REVOCATION_MARGIN_SECONDS) + if cache.revoke(session['token_hash']): + try: + store.clear_revocations([session['token_hash']]) + except psycopg.Error: + pass diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 00000000..bdc3e275 --- /dev/null +++ b/src/cache.py @@ -0,0 +1,89 @@ +from datetime import datetime, timezone +import json +import logging +import threading +import time + +import redis + +from config import settings + +log = logging.getLogger('messenger.cache') + +BYPASS_SECONDS = 5 +REVOKED = 'revoked' +FAILED = object() + + +class SessionCache: + def __init__(self, url, max_ttl): + self.client = redis.Redis.from_url(url, socket_timeout=0.2, socket_connect_timeout=0.2, + decode_responses=True) if url else None + self.max_ttl = max_ttl + self.bypass_until = 0.0 + self.lock = threading.Lock() + self.errors = 0 + self.synced = False + self.synced_at = 0.0 + + def _run(self, action, force=False): + if self.client is None: + return None + if not force and time.monotonic() < self.bypass_until: + return FAILED + try: + return action(self.client) + except redis.RedisError as error: + log.warning('session cache unavailable (%s), using PostgreSQL for %ds', type(error).__name__, BYPASS_SECONDS) + self.bypass_until = time.monotonic() + BYPASS_SECONDS + with self.lock: + self.errors += 1 + self.synced = False + return FAILED + + @staticmethod + def key(token_hash): + return 'session:' + token_hash.hex() + + def mark_synced(self, errors_before): + with self.lock: + if self.errors == errors_before: + self.synced = True + self.synced_at = time.monotonic() + + def trusted(self): + return self.synced and time.monotonic() - self.synced_at < settings.revocation_sync_seconds + + def get(self, token_hash): + if not self.trusted(): + return None + raw = self._run(lambda c: c.get(self.key(token_hash))) + if raw is FAILED or not raw or raw == REVOKED: + return REVOKED if raw == REVOKED else None + entry = json.loads(raw) + if entry['expires_at'] <= time.time(): + return None + return entry['user'] + + def put(self, token_hash, user, expires_at: datetime): + remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds()) + ttl = min(self.max_ttl, remaining) + if ttl <= 0: + return + value = json.dumps({'user': user, 'expires_at': expires_at.timestamp()}, ensure_ascii=False) + self._run(lambda c: c.set(self.key(token_hash), value, ex=ttl, nx=True)) + + def revoke(self, token_hash): + return self._run(lambda c: c.set(self.key(token_hash), REVOKED, ex=self.max_ttl), force=True) is not FAILED + + def revoke_all(self, token_hashes): + def write(client): + with client.pipeline(transaction=False) as pipe: + for token_hash in token_hashes: + pipe.set(self.key(token_hash), REVOKED, ex=self.max_ttl) + pipe.execute() + return True + return not token_hashes or self._run(write) is not FAILED + + +cache = SessionCache(settings.cache_url, settings.session_cache_ttl) diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..acd5a689 --- /dev/null +++ b/src/config.py @@ -0,0 +1,54 @@ +import os + + +def setting(name, default=None): + path = os.environ.get(name + '_FILE') + if path: + with open(path, encoding='utf-8') as file: + return file.read().strip() + return os.environ.get(name, default) + + +def positive_int(name, default): + value = int(os.environ.get(name, default)) + if value <= 0: + raise SystemExit('%s must be a positive integer' % name) + return value + + +class Settings: + def __init__(self): + self.host = os.environ.get('APP_HOST', '0.0.0.0') + self.port = positive_int('APP_PORT', '8080') + self.env = os.environ.get('APP_ENV', 'lab') + self.shutdown_timeout = positive_int('APP_SHUTDOWN_TIMEOUT', '5') + self.database_url = setting('DATABASE_URL') + self.cache_url = setting('CACHE_URL') or None + self.session_ttl = positive_int('SESSION_TTL_SECONDS', '3600') + self.session_cache_ttl = positive_int('SESSION_CACHE_TTL_SECONDS', '60') + self.db_pool_min = positive_int('DB_POOL_MIN', '1') + self.db_pool_max = positive_int('DB_POOL_MAX', '10') + self.db_pool_timeout = float(os.environ.get('DB_POOL_TIMEOUT_SECONDS', '2')) + self.db_statement_timeout_ms = positive_int('DB_STATEMENT_TIMEOUT_MS', '3000') + self.db_request_timeout = float(os.environ.get('DB_REQUEST_TIMEOUT_SECONDS', '4')) + self.drain_seconds = float(os.environ.get('APP_DRAIN_SECONDS', '3')) + self.idempotency_ttl = positive_int('IDEMPOTENCY_TTL_SECONDS', '86400') + self.revocation_sync_seconds = float(os.environ.get('REVOCATION_SYNC_SECONDS', '1')) + self.password_hash_concurrency = positive_int('PASSWORD_HASH_CONCURRENCY', '4') + self.s3_endpoint = os.environ.get('S3_ENDPOINT', 'http://s3:8333') + self.s3_region = os.environ.get('S3_REGION', 'us-east-1') + self.s3_bucket = os.environ.get('S3_BUCKET', 'messenger-attachments') + self.s3_access_key = setting('S3_ACCESS_KEY_ID') + self.s3_secret_key = setting('S3_SECRET_ACCESS_KEY') + self.s3_timeout = float(os.environ.get('S3_TIMEOUT_SECONDS', '15')) + self.max_upload_bytes = positive_int('MAX_UPLOAD_BYTES', '10485760') + self.max_image_pixels = positive_int('MAX_IMAGE_PIXELS', '40000000') + self.upload_memory_bytes = positive_int('UPLOAD_MEMORY_BYTES', '1048576') + self.upload_concurrency = positive_int('UPLOAD_CONCURRENCY', '4') + self.upload_queue_seconds = float(os.environ.get('UPLOAD_QUEUE_SECONDS', '30')) + self.faults = set(filter(None, os.environ.get('FAULTS', '').split(','))) + if not self.database_url: + raise SystemExit('DATABASE_URL or DATABASE_URL_FILE is required') + + +settings = Settings() diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..9175928e --- /dev/null +++ b/src/db.py @@ -0,0 +1,133 @@ +from contextlib import contextmanager +import contextvars +from datetime import timezone +import itertools +import logging +import os +import socket +import threading +import time + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from config import settings + +log = logging.getLogger('messenger.db') + +request_deadline = contextvars.ContextVar('request_deadline', default=None) + +CHECK_SECONDS = 1 + + +def abort(conn): + try: + sock = socket.socket(fileno=os.dup(conn.fileno())) + except (OSError, psycopg.Error): + return + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + finally: + sock.close() + + +class Watchdog: + def __init__(self): + self.cond = threading.Condition() + self.armed = {} + self.ids = itertools.count() + threading.Thread(target=self.loop, name='db-watchdog', daemon=True).start() + + @contextmanager + def guard(self, conn, seconds): + key = next(self.ids) + with self.cond: + self.armed[key] = (time.monotonic() + seconds, conn) + self.cond.notify() + try: + yield + finally: + with self.cond: + self.armed.pop(key, None) + + def loop(self): + with self.cond: + while True: + now = time.monotonic() + for key, (deadline, conn) in list(self.armed.items()): + if deadline <= now: + del self.armed[key] + log.warning('database did not answer in time, aborting connection') + abort(conn) + wake = min((deadline for deadline, _ in self.armed.values()), default=now + 60) + self.cond.wait(max(wake - now, 0.01)) + + +watchdog = Watchdog() + + +def check(conn): + with watchdog.guard(conn, CHECK_SECONDS): + ConnectionPool.check_connection(conn) + + +pool = ConnectionPool( + settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + timeout=settings.db_pool_timeout, + check=check, + open=False, + name='messenger', + kwargs={ + 'connect_timeout': 3, + 'row_factory': dict_row, + 'options': '-c timezone=UTC -c statement_timeout=%d' % settings.db_statement_timeout_ms, + }, +) + + +def restart_budget(): + budget = request_deadline.get() + if budget is not None: + budget['deadline'] = None + + +def remaining(limit=None): + budget = request_deadline.get() + if budget is None: + left = settings.db_request_timeout + else: + if budget['deadline'] is None: + budget['deadline'] = time.monotonic() + settings.db_request_timeout + left = budget['deadline'] - time.monotonic() + if limit is not None: + left = min(left, limit) + if left <= 0: + raise psycopg.OperationalError('request deadline exceeded') + return left + + +@contextmanager +def connection(limit=None): + conn = pool.getconn(timeout=min(settings.db_pool_timeout, remaining(limit))) + try: + with watchdog.guard(conn, remaining(limit)): + try: + yield conn + except BaseException: + try: + conn.rollback() + except psycopg.Error: + pass + raise + conn.commit() + finally: + pool.putconn(conn) + + +def iso(value): + return value.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ') diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..5cbdbba5 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,37 @@ +from fastapi import Request +from fastapi.responses import JSONResponse + +CODES = { + 400: 'invalid_request', + 401: 'unauthorized', + 404: 'not_found', + 409: 'conflict', + 413: 'payload_too_large', + 415: 'unsupported_media_type', + 429: 'rate_limited', + 500: 'internal_error', + 503: 'unavailable', +} + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiError(Exception): + def __init__(self, status, message, headers=None): + super().__init__(message) + self.status = status + self.message = message + self.headers = headers + + +def unauthorized(challenge, message='Нужна действительная сессия'): + return ApiError(401, message, {'WWW-Authenticate': challenge}) + + +def error_response(request: Request, status, message, headers=None): + request_id = getattr(request.state, 'request_id', '-') + body = {'error': {'code': CODES.get(status, 'invalid_request'), 'message': message, 'request_id': request_id}} + if status == 401 and not headers: + headers = {'WWW-Authenticate': BEARER_CHALLENGE} + return JSONResponse(body, status_code=status, headers=headers) diff --git a/src/images.py b/src/images.py new file mode 100644 index 00000000..ae081395 --- /dev/null +++ b/src/images.py @@ -0,0 +1,139 @@ +import struct +import zlib + +from PIL import Image + +from config import settings +from errors import ApiError + +SIGNATURES = {'image/png': b'\x89PNG\r\n\x1a\n', 'image/jpeg': b'\xff\xd8\xff'} +FORMATS = {'image/png': 'PNG', 'image/jpeg': 'JPEG'} +FORBIDDEN_FILENAME = set('/\\\x7f') | {chr(code) for code in range(32)} | {chr(code) for code in range(0x80, 0xa0)} + +Image.MAX_IMAGE_PIXELS = settings.max_image_pixels + + +def check_filename(filename): + if filename is None or not 1 <= len(filename) <= 255 or not filename.strip() \ + or any(char in FORBIDDEN_FILENAME for char in filename): + raise ApiError(400, 'Некорректный filename: 1–255 символов без /, \\ и управляющих символов') + try: + filename.encode('utf-8') + except UnicodeEncodeError: + raise ApiError(400, 'Некорректный filename') from None + + +def check_content_type(header): + content_type = (header or '').split(';')[0].strip().lower() + if content_type not in SIGNATURES: + raise ApiError(415, 'Принимаются только сырые байты image/png или image/jpeg') + return content_type + + +CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) + + +def png_rows(ihdr): + width, height, depth, color, _, _, interlace = struct.unpack('>IIBBBBB', ihdr) + bits = depth * CHANNELS[color] + if interlace: + passes = [(-(-(width - x0) // dx), -(-(height - y0) // dy)) for x0, y0, dx, dy in ADAM7] + else: + passes = [(width, height)] + return [((w * bits + 7) // 8 + 1, h) for w, h in passes if w > 0 and h > 0] + + +class PixelData: + def __init__(self, ihdr, piece): + self.rows = png_rows(ihdr) + self.expected = sum(length * count for length, count in self.rows) + self.inflater = zlib.decompressobj() + self.piece = piece + self.total = 0 + self.segment = 0 + self.offset = 0 + + def feed(self, data): + while data: + if self.inflater.eof: + return + out = self.inflater.decompress(data, self.piece) + data = self.inflater.unconsumed_tail + self.check(out) + + def check(self, out): + self.total += len(out) + if self.total > self.expected: + raise ValueError('PNG pixel data is longer than IHDR allows') + while out: + length, count = self.rows[self.segment] + take = min(len(out), length * count - self.offset) + part, out = out[:take], out[take:] + filters = part[(-self.offset) % length::length] + if filters and max(filters) > 4: + raise ValueError('PNG row has an invalid filter type') + self.offset += take + if self.offset == length * count: + self.segment, self.offset = self.segment + 1, 0 + + def finish(self): + if not self.inflater.eof or self.total != self.expected: + raise ValueError('PNG pixel data is truncated') + + +def verify_png(fileobj, piece=64 * 1024): + """Потоково: CRC каждого чанка и распаковка IDAT кусками по piece байт без хранения пикселей.""" + fileobj.seek(8) + pixels = None + while True: + header = fileobj.read(8) + if len(header) < 8: + raise ValueError('PNG ended before IEND') + length, kind = struct.unpack('>I4s', header) + if pixels is None and kind != b'IHDR': + raise ValueError('PNG must start with IHDR') + crc = zlib.crc32(kind) + left = length + while left: + data = fileobj.read(min(piece, left)) + if not data: + raise ValueError('PNG chunk is truncated') + crc = zlib.crc32(data, crc) + left -= len(data) + if kind == b'IHDR': + pixels = PixelData(data, piece) + elif kind == b'IDAT': + pixels.feed(data) + stored = fileobj.read(4) + if len(stored) < 4 or struct.unpack('>I', stored)[0] != crc & 0xffffffff: + raise ValueError('PNG chunk CRC mismatch') + if kind == b'IEND': + pixels.finish() + return + + +def inspect(fileobj, content_type): + fileobj.seek(0) + if not fileobj.read(8).startswith(SIGNATURES[content_type]): + raise ApiError(415, 'Содержимое файла не соответствует ' + content_type) + fileobj.seek(0) + try: + with Image.open(fileobj, formats=[FORMATS[content_type]]) as image: + width, height = image.size + if width * height > settings.max_image_pixels: + raise ApiError(413, 'Изображение больше %d пикселей' % settings.max_image_pixels) + if content_type == 'image/png': + verify_png(fileobj) + else: + image.draft('L', (max(1, width // 8), max(1, height // 8))) + image.load() + except ApiError: + raise + except Image.DecompressionBombError: + raise ApiError(413, 'Изображение больше %d пикселей' % settings.max_image_pixels) from None + except Exception: + raise ApiError(415, 'Файл повреждён или не является изображением ' + content_type) from None + finally: + fileobj.seek(0) + return width, height diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..088b97b0 --- /dev/null +++ b/src/main.py @@ -0,0 +1,34 @@ +import logging +import sys +import threading + +import uvicorn + +from config import settings + + +class DrainingServer(uvicorn.Server): + def handle_exit(self, sig, frame): + from app import state + if not state['ready'] or settings.drain_seconds <= 0: + return super().handle_exit(sig, frame) + state['ready'] = False + logging.getLogger('messenger').info('signal %s: readiness off, draining for %.1fs', sig, settings.drain_seconds) + threading.Timer(settings.drain_seconds, super().handle_exit, (sig, frame)).start() + + +def main(): + logging.basicConfig(stream=sys.stdout, level=logging.INFO, + format='%(asctime)s %(levelname)s %(name)s: %(message)s') + logging.getLogger('messenger').info('config host=%s port=%d env=%s session_ttl=%ds session_cache_ttl=%ds', + settings.host, settings.port, settings.env, + settings.session_ttl, settings.session_cache_ttl) + + from app import app + config = uvicorn.Config(app, host=settings.host, port=settings.port, access_log=False, log_config=None, + timeout_graceful_shutdown=settings.shutdown_timeout) + DrainingServer(config).run() + + +if __name__ == '__main__': + main() diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..7ac81bac --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,47 @@ +import logging +from pathlib import Path +import sys +import time + +import psycopg + +from config import settings + +MIGRATIONS = Path(__file__).resolve().parent.parent / 'migrations' +LOCK_ID = 20260002 + +log = logging.getLogger('migrate') + + +def connect(attempts=30): + for attempt in range(1, attempts + 1): + try: + return psycopg.connect(settings.database_url, autocommit=True, connect_timeout=3) + except psycopg.OperationalError: + if attempt == attempts: + raise + log.info('database is not ready, retry %d/%d', attempt, attempts) + time.sleep(1) + + +def main(): + logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s %(name)s: %(message)s') + with connect() as conn: + conn.execute('SELECT pg_advisory_lock(%s)', (LOCK_ID,)) + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (' + 'version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())') + applied = {row[0] for row in conn.execute('SELECT version FROM schema_migrations')} + for path in sorted(MIGRATIONS.glob('*.sql')): + if path.stem in applied: + log.info('skip %s: already applied', path.stem) + continue + with conn.transaction(): + conn.execute(path.read_text(encoding='utf-8')) + conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.stem,)) + log.info('applied %s', path.stem) + conn.execute('SELECT pg_advisory_unlock(%s)', (LOCK_ID,)) + log.info('schema is up to date') + + +if __name__ == '__main__': + main() diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..f8349149 --- /dev/null +++ b/src/models.py @@ -0,0 +1,62 @@ +import uuid + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra='forbid', strict=True) + + +def not_blank(value): + if not value.strip(): + raise ValueError('must not be blank') + try: + value.encode('utf-8') + except UnicodeEncodeError: + raise ValueError('must be valid Unicode') from None + return value + + +class CreateUser(StrictModel): + username: str = Field(pattern=r'^[a-z0-9_]{3,32}$') + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator('display_name', 'password') + @classmethod + def not_blank_fields(cls, value): + return not_blank(value) + + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator('title') + @classmethod + def title_not_blank(cls, value): + return not_blank(value) + + +class CreateMessage(StrictModel): + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator('attachment_ids') + @classmethod + def valid_attachment_ids(cls, value): + try: + ids = [str(uuid.UUID(item)) for item in value] + except ValueError: + raise ValueError('attachment_ids must be UUIDs') from None + if len(set(ids)) != len(ids): + raise ValueError('attachment_ids must be unique') + return ids + + @model_validator(mode='after') + def text_or_attachments(self): + if not self.attachment_ids: + not_blank(self.text) + else: + self.text.encode('utf-8') + return self diff --git a/src/pagination.py b/src/pagination.py new file mode 100644 index 00000000..81b08760 --- /dev/null +++ b/src/pagination.py @@ -0,0 +1,31 @@ +import base64 +from datetime import datetime +import json +import uuid + +from errors import ApiError + + +def encode_cursor(scope, item): + raw = json.dumps({'s': scope, 't': item['created_at'], 'i': item['id']}, separators=(',', ':')) + return base64.urlsafe_b64encode(raw.encode()).decode().rstrip('=') + + +def decode_cursor(scope, cursor): + try: + padded = cursor + '=' * (-len(cursor) % 4) + data = json.loads(base64.urlsafe_b64decode(padded.encode('ascii'))) + if data['s'] != scope or not isinstance(data['t'], str) or not isinstance(data['i'], str): + raise ValueError + created_at = datetime.fromisoformat(data['t']) + if created_at.tzinfo is None: + raise ValueError + return created_at, uuid.UUID(data['i']) + except (ValueError, TypeError, KeyError, AttributeError): + raise ApiError(400, 'Некорректный cursor') from None + + +def page(items, scope, limit): + chunk = items[:limit] + has_more = len(items) > limit + return {'items': chunk, 'next_cursor': encode_cursor(scope, chunk[-1]) if has_more else None} diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..1d15f91d --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,165 @@ +"""Сверка PostgreSQL (метаданные вложений) и S3 (байты). + + python src/reconcile.py # dry-run: только отчёт + python src/reconcile.py --apply # исправить то, что безопасно исправлять + python src/reconcile.py --grace-seconds 900 --report /tmp/report.json + python src/reconcile.py --apply --interval 600 # периодически (сервис reconciler в compose) + +Категории: + ok — ready-строка и объект на месте (размер совпадает); + orphan — объект без строки, старше grace → удалить объект; + fresh_orphan — объект без строки, моложе grace → не трогать (загрузка может идти прямо сейчас); + stale_upload — строка 'uploading' старше grace → удалить строку и объект (если он есть); + fresh_upload — строка 'uploading' моложе grace → не трогать; + missing — ready-строка без объекта → только отчёт (восстановить нечего, удалять метаданные нельзя); + size_mismatch — ready-строка и объект разного размера → только отчёт. +Сначала читается список объектов, потом таблица: объект появляется в S3 только после строки 'uploading', +поэтому у любого увиденного объекта, загруженного API, строка уже есть или была удалена компенсацией. +""" +import argparse +from datetime import timezone +import json +import logging +import resource +import signal +import sys +import threading +import time + +import psycopg +from psycopg.rows import dict_row + +from config import setting, settings +from s3 import make_client + +PREFIX = 'attachments/' +HEARTBEAT = '/tmp/reconciler.alive' # обновляется после каждого прохода цикла; по нему healthcheck + +log = logging.getLogger('reconcile') + + +def list_objects(client): + objects, pages = {}, 0 + for page in client.get_paginator('list_objects_v2').paginate(Bucket=settings.s3_bucket, Prefix=PREFIX): + pages += 1 + for item in page.get('Contents', []): + objects[item['Key']] = {'size': item['Size'], 'modified': item['LastModified'].astimezone(timezone.utc)} + return objects, pages + + +def classify(objects, rows, now, grace): + found = {name: [] for name in ('ok', 'orphan', 'fresh_orphan', 'stale_upload', 'fresh_upload', 'missing', + 'size_mismatch')} + by_key = {row['object_key']: row for row in rows} + for key, info in objects.items(): + if key not in by_key: + age = (now - info['modified']).total_seconds() + found['orphan' if age > grace else 'fresh_orphan'].append({'object_key': key, 'age_s': round(age)}) + for row in rows: + age = (now - row['created_at']).total_seconds() + item = {'id': str(row['id']), 'object_key': row['object_key'], 'age_s': round(age), + 'referenced': row['referenced'], 'object_exists': row['object_key'] in objects} + if row['status'] == 'uploading': + found['stale_upload' if age > grace else 'fresh_upload'].append(item) + elif row['object_key'] not in objects: + found['missing'].append(item) + elif objects[row['object_key']]['size'] != row['size_bytes']: + found['size_mismatch'].append(item) + else: + found['ok'].append(item) + return found + + +def apply(found, client, db): + done = {'orphan_objects_deleted': 0, 'stale_rows_deleted': 0, 'stale_objects_deleted': 0, 'skipped': 0} + for item in found['orphan']: + if db.execute('SELECT 1 FROM attachments WHERE object_key = %s', (item['object_key'],)).fetchone(): + done['skipped'] += 1 + continue + client.delete_object(Bucket=settings.s3_bucket, Key=item['object_key']) + done['orphan_objects_deleted'] += 1 + for item in found['stale_upload']: + deleted = db.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading' " + "AND NOT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = %s) RETURNING 1", + (item['id'], item['id'])).fetchone() + db.commit() + if not deleted: + done['skipped'] += 1 + continue + done['stale_rows_deleted'] += 1 + if item['object_exists']: + client.delete_object(Bucket=settings.s3_bucket, Key=item['object_key']) + done['stale_objects_deleted'] += 1 + return done + + +def run_once(client, args): + started = time.monotonic() + with psycopg.connect(settings.database_url, row_factory=dict_row, connect_timeout=5) as db: + objects, pages = list_objects(client) + listed = time.monotonic() + now = db.execute('SELECT now() AS now').fetchone()['now'].astimezone(timezone.utc) + rows = db.execute('SELECT a.id, a.object_key, a.status, a.size_bytes, a.created_at, ' + 'EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) AS referenced ' + 'FROM attachments a').fetchall() + db.commit() + scanned = time.monotonic() + found = classify(objects, rows, now, args.grace_seconds) + done = apply(found, client, db) if args.apply else None + report = { + 'mode': 'apply' if args.apply else 'dry-run', 'grace_seconds': args.grace_seconds, + 'objects_listed': len(objects), 'list_pages': pages, 'rows_scanned': len(rows), + 'counts': {name: len(items) for name, items in found.items()}, + 'actions': done, + 'seconds': {'list_s3': round(listed - started, 3), 'scan_db': round(scanned - listed, 3), + 'total': round(time.monotonic() - started, 3)}, + 'max_rss_mib': round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1), + 'missing': found['missing'], 'size_mismatch': found['size_mismatch'], + } + if args.details: + report['items'] = {name: items for name, items in found.items() if name != 'ok'} + return report + + +def run_forever(client, args): + """Периодический режим (сервис reconciler): прогон сразу и далее каждые --interval секунд.""" + stop = threading.Event() + signal.signal(signal.SIGTERM, lambda *_: stop.set()) + log.info('periodic mode: every %.0fs, %s, grace %.0fs', args.interval, + 'apply' if args.apply else 'dry-run', args.grace_seconds) + while not stop.is_set(): + try: + report = run_once(client, args) + log.info('run: objects %d, rows %d, counts %s, actions %s, %.2fs', report['objects_listed'], + report['rows_scanned'], {k: v for k, v in report['counts'].items() if v}, + report['actions'], report['seconds']['total']) + except Exception as error: + log.warning('run failed, will retry next time: %s: %s', type(error).__name__, error) + with open(HEARTBEAT, 'w') as file: + file.write(str(time.time())) + stop.wait(args.interval) + log.info('stopped') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--apply', action='store_true') + parser.add_argument('--grace-seconds', type=float, default=900) + parser.add_argument('--interval', type=float, default=0, help='repeat every N seconds (0 = run once)') + parser.add_argument('--report') + parser.add_argument('--details', action='store_true', help='include every item in the report') + args = parser.parse_args() + logging.basicConfig(stream=sys.stderr, level=logging.INFO, format='%(asctime)s %(name)s: %(message)s') + client = make_client(setting('S3_RECONCILER_ACCESS_KEY_ID'), setting('S3_RECONCILER_SECRET_ACCESS_KEY'), timeout=30) + if args.interval > 0: + run_forever(client, args) + return + text = json.dumps(run_once(client, args), ensure_ascii=False, indent=2, default=str) + if args.report: + with open(args.report, 'w', encoding='utf-8') as file: + file.write(text) + print(text) + + +if __name__ == '__main__': + main() diff --git a/src/s3.py b/src/s3.py new file mode 100644 index 00000000..ea5636ab --- /dev/null +++ b/src/s3.py @@ -0,0 +1,65 @@ +import logging + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from config import settings +from errors import ApiError + +log = logging.getLogger('messenger.s3') + +CHUNK = 64 * 1024 + + +def make_client(access_key, secret_key, timeout=None): + return boto3.client( + 's3', endpoint_url=settings.s3_endpoint, region_name=settings.s3_region, + aws_access_key_id=access_key, aws_secret_access_key=secret_key, + config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, + connect_timeout=timeout or settings.s3_timeout, read_timeout=timeout or settings.s3_timeout, + retries={'max_attempts': 2, 'mode': 'standard'}, + request_checksum_calculation='when_required', response_checksum_validation='when_required')) + + +client = make_client(settings.s3_access_key, settings.s3_secret_key) if settings.s3_access_key else None + + +def unavailable(error): + log.warning('object storage unavailable: %s', type(error).__name__) + return ApiError(503, 'Хранилище файлов временно недоступно') + + +def put(key, fileobj, size, content_type): + try: + client.put_object(Bucket=settings.s3_bucket, Key=key, Body=fileobj, ContentLength=size, + ContentType=content_type) + except (BotoCoreError, ClientError) as error: + raise unavailable(error) from None + + +def delete(key): + try: + client.delete_object(Bucket=settings.s3_bucket, Key=key) + return True + except (BotoCoreError, ClientError) as error: + log.warning('could not delete object %s: %s', key, type(error).__name__) + return False + + +def open_stream(key): + try: + body = client.get_object(Bucket=settings.s3_bucket, Key=key)['Body'] + except ClientError as error: + if error.response.get('Error', {}).get('Code') in ('NoSuchKey', '404'): + log.error('object %s is missing for a ready attachment', key) + raise unavailable(error) from None + except BotoCoreError as error: + raise unavailable(error) from None + + def chunks(): + try: + yield from body.iter_chunks(CHUNK) + finally: + body.close() + return chunks() diff --git a/src/s3_bootstrap.py b/src/s3_bootstrap.py new file mode 100644 index 00000000..8601e9a0 --- /dev/null +++ b/src/s3_bootstrap.py @@ -0,0 +1,55 @@ +import logging +import sys +import time +import urllib.error +import urllib.request + +from botocore.exceptions import BotoCoreError, ClientError + +from config import setting, settings +from s3 import make_client + +log = logging.getLogger('s3-bootstrap') + + +def wait_for(client, attempts=30): + for attempt in range(1, attempts + 1): + try: + return client.list_buckets() + except (BotoCoreError, ClientError) as error: + if attempt == attempts: + raise + log.info('object storage is not ready (%s), retry %d/%d', type(error).__name__, attempt, attempts) + time.sleep(1) + + +def anonymous_status(path): + try: + urllib.request.urlopen(settings.s3_endpoint + path, timeout=5) + return 200 + except urllib.error.HTTPError as error: + return error.code + + +def main(): + logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s %(name)s: %(message)s') + admin = make_client(setting('S3_ADMIN_ACCESS_KEY_ID'), setting('S3_ADMIN_SECRET_ACCESS_KEY')) + buckets = {bucket['Name'] for bucket in wait_for(admin).get('Buckets', [])} + if settings.s3_bucket in buckets: + log.info('bucket %s already exists, nothing to do', settings.s3_bucket) + else: + admin.create_bucket(Bucket=settings.s3_bucket) + log.info('bucket %s created', settings.s3_bucket) + started = time.monotonic() + admin.put_object(Bucket=settings.s3_bucket, Key='bootstrap/warmup', Body=b'warmup') + admin.delete_object(Bucket=settings.s3_bucket, Key='bootstrap/warmup') + log.info('warm-up write done in %.1fs (storage volumes allocated before first upload)', time.monotonic() - started) + for path in ('/' + settings.s3_bucket, '/' + settings.s3_bucket + '/probe'): + status = anonymous_status(path) + if status not in (401, 403): + raise SystemExit('anonymous access to %s returned %d, bucket is not private' % (path, status)) + log.info('anonymous list/get denied: bucket %s is private', settings.s3_bucket) + + +if __name__ == '__main__': + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..d4d4eddc --- /dev/null +++ b/src/store.py @@ -0,0 +1,246 @@ +import uuid + +from psycopg import errors + +from db import connection, iso +from errors import ApiError +from pagination import decode_cursor, page + +USER_COLUMNS = 'u.id, u.username, u.display_name, u.created_at' +ATTACHMENT_IDS = ('ARRAY(SELECT ma.attachment_id FROM message_attachments ma ' + 'WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids') +MESSAGE_COLUMNS = 'm.id, m.chat_id, m.sender_id, m.text, m.created_at, ' + ATTACHMENT_IDS +ATTACHMENT_COLUMNS = 'a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, a.status, a.created_at, a.updated_at' +CHAT_SELECT = ('SELECT c.id, c.title, c.created_at, ' + 'ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids ' + 'FROM chats c ') + + +def utf8(value): + return value.encode('utf-8') + + +def unicode(value): + return bytes(value).decode('utf-8') + + +def user_out(row): + return {'id': str(row['id']), 'username': row['username'], + 'display_name': unicode(row['display_name']), 'created_at': iso(row['created_at'])} + + +def chat_out(row): + return {'id': str(row['id']), 'title': unicode(row['title']), + 'member_ids': [str(m) for m in row['member_ids']], 'created_at': iso(row['created_at'])} + + +def message_out(row): + return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'sender_id': str(row['sender_id']), + 'text': unicode(row['text']), 'created_at': iso(row['created_at']), + 'attachment_ids': [str(a) for a in row['attachment_ids']]} + + +def attachment_out(row): + return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']), + 'filename': row['filename'], 'content_type': row['content_type'], 'size_bytes': row['size_bytes'], + 'sha256': bytes(row['sha256']).hex(), 'status': row['status'], 'variants': [], 'error_code': None, + 'created_at': iso(row['created_at']), 'updated_at': iso(row['updated_at'])} + + +def create_user(username, display_name, password_hash): + try: + with connection() as conn: + row = conn.execute( + 'INSERT INTO users AS u (username, display_name, password_hash) VALUES (%s, %s, %s) ' + 'RETURNING ' + USER_COLUMNS, + (username, utf8(display_name), password_hash)).fetchone() + except errors.UniqueViolation: + raise ApiError(409, 'Имя пользователя уже занято') from None + return user_out(row) + + +def find_credentials(username): + with connection() as conn: + return conn.execute('SELECT ' + USER_COLUMNS + ', u.password_hash FROM users u WHERE u.username = %s', + (username,)).fetchone() + + +def create_session(user_id, token_hash, ttl_seconds): + with connection() as conn: + conn.execute('DELETE FROM sessions WHERE user_id = %s AND expires_at <= now()', (user_id,)) + return conn.execute( + 'INSERT INTO sessions (token_hash, user_id, expires_at) ' + 'VALUES (%s, %s, now() + make_interval(secs => %s)) RETURNING expires_at', + (token_hash, user_id, ttl_seconds)).fetchone()['expires_at'] + + +def session_user(token_hash): + with connection() as conn: + return conn.execute( + 'SELECT ' + USER_COLUMNS + ', s.expires_at FROM sessions s JOIN users u ON u.id = s.user_id ' + 'WHERE s.token_hash = %s AND s.expires_at > now()', + (token_hash,)).fetchone() + + +def revoke_session(token_hash, keep_seconds): + with connection() as conn: + conn.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,)) + conn.execute('INSERT INTO session_revocations (token_hash, expires_at) ' + 'VALUES (%s, now() + make_interval(secs => %s)) ' + 'ON CONFLICT (token_hash) DO UPDATE SET expires_at = EXCLUDED.expires_at', + (token_hash, keep_seconds)) + + +def pending_revocations(): + with connection() as conn: + conn.execute('DELETE FROM session_revocations WHERE expires_at <= now()') + return [bytes(r['token_hash']) for r in conn.execute('SELECT token_hash FROM session_revocations').fetchall()] + + +def clear_revocations(token_hashes): + with connection() as conn: + conn.execute('DELETE FROM session_revocations WHERE token_hash = ANY(%s)', (list(token_hashes),)) + + +def create_chat(creator_id, title, member_ids): + try: + members = [str(uuid.UUID(m)) for m in member_ids] + except ValueError: + raise ApiError(400, 'Неизвестный участник чата') from None + if len(set(members)) != len(members): + raise ApiError(400, 'Участники не должны повторяться') + if creator_id not in members: + raise ApiError(400, 'Создатель должен быть участником чата') + try: + with connection() as conn, conn.transaction(): + chat = conn.execute('INSERT INTO chats (title, created_by) VALUES (%s, %s) RETURNING id, title, created_at', + (utf8(title), creator_id)).fetchone() + conn.execute('INSERT INTO chat_members (chat_id, user_id, position) ' + 'SELECT %s, member, position FROM unnest(%s::uuid[]) WITH ORDINALITY AS t(member, position)', + (chat['id'], members)) + except errors.ForeignKeyViolation: + raise ApiError(400, 'Неизвестный участник чата') from None + return chat_out({**chat, 'member_ids': members}) + + +def get_chat(chat_id, user_id): + with connection() as conn: + row = conn.execute(CHAT_SELECT + 'JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s ' + 'WHERE c.id = %s', (user_id, chat_id)).fetchone() + if row is None: + raise ApiError(404, 'Чат не найден') + return chat_out(row) + + +def list_chats(user_id, cursor, limit): + scope = 'user:' + user_id + sql = CHAT_SELECT + 'JOIN chat_members me ON me.chat_id = c.id WHERE me.user_id = %s ' + params = [user_id] + if cursor: + sql += 'AND (c.created_at, c.id) > (%s, %s) ' + params += decode_cursor(scope, cursor) + sql += 'ORDER BY c.created_at, c.id LIMIT %s' + with connection() as conn: + rows = conn.execute(sql, params + [limit + 1]).fetchall() + return page([chat_out(r) for r in rows], scope, limit) + + +def is_member(conn, chat_id, user_id): + return conn.execute('SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s', + (chat_id, user_id)).fetchone() is not None + + +def create_message(chat_id, sender_id, text, attachment_ids, key, request_hash, key_ttl_seconds): + scope = 'POST /api/v1/chats/%s/messages' % chat_id + with connection() as conn, conn.transaction(): + conn.execute('DELETE FROM idempotency_keys WHERE user_id = %s AND created_at < now() - make_interval(secs => %s)', + (sender_id, key_ttl_seconds)) + claimed = conn.execute( + 'INSERT INTO idempotency_keys (user_id, scope, key, request_hash) VALUES (%s, %s, %s, %s) ' + 'ON CONFLICT DO NOTHING RETURNING 1', + (sender_id, scope, key, request_hash)).fetchone() + if claimed is None: + if not is_member(conn, chat_id, sender_id): + raise ApiError(404, 'Чат не найден') + stored = conn.execute( + 'SELECT k.request_hash, ' + MESSAGE_COLUMNS + ' ' + 'FROM idempotency_keys k JOIN messages m ON m.id = k.message_id ' + 'WHERE k.user_id = %s AND k.scope = %s AND k.key = %s', + (sender_id, scope, key)).fetchone() + if bytes(stored['request_hash']) != request_hash: + raise ApiError(409, 'Idempotency-Key уже использован с другим запросом') + return message_out(stored) + row = conn.execute( + 'INSERT INTO messages AS m (chat_id, sender_id, text) SELECT %s, %s, %s ' + 'WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s) ' + 'RETURNING m.id, m.chat_id, m.sender_id, m.text, m.created_at', + (chat_id, sender_id, utf8(text), chat_id, sender_id)).fetchone() + if row is None: + raise ApiError(404, 'Чат не найден') + if attachment_ids: + found = conn.execute("SELECT count(*) AS n FROM attachments WHERE id = ANY(%s::uuid[]) " + "AND chat_id = %s AND status = 'ready'", (attachment_ids, chat_id)).fetchone()['n'] + if found != len(attachment_ids): + raise ApiError(404, 'Вложение не найдено в этом чате') + conn.execute('INSERT INTO message_attachments (message_id, attachment_id, position) ' + 'SELECT %s, attachment, position FROM unnest(%s::uuid[]) WITH ORDINALITY AS t(attachment, position)', + (row['id'], attachment_ids)) + row = dict(row, attachment_ids=attachment_ids) + conn.execute('UPDATE idempotency_keys SET message_id = %s WHERE user_id = %s AND scope = %s AND key = %s', + (row['id'], sender_id, scope, key)) + return message_out(row) + + +def list_messages(chat_id, user_id, cursor, limit): + scope = 'chat:' + chat_id + sql = 'SELECT ' + MESSAGE_COLUMNS + ' FROM messages m WHERE m.chat_id = %s ' + params = [chat_id] + if cursor: + sql += 'AND (m.created_at, m.id) > (%s, %s) ' + params += decode_cursor(scope, cursor) + sql += 'ORDER BY m.created_at, m.id LIMIT %s' + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise ApiError(404, 'Чат не найден') + rows = conn.execute(sql, params + [limit + 1]).fetchall() + return page([message_out(r) for r in rows], scope, limit) + + +def start_attachment(attachment_id, chat_id, owner_id, filename, content_type, size, sha256, object_key): + with connection() as conn: + if not is_member(conn, chat_id, owner_id): + raise ApiError(404, 'Чат не найден') + conn.execute("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'uploading')", + (attachment_id, chat_id, owner_id, filename, content_type, size, sha256, object_key)) + + +def finish_attachment(attachment_id): + with connection() as conn: + row = conn.execute("UPDATE attachments AS a SET status = 'ready', updated_at = clock_timestamp() " + "WHERE a.id = %s AND a.status = 'uploading' RETURNING " + ATTACHMENT_COLUMNS, + (attachment_id,)).fetchone() + if row is None: + raise ApiError(503, 'Загрузка не завершена, повторите') + return attachment_out(row) + + +def forget_attachment(attachment_id): + with connection() as conn: + conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,)) + + +def check_member(chat_id, user_id): + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise ApiError(404, 'Чат не найден') + + +def get_attachment(attachment_id, user_id): + with connection() as conn: + row = conn.execute("SELECT " + ATTACHMENT_COLUMNS + " FROM attachments a " + "JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s " + "WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id)).fetchone() + if row is None: + raise ApiError(404, 'Вложение не найдено') + return row -- 2.54.0 From f84831d0bb709632e75fe3ee62a457969aaa7791 Mon Sep 17 00:00:00 2001 From: Pavel Grigorev Date: Fri, 13 Nov 2026 16:22:54 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D1=82=D0=B5=D1=81=D1=82=D1=8B=20=D0=B2=D0=BB=D0=BE?= =?UTF-8?q?=D0=B6=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=B8=20=D1=85=D1=80=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B0,=20=D0=BF=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B5=D1=80=D0=BA=D1=83=20=D1=81=D0=BE=D0=B3=D0=BB=D0=B0?= =?UTF-8?q?=D1=81=D0=BE=D0=B2=D0=B0=D0=BD=D0=BD=D0=BE=D1=81=D1=82=D0=B8=20?= =?UTF-8?q?=D0=B8=20=D0=B8=D1=81=D1=81=D0=BB=D0=B5=D0=B4=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D1=81=D0=B2=D0=B5=D1=80=D1=89=D0=B8=D0=BA?= =?UTF-8?q?=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/app.yml | 31 +++ research/s3ops.py | 43 ++++ research/track2.py | 282 ++++++++++++++++++++++++++ student-tests/consistency_check.py | 156 ++++++++++++++ student-tests/expiry_check.py | 56 +++++ student-tests/failover_series.py | 71 +++++++ student-tests/failure_check.py | 127 ++++++++++++ student-tests/fixtures/sample.jpg | Bin 0 -> 19881 bytes student-tests/helpers.py | 191 +++++++++++++++++ student-tests/imagegen.py | 66 ++++++ student-tests/rolling_update_check.py | 60 ++++++ student-tests/sigterm_check.py | 79 ++++++++ student-tests/test_api.py | 157 ++++++++++++++ student-tests/test_attachments.py | 192 ++++++++++++++++++ student-tests/test_auth.py | 132 ++++++++++++ student-tests/test_concurrency.py | 68 +++++++ student-tests/test_db.py | 58 ++++++ student-tests/test_idempotency.py | 96 +++++++++ student-tests/test_replicas.py | 41 ++++ student-tests/test_storage.py | 81 ++++++++ student-tests/upload_memory.py | 118 +++++++++++ 21 files changed, 2105 insertions(+) create mode 100644 .github/workflows/app.yml create mode 100644 research/s3ops.py create mode 100644 research/track2.py create mode 100644 student-tests/consistency_check.py create mode 100644 student-tests/expiry_check.py create mode 100644 student-tests/failover_series.py create mode 100644 student-tests/failure_check.py create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/helpers.py create mode 100644 student-tests/imagegen.py create mode 100644 student-tests/rolling_update_check.py create mode 100644 student-tests/sigterm_check.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_auth.py create mode 100644 student-tests/test_concurrency.py create mode 100644 student-tests/test_db.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_replicas.py create mode 100644 student-tests/test_storage.py create mode 100644 student-tests/upload_memory.py diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..b9ec4641 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,31 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + stack: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: cp .env.example .env + - run: python scripts/init_secrets.py + - run: docker compose config --quiet + - run: docker compose up -d --build --wait + - run: make test BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: make student-test BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: python student-tests/expiry_check.py --ttl 2 + - run: python student-tests/failure_check.py + - run: python student-tests/consistency_check.py + - run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: docker compose start api && sleep 15 + - run: python student-tests/failover_series.py + - run: python student-tests/rolling_update_check.py --no-build + - if: failure() + run: docker compose ps -a && docker compose logs --no-color + - if: always() + run: docker compose down -v diff --git a/research/s3ops.py b/research/s3ops.py new file mode 100644 index 00000000..cd2495e2 --- /dev/null +++ b/research/s3ops.py @@ -0,0 +1,43 @@ +"""Операции с S3 для внедрения сбоев в треке 2. Запускается внутри контейнера reconcile (ключи сверщика): + python research/s3ops.py put SIZE < keys # создать объекты указанного размера + python research/s3ops.py delete < keys # удалить объекты + python research/s3ops.py exists < keys # вывести число существующих +""" +from concurrent.futures import ThreadPoolExecutor +import sys + +sys.path.insert(0, 'src') + +from botocore.exceptions import ClientError # noqa: E402 + +from config import setting, settings # noqa: E402 +from s3 import make_client # noqa: E402 + +client = make_client(setting('S3_RECONCILER_ACCESS_KEY_ID'), setting('S3_RECONCILER_SECRET_ACCESS_KEY'), timeout=30) + + +def exists(key): + try: + client.head_object(Bucket=settings.s3_bucket, Key=key) + return True + except ClientError: + return False + + +def main(): + command = sys.argv[1] + keys = [line.strip() for line in sys.stdin if line.strip()] + with ThreadPoolExecutor(max_workers=16) as pool: + if command == 'put': + body = b'x' * int(sys.argv[2]) + list(pool.map(lambda k: client.put_object(Bucket=settings.s3_bucket, Key=k, Body=body), keys)) + print(len(keys)) + elif command == 'delete': + list(pool.map(lambda k: client.delete_object(Bucket=settings.s3_bucket, Key=k), keys)) + print(len(keys)) + elif command == 'exists': + print(sum(pool.map(exists, keys))) + + +if __name__ == '__main__': + main() diff --git a/research/track2.py b/research/track2.py new file mode 100644 index 00000000..3a0877e2 --- /dev/null +++ b/research/track2.py @@ -0,0 +1,282 @@ +"""Трек 2: согласованность PostgreSQL и S3 — исследование сверщика (src/reconcile.py). + +Стенд запущен (docker compose up -d). BASE_URL — адрес LB (по умолчанию http://127.0.0.1:8080). + python research/track2.py scenario # H1/H2: внедрённые сбои → dry-run → apply → повторный apply + python research/track2.py race # H2: apply в цикле под непрерывными загрузками + python research/track2.py cost # H3: время/память обхода при 1k…20k объектов + python research/track2.py cleanup # удалить синтетические данные исследования + python research/track2.py report # evidence/track2/results.md + python research/track2.py all +Синтетические строки живут в отдельном чате пользователя track2_*; cleanup удаляет их и их объекты. +На время запуска периодический сервис reconciler останавливается и затем запускается снова. +""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import statistics +import subprocess +import sys +import threading +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'student-tests')) + +from helpers import call, compose, new_chat, new_user, sql # noqa: E402 +from imagegen import SAMPLE_PNG # noqa: E402 +from test_attachments import download, upload # noqa: E402 + +OUT = ROOT / 'evidence' / 'track2' +STATE = OUT / 'state.json' +GRACE = 30 + + +def save(name, data): + OUT.mkdir(parents=True, exist_ok=True) + (OUT / name).write_text(json.dumps(data, ensure_ascii=False, indent=2, default=str) + '\n', encoding='utf-8') + print('saved evidence/track2/' + name) + + +def load(name): + return json.loads((OUT / name).read_text(encoding='utf-8')) + + +def s3ops(command, keys, *args): + result = subprocess.run(['docker', 'compose', 'run', '--rm', '-T', '-v', str(ROOT / 'research') + ':/app/research:ro', + '--entrypoint', 'python', 'reconcile', 'research/s3ops.py', command, *map(str, args)], + cwd=ROOT, input='\n'.join(keys), capture_output=True, text=True, encoding='utf-8') + if result.returncode: + raise SystemExit(result.stderr[-2000:]) + return int(result.stdout.strip().splitlines()[-1]) + + +def reconcile(*args): + return json.loads(compose('run', '--rm', 'reconcile', *map(str, args))) + + +def research_chat(): + if STATE.exists(): + return json.loads(STATE.read_text(encoding='utf-8')) + user = new_user('track2') + chat = new_chat(user) + state = {'user_id': user['id'], 'chat_id': chat['id'], 'token': None} + OUT.mkdir(parents=True, exist_ok=True) + STATE.write_text(json.dumps(state), encoding='utf-8') + return state + + +def insert_rows(state, count, status='ready', age_seconds=0, size=1024): + """Синтетические строки attachments в чате исследования; возвращает ключи объектов.""" + if count <= 0: + return [] + out = sql("WITH new AS (INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "object_key, status, created_at, updated_at) SELECT id, '%s', '%s', 'synthetic.png', 'image/png', %d, " + "decode(repeat('00', 32), 'hex'), 'attachments/' || id, '%s', now() - make_interval(secs => %d), now() " + "FROM (SELECT gen_random_uuid() AS id FROM generate_series(1, %d)) g RETURNING object_key) " + "SELECT string_agg(object_key, ' ') FROM new" + % (state['chat_id'], state['user_id'], size, status, age_seconds, count)) + return out.split() + + +def reference(state, keys): + """Привязать вложения к сообщению (используемые вложения).""" + message = sql("INSERT INTO messages (chat_id, sender_id, text) VALUES ('%s', '%s', convert_to('track2', 'UTF8')) " + "RETURNING id" % (state['chat_id'], state['user_id'])).splitlines()[0] + sql("INSERT INTO message_attachments (message_id, attachment_id, position) SELECT '%s', a.id, " + "row_number() OVER () FROM attachments a WHERE a.object_key IN (%s)" + % (message, ','.join("'%s'" % k for k in keys))) + + +def orphan_keys(count): + return ['attachments/%s' % uuid.uuid4() for _ in range(count)] + + +def scenario(): + state = research_chat() + real = new_user('track2real') + real_chat = new_chat(real) + real_att = upload(real, real_chat, SAMPLE_PNG)[2] + call('POST', '/api/v1/chats/%s/messages' % real_chat['id'], {'text': '', 'attachment_ids': [real_att['id']]}, + real['token']) + baseline = reconcile('--grace-seconds', GRACE) + + ok = insert_rows(state, 200) + s3ops('put', ok, 1024) + referenced = ok[:20] + reference(state, referenced) + missing = ok[20:30] + referenced[:5] + s3ops('delete', missing) + mismatch = ok[40:50] + s3ops('put', mismatch, 2048) + stale_with_object = insert_rows(state, 25, 'uploading', 3600) + s3ops('put', stale_with_object, 1024) + stale_without_object = insert_rows(state, 25, 'uploading', 3600) + old_orphans = orphan_keys(25) + s3ops('put', old_orphans, 1024) + print('waiting %ds so that injected orphans become older than grace…' % (GRACE + 3)) + time.sleep(GRACE + 3) + fresh_uploads = insert_rows(state, 10, 'uploading', 0) + s3ops('put', fresh_uploads, 1024) + fresh_orphans = orphan_keys(10) + s3ops('put', fresh_orphans, 1024) + + expected = {'orphan': 25, 'fresh_orphan': 10, 'stale_upload': 50, 'fresh_upload': 10, 'missing': 15, + 'size_mismatch': 10} + base = baseline['counts'] + dry = reconcile('--grace-seconds', GRACE) + dry_again = reconcile('--grace-seconds', GRACE) + applied = reconcile('--apply', '--grace-seconds', GRACE) + rerun = reconcile('--apply', '--grace-seconds', GRACE) + after = reconcile('--grace-seconds', GRACE) + detected = {k: dry['counts'][k] - base.get(k, 0) for k in expected} + real_download = download(real, real_att['id']) + checks = { + 'detected_equals_injected': detected == expected, + 'dry_run_changes_nothing': dry_again['counts'] == dry['counts'] and dry_again['objects_listed'] == dry['objects_listed'], + 'apply_actions': applied['actions'], + 'apply_deleted_exactly_old_orphans_and_stale': applied['actions'] == { + 'orphan_objects_deleted': 25, 'stale_rows_deleted': 50, 'stale_objects_deleted': 25, 'skipped': 0}, + 'rerun_is_noop': not any(rerun['actions'][k] for k in ('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')), + 'fresh_orphans_kept': s3ops('exists', fresh_orphans) == 10, + 'fresh_uploads_kept': s3ops('exists', fresh_uploads) == 10 and after['counts']['fresh_upload'] >= 10, + 'old_orphans_gone': s3ops('exists', old_orphans) == 0, + 'stale_objects_gone': s3ops('exists', stale_with_object) == 0, + 'referenced_rows_kept': int(sql("SELECT count(*) FROM attachments WHERE object_key IN (%s)" + % ','.join("'%s'" % k for k in referenced))) == 20, + 'missing_still_reported_not_deleted': after['counts']['missing'] - base.get('missing', 0) == 15, + 'real_attachment_intact': real_download[0] == 200 and hashlib.sha256(real_download[2]).hexdigest() == real_att['sha256'], + } + passed = all(v for k, v in checks.items() if k != 'apply_actions') + save('scenario.json', {'grace_seconds': GRACE, 'injected': expected, 'baseline_counts': base, + 'dry_run': dry, 'apply': applied, 'rerun': rerun, 'after': after, + 'detected': detected, 'checks': checks, 'passed': passed}) + for name, value in checks.items(): + print(' %-48s %s' % (name, value)) + print('PASS' if passed else 'FAIL') + + +def race(seconds=40): + user = new_user('track2race') + chat = new_chat(user) + uploaded, errors, stop = [], [], threading.Event() + + def uploader(): + while not stop.is_set(): + status, _, body = upload(user, chat, SAMPLE_PNG) + (uploaded.append(body['id']) if status == 201 else errors.append(status)) + + preclean = reconcile('--apply', '--grace-seconds', 0)['actions'] + threads = [threading.Thread(target=uploader) for _ in range(4)] + for thread in threads: + thread.start() + runs, started = [], time.monotonic() + while time.monotonic() - started < seconds: + report = reconcile('--apply', '--grace-seconds', GRACE) + runs.append({'counts': report['counts'], 'actions': report['actions'], 'total_s': report['seconds']['total']}) + stop.set() + for thread in threads: + thread.join() + intact = 0 + for attachment_id in uploaded: + status, _, body = download(user, attachment_id) + intact += status == 200 and body == SAMPLE_PNG + result = {'seconds': seconds, 'grace_seconds': GRACE, 'preclean_before_load': preclean, + 'reconcile_runs': len(runs), 'uploads_201': len(uploaded), + 'upload_errors': errors, 'downloaded_intact': intact, + 'deleted_by_reconciler': sum(r['actions']['orphan_objects_deleted'] + r['actions']['stale_rows_deleted'] + for r in runs), + 'fresh_uploads_seen_in_flight': sum(r['counts']['fresh_upload'] for r in runs), 'runs': runs} + result['passed'] = intact == len(uploaded) and result['deleted_by_reconciler'] == 0 + save('race.json', result) + print('race: %d uploads, %d intact, reconciler runs %d, deleted %d, in-flight uploads seen %d → %s' + % (len(uploaded), intact, len(runs), result['deleted_by_reconciler'], result['fresh_uploads_seen_in_flight'], + 'PASS' if result['passed'] else 'FAIL')) + + +def cost(sizes=(1000, 5000, 10000, 20000), repeats=3): + state = research_chat() + rows = [] + for target in sizes: + current = int(sql("SELECT count(*) FROM attachments")) + if target > current: + keys = insert_rows(state, target - current) + started = time.monotonic() + for start in range(0, len(keys), 2000): + s3ops('put', keys[start:start + 2000], 256) + print(' +%d objects in %.1fs' % (len(keys), time.monotonic() - started)) + reports = [reconcile('--grace-seconds', GRACE) for _ in range(repeats)] + row = {'objects': reports[-1]['objects_listed'], 'rows': reports[-1]['rows_scanned'], + 'list_pages': reports[-1]['list_pages'], + 'list_s3_s': statistics.median(r['seconds']['list_s3'] for r in reports), + 'scan_db_s': statistics.median(r['seconds']['scan_db'] for r in reports), + 'total_s': statistics.median(r['seconds']['total'] for r in reports), + 'max_rss_mib': max(r['max_rss_mib'] for r in reports)} + row['us_per_object'] = round(row['total_s'] / row['objects'] * 1e6, 1) + rows.append(row) + print(' %(objects)6d objects: list %(list_s3_s).2fs, db %(scan_db_s).3fs, total %(total_s).2fs, ' + 'rss %(max_rss_mib).0f MiB, %(us_per_object).0f µs/object' % row) + save('cost.json', {'repeats': repeats, 'rows': rows}) + + +def cleanup(): + if not STATE.exists(): + print('nothing to clean') + return + state = json.loads(STATE.read_text(encoding='utf-8')) + keys = sql("SELECT coalesce(string_agg(object_key, ' '), '') FROM attachments WHERE chat_id = '%s'" % state['chat_id']).split() + for start in range(0, len(keys), 2000): + s3ops('delete', keys[start:start + 2000]) + sql("DELETE FROM message_attachments WHERE message_id IN (SELECT id FROM messages WHERE chat_id = '%s')" % state['chat_id']) + sql("DELETE FROM messages WHERE chat_id = '%s'" % state['chat_id']) + sql("DELETE FROM attachments WHERE chat_id = '%s'" % state['chat_id']) + leftovers = reconcile('--apply', '--grace-seconds', 0) + STATE.unlink() + print('cleanup: %d research objects deleted; reconciler afterwards: %s' % (len(keys), leftovers['counts'])) + + +def report(): + s, r, c = load('scenario.json'), load('race.json'), load('cost.json') + lines = ['# Трек 2: результаты', '', '## H1/H2. Внедрённые сбои (grace %d с)' % s['grace_seconds'], '', + '| категория | внедрено | найдено dry-run | действие apply |', '|---|---|---|---|'] + actions = {'orphan': 'удалено объектов: %d' % s['apply']['actions']['orphan_objects_deleted'], + 'stale_upload': 'удалено строк: %d, объектов: %d' % (s['apply']['actions']['stale_rows_deleted'], + s['apply']['actions']['stale_objects_deleted']), + 'fresh_orphan': 'не тронуто', 'fresh_upload': 'не тронуто', 'missing': 'только отчёт', + 'size_mismatch': 'только отчёт'} + for name, count in s['injected'].items(): + lines.append('| %s | %d | %d | %s |' % (name, count, s['detected'][name], actions[name])) + lines += ['', '| проверка | результат |', '|---|---|'] + lines += ['| %s | %s |' % (k, v) for k, v in s['checks'].items() if k != 'apply_actions'] + lines += ['', '## H2. Сверщик под нагрузкой загрузок', '', + '%d с, 4 потока загрузок, apply с grace %d с в цикле: запусков сверщика %d, загрузок 201 — %d, ' + 'скачано без изменений — %d, удалено сверщиком — %d, «свежих» загрузок в полёте увидено — %d.' + % (r['seconds'], r['grace_seconds'], r['reconcile_runs'], r['uploads_201'], r['downloaded_intact'], + r['deleted_by_reconciler'], r['fresh_uploads_seen_in_flight']), + '', '## H3. Стоимость обхода (медиана из %d)' % c['repeats'], '', + '| объектов | строк | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB |', + '|---|---|---|---|---|---|---|---|'] + for row in c['rows']: + lines.append('| %(objects)d | %(rows)d | %(list_pages)d | %(list_s3_s).2f | %(scan_db_s).3f | %(total_s).2f | ' + '%(us_per_object).0f | %(max_rss_mib).0f |' % row) + (OUT / 'results.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + print('saved evidence/track2/results.md') + + +COMMANDS = {'scenario': scenario, 'race': race, 'cost': cost, 'cleanup': cleanup, 'report': report} + +if __name__ == '__main__': + command = sys.argv[1] if len(sys.argv) > 1 else 'all' + # Периодический сервис reconciler (--apply, grace 900 с) удалил бы состаренные синтетические заготовки + # посреди эксперимента; на время исследования он остановлен, сверщик запускается явно. + paused = 'reconciler' in compose('ps', '--status', 'running', '--services') + if paused: + compose('stop', 'reconciler') + try: + for step in (['scenario', 'race', 'cost', 'report', 'cleanup'] if command == 'all' else [command]): + print('==', step) + COMMANDS[step]() + finally: + if paused: + compose('start', 'reconciler') diff --git a/student-tests/consistency_check.py b/student-tests/consistency_check.py new file mode 100644 index 00000000..a0b46121 --- /dev/null +++ b/student-tests/consistency_check.py @@ -0,0 +1,156 @@ +"""Usage (стенд запущен через docker compose up -d): + python student-tests/consistency_check.py + +Проверяет, что сбои при загрузке не оставляют вечных неучтённых данных: + 1) клиент обрывает загрузку посреди тела → ни строки, ни объекта; + 2) S3 остановлен → upload/download 503, ни строки, ни объекта; метаданные, текст и ready работают; + 3) процесс «падает» после S3 PUT до COMMIT ready (FAULTS=after_s3_put, только в этом тесте) → + остаётся строка uploading + объект; сверщик не трогает их в пределах grace и убирает после; + 4) объект без строки в БД (orphan) → свежий не трогается, старый удаляется; повторный запуск ничего не делает; + 5) периодический сервис reconciler сам удаляет заготовку старше grace (без ручного запуска). +""" +import json +import os +import socket +import time +import urllib.parse +import uuid + +from helpers import BASE_URL, call, compose, new_chat, new_user, sql, wait_ready +from imagegen import SAMPLE_PNG +from test_attachments import download, upload + +failures = [] + + +def check(title, ok, detail=''): + print(' %-62s %s %s' % (title, 'OK' if ok else 'FAIL', detail)) + if not ok: + failures.append(title) + + +def reconcile(*args): + return json.loads(compose('run', '--rm', 'reconcile', *args)) + + +def s3_python(code): + return compose('run', '--rm', '--entrypoint', 'python', 'reconcile', '-c', + 'import sys; sys.path.insert(0, "src")\n' + code) + + +def counts(): + return sql("SELECT count(*) FILTER (WHERE status = 'ready') || '/' || count(*) FILTER (WHERE status = 'uploading') " + "FROM attachments") + + +def object_exists(key): + return s3_python('from config import setting, settings\nfrom s3 import make_client\n' + 'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n' + 'keys = [o["Key"] for o in c.list_objects_v2(Bucket=settings.s3_bucket, Prefix=%r).get("Contents", [])]\n' + 'print(len(keys))' % key).strip() == '1' + + +def restart_api(faults=None): + env = dict(os.environ) + env['FAULTS'] = faults or '' + compose('up', '-d', '--no-deps', '--no-build', 'api', env=env) + time.sleep(3) + wait_ready() + time.sleep(3) + + +def interrupted_upload(alice, chat): + target = urllib.parse.urlsplit(BASE_URL) + with socket.create_connection((target.hostname, target.port), timeout=10) as sock: + head = ('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n' + 'Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n' + % (chat['id'], alice['token'], 5 * 1024 * 1024)) + sock.sendall(head.encode() + SAMPLE_PNG + bytes(512 * 1024)) + time.sleep(2) + + +def main(): + alice = new_user('consistency') + chat = new_chat(alice) + + print('1. Обрыв загрузки посреди тела') + before = counts() + interrupted_upload(alice, chat) + check('строк не прибавилось (ready/uploading)', counts() == before, '%s -> %s' % (before, counts())) + + print('2. S3 остановлен: загрузка, скачивание, остальной API') + stored = upload(alice, chat, SAMPLE_PNG)[2] + before = counts() + compose('stop', 's3') + try: + started = time.monotonic() + status = upload(alice, chat, SAMPLE_PNG)[0] + check('upload при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started)) + check('заготовка удалена компенсацией', counts() == before, '%s -> %s' % (before, counts())) + started = time.monotonic() + status = download(alice, stored['id'])[0] + check('download при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started)) + check('метаданные вложения доступны (они в PostgreSQL)', + call('GET', '/api/v1/attachments/' + stored['id'], token=alice['token'])[0] == 200) + check('текстовое сообщение → 201', call('POST', '/api/v1/chats/%s/messages' % chat['id'], + {'text': 'без S3'}, alice['token'])[0] == 201) + check('/health/ready → 200 (S3 не критичен для текстового API)', call('GET', '/health/ready')[0] == 200) + finally: + compose('up', '-d', '--wait', 's3') + check('после старта S3 upload снова 201', upload(alice, chat, SAMPLE_PNG)[0] == 201) + + print('3. Сбой после S3 PUT до фиксации ready (FAULTS=after_s3_put)') + restart_api('after_s3_put') + try: + status = upload(alice, chat, SAMPLE_PNG, filename='fault-%s.png' % uuid.uuid4().hex[:6])[0] + row = sql("SELECT id || ' ' || object_key FROM attachments WHERE status = 'uploading' " + "AND filename LIKE 'fault-%%' ORDER BY created_at DESC LIMIT 1").split() + check('клиент получил ошибку (не 201)', status != 201, str(status)) + check('осталась строка uploading и объект в S3', bool(row) and object_exists(row[1]), ' '.join(row)) + finally: + restart_api() + fresh = reconcile('--grace-seconds', '900') + check('dry-run, grace 900 с: заготовка свежая, не трогается', fresh['counts']['fresh_upload'] >= 1 + and fresh['counts']['stale_upload'] == 0, str(fresh['counts'])) + applied = reconcile('--apply', '--grace-seconds', '0') + check('apply, grace 0: строка и объект удалены', applied['actions']['stale_rows_deleted'] >= 1 + and applied['actions']['stale_objects_deleted'] >= 1, str(applied['actions'])) + check('объекта больше нет', not object_exists(row[1])) + + print('4. Объект без строки в БД (orphan)') + key = 'attachments/%s' % uuid.uuid4() + s3_python('from config import setting, settings\nfrom s3 import make_client\n' + 'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n' + 'c.put_object(Bucket=settings.s3_bucket, Key=%r, Body=b"orphan")' % key) + fresh = reconcile('--grace-seconds', '900') + check('dry-run, grace 900 с: свежая сирота не трогается', fresh['counts']['fresh_orphan'] >= 1 + and fresh['counts']['orphan'] == 0, str(fresh['counts'])) + applied = reconcile('--apply', '--grace-seconds', '0') + check('apply, grace 0: сирота удалена', applied['actions']['orphan_objects_deleted'] >= 1, str(applied['actions'])) + again = reconcile('--apply', '--grace-seconds', '0') + check('повторный запуск ничего не делает', not any(again['actions'][k] for k in + ('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')), str(again['actions'])) + check('рабочие вложения не пострадали (missing = 0)', again['counts']['missing'] == 0, str(again['counts'])) + + print('5. Периодический сервис reconciler убирает зависшую заготовку сам') + stale = str(uuid.uuid4()) + sql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, " + "status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'stale.png', 'image/png', 1, " + "decode(repeat('00', 32), 'hex'), 'attachments/%s', 'uploading', now() - interval '1 hour', " + "now() - interval '1 hour')" % (stale, chat['id'], alice['id'], stale)) + running = 'reconciler' in compose('ps', '--status', 'running', '--services') + check('сервис reconciler запущен (docker compose ps)', running) + if running: + compose('restart', 'reconciler') # проход выполняется сразу после старта, дальше — раз в интервал + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) != '0': + time.sleep(1) + check('заготовка старше grace удалена без ручного запуска', + sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) == '0') + + print('FAIL: ' + ', '.join(failures) if failures else 'PASS: сбои загрузки не оставляют неучтённых данных') + raise SystemExit(1 if failures else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/expiry_check.py b/student-tests/expiry_check.py new file mode 100644 index 00000000..7bde51ec --- /dev/null +++ b/student-tests/expiry_check.py @@ -0,0 +1,56 @@ +"""Usage (стенд уже запущен через docker compose up -d): + python student-tests/expiry_check.py --ttl 2 + +Скрипт пересоздаёт только api с SESSION_TTL_SECONDS=, проверяет истечение сессии, +затем возвращает api с обычным TTL из .env. Volumes и данные не трогаются. +""" +import argparse +import hashlib +import os +import time + +from helpers import call, compose, new_user, wait_ready + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop('SESSION_TTL_SECONDS', None) + else: + env['SESSION_TTL_SECONDS'] = str(ttl) + compose('up', '-d', '--no-deps', '--no-build', 'api', env=env) + wait_ready() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--ttl', type=int, default=2) + args = parser.parse_args() + + print('пересоздаю api с SESSION_TTL_SECONDS=%d' % args.ttl) + restart_api(args.ttl) + try: + alice = new_user('expiry') + status = call('GET', '/api/v1/users/me', token=alice['token'])[0] + print('сразу после входа: /users/me -> %d' % status) + assert status == 200 + key = 'session:' + hashlib.sha256(alice['token'].encode()).hexdigest() + cache_ttl = int(compose('exec', '-T', 'valkey', 'valkey-cli', 'TTL', key)) + print('TTL ключа этой сессии в Valkey: %d с (-2 = ключа нет)' % cache_ttl) + assert cache_ttl <= args.ttl, 'кэш живёт дольше сессии' + + time.sleep(args.ttl + 1.5) + status, headers, _ = call('GET', '/api/v1/users/me', token=alice['token']) + print('через %.1f с: /users/me -> %d, WWW-Authenticate: %s' % (args.ttl + 1.5, status, headers['WWW-Authenticate'])) + assert status == 401 and headers['WWW-Authenticate'] == 'Bearer realm="messenger"' + status = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0] + print('logout истёкшим токеном -> %d' % status) + assert status == 401 + print('PASS: сессия истекла по SESSION_TTL_SECONDS=%d, кэш не продлил её' % args.ttl) + finally: + print('возвращаю api с TTL из .env') + restart_api() + + +if __name__ == '__main__': + main() diff --git a/student-tests/failover_series.py b/student-tests/failover_series.py new file mode 100644 index 00000000..bbe15e39 --- /dev/null +++ b/student-tests/failover_series.py @@ -0,0 +1,71 @@ +"""Usage (стенд запущен через docker compose up -d, обе реплики в ротации): + python student-tests/failover_series.py --output evidence/failover-series.json + +Для каждой API-реплики по очереди и в двух режимах (docker stop — штатная остановка с выводом из ротации, +docker kill — внезапная смерть процесса) под нагрузкой: останавливает реплику, меряет ошибки, RPS, +время до последнего ответа остановленной реплики, восстанавливает её (docker compose start api) +и ждёт возврата в ротацию LB. Volumes, БД и LB не трогаются. +""" +import argparse +import json +import subprocess +import time + +from helpers import Load, api_containers, compose, new_chat, new_user, wait_in_rotation + +WARMUP, AFTER = 5, 10 + + +def run(container, mode): + alice = new_user('series') + chat = new_chat(alice) + load = Load(alice['token'], chat['id']).start() + time.sleep(WARMUP) + acted = time.monotonic() + command = ['docker', 'stop', '-t', '15', container] if mode == 'stop' else ['docker', 'kill', container] + subprocess.run(command, capture_output=True, check=True) + action_seconds = time.monotonic() - acted + time.sleep(AFTER) + load.stop() + instance = container[:12] + target = [r for r in load.records if r[2] == 201 and r[3] == instance and r[0] >= acted] + others = [r for r in load.records if r[2] == 201 and r[3] != instance and r[0] >= acted] + restored = time.monotonic() + compose('start', 'api') + back = wait_in_rotation(container) + result = { + 'replica': instance, 'mode': mode, 'action_seconds': round(action_seconds, 2), + 'target_served_until_s': round(max(r[1] for r in target) - acted, 2) if target else 0.0, + 'first_success_on_other_s': round(min(r[1] for r in others) - acted, 2) if others else None, + 'before': load.window(acted - WARMUP, acted), + 'during_10s': load.window(acted, acted + AFTER), + 'back_in_rotation_s': round(back - restored, 2), + 'data': load.verify(alice['token']), + } + print('%-12s %-4s errors=%-3d rps %5.1f -> %5.1f target last answer +%.2fs restore %.1fs lost=%d dup=%d' + % (instance, mode, result['during_10s']['errors'], result['before']['rps'], result['during_10s']['rps'], + result['target_served_until_s'], result['back_in_rotation_s'], result['data']['lost'], + result['data']['duplicate_texts'])) + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--modes', default='stop,kill') + args = parser.parse_args() + replicas = api_containers() + assert len(replicas) >= 2, 'need at least two running API replicas' + results = [run(container, mode) for mode in args.modes.split(',') for container in replicas] + bad = [r for r in results if r['data']['lost'] or r['data']['duplicate_texts'] or r['data']['duplicate_ids']] + summary = {'replicas': [c[:12] for c in replicas], 'warmup_s': WARMUP, 'observe_s': AFTER, 'workers': 4, + 'lb_check': 'GET /health/ready every 1s, fall 2, rise 2', 'results': results} + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + json.dump(summary, file, ensure_ascii=False, indent=2) + print('FAIL: data loss or duplicates' if bad else 'PASS: every replica stopped once per mode; no loss, no duplicates') + raise SystemExit(1 if bad else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/failure_check.py b/student-tests/failure_check.py new file mode 100644 index 00000000..a6ffa7ce --- /dev/null +++ b/student-tests/failure_check.py @@ -0,0 +1,127 @@ +"""Usage (стенд уже запущен через docker compose up -d): + python student-tests/failure_check.py + +Останавливает и замораживает Valkey и PostgreSQL (docker compose stop/pause, без удаления volumes), +проверяет поведение API и восстановление. +""" +import time +import uuid + +from helpers import basic, call, compose, new_chat, new_user, wait_ready + +failures = [] + + +def timed(method, path, token=None, body=None, headers=None): + started = time.monotonic() + try: + status = call(method, path, body=body, token=token, headers=headers, timeout=10)[0] + except OSError as error: + status = type(error).__name__ + return status, time.monotonic() - started + + +def expect(title, result, allowed, max_seconds): + status, seconds = result + ok = status in allowed and seconds <= max_seconds + print(' %-48s -> %-4s за %.2f с %s' % (title, status, seconds, 'OK' if ok else 'FAIL')) + if not ok: + failures.append(title) + + +def main(): + alice = new_user('failure') + chat = new_chat(alice) + history = '/api/v1/chats/%s/messages' % chat['id'] + assert call('POST', history, {'text': 'до отказа'}, alice['token'])[0] == 201 + fake = uuid.uuid4().hex * 2 + + print('1. Valkey остановлен: кэш обходится, всё работает через PostgreSQL') + compose('stop', 'valkey') + try: + expect('первый запрос: обнаружение отказа кэша', timed('GET', '/api/v1/users/me', alice['token']), [200], 1.5) + expect('GET /users/me действующим токеном', timed('GET', '/api/v1/users/me', alice['token']), [200], 1) + expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'без кэша'}), [201], 1) + expect('неизвестный токен', timed('GET', '/api/v1/users/me', fake), [401], 1) + expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1) + expect('/health/ready (Valkey некритичен)', timed('GET', '/health/ready'), [200], 1) + started = time.monotonic() + other = new_user('nocache') + expect('регистрация и вход без кэша', (200, time.monotonic() - started), [200], 1) + expect('logout без кэша', timed('DELETE', '/api/v1/auth/sessions/current', other['token']), [204], 1.5) + expect('отозванный токен', timed('GET', '/api/v1/users/me', other['token']), [401], 1) + finally: + compose('up', '-d', '--wait', 'valkey') + + print('1б. Valkey на паузе: logout, затем unpause — отозванный токен не оживает из кэша') + revoked = new_user('paused') + call('GET', '/api/v1/users/me', token=revoked['token']) + compose('pause', 'valkey') + try: + expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', revoked['token']), [204], 1.5) + finally: + compose('unpause', 'valkey') + time.sleep(6) + expect('через 6 с после unpause: /users/me', timed('GET', '/api/v1/users/me', revoked['token']), [401], 1) + expect('через 6 с после unpause: сообщение', timed('POST', history, revoked['token'], {'text': 'после logout'}), [401], 1) + + print('1в. Valkey на паузе: logout, рестарт API, unpause — отзыв переживает рестарт процесса') + restarted = new_user('restart') + call('GET', '/api/v1/users/me', token=restarted['token']) + compose('pause', 'valkey') + try: + expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', restarted['token']), [204], 1.5) + compose('restart', 'api') + wait_ready() + finally: + compose('unpause', 'valkey') + time.sleep(6) + expect('после рестарта API и unpause: /users/me', timed('GET', '/api/v1/users/me', restarted['token']), [401], 1) + expect('после рестарта API и unpause: сообщение', timed('POST', history, restarted['token'], {'text': 'после рестарта'}), [401], 1) + + print('2. PostgreSQL остановлен: ограниченная ошибка 503, анонимный запрос не проходит') + compose('stop', 'postgres') + stopped_at = time.monotonic() + try: + expect('GET история', timed('GET', history, alice['token']), [503], 3) + expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при отказе БД'}), [503], 3) + expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 3) + expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1) + expect('вход по Basic верным паролем', timed('POST', '/api/v1/auth/sessions', + headers=basic(alice['username'], alice['password'])), [503], 3) + expect('/health/live', timed('GET', '/health/live'), [200], 1) + expect('/health/ready', timed('GET', '/health/ready'), [503], 3) + finally: + compose('up', '-d', '--wait', 'postgres') + ready_at = wait_ready() + print('3. PostgreSQL запущен: готовность через %.1f с после остановки' % (ready_at - stopped_at)) + status = call('GET', '/api/v1/users/me', token=alice['token'])[0] + texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']] + print(' прежний токен -> %d; история: %s' % (status, texts)) + if status != 200 or texts != ['до отказа', 'без кэша']: + failures.append('восстановление') + + print('4. PostgreSQL заморожен (pause): запросы получают 503 в пределах общего срока, а не зависают') + compose('pause', 'postgres') + try: + expect('GET история', timed('GET', history, alice['token']), [503], 4.5) + expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при заморозке'}), [503], 4.5) + expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 4.5) + expect('вход по Basic', timed('POST', '/api/v1/auth/sessions', headers=basic(alice['username'], alice['password'])), [503], 4.5) + expect('/health/ready', timed('GET', '/health/ready'), [503], 2) + expect('/health/live', timed('GET', '/health/live'), [200], 1) + finally: + compose('unpause', 'postgres') + unpaused = time.monotonic() + wait_ready() + texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']] + print(' готовность через %.1f с после unpause; история: %s' % (time.monotonic() - unpaused, texts)) + if texts != ['до отказа', 'без кэша']: + failures.append('после заморозки') + + print('FAIL: ' + ', '.join(failures) if failures else 'PASS: отказы Valkey и PostgreSQL обработаны') + raise SystemExit(1 if failures else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..48428645585f37b25e4daefb54550a2048688977 GIT binary patch literal 19881 zcmd?Qc|4R4^gsGAF_MT>qcE~$X+i5)#-QwF?1U`QM`YiZ7?h6^Gxky#SsG-C>|#h6 zOBrP+;WNb8x512=ai6}w-|ODj>;84`{r5gz<_~6`dCqyxbIy6c-{%~R9!vwr40R23 z0S*oT-~fMsg9+d!zy&#U=n#YpeBt8a;)e3^K*2-c$dSW*f&xN9f&zkq$4`o$Ixc+r zgrFeoEbO$H7#t2ieo9hWQe0Y894`L%Bpl#Jq1;e@9v*&iVL@T>|JTPs8z9I7ba7}x zI3xj1K@Nx@$H5l>4qhi0$NwC_|2{Z4A&0=L|K z;E*7f&{<_I?&GG;P)R@G%P}c$d8D+f+E1AMSeL$X-#_*+@5xgzk<&8gWaZ=)u3o!- z1EHdN^OlaTp1y&hxrL>bwar~y7uN^>xIJ|D2nY-c4hanlfATc$-}q2t@Iubrf>?w+3mgG0k3qhsSU|50Y={>(2>7dJMy zwrM-NdvwNMyEp*I|1t}F{a=Rt-|P|u?cxM)%OU7ryEr&Qz+Z^qA+EE^+(KHWP-nm6 zl9yw6gtb%NR<$3Nx?;9|;=cb6-jmW-XJj`1n)W}I{Xa7-_WzS*|EFR9mtErkAA|$E zc@RMW1?>GnY+1R~kv&p}FHjPOYU}zCm4t48pGv!%r=HDSp;VJSK8okyXWnmqvif%1 zIrX;N!xh@qFZ=fMYw02j*-FwRChI*9t#ct+A?B%up%*UR=v6rO zwE2cS@pZzTFb9&CYIUUJto1(fHO1po7JD&}xnZv7<$;-dIC6iTbA^8|IB6i zYbVAcNJa^@t&rtpY5q(&sqs=(O8wT=?Ry3OCxRgOW_%aKrZ0)-eP1dT|IhdE=Jz$9 zaz6Eq2b~#9A&He70GTH2n+HI`d*z97JpF;dENZXR_gU8w(kB$*`;%n=u&67~etZDz z9RLTwgL|-0lO!FHffs1Ep4tN-HmI|pE|HAoI{b>%~8~g47K*6(;3?6a& z?Gyj!&n0~~VEq!EH-%0e0A)I%2>WX?1{TGYkuR;ty4UV4KWqKYt8|y|pfj4KSE5MO@r`|9Xv4UgP#Iu>Wjgl{J zdzR4-0is#t>9CTU!VkXv{K7M>pN*e08ts2aOQ>G^?)E2}z~!AA$q)LNO`PfZ(Se`& z$Rsj`Zj5!ASeJkZTjtkyd9-e|fT!mSVpsm~_eg z{H4{UpWi!8iSN~%%}rxZS^n~LkqB2vqw@yeza5tOEQ{JCafM;A@iqIEcT@s}fm<)R z^-_tiE8b@}28{S6^Vf+|-$P&rz>NzEdXW=JWi6fufK+vk(E(s}HmMxV{1DIHJMqu8 zI@IQx0BcEby6mJ!MOnAnqQXK>d1g3lja6^%SIM(4RClow39>}2`Z^F_nd_pFhv zsNh<}*fKxz&-8U0Tc1pevxkGd z%`w-6UvIWx$AerM#IDMnh-;lgVOLm#RQiAKu;YI%J4{?lVPA7C>k)3&;b~V_{LS2A zm={2NB_GAhrhSxx=gNiJUsbTR$j^Q4|223_MFrKot|mVe{Hy%~4pFZ1YC^La*b<%b zR}Fvt#P9{;zp~3=yKC*AA3u>AH~=bNC*~I2zGQt$&oJY3LS$3@?!)HmSsTnn^AXp} zo!_{AjRJhB5f=Fd(k@~j#^85GXAgk*trA(QRA(;{);J|t4bQIMDiP>q<9v1*Y63Gm z;tQ-$wHwBcz3@wGn??@Bce*E&%arWIij2B(P0+3FlD1bRZhnWNo;+C^ zsfbFSd+t?7It@*yLGy9pq0ddXE>Yi6=Ezzsjf~g+`8ik9g?UP09ushJ)Dh=<1v=pa-R=qfi9fWX=L- z$t(Cq7`-_NNi^(zeYFy{U*hf)7k>9Dab4aJO%K-dQu&pD7?x;EkRzMHGG!OIkKXy0 zm!Aq#+;G+Xw0{63sYq@P`Mx;Z864Aya~xp)DO8T`ZI0E27-V`zs)dxL7a5Dbzw@c( zQrSy<3(sOrc+Q8MF=0PBzT&ERbx&>5mb%5h%@x<`3_t)6-LPZ~)cM}@rbi=^!E*-& zl_;6G$b@yV;m*-2l0|pSu(^wlA$9?JtGh2BxjzC=)ir!9`dC7Api?wAYF#1YM6Wk4 zc-mKccAlEZ8~#NK!YeE|>UAhYN=1c`&X)XyYx+KjGY(s$+SD&+l)Kg&5!~QEXk)(_ zI|DgglqBKzou>{_9zt-v=iX_WO($kRQ!+_3cu%hICSkSf@>?u7Mo4YnydU_HYmpp5 zIRKusYXx6FF?g7U-VWXjF`ioI@4B5Zu9R|0p#TzVop=CDNxpl2_t>HCe<-oW$h~u! zay@NCIr|SH6S$Eh%*Dv)EBaxip5}PWmq@hjpSPE6eNz`+Qra>%%P^AFb%uOn%5-T3)F_a&izqeBl(%vg{9Br^Q(&nsoUpwc zR!K6jE1#27Y+EL%r4-)usESnq3tH~2Cv?7lisJO0&c2G}7ja#oV=kDdN}RZt3_vd) zrj8VomJ%}3u!ih<#y5o(fULJG$sO@}T ziFbn@eC_gu^$MEv1t-^(!AKn>MDl^(Riw=^#C*dV+Hy1fj`!VU{~lvQu&sEn@o?aA z&xX6hVvj+{z+`cAO`2`_g}(6sB8_Ep->zMC^@X4TR(Tgah>6qqsfWEO8vS0rof|(J z&-v&9M>IVpf`vQ)$iw$v1o*V*KG4dLDiP0k9q@F>_IlE-%2npIQ+B~GmkK7Y)E!6b z)z4Zt*V)|t=b5wUbfr^dDJvfFZoG2*V(ul42NYSN&oI6yAYni;M^z*db7okT)S@SUX%LAaa$bOyg zYjr@~F$5FCph>-Vx3pZgqOIMGL*^Yt zCa+l<`(Da~ameMsJRvD5PVEW`&Li`y`U|T!-iBDj+V~5d^MtN8zKjWSBZ>C@sJI!A z?r-D&)UX}&*CdGPz~}z-C^@a#B^5csrpL~vhL-VW6fs3qeka1I7pIIp#lUZ^n0>+j zV~M@?t?V*C!gR@O#oV`#wCkHFP#M*-MVd2ye)rUO)KU>jQ3UL(hU}=i2Ij#gcg{6Q zugG7@9(dV&YPh#l&t5jNi-L1wARFlayL3qF3&Umf(gJ_0F1=OWXfi2P&ApEzrE`llwOY`3gObVM zpCVJuZspqYDW=B9#Ju!bKLFGR^JSrbBzhCKc3t4${jM}N5$JIvOOpZe`Zgw&1z0+qk zOFlZ;^QD_(m6>`!3waP1@`@z_)vhe7<{kj-(r!Z~s{wR-&_Y>rVLtKoVUJ*K7s*5} z8KqN*&&<4lN!&K!^IHpmGZ7=uogL5__WgFk-RQF~_@kq30$2Jmw&D3P%^F!}8=JYRAM1yn~{$ zN~E4)uRFQKiMp{lRU{~lgwlUCFMkr{^n3CP^O3pqM*pk3+W^hTc$~x*xilUi6$hAq z@YQgxPZa3bSoE;aj0QN9m@YB6O8B?Im=M+2Gl$lZI5R{Sb(Ht zG0S;v8gcJ&J#G1AzS`N#a#EvJLcHr}hzOOvim}ZU9&H0Izu(nt@1}JM5L}Go^flk= z$q&H#jFH?JDRa-y*>PY`8MlbMzpUQ zBavSd_%fThkhYL|Terfowof2W^S5>?LyDb97ExJgal{@4N zLcS5Ra`&Rp+JnJm&|j2^t}|+!8jijkaQSN&9-h=JZ9sDKcS2ApjDgq_up`mB+`w$4 z>J9Jg!I&GI2!$)Tm!S>?rFg%ejp>74!et{fyXE`M?gRPv`m;`-HRCMdgIe4xjI`b; zxo>jHKN=8KzX!gb$?`CX=QzS2P;>kM$g7`gXS&a*OlETF8oBVbW{Hu|?AnV?t4)C` zd)rnad*$!_^d5ly$5kH7=Ynq0)E;~X_~@7wM%?v#X1^`9TEBe2r>UQ+AA;5HQRMQ5^w+(iituksKZ#1hpqm~%#7H%Dp$yb9Na`1rs z`T=IRManx&n0NEN(1&fEQBOw33wd};)r6k1SB)1V1?cm1eZdz(P{ys$E{6~hfLflG zF0Wo`+5O~|pHXYI>!J4`f-q)`8KJQxg|%S&x1=few_*W`Hnd{F`=LkJL+zo6j%kOWlx!wkDxN`JEx{6T9n#+l)Nmx~<1Aq}{W5LO? zahum9(J_W{J-kIX&<_yZVY=>v!<;f-WPK4tkhFohT6q#j(7`gnR3o)ZtZd7}xLk&LpSxKXXu}g+$sLHrH zUOfOR(wU!J1wcRg6p6^(8PmG@dLgZ*uqnXUVcF5{Qi+&v&aM;|iC@^E%r7jX;zuI} zQzT4Wo|fj#6v@I<*tL>hgkdJp35^-?i}Eu!6`JLF1Xd}lBoG+nqVT92GBqD<-K+1J zMGB3Z?G-5oU)TSetddR~-^DcH5K~yY1vLz+$UrFUp;D`%I5+n_Vh3}}>{&Ma)sstC z$dic}JiA3j6NR|n*k^;w+IQBYGv!rOz3$yDud^Coio96#^U?m#^p=6T>{XWuJ{zuy zoaIy0DTZYm=dUXn@TNq5w8H@~B0OQU^*{i=*$+O)ulmNcxL$m(m)>qCZ-2cYy?FDe zp;4iWmzwUKi%s2?doGlMs}qt5f^%QJIeH&|-(F!ymf9zET8hZdzMw9#FsD;qX6c)S zz~uub65&f`C&s_zsh}qb`;%n)_S@dEOE%4NE8ahvtmyJaeP*~+HTk|-qtp5H$ImGH zd-HzFC^DwVNGRp3xBAa3Wc?8^7L$55m$PPNZ8i~$30AyzPm3os^ z)NqKEhFu1RR)Xjex5L<-w{v*?DBr$cO@2`FmZd0QU5(ArgRvFo%t|wQy`*Eh`}9Sq zxl>xd@gJt2G_M9Ugfkcd&Mc2R%?1dRqQ)!ez&`QYhRwdQQA_!rPu>1q>g<0tRTIW8 zg226K0GQ=qP&QG@Oa;qlsRG)eP*fL9o#Es(kg0Ti8sJ8B0!XX?x@7)!=8TRk}hNsI32)TUt2R z7pbSov5e8k7Vt2gs8e3S;%`UclLGn*<=dC2EKK{~IxcyyI4@ebxG%J=Tcs1(f;#>B zngo9qz>(Gs$rac;zD*#VTD5S2*;8Q4VGAgAfpSB^>X4%yB{ULkL3K7_u;aDTi%lyQ zrjmHG?)X{D@#_H4ryK)f6sbNmLfxV?OGN}G7S@CNqn7KhQevqx?yvjH6+2Zzax^uz zJgd17>6aLuyT`yH${d?Fkf*2z0MwyxLagSGo~W}iRHS{JJM;hudCOEAEPmAvN8LUa z)pBN{fA9s&1f*ea9Mc1WsZ3If{;4_X={gRzM&`cY+QfP>@f^dB4gYxnC_cZwt>=2~ z*0W|F*pVMtr7H`1-g>6YMUw_Il7gN(E(x5wU_HQ?a_+?DKjRIrukh{YwvFE^p>B%eS|$PCvao_e^3&bO&U& zh-BIc)vxKMwPC=FDm-Xc96#G#^+v?>oE`$c1l_}Jjc4dKF%Y<#W}9)a^9~p)oO-C%wY*mJKo?~xA0xKs~^;y;=x8CT;u;smW%u?6m<6ABAK7cbvXs+GACU} zn99r;k#}RlCb&!IefjPF_z--+o_W^$a&b3G+7a!zv^SXesYg4lfo_!h3#?|g$rZyg z96eDhsNbs?auL5nxBu5My+^@GKkK^<9sJWDgWO<nQvaseXoi?cBG21-6{#o}(h073teM#r}cg!5{Cb-9_fq z)Rm|Of<5ysuvR2tuWk-mqF>@OBDyf2v-3{4b@%z&3;e*4_Xc`62@#_6{MHA6CVZYT zKxS>#<6iDz#>CkNz^sdHQ^G>3<495M(z~X2MY0pdx|Zo-`-;pX42!z1gbzZ8l?&7W zH9s~t-m|RkL(_T>R%^GdBtsV_Ki+;qWi2fY*ESsLZDanVu0#hsU;1q72qvIpVJmG*nqlT@&46q7Q9MH%DE@6x>arNN(o-id4cpW5Z|AmeW_bEv52Am-_%Fn*Q}4y7P$vbm~Oq z1J|?_n2{@pm8FuduPEekL7hpH4$Bc+?8rZH!d{=_MiE{6(YqS+1I(8S%AQWAq+R(F zLn+wLMLNKHevV#O?3KwXW;}@SGE_E=26_ipYhT{GI@}`!_ROFB_Pr)Zy-d_H&U}r1 ztCz>hvi)dIo}yZ0JQ1YKCG|~8lZrzG(z)1hr^rY&T`11NyeCypwhjrc->R@Rc~>&Y zWD=*@w(e0j6$=)F&!rrKOOp!bIs(N&O6)6!R%(%C?Z(UficjT%LrgU1wsk@xfhmq#Uw6;Hmx4JvE=;0+Iq)n2e zPq~Y5sU$8`viVU_U$%hO&L^Vaa{@3=Bqr0tWzEYx~;u?~QE;U_Q|eNdS8bk%CU}a6osjk`ot_C!Odh3(VrY%xbYU7gBc>0a z$0qIvFyONumL`|&xy1Z(>9Zj0!#RYN8s8XFTx!j%9kJ7hcRiiHDiAAf35 zlcEb>iOwAONp|1Sb`>P8)Qd8S9qfLZ z_|&X%iiCvFsiU};IZt=RNdwII&LRt^r=E%7@-%T$kzq=rv&A|Ojn?BDmC8EjsBD^4mdg)TBQH4##JYoqF`D$cVb!nTz5D7{HopBdzDqrvAtl$ePgi(XkeY0%v+Vov|3 zAfq9o4RD~{^bXu@Z7ImCI-XPD_08z(f+(Df=Jdvw*oZ1H4zGx|X+HgXS?f8!svB&n} z_1Y}3-b(35-+P*q=LI7J{v?J){&knnGtUq41>Q%9sR{Ju%*L5;Cd!d+yZIpITPbFx zKw&*n3$ciKQQv?6ts)5c0Imb|e8tAl{!O#p`sSovR z%70c&&ybDw7eG1SvzQ<6mNdv*vKyadgj08=_jSC3KRXpSYjSAEw5{OQ*^#Q|pR#l5 zOppMmoT?u#j59!P;qBoEfFseTTtCVuMKw^`FFRN~ItuTA4U+G}Uu`woG>CHST?{sV z(|G!pX+US2${{}-b%5}n`(l=Jv-5nmI4ILDg1QVZCRI?c=ZIiiCyJ$sp^-gaI`#b) zR-z;(tETPkv(^3V%NzFVYRik(Ds@5#KDvx|A?8^^dE=bZe7=wkGGv`JLn{duGJ4!% zu!vM`1l2WbJo}gWxY`7P*j?CY$_z*>&OJY`sO~ zSN-DwEUT$ET z4}+%mC-~mpL^i>D@v|6+dDz)OPZJoQ%g6*4EL_DPU6=*lj@#K_mDys9TtvIe%k{d| z)59xCt3FNHKSVvc3W0{j%LwsL;lWb1w~x!R%!9-Pkf>KHi%1U>rQp17N-~J9!*icT z_!*wcK6xK~#)OT4=jzQRh?P`0JSrl*d`CoiFDRtGNDQPc`XZZg+G-XFMVg+jX~{hB8=&6qhBLjpRs8iH#>&V` zW3Cb%tSa^)3kE1o0({K8?d6V23bRy4w{IZUuSPIBgx`WKS!i*#^QW=Cg2AFDWlJ>N z&urR@H>-UntP}HG4_VFg-6e7&$~I{8o}y2>)eG~w<5Ef1=i4{8vRyf98&H+PG+kbU zMDOv#^Qgn;~UK~5g#k5T=1cDbpg)#H1)SgNhg zHqoN5C*aJSzW=jmfgH_4m}5P-p6n_W-~G^ENh5X2%taf^k7u4F8>;NU&YMka34xpk z$nEqm3KdxyWz~MloY1jFUVntqm=OC={9}t&*Y1A0s$H<&$DV^RICFpNvR(9au7HmH zY3%O{Ux>8tsBP7{uJ@lfGPyR9SylLX$B?)$7NMbrhSw9FXlO$7lw*+NohVy)mtblm zpJ{Z=32d-}i&%FzOTp4~x7}*Cp$DHo*zn+f2$w`X;FhvkLZCRS1(5DXA|T>%rG*p& zy9e;z<(?6`k&GM8L|SVXcOPNi``h&`zIh8S341S8MHVt~A2KMHc(JOeBk?I5_`>ya$e8OwDe^ zDsgl00+hFqY_}0MnHBOsN$Er|QCVOU{=nGMFOf6PO^k@|cRPm{b~<)FCR)RCw$dA=9z6Rx z!(64!L03T34c;rxg0?sxk*N+th^?)i- zemonrWeEvBq^Rp>R#S56Y=`VfXt{ngWv+v3C6|Md+e)mwZ4t0p+gnuWb#27@0d;n& z)a`*va-2znsN8f)XJkBqzV_4AH2Mwi#+J*6WhBud60P@)Z7EkUhnVaa$lGDkY3g@G zAAOW%ITAN8G$Q%->rURsJ602O-c36T<6A>@!Jpsq61uJ@-$(`^c`Iza1HfKim@8Qh zBw9$`#QDkt;Dot?(zm2eU4Z1Xh1-fTmVeP0F%&wj&b2 zVxsQKXF)kpkvYj&%q4#hUg}?Y+9|CE$(x9i5SU^3fsWc%7qPk&qdgD5q^iV3kqIio zrDbI&umbBNmmcY>JP@{zAnEwGBC0cu@sE$x|2*2|E>3n zN?zY>CwU(L#U(CUBAyk%!r?_Vfw^sUtUMJ4;XN$j4SDNG0vp!K{@3Ypz5E=~d|XR= za1u>CBi7@vT~Ph6?UA6sn%10fnTf+Gx-q(KDpI0;Sd8%jpsiptq{$IHjpY#w&*oq@ zO+P_f97In0-j6~(6|pk zl5?8GiN#D^1(PRdG>?#+2dGoP^bYJ?VUiNJrKY`q!PzK0x&F{VLE`GDu&a1ZR2Iux zcKn0St@75~sGMcmfX-|kcpE$)D%T_xAi5Slu^T-7*35LX=-Mk}KrqqmonJq%Adwp;)6G00f8tH5hqHk03?OP4*x+%z4TmXgm!40{TPxUAz?<>E&0gkx&JI>nBbuYo?^Yk_J<1xJW+f0)K|&9Q@0q9X|=@ z`gOClJRC2%%0kZ?Af!;g4}kyl=Fpp6@T(FTNUSPmA6_1Cb-FCg=zG*D>!y_;wqB6x zQzPE!;|O>NA3}^M@ZU8hD<>Xp-Bl=`njikYMU?Pdq6BbQsQ|VDwrAZ$)s*{O=q#*3 zpY@^udQYT`W}RTQ9Ui3-DdZ6EdG1z9(wk8E>?qi>K>gy7oS8-AT0ccXz;t8)rNdcd zqSbpmIiS-Q#UFV9tfd%QvvL}CsEl9|TW!sLtAy7#d})FaMF7W?Jo2L<(Rr7Yn`2g3 zs+jO1USFt8)nKJVJTVHNM=LprO3~$8JYc zOkkXy7%+>;#S=ahjdR8cMFX9jAQ!8C0OSlwKGR!YUE0NB$e58S5rqUXGdgwTqt`gY ze&uP`&&$hs5`r*HsoJdBVq?ez>I7Ud=S8O( z!O-Ms#qTx7LdZ|AYRB_Rd8Bo$^2+w^_S#zSf0hqFe{_!D#W|v>jDY?ZN?%`@xa{e{ zb{G+iklndEb*p0_VqjrLTBCf7>%{X73s0<_xf(c7T6eze0f(NMwP-{{q$<00`gXC6 z@2w|3kcREu9VSxq+jHSk-NVi)?eokX7P1-)+YFJX$FU^%d_8@_ULJA&Zqc9Cxxk!} zG{?7-yiXs&+O*mhAp9!eml;PMH8}0PDO<(uD73AgJt$AHHc>C(FPYpQa`XpYjQpHJEn&=?vRAK}fHWu& ztvm`6TiXl2hZG1BO5RWvF{1btocDXPvM?Q(7?h)dsj>^E&z2G^jZuDqfz&9%FU*wg z^MIR1_uMz>?(~p;aKiY^#$BNpdqBTqq2+|MS z9C}o9QeNc9Zom}$zgo%CoPuwkML}OdKy@aPogsZ+$RgvH0W@t1$CeuocXgqPmhpIih-j2&OmNIGPr-h{#($xIUjaeGo2R*l2a-B{v;=eLsP8tG5`aw)!mK)Z+C!;cT(@J%w~ znueX{8fAZK9~o6-7^mM)K)ffwCPRKi$!^RloF9#ml2krZBK7bHc1jNZz0~B>oQ*y) zR9>Bu#*7_oweK?K*G>hevs(^;FSs83T(ZVPErkftzyn|hW&6jy{qDJ!ubG!BgT|Mb z*u@~8%oD&lUyW6muUZuRi-%KL+p(ufzxZLn6pb&%O!;1zCLrXjt2`lg0Hj3)Nq*tQ zUlC2qt{fD{uV1;lSSeh-u+4nTkZ$20z5QJ3%=LA11XFnw?lV+sQdkl&^|n(}yrA-3 zR05Y_-jhsOiTT9oC{b`iaY{11a$n7FWndc|OAh(nIBzCAO&h7-{_Ebx=RPNcF>RO~ z!;d~bIoX(yzplNI-`9s~23=~g(##=+_`C^L-H!w7=k0C&&pGUXSr^OST?uNT{hZ6( zfb0ldaSz^8h=rx>uBCY1{6^Y#%r0$$xIr?{b_!-F&YKm=aC4T5 zI)`?XZ?ip!Ir^m>eKtoM_eUjyrz4JGrmwwtdLcdkWvOF6dVOmlc%b-Jl|r<(yvdcE zLXH6uG80NG$;Vz&dZs*p*jXZfV7|_k0=7;3vN@vBeq<(3lw^^m+7q{bS3^ZuNL+s{ zuTo-d0(F4KZe9-7e(Qk1jCCTLz_fJ$VD|^}rOlHHd=YJ(dVe(V$+QwF?F63N{!lH_ zALu)cg5U3S#{H5DkRgI=BvUN@U*vi0vDnzwTnwl!foVirrfvMytP5}h!~VcR-qAn| zX=jcXBu1@?#}nz)HM6H;gB&DF#x5KOH>H|Mw2{9RH1^Ga1&U1DKKEv}Wt__Ldazpq z!d@w?qLzU86wAk%two4CoYY@Rfp^d`iUV62Qotn_GnKi^%4|&*Qcuz3N3rk3OQI(x z7saxXQn?13dgQ&k(iGJqJ~6sNP_E*^DxZ#cTrk0eQb$@|TkLIys zFI=^@jv@Ng*p4sh3wtv!f;p>17NG-7wnM@33r6A`_r66$U_b(eQGNT3anwIje>#U2 zQI(jH!Hp4B3(AfVGiqnK)!={P>8wqLuB}8t)kPcz93WHkaFmAtC!kAH69jaB?&m^~ z`{8h*j7Yn^WyaH%74DTOX(?zTR;K!cRv7%9A7|swf3p5X>``2I%$hKMEA$ySq`k z-NQ@5#G<-*SB0<)N<6BAE-*&|*F-ofvNsEv8lUt7rpd?ry;MoxdRuH-#pfIGDf3SB^g6{%GRDQO=)VUnAL zgRL-JF;mwh^LVG^&L?gXJ7|9;q_-Atef4Ets(VAFkXPuL%wi7#D`I)X-N>CSa3JBV zez%$BeQ8`e2OV>b*;1i|0=9c!<0R;C(vcC zUK<*cKl6DTvi7(D_dN+MphkqVa_c|;CsbN8XfWmG)z{E#R4{KB`tc(hoItF$d=v3_ znX*sZmoa#7GT4tb#Y!c3jUE>1Je3J$<>F&(8MiN;PAx38ux{;bWIjJ#v2)9GqBHuG zlM;tU10B-_stdkf3LlqEX!b$hT>+=LUcb6kynlW5zCNX18nfIsm;mGJ8tpsdfsKGK z4vkaIv8l0+=UFRbkv;R4s<=iW&10^gl|K1Tf;e$Gs%C@tUyeg=Sk-I_r2Yp+HRG(D z(g_pFkxt`qw=GK$u$b`sw|S9&w?pb7qfbBwl-ZwkEfQ`qV+mp6@i*&ov}K-Ka0(j6 zWUol@zv{zGqss3H`}sZ9!e)``9U|jr2RC5Q{3nUie4G(j%=FET6AC52&ZyrAXd_dX zqvDE9pFSwFHpW-drd+B|@VziG`_+{vRMOx!DuX_KXquz(O|+iZH-W9f=Qef{Vz6V8 zoo$$+yFm}a_nC{M{#pdQClLn7}*W0c=l3^{u;Ubfx333}Sea6aMywA(} zR;wZ%-Xn4xm-UgOK@^Q|?sJ?cVKawi&}W``ZtNXL^B&9Iw0=TDh{ad-xHp#Cdj9Vb z?xC8@mh^eQXECR|wVj+O#Ve^1f#znCvvcN0a&`R{{qPktdxNpa_x1TP|B7iso@56M zK!sUPIgLbPw!(1^IV$Z;57F$9A2M?hk9Qml&hIpi8M|s-pFZacJrjx`?FEqi%oRfJ zHOR_}p;&h{T25y_LqmDB=CQ)_4DqmK56uJtNXp2>V&q5cuhZ>DnVVygQ|nRt?%wf% z=ai3?R-A%XWBJ1|wAQ@SDJCvwqeaHgfSs8J4-M1@S4=a;8|c-($^-EJjlZaoqgz4_ z(JjFWpfNT%%4nwOUUIJkc6emyVX%4DX5cLs2Cv8*{1Q>iXs2Vwre3gZZf? zM!-V&m(*F?vF&<{BI^5E#Jgbgk`lg$$KJo?I84pOvwXG|ZvD`;@>lW=6UE>O+sm%E z`203!7y>u98H}j-*uBA%&!$~x;ai}D27;+aa<$Mip06>f!S`|o9Dk}gUp}uhCgOsH ziqOE}EKv2fN$b_ro_q5y9FvRb>OL>*)@5=fwhg|8^8w`rEhWdSH-DCD{rUtzf&2!(x;>*4~>@1eC z`;72+$+d@fBlVt!{qC=i4k%IIS||i z9DMBG(evter{t}g2c zhbBr#25)xM_fAZmH_MOXN(Zu}#g|n#@c&%hj5)&R`!to1^yPXe^DrTf$2jbKJ0+XH zV`T2?uc3?AAB9YGhp{g%j{n_9Y51b~qW-LslHXu(H?m2-M9|;AB#tbFpdrX3zDy4- z@u7(ynjQRk9PNE7)F%m1pE+c(n)DXEO5^Nl_BDQEwT0BpmevSG>5P4QaC%wdjqh;; zByp^Uml~Qu$_Z2a9CWSj>#MVMbeK0rMv#qJmK40!!^@DI%6|fb#BO{Hx7cl z48Y}4nF{8D-P*#J{@`S{m|MoYR~qXA;t_{vFrOZLi@{P+^Fn2F__nQdWkrtJUx~s` zr~V03{TQ-A0`jYbQ2JDXn@7}1TtGC+C6O`MeWuvbw2J~kaACd#5hP{E}GQHV-?JLCMVy?xKvufU`ZCAY0sZv7cWL-Om;s z7*_YSUUQS+I{-eZZVX5kj$cW&_B&H;4FNNbwkjjSj3|>k3??*ea9QyAs^T0b^%mv7-m|^{05Qp-JM#nm zI}9sc)7U?$Ug7RrOFqqaffA~ zMTXr=fz7q1@{w`yfx2(wgwZc+8Js5-kjrWwz>!M7w@8^WF#m7WeCbq9h zyT|b=9bSM%{#3>FGE=_mr@Z={GYUyoi-N2(u<;4WioMZau>j1Wg|rvM5Fd0ulu@?t zKD#}nhV**(X2+=dc;jh_&SSnxZTz7i1Vw!~F&Z7?dIniCc!8+m3!CV0BWpX!recEbmEw+C&T;%-PXP_S?V0B-egK8PO8< zTira*>82#>4ylXfiD#|)u`wLzB+D>_L}Wj=E;4e9!BXv>OLTeYTwmwJegG(ji7dTF8*6ygz-Ed<0vQ8da125xi27Cmf(fh zI_Mw3+U!<&V*qVk{$YLHq<->iPLqQo>GMN~Wod8I$_zjqdvOG(i} zzL5yT_A-~su~OIAQf~h%3u-IaqdwHdEIa&|h2FV^)PED$$qo`B&t`hbB ziWC=YZVT3{^r?kY*4u>l22k?%8mFU`6KS2{#|nxJ6L@9}?0uEH#o~<%x7N^t6~U>8 z`+jPPkgbqNRkQT`GD_WV;<}^WCMfHqNOGz^J+#5@Tjd5;V$A4aF#SG#89mLzYI;<( ziMF?;bF_^d$-E7SRIt?H=XN~v;qdA)*QnYg%A*@CwG)xKO%Cj(0liHlu&jM^c~lPo z-(n&H?J=qm#X=`c03ierOqbvIdZyUydu(Y7=99mNGjFn^EFy#%P*1lq-^q{)m8rat zOYB)u>>S?I^HP28{rCXzI_dQv6SHkY`25E5#LGDjKWyUz(w=#FR)&-bH#eCX>NMEp zI@X{tdu|m!>B+m_WJy)^O>G&P7^ntUk`49QnoqMt^*Jj`rWL%}0#A&ue5+Usr+&4eGZI)CJZ>fr?lN(UEhpm}4(fcjoQV|BJ zS{f7wh0%x-x%b!hJlU)YhSwOliUUi@fQeE#{67T@V8=wm*A;B~|B7BBTumHZR0h1` zbvOyj%UYi000+7pR9!9J==vM3N(CEE2TU#KIifwRXFrbr#>@{7uUslG4A%bVPAO3O zO%u3ffGiC(PyQsw*A+0O4mL3+DgX4K8T0VXYyVivJ)m<4c2z(t>0eck4!k{8UCXk* zIy$sK;E&m9HevI^Cv)0m%Z z{M#{qtEv9l#7|jpU-N2GaI!!T(96*W)mGc+2j~Cw3+Dcr2_xATli58Oa=HEqJ?ZIK z0i<9v4pcXhR1%v$LHf?rAPs^IV+yDZ5yVz}_E7$|Eo%{V^a{$PameYa;)%xa=mPgn zo)wx{i^nZ48%XT`5upfP_Xs#R;Ew!%G5Cs=Vah1lIU8}<{{XUm$7B9W(lfVi3xoG^ zaCdv2&HVoWDkP^Cn$!B(+h0zp+S{&v%5?}tO0R`HU>pyD$El~9q4=c(fW4t;$xKAxac)-V@n0E}RaXFh;u`E?no0s?;U z<+#btI%Dd3dLOMrymU(5T|cY&b$TR@%5x$_0B`^UoPpE09G~;p_NhXiPhxiCAMJnj z`eX1lBB#F;n%;v;q|E_i!i9iRq2krA8+V?3XmZK8+UR@=c&)*>DwpV zQ46R=_Fv_u$7CuIq5s>!;%VVeUaf)9WD)6({{TV99Gc2gQCeyJbnDq`eJ`^+Y0Qty8$%!?9Y8th#~)8# zgEcx0u@(vr2Xn?h!}UFB@y59TDB8mq;Pn1~$a79a$}g_&Ip;pziTCI`bga~!Hr#Y= zbnm9STU)1VYodKl$WG%Th$`-Ej1k+M_89f_9lo_7M%r>P8-T0d*dOM9{dFR$?gu-E zMFWibf%@eC0QKol&HmPS`FP#_$n+ln0OVmhu%`nZ9B#%bvfKf&?#OO zax>nEbF_jmFh_m9oOdRrSaOOsjz)3V`=9nlVg5^%9|IU}rFRT-{qybV z{{ZTuT+vpxpPt%V^6#_WGJTZhMuS1daKKK3{tLH_{PsZt@ZC?^CJ$o}X*G4IsV z(mdcPE5Qm_XA9J~h}EZmEU;DS*CWvWJ;I; z%BWHM!-0$s;T(Q{ok+1bQh<@mPm&}01dr^39cIQ}U} zJo^lOhOyH2mG$fEYx(tzt zAi`yd+Yxd2{{Z^w{c-;QUgP>!Gp!gWJ)2KYURL(gNnh6YJ88`D3seDBn1BF?l6zF> zry+Py8Ib3`dH#5zBF&Va_0B)!YRsxo%klyvkz5a7^NRffgLLRc?5%%Q%{esutqz;) ze;>wof7h4!kVK#&l~J95uJ2UiALr7h!YS%Xj(swX`kWqrDv&3Z?VRE{GIBBkk&ZuF ze$ayq5AM##&p#>Xe-G(LGo=eQmA^-?%b7|}{UCNicVtN!Rtf+N2t&t0KI#6SolJ|= zpa+6N=l=R%>(kzyF9d?zgm%3le-E-4aENd3l5F+Kj)=m88|yVJ9&PK z&g$IPZ%Cc6oygKP+CV=j11twP9SHBoL;f`)kCmH+=N)pRKl>^F06w)KRUDIo2t4#g z?mmb7_opBG=Wic3@cSQ8@BT&=Dn^`lwZGu~$!hn#?%U?Ok99fR%7#4c{{XLX{VH^X z28~Lbya7k#5A>pn)`ciIMon(wzWF5o00w+~WbR26Ylw>`1E-hxgMs|NT9PQhRcAl| z9#b52>4W^gS}3Jtl6KlLdoA2!FVFL9wYzD_k4W|cSqU-xjsWVtznAInQlvl-p>kDl za55P^ze*^i=|&naPr6c%{@A{E{%3y5a%_?~`2kg2Y{H-KFQ?c1Dp*lY+$cP385qd^ z>GVJ6>QPFX)(`o=`%nH$t@c;F<$lOXi5y1U*k`Er zJdeVPD|u6lWY=r<=;FVxmqd}*PE2}60+YZ1Z3olWsQnMMOtB6Q-JlE}Gqiepemrs3 ziYuO)QjPm0{{RdB0KoK1vD*nT>X;b{q;2B^p#6B{dXJ?|8Y$?bp#uZAJa_c>{3xQP z66LC*wzdBNuep84Y#XrHV$R+@2lC_p0ImN3j?~KNiYtZ@il(J) zTd~~(jBNq2k6u2WoDR4g^r@0H3={##!kmBlvO0PWv{6^el}=lAiu>*Uf7OxM zPENy;Fa~%W@t$%$I&<_KaZ)NOup|XlBpiTnIO<0q@Dc6rMHS6A$xX`JB>w<6`hG0! zrrpU>G6&sLwRz7UpP=^t0PEDTBAvKUc;h1(9;eZLzsV@A;<;+VD`)yiNAmoAT77Vn zu#v^Pw(Rlp@w@$z=m)R)6D(=-Wak)Dw0-YIUvKmDD6b|hMx1A4?|uIO!_E1>yVh6q zAl=CXV+U|hR1Y(rgR+my{P9w(Yq%=~1dYvsjj8?L^Zh8IwsBEALojHmKYQAsFi{z9R_j!N3|4JG;KOka$7IWL;nDlOYrQD z*fyW_{d6ReK#F+)P&h@v>5u8sr;!)^BMJu4pkQ$8Q5f)4b zPcQKY1NnZnBvF8>&VT?sra0--2l;-qQA)`q?X+U{Te!wwpXS$VcGHs{k?aJr5@Y!s z0o8kdFVo(oNPr start else 0, + 'p50_ms': p(0.5), 'p95_ms': p(0.95), 'p99_ms': p(0.99), + 'error_kinds': sorted({str(r[2]) for r in rows if r[2] != 201})} + + def verify(self, token): + stored, cursor = [], None + while True: + query = '?limit=100' + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '') + page = call('GET', self.path + query, token=token)[2] + stored += page['items'] + cursor = page['next_cursor'] + if cursor is None: + break + ids = [m['id'] for m in stored] + texts = [m['text'] for m in stored] + return {'confirmed': len(self.confirmed), 'stored': len(stored), + 'lost': len({m['id'] for m in self.confirmed} - set(ids)), + 'duplicate_ids': len(ids) - len(set(ids)), 'duplicate_texts': len(texts) - len(set(texts)), + 'uncertain_at_stop': self.uncertain} diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 00000000..8e19ffbe --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,66 @@ +import os +from pathlib import Path +import struct +import zlib + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE_PNG = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes() +SAMPLE_JPG = (Path(__file__).resolve().parent / 'fixtures' / 'sample.jpg').read_bytes() + + +def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff) + + +def png(width, height, rows, color_type=0, level=9): + """rows — итератор байтовых строк пикселей (без байта фильтра).""" + compressor = zlib.compressobj(level) + parts = [] + for row in rows: + parts.append(compressor.compress(b'\x00' + row)) + parts.append(compressor.flush()) + header = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'') + + +def bomb(width=10000, height=5000): + """50 Мп чёрного: в распакованном виде 50 МБ, в файле — несколько десятков КБ.""" + row = bytes(width) + return png(width, height, (row for _ in range(height))) + + +def noise_png(target_bytes): + """Шумный RGB PNG (почти не сжимается) размером чуть меньше target_bytes.""" + width = 1024 + height = max(1, (target_bytes - 4096) // (width * 3 + 1)) + return png(width, height, (os.urandom(width * 3) for _ in range(height)), color_type=2, level=0) + + +def broken_crc(data): + """Испортить один байт внутри IDAT, не пересчитывая CRC.""" + position = data.index(b'IDAT') + 100 + return data[:position] + bytes([data[position] ^ 0xff]) + data[position + 1:] + + +def with_idat(data, transform): + """Заменить пиксельные данные PNG (все IDAT) на transform(idat), с правильными CRC.""" + position, chunks = 8, [] + while position < len(data): + length, kind = struct.unpack('>I4s', data[position:position + 8]) + chunks.append((kind, data[position + 8:position + 8 + length])) + position += 12 + length + idat = b''.join(body for kind, body in chunks if kind == b'IDAT') + first = next(index for index, (kind, _) in enumerate(chunks) if kind == b'IDAT') + kept = [(kind, body) for kind, body in chunks if kind != b'IDAT'] + kept.insert(first, (b'IDAT', transform(idat))) + return data[:8] + b''.join(chunk(kind, body) for kind, body in kept) + + +def garbage_idat(data): + """Мусор вместо сжатых пикселей, CRC верные.""" + return with_idat(data, lambda idat: idat[:2] + os.urandom(len(idat) - 2)) + + +def short_idat(data): + """Корректный zlib-поток, но пикселей меньше, чем требует IHDR.""" + return with_idat(data, lambda idat: zlib.compress(zlib.decompress(idat)[:-500])) diff --git a/student-tests/rolling_update_check.py b/student-tests/rolling_update_check.py new file mode 100644 index 00000000..eabfedcc --- /dev/null +++ b/student-tests/rolling_update_check.py @@ -0,0 +1,60 @@ +"""Usage (стенд запущен, обе реплики в ротации): + python student-tests/rolling_update_check.py --output evidence/rolling-update.json + +Под непрерывной нагрузкой (4 потока, повтор с тем же Idempotency-Key) выполняет scripts/rolling_update.py +и проверяет: ошибки клиента, RPS, что все реплики заменены, что подтверждённые сообщения не потеряны и не задвоены. +""" +import argparse +import json +from pathlib import Path +import subprocess +import sys +import time + +from helpers import Load, api_containers, new_chat, new_user + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--no-build', action='store_true') + args = parser.parse_args() + before = api_containers() + alice = new_user('rolling') + chat = new_chat(alice) + load = Load(alice['token'], chat['id']).start() + time.sleep(5) + started = time.monotonic() + command = [sys.executable, str(ROOT / 'scripts' / 'rolling_update.py')] + (['--no-build'] if args.no_build else []) + update = subprocess.run(command, capture_output=True, text=True, encoding='utf-8') + finished = time.monotonic() + time.sleep(5) + load.stop() + print(update.stdout.strip()) + after = api_containers() + result = { + 'replicas_before': [c[:12] for c in before], 'replicas_after': [c[:12] for c in after], + 'update_seconds': round(finished - started, 1), 'update_exit_code': update.returncode, + 'before_5s': load.window(started - 5, started), + 'during_update': load.window(started, finished), + 'after_5s': load.window(finished, finished + 5), + 'data': load.verify(alice['token']), + 'log': update.stdout.strip().splitlines(), + } + replaced = not set(before) & set(after) and len(after) == len(before) + ok = update.returncode == 0 and replaced and not result['data']['lost'] and not result['data']['duplicate_texts'] + print('rolling update %.1fs: errors=%d, rps %.1f -> %.1f -> %.1f, lost=%d, duplicates=%d, all replaced=%s' + % (result['update_seconds'], result['during_update']['errors'], result['before_5s']['rps'], + result['during_update']['rps'], result['after_5s']['rps'], result['data']['lost'], + result['data']['duplicate_texts'], replaced)) + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + json.dump(result, file, ensure_ascii=False, indent=2) + print('PASS' if ok else 'FAIL: ' + update.stderr.strip()[-500:]) + raise SystemExit(0 if ok else 1) + + +if __name__ == '__main__': + main() diff --git a/student-tests/sigterm_check.py b/student-tests/sigterm_check.py new file mode 100644 index 00000000..5d09b0ee --- /dev/null +++ b/student-tests/sigterm_check.py @@ -0,0 +1,79 @@ +"""Usage (стенд запущен через docker compose up -d): + python student-tests/sigterm_check.py + docker compose start api + +Останавливает одну API-реплику (docker stop = SIGTERM) и всё это время опрашивает API через LB: +реплика сначала выходит из ротации (readiness 503), потом завершается; клиенты не должны получить ошибок. +""" +import argparse +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid + +from helpers import BASE_URL, api_containers, call, new_chat, new_user + + +def docker(*args): + return subprocess.run(['docker', *args], capture_output=True, text=True, encoding='utf-8').stdout.strip() + + +def probe_forever(results, stop_event): + while not stop_event.is_set(): + started = time.monotonic() + try: + with urllib.request.urlopen(BASE_URL + '/health/live', timeout=1) as response: + outcome = str(response.status) + except urllib.error.HTTPError as error: + outcome = str(error.code) + except Exception as error: + outcome = type(error).__name__ + results.append((started, outcome)) + time.sleep(0.02) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--container', default=None) + parser.add_argument('--stop-timeout', type=int, default=10, help='секунд до SIGKILL у docker stop') + args = parser.parse_args() + args.container = args.container or api_containers()[0] + + marker = 'PRIVATE-' + uuid.uuid4().hex + alice = new_user('alice') + chat = new_chat(alice) + status = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': marker}, alice['token'])[0] + logs_before = docker('logs', args.container) + print('сообщение с маркером отправлено: HTTP %d; маркер в логах: %s' % (status, marker in logs_before)) + + results, stop_event = [], threading.Event() + prober = threading.Thread(target=probe_forever, args=(results, stop_event)) + prober.start() + time.sleep(1) + + sigterm_at = time.monotonic() + docker('stop', '-t', str(args.stop_timeout), args.container) + stopped_at = time.monotonic() + time.sleep(1) + stop_event.set() + prober.join() + + before = [r for t, r in results if t < sigterm_at] + after = [r for t, r in results if t >= sigterm_at] + exit_code = docker('inspect', '--format', '{{.State.ExitCode}}', args.container) + print('docker stop занял: %.2f с (лимит %d с)' % (stopped_at - sigterm_at, args.stop_timeout)) + print('код выхода контейнера: %s (0 — сам завершился по SIGTERM; 137 — убит SIGKILL)' % exit_code) + print('запросы до SIGTERM: %d, все 200: %s' % (len(before), set(before) == {'200'})) + print('запросы после SIGTERM: %d, результаты: %s' % (len(after), sorted(set(after)))) + print('ответов 5xx за всё время: %d' % sum(r.startswith('5') for _, r in results)) + + print('\nлоги остановки:') + for line in docker('logs', args.container).splitlines(): + if any(word in line for word in ('draining', 'Shutting down', 'stopping', 'shutdown', 'Finished server')): + print(' ' + line) + + +if __name__ == '__main__': + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..be589b60 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,157 @@ +import unittest +import uuid + +from helpers import call, login, new_chat, new_user, read_all_messages + + +class ErrorFormatMixin: + def assertError(self, result, status, code): + actual, headers, body = result + self.assertEqual(actual, status) + self.assertEqual(body['error']['code'], code) + self.assertEqual(body['error']['request_id'], headers['X-Request-Id']) + self.assertNotIn('Traceback', body['error']['message']) + + +class InvalidJsonTest(ErrorFormatMixin, unittest.TestCase): + def test_malformed_json(self): + """Битый JSON -> 400 invalid_request""" + self.assertError(call('POST', '/api/v1/users', raw=b'{"username": "abc",'), 400, 'invalid_request') + + def test_json_not_object(self): + """JSON-массив вместо объекта -> 400""" + self.assertError(call('POST', '/api/v1/users', raw=b'[1, 2, 3]'), 400, 'invalid_request') + + def test_wrong_type(self): + """Число вместо строки -> 400 (без тихого приведения типов)""" + self.assertError(call('POST', '/api/v1/users', {'username': 12345, 'display_name': 'x', 'password': 'Long-enough-pass-1'}), 400, 'invalid_request') + + def test_extra_field(self): + """Лишнее поле в теле -> 400""" + body = {'username': 'u_' + uuid.uuid4().hex[:8], 'display_name': 'x', 'password': 'Long-enough-pass-1', 'is_admin': True} + self.assertError(call('POST', '/api/v1/users', body), 400, 'invalid_request') + + +class DuplicateUsernameTest(ErrorFormatMixin, unittest.TestCase): + def test_duplicate_username(self): + """Повторная регистрация того же username -> 409 conflict""" + user = new_user('dup') + again = call('POST', '/api/v1/users', {'username': user['username'], 'display_name': 'Другой', + 'password': 'Other-password-123'}) + self.assertError(again, 409, 'conflict') + + +class ForeignChatTest(ErrorFormatMixin, unittest.TestCase): + def setUp(self): + self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve') + self.chat = new_chat(self.alice, self.bob) + self.path = '/api/v1/chats/' + self.chat['id'] + + def test_outsider_cannot_read_chat(self): + """Посторонний читает чат -> 404, как будто чата нет""" + self.assertError(call('GET', self.path, token=self.eve['token']), 404, 'not_found') + missing = call('GET', '/api/v1/chats/' + str(uuid.uuid4()), token=self.eve['token']) + self.assertEqual(missing[2]['error']['message'], call('GET', self.path, token=self.eve['token'])[2]['error']['message']) + + def test_outsider_cannot_write_or_read_history(self): + """Посторонний пишет/читает историю -> 404""" + self.assertError(call('POST', self.path + '/messages', {'text': 'чужое'}, self.eve['token']), 404, 'not_found') + self.assertError(call('GET', self.path + '/messages', token=self.eve['token']), 404, 'not_found') + + def test_outsider_does_not_see_chat_in_list(self): + """Чужой чат не попадает в список чатов""" + status, _, page = call('GET', '/api/v1/chats?limit=100', token=self.eve['token']) + self.assertEqual(status, 200) + self.assertNotIn(self.chat['id'], [c['id'] for c in page['items']]) + + def test_member_has_access(self): + """Участник видит чат (контроль: 404 выше не из-за ошибки в тесте)""" + self.assertEqual(call('GET', self.path, token=self.bob['token'])[0], 200) + + +class EmptyMessageTest(ErrorFormatMixin, unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.path = '/api/v1/chats/%s/messages' % new_chat(self.alice)['id'] + + def test_empty_text(self): + """Пустой текст -> 400""" + self.assertError(call('POST', self.path, {'text': ''}, self.alice['token']), 400, 'invalid_request') + + def test_whitespace_text(self): + """Текст только из пробельных символов -> 400""" + self.assertError(call('POST', self.path, {'text': ' \t\n '}, self.alice['token']), 400, 'invalid_request') + + def test_missing_text(self): + """Тело без поля text -> 400""" + self.assertError(call('POST', self.path, {}, self.alice['token']), 400, 'invalid_request') + + def test_text_length_limit(self): + """4000 символов можно (эмодзи = 1 символ), 4001 — 400""" + self.assertEqual(call('POST', self.path, {'text': '😀' * 4000}, self.alice['token'])[0], 201) + self.assertError(call('POST', self.path, {'text': 'x' * 4001}, self.alice['token']), 400, 'invalid_request') + + +class PaginationTest(ErrorFormatMixin, unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.chat = new_chat(self.alice) + self.sent = [call('POST', '/api/v1/chats/%s/messages' % self.chat['id'], + {'text': 'сообщение %d' % i}, self.alice['token'])[2] for i in range(7)] + + def test_walk_several_pages(self): + """7 сообщений по 3: три страницы, без потерь, повторов и перестановок""" + items, pages = read_all_messages(self.chat['id'], self.alice['token'], limit=3) + self.assertEqual(pages, 3) + self.assertEqual([m['id'] for m in items], [m['id'] for m in self.sent]) + keys = [(m['created_at'], m['id']) for m in items] + self.assertEqual(keys, sorted(keys)) + + def test_cursor_from_other_chat(self): + """cursor от другого чата -> 400""" + _, _, page = call('GET', '/api/v1/chats/%s/messages?limit=1' % self.chat['id'], token=self.alice['token']) + other = new_chat(self.alice, title='Другой') + self.assertError(call('GET', '/api/v1/chats/%s/messages?cursor=%s' % (other['id'], page['next_cursor']), + token=self.alice['token']), 400, 'invalid_request') + + def test_invalid_cursor_and_limit(self): + """Мусорный cursor и limit вне 1..100 -> 400""" + path = '/api/v1/chats/%s/messages' % self.chat['id'] + for query in ('?cursor=not-a-cursor', '?limit=0', '?limit=101', '?limit=abc'): + with self.subTest(query=query): + self.assertError(call('GET', path + query, token=self.alice['token']), 400, 'invalid_request') + + +class IdentityTest(ErrorFormatMixin, unittest.TestCase): + def test_sender_cannot_be_forged(self): + """sender_id в теле запрещён: отправитель только из сессии""" + alice, bob = new_user('alice'), new_user('bob') + chat = new_chat(alice, bob) + forged = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': 'hi', 'sender_id': bob['id']}, alice['token']) + self.assertError(forged, 400, 'invalid_request') + + def test_nul_character_is_preserved(self): + """U+0000 — допустимый code point: сообщение, display_name и title сохраняются и читаются без изменений""" + username, password = 'nul_' + uuid.uuid4().hex[:12], 'Nul-password-123' + status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'a\u0000lice', 'password': password}) + self.assertEqual((status, user['display_name']), (201, 'a\u0000lice')) + alice = dict(user, token=login(username, password)['token']) + self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[2]['display_name'], 'a\u0000lice') + status, _, chat = call('POST', '/api/v1/chats', {'title': 't\u0000', 'member_ids': [alice['id']]}, alice['token']) + self.assertEqual((status, chat['title']), (201, 't\u0000')) + path = '/api/v1/chats/%s/messages' % chat['id'] + status, _, message = call('POST', path, {'text': 'a\u0000b'}, alice['token']) + self.assertEqual((status, message['text']), (201, 'a\u0000b')) + self.assertEqual(call('GET', path, token=alice['token'])[2]['items'][0]['text'], 'a\u0000b') + self.assertEqual(call('GET', '/api/v1/chats/' + chat['id'], token=alice['token'])[2]['title'], 't\u0000') + + def test_lone_surrogate_is_rejected(self): + """Одиночный суррогат не кодируется в UTF-8 -> 400, а не 500""" + alice = new_user('alice') + chat = new_chat(alice) + self.assertError(call('POST', '/api/v1/chats/%s/messages' % chat['id'], raw=b'{"text": "a\\ud800b"}', + token=alice['token']), 400, 'invalid_request') + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..7b2397bc --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,192 @@ +import hashlib +import http.client +import unittest +import urllib.parse +import uuid + +from helpers import BASE_URL, call, new_chat, new_user +from imagegen import SAMPLE_JPG, SAMPLE_PNG, bomb, broken_crc, garbage_idat, noise_png, short_idat + +MAX = 10 * 1024 * 1024 + + +def upload(user, chat, data, content_type='image/png', filename='photo.png', token=True): + query = '' if filename is None else '?filename=' + urllib.parse.quote(filename, safe='') + return call('POST', '/api/v1/chats/%s/attachments%s' % (chat['id'], query), raw=data, + token=user['token'] if token else None, headers={'Content-Type': content_type}) + + +def download(user, attachment_id): + target = urllib.parse.urlsplit(BASE_URL) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=30) + connection.request('GET', '/api/v1/attachments/%s/content' % attachment_id, + headers={'Authorization': 'Bearer ' + user['token']}) + response = connection.getresponse() + body = response.read() + connection.close() + return response.status, response.headers, body + + +def send(user, chat, body, key=None): + return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'], + headers={'Idempotency-Key': key or uuid.uuid4().hex}) + + +class UploadTest(unittest.TestCase): + def setUp(self): + self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve') + self.chat = new_chat(self.alice, self.bob) + + def assertCode(self, result, status, code=None): + self.assertEqual(result[0], status, result[2]) + if code: + self.assertEqual(result[2]['error']['code'], code) + + def test_png_and_jpeg_are_stored_and_downloaded_byte_exact(self): + """PNG и JPEG → 201 ready; второй участник скачивает те же байты с верным Content-Type""" + for data, content_type in ((SAMPLE_PNG, 'image/png'), (SAMPLE_JPG, 'image/jpeg')): + with self.subTest(content_type=content_type): + status, _, att = upload(self.alice, self.chat, data, content_type) + self.assertEqual(status, 201) + self.assertEqual((att['status'], att['variants'], att['error_code']), ('ready', [], None)) + self.assertEqual((att['content_type'], att['size_bytes'], att['sha256']), + (content_type, len(data), hashlib.sha256(data).hexdigest())) + self.assertEqual((att['chat_id'], att['owner_id']), (self.chat['id'], self.alice['id'])) + self.assertEqual(call('GET', '/api/v1/attachments/' + att['id'], token=self.bob['token'])[2], att) + status, headers, body = download(self.bob, att['id']) + self.assertEqual((status, headers.get_content_type(), body), (200, content_type, data)) + self.assertEqual(int(headers['Content-Length']), len(data)) + + def test_download_through_every_replica(self): + """Файл скачивается через любую API-реплику (байты не лежат локально у загрузившей)""" + att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + replicas = set() + for _ in range(6): + status, headers, body = download(self.bob, att['id']) + self.assertEqual((status, body), (200, SAMPLE_PNG)) + replicas.add(headers['X-Instance-Id']) + self.assertGreaterEqual(len(replicas), 2) + + def test_filename_is_display_only_and_validated(self): + """Unicode-имя сохраняется как есть; нет имени, пустое, /, \\, NUL, управляющий символ, >255 → 400""" + status, _, att = upload(self.alice, self.chat, SAMPLE_PNG, filename='фото отпуска 😀.png') + self.assertEqual((status, att['filename']), (201, 'фото отпуска 😀.png')) + for name in (None, '', ' ', '../etc/passwd', 'a\\b.png', 'a\x00b.png', 'bell\x07.png', 'x' * 256): + with self.subTest(name=name): + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, filename=name), 400, 'invalid_request') + + def test_wrong_declared_type(self): + """Content-Type не image/png|jpeg (octet-stream, multipart, gif) → 415""" + for content_type in ('application/octet-stream', 'multipart/form-data; boundary=x', 'image/gif', 'text/plain'): + with self.subTest(content_type=content_type): + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, content_type), 415, 'unsupported_media_type') + + def test_mime_spoofing(self): + """Подмена MIME: PNG как jpeg, JPEG как png, текст и GIF под видом png → 415""" + for data, content_type in ((SAMPLE_PNG, 'image/jpeg'), (SAMPLE_JPG, 'image/png'), + (b'hello, not an image', 'image/png'), (b'GIF89a' + bytes(100), 'image/png'), + (b'', 'image/png')): + with self.subTest(content_type=content_type, head=data[:6]): + self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type') + + def test_corrupted_images(self): + """Обрезанный PNG, битый CRC, мусор или нехватка пикселей в IDAT при верных CRC, обрезанный JPEG → 415""" + for data, content_type in ((SAMPLE_PNG[:5000], 'image/png'), (broken_crc(SAMPLE_PNG), 'image/png'), + (garbage_idat(SAMPLE_PNG), 'image/png'), (short_idat(SAMPLE_PNG), 'image/png'), + (SAMPLE_JPG[:3000], 'image/jpeg')): + with self.subTest(size=len(data), content_type=content_type): + self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type') + + def test_too_large_file(self): + """10 MiB + 1 байт → 413 (и по Content-Length, и chunked без длины); файл ~10 MiB принимается""" + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG + bytes(MAX + 1 - len(SAMPLE_PNG))), 413, 'payload_too_large') + target = urllib.parse.urlsplit(BASE_URL) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=60) + chunks = (SAMPLE_PNG if i == 0 else bytes(1024 * 1024) for i in range(12)) + connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % self.chat['id'], body=chunks, + headers={'Authorization': 'Bearer ' + self.alice['token'], 'Content-Type': 'image/png'}, + encode_chunked=True) + self.assertEqual(connection.getresponse().status, 413) + connection.close() + big = noise_png(MAX - 50000) + self.assertLessEqual(len(big), MAX) + status, _, att = upload(self.alice, self.chat, big) + self.assertEqual((status, att['size_bytes']), (201, len(big))) + self.assertEqual(download(self.bob, att['id'])[2], big) + + def test_decompression_bomb(self): + """PNG 10000×5000 (50 Мп) весом в десятки КБ → 413 без распаковки""" + data = bomb() + self.assertLess(len(data), 1024 * 1024) + self.assertCode(upload(self.alice, self.chat, data), 413, 'payload_too_large') + + def test_access_rules(self): + """Без токена 401; посторонний не может загрузить в чужой чат и не видит метаданные/байты (404)""" + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, token=False), 401, 'unauthorized') + self.assertCode(upload(self.eve, self.chat, SAMPLE_PNG), 404, 'not_found') + self.assertCode(upload(self.alice, {'id': str(uuid.uuid4())}, SAMPLE_PNG), 404, 'not_found') + att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + self.assertCode(call('GET', '/api/v1/attachments/' + att['id'], token=self.eve['token']), 404, 'not_found') + self.assertEqual(download(self.eve, att['id'])[0], 404) + self.assertCode(call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), token=self.bob['token']), 404) + self.assertCode(call('GET', '/api/v1/attachments/not-a-uuid', token=self.bob['token']), 400) + self.assertCode(call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att['id'], + token=self.bob['token']), 400) + + +class MessageAttachmentsTest(unittest.TestCase): + def setUp(self): + self.alice, self.bob = new_user('alice'), new_user('bob') + self.chat = new_chat(self.alice, self.bob) + self.other_chat = new_chat(self.alice, title='Другой') + self.att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + + def history(self): + return call('GET', '/api/v1/chats/%s/messages?limit=100' % self.chat['id'], token=self.bob['token'])[2]['items'] + + def test_message_with_attachment_only(self): + """text="" + одно вложение → 201; attachment_ids видны в истории; обычное сообщение получает []""" + status, _, message = send(self.alice, self.chat, {'text': '', 'attachment_ids': [self.att['id']]}) + self.assertEqual((status, message['attachment_ids'], message['text']), (201, [self.att['id']], '')) + plain = send(self.alice, self.chat, {'text': 'без вложений'})[2] + self.assertEqual(plain['attachment_ids'], []) + self.assertEqual([m['attachment_ids'] for m in self.history()], [[self.att['id']], []]) + + def test_other_member_can_reference_attachment(self): + """Боб прикрепляет вложение, загруженное Алисой в этот же чат""" + status, _, message = send(self.bob, self.chat, {'text': 'смотри', 'attachment_ids': [self.att['id']]}) + self.assertEqual((status, message['attachment_ids']), (201, [self.att['id']])) + + def test_foreign_or_unknown_attachment(self): + """Вложение другого чата или несуществующее → 404, сообщение не создаётся""" + foreign = upload(self.alice, self.other_chat, SAMPLE_PNG)[2] + for ids in ([foreign['id']], [str(uuid.uuid4())], [self.att['id'], foreign['id']]): + with self.subTest(ids=ids): + self.assertEqual(send(self.alice, self.chat, {'text': 'x', 'attachment_ids': ids})[0], 404) + self.assertEqual(self.history(), []) + + def test_invalid_attachment_lists(self): + """Повтор id, 11 вложений, не-UUID, пустой текст без вложений, пробелы без вложений → 400""" + for body in ({'text': 'x', 'attachment_ids': [self.att['id'], self.att['id']]}, + {'text': 'x', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]}, + {'text': 'x', 'attachment_ids': ['nope']}, + {'text': ''}, {'text': ' '}, {'text': '', 'attachment_ids': []}): + with self.subTest(body=body): + self.assertEqual(send(self.alice, self.chat, body)[0], 400) + + def test_idempotency_includes_attachments(self): + """Тот же ключ: те же вложения → тот же Message; другие вложения → 409; нет поля = [] для текста""" + key = uuid.uuid4().hex + first = send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [self.att['id']]}, key) + again = send(self.alice, self.chat, {'attachment_ids': [self.att['id']], 'text': 'фото'}, key) + self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2])) + other = upload(self.alice, self.chat, SAMPLE_JPG, 'image/jpeg', 'b.jpg')[2] + self.assertEqual(send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [other['id']]}, key)[0], 409) + key = uuid.uuid4().hex + plain = send(self.alice, self.chat, {'text': 'текст'}, key) + same = send(self.alice, self.chat, {'text': 'текст', 'attachment_ids': []}, key) + self.assertEqual(plain[2]['id'], same[2]['id']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..3a56ee20 --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,132 @@ +from datetime import datetime, timezone +import unittest +import uuid + +from helpers import basic, call, login, new_user, register + + +class RegistrationTest(unittest.TestCase): + def body(self, password): + return {'username': 'reg_' + uuid.uuid4().hex[:12], 'display_name': 'Регистрация', 'password': password} + + def test_password_is_not_returned(self): + """Ответ регистрации и /users/me не содержат password/hash""" + status, _, user = call('POST', '/api/v1/users', self.body('Secret-password-1')) + self.assertEqual(status, 201) + self.assertEqual(set(user), {'id', 'username', 'display_name', 'created_at'}) + + def test_password_length_limits(self): + """Пароль 12..128 символов; 11 и 129 -> 400; эмодзи считается одним символом""" + self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 11))[0], 400) + self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 129))[0], 400) + self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 128))[0], 201) + self.assertEqual(call('POST', '/api/v1/users', self.body('🔑' * 12))[0], 201) + + def test_password_required_and_not_blank(self): + """Без password или из одних пробелов -> 400""" + body = self.body('x') + del body['password'] + self.assertEqual(call('POST', '/api/v1/users', body)[0], 400) + self.assertEqual(call('POST', '/api/v1/users', self.body(' ' * 20))[0], 400) + + +class LoginTest(unittest.TestCase): + def setUp(self): + self.user, self.password = register('login') + + def assertBasicChallenge(self, result): + status, headers, body = result + self.assertEqual(status, 401) + self.assertEqual(headers['WWW-Authenticate'], 'Basic realm="messenger", charset="UTF-8"') + self.assertEqual(body['error']['code'], 'unauthorized') + return body['error']['message'] + + def test_login_returns_opaque_session(self): + """Вход: 201, token_type=Bearer, токен случайный и не содержит username/UUID""" + first = login(self.user['username'], self.password) + second = login(self.user['username'], self.password) + self.assertEqual(first['token_type'], 'Bearer') + self.assertEqual(first['user'], self.user) + self.assertGreaterEqual(len(first['token']), 43) + self.assertNotEqual(first['token'], second['token']) + self.assertNotIn(self.user['username'], first['token']) + self.assertNotIn(self.user['id'], first['token']) + expires = datetime.fromisoformat(first['expires_at'].replace('Z', '+00:00')) + self.assertGreater(expires, datetime.now(timezone.utc)) + + def test_wrong_password_and_unknown_user_look_the_same(self): + """Неверный пароль и несуществующий пользователь -> одинаковый 401 с Basic challenge""" + wrong = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', + headers=basic(self.user['username'], 'Wrong-password-1'))) + unknown = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', + headers=basic('nobody_' + uuid.uuid4().hex[:8], 'Wrong-password-1'))) + self.assertEqual(wrong, unknown) + + def test_malformed_basic(self): + """Нет заголовка, не base64, нет двоеточия, Bearer вместо Basic -> 401 Basic""" + for header in (None, 'Basic', 'Basic !!!not-base64!!!', 'Basic ' + 'bm9jb2xvbg==', 'Bearer abc'): + with self.subTest(header=header): + headers = {'Authorization': header} if header else {} + self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', headers=headers)) + + def test_password_with_colon_and_unicode(self): + """Двоеточие и не-ASCII в пароле допустимы: делим только по первому ':'""" + user, password = register('colon', password='пароль:с:двоеточием🔑') + self.assertEqual(login(user['username'], password)['user']['id'], user['id']) + + def test_login_rejects_body(self): + """POST /auth/sessions принимает только заголовок, JSON body -> 400""" + status = call('POST', '/api/v1/auth/sessions', {'username': self.user['username']}, + headers=basic(self.user['username'], self.password))[0] + self.assertEqual(status, 400) + + +class BearerTest(unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.eve = new_user('eve') + + def assertBearerChallenge(self, headers): + status, response_headers, _ = call('GET', '/api/v1/users/me', headers=headers) + self.assertEqual(status, 401) + self.assertEqual(response_headers['WWW-Authenticate'], 'Bearer realm="messenger"') + + def test_only_bearer_is_accepted(self): + """Нет заголовка, Basic, X-User-Id, мусорный токен -> 401 Bearer""" + self.assertBearerChallenge({}) + self.assertBearerChallenge(basic(self.alice['username'], self.alice['password'])) + self.assertBearerChallenge({'X-User-Id': self.alice['id']}) + self.assertBearerChallenge({'Authorization': 'Bearer ' + uuid.uuid4().hex}) + self.assertBearerChallenge({'Authorization': 'Bearer'}) + + def test_authentication_before_validation(self): + """Аноним с битым JSON на защищённом endpoint получает 401, а не 400; с сессией — 400""" + self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad')[0], 401) + self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad', token=self.alice['token'])[0], 400) + self.assertEqual(call('POST', '/api/v1/users', raw=b'{bad')[0], 400) + + def test_scheme_is_case_insensitive(self): + """Схема аутентификации нечувствительна к регистру (RFC 9110)""" + status, _, me = call('GET', '/api/v1/users/me', headers={'Authorization': 'bearer ' + self.alice['token']}) + self.assertEqual((status, me['id']), (200, self.alice['id'])) + + def test_x_user_id_does_not_override_session(self): + """X-User-Id игнорируется: владелец определяется только токеном""" + status, _, me = call('GET', '/api/v1/users/me', token=self.alice['token'], headers={'X-User-Id': self.eve['id']}) + self.assertEqual((status, me['id']), (200, self.alice['id'])) + + +class LogoutTest(unittest.TestCase): + def test_logout_revokes_only_current_session(self): + """Logout -> 204 без тела; этот токен -> 401, вторая сессия того же пользователя работает""" + alice = new_user('alice') + other = login(alice['username'], alice['password'])['token'] + status, _, body = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token']) + self.assertEqual((status, body), (204, None)) + self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[0], 401) + self.assertEqual(call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0], 401) + self.assertEqual(call('GET', '/api/v1/users/me', token=other)[0], 200) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..4d258446 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,68 @@ +from concurrent.futures import ThreadPoolExecutor +import threading +import unittest +import uuid + +from helpers import basic, call, new_chat, new_user, read_all_messages + +CLIENTS = 10 +REGISTRATIONS = 20 +PER_CLIENT = 10 + + +class ConcurrencyTest(unittest.TestCase): + def test_100_messages_from_10_clients(self): + """100 параллельных сообщений от 10 клиентов -> ровно 100 уникальных в истории""" + users = [new_user('client%d' % i) for i in range(CLIENTS)] + chat = new_chat(users[0], *users[1:], title='Нагрузка') + path = '/api/v1/chats/%s/messages' % chat['id'] + barrier = threading.Barrier(CLIENTS) + + def client(index): + barrier.wait() + results = [] + for n in range(PER_CLIENT): + status, _, message = call('POST', path, {'text': 'client %d msg %d' % (index, n)}, users[index]['token']) + results.append((status, message)) + return results + + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + responses = [r for batch in pool.map(client, range(CLIENTS)) for r in batch] + + self.assertEqual([s for s, _ in responses], [201] * CLIENTS * PER_CLIENT) + sent_ids = {m['id'] for _, m in responses} + items, pages = read_all_messages(chat['id'], users[0]['token'], limit=30) + read_ids = [m['id'] for m in items] + keys = [(m['created_at'], m['id']) for m in items] + + self.assertEqual(len(sent_ids), 100, 'сервер выдал повторяющиеся id') + self.assertEqual(len(read_ids), 100, 'в истории не ровно 100 сообщений') + self.assertEqual(set(read_ids), sent_ids, 'история не совпадает с подтверждёнными сообщениями') + self.assertEqual(len({m['text'] for m in items}), 100, 'потерялся или задвоился текст') + self.assertEqual(keys, sorted(keys), 'история не упорядочена по (created_at, id)') + print('\n отправлено 201: %d, уникальных id: %d, прочитано: %d за %d страниц(ы)' + % (len(responses), len(sent_ids), len(read_ids), pages)) + + def test_parallel_registration_of_same_username(self): + """20 одновременных регистраций одного username -> ровно один 201, остальные 409; вход только с паролем победителя""" + username = 'race_' + uuid.uuid4().hex[:12] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + password = 'Race-password-%02d' % index + barrier.wait() + status = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Гонка', 'password': password})[0] + return status, password + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + results = list(pool.map(register, range(REGISTRATIONS))) + statuses = sorted(status for status, _ in results) + self.assertEqual(statuses, [201] + [409] * (REGISTRATIONS - 1)) + winner = [password for status, password in results if status == 201][0] + loser = [password for status, password in results if status == 409][0] + self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, winner))[0], 201) + self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, loser))[0], 401) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 00000000..5997fd83 --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,58 @@ +import hashlib +import time +import unittest +import uuid + +from helpers import call, compose, new_chat, new_user, sql + + +class DatabaseTest(unittest.TestCase): + def test_chat_creation_is_rolled_back(self): + """Неизвестный участник: INSERT chats выполнен, INSERT chat_members упал по FK, всё откатано""" + alice = new_user('alice') + chats_before = int(sql('SELECT count(*) FROM chats')) + members_before = int(sql('SELECT count(*) FROM chat_members')) + ghost = str(uuid.uuid4()) + since = sql("SELECT to_char(now() - interval '1 second', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')") + status, _, body = call('POST', '/api/v1/chats', {'title': 'Откат', 'member_ids': [alice['id'], ghost]}, + alice['token']) + self.assertEqual(status, 400, body) + time.sleep(0.5) + log = compose('logs', '--no-log-prefix', '--since', since, 'postgres') + self.assertIn('violates foreign key constraint', log) + self.assertIn(ghost, log) + self.assertEqual(int(sql('SELECT count(*) FROM chats')), chats_before) + self.assertEqual(int(sql('SELECT count(*) FROM chat_members')), members_before) + + def test_secrets_are_not_stored_in_plaintext(self): + """В users лежит argon2id-хеш, в sessions — sha256 токена, не сам токен""" + alice = new_user('alice') + password_hash = sql("SELECT password_hash FROM users WHERE id = '%s'" % alice['id']) + self.assertTrue(password_hash.startswith('$argon2id$'), password_hash[:10]) + self.assertNotIn(alice['password'], password_hash) + digest = hashlib.sha256(alice['token'].encode()).hexdigest() + self.assertEqual(sql("SELECT count(*) FROM sessions WHERE token_hash = '\\x%s'" % digest), '1') + dump = sql("SELECT string_agg(encode(token_hash, 'escape'), '') FROM sessions WHERE user_id = '%s'" % alice['id']) + self.assertNotIn(alice['token'], dump) + + def test_migrations_are_idempotent(self): + """Повторный запуск миграций ничего не меняет и завершается успешно""" + before = sql('SELECT version, applied_at FROM schema_migrations ORDER BY version') + users = sql('SELECT count(*) FROM users') + output = compose('run', '--rm', '--no-deps', 'migrate') + self.assertIn('skip 0001_init: already applied', output) + self.assertEqual(sql('SELECT version, applied_at FROM schema_migrations ORDER BY version'), before) + self.assertEqual(sql('SELECT count(*) FROM users'), users) + + def test_history_query_uses_index(self): + """Запрос страницы истории идёт по индексу (chat_id, created_at, id), без Sort""" + alice = new_user('alice') + chat = new_chat(alice) + plan = sql("SET enable_seqscan = off; SET enable_bitmapscan = off; EXPLAIN SELECT id FROM messages WHERE chat_id = '%s' " + "ORDER BY created_at, id LIMIT 51" % chat['id']) + self.assertIn('messages_chat_history_idx', plan) + self.assertNotIn('Sort', plan) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..4ed9185b --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,96 @@ +from concurrent.futures import ThreadPoolExecutor +import threading +import unittest +import uuid + +from helpers import call, new_chat, new_user, read_all_messages + +PARALLEL = 20 + + +def post(chat, user, key, body=None, raw=None): + return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'], raw=raw, + headers={'Idempotency-Key': key}) + + +class IdempotencyTest(unittest.TestCase): + def setUp(self): + self.alice, self.bob = new_user('alice'), new_user('bob') + self.chat = new_chat(self.alice, self.bob) + + def history(self): + return read_all_messages(self.chat['id'], self.alice['token'], limit=100)[0] + + def test_replay_returns_same_message(self): + """Повтор того же запроса с тем же ключом -> тот же 201 и тот же Message, дубля нет""" + key = uuid.uuid4().hex + first = post(self.chat, self.alice, key, {'text': 'один раз'}) + second = post(self.chat, self.alice, key, {'text': 'один раз'}) + self.assertEqual((first[0], second[0]), (201, 201)) + self.assertEqual(first[2], second[2]) + self.assertEqual([m['id'] for m in self.history()], [first[2]['id']]) + + def test_json_formatting_does_not_matter(self): + """Порядок/пробелы в JSON семантически равны: повтор с другим форматированием -> тот же Message""" + key = uuid.uuid4().hex + first = post(self.chat, self.alice, key, raw='{"text":"формат"}'.encode()) + second = post(self.chat, self.alice, key, raw='{ "text" : "формат" }'.encode()) + self.assertEqual((first[0], second[0]), (201, 201)) + self.assertEqual(first[2]['id'], second[2]['id']) + + def test_other_body_with_same_key_conflicts(self): + """Тот же ключ с другим телом -> 409, второе сообщение не создаётся""" + key = uuid.uuid4().hex + self.assertEqual(post(self.chat, self.alice, key, {'text': 'первое'})[0], 201) + status, _, body = post(self.chat, self.alice, key, {'text': 'другое'}) + self.assertEqual((status, body['error']['code']), (409, 'conflict')) + self.assertEqual(len(self.history()), 1) + + def test_missing_or_invalid_key(self): + """Нет ключа, пустой, недопустимые символы, длиннее 128 -> 400""" + for key in (None, '', 'bad key', 'bad/key', 'café', 'x' * 129): + with self.subTest(key=key): + self.assertEqual(post(self.chat, self.alice, key, {'text': 'x'})[0], 400) + self.assertEqual(post(self.chat, self.alice, 'A.b_c:d-1' + 'x' * 119, {'text': 'граница'})[0], 201) + + def test_key_scope_is_user_and_chat(self): + """Одинаковый ключ у разных пользователей и в разных чатах — независимые сообщения""" + key = uuid.uuid4().hex + other_chat = new_chat(self.alice, title='Другой') + a = post(self.chat, self.alice, key, {'text': 'от Алисы'}) + b = post(self.chat, self.bob, key, {'text': 'от Боба'}) + c = post(other_chat, self.alice, key, {'text': 'от Алисы'}) + self.assertEqual([a[0], b[0], c[0]], [201, 201, 201]) + self.assertEqual(len({a[2]['id'], b[2]['id'], c[2]['id']}), 3) + + def test_replay_rechecks_session_and_access(self): + """Повтор снова проверяет сессию и доступ: после logout -> 401, посторонний с тем же ключом -> 404""" + key = uuid.uuid4().hex + self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 201) + eve = new_user('eve') + self.assertEqual(post(self.chat, eve, key, {'text': 'до выхода'})[0], 404) + call('DELETE', '/api/v1/auth/sessions/current', token=self.alice['token']) + self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 401) + + def test_parallel_retries_through_replicas_create_one_message(self): + """20 одновременных POST с одним ключом через LB (разные реплики) -> ровно одно сообщение, все 201 с одним id""" + key = uuid.uuid4().hex + barrier = threading.Barrier(PARALLEL) + + def send(_): + barrier.wait() + status, headers, body = post(self.chat, self.alice, key, {'text': 'конкурентный повтор'}) + return status, headers['X-Instance-Id'], body + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(send, range(PARALLEL))) + self.assertEqual({status for status, _, _ in results}, {201}) + self.assertEqual(len({body['id'] for _, _, body in results}), 1) + self.assertGreaterEqual(len({instance for _, instance, _ in results}), 2, 'запросы не попали на разные реплики') + self.assertEqual(len(self.history()), 1) + print('\n 20 повторов: реплики %s, сообщений в истории: %d' + % (sorted({instance for _, instance, _ in results}), len(self.history()))) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_replicas.py b/student-tests/test_replicas.py new file mode 100644 index 00000000..84149108 --- /dev/null +++ b/student-tests/test_replicas.py @@ -0,0 +1,41 @@ +import unittest +import uuid + +from helpers import call, new_chat, new_user + + +class ReplicasTest(unittest.TestCase): + def test_one_session_is_served_by_several_replicas(self): + """Один токен без повторного входа обслуживается разными репликами (X-Instance-Id), все ответы 200""" + alice = new_user('alice') + results = [call('GET', '/api/v1/users/me', token=alice['token']) for _ in range(10)] + self.assertEqual({status for status, _, _ in results}, {200}) + self.assertGreaterEqual(len({headers['X-Instance-Id'] for _, headers, _ in results}), 2) + + def test_write_on_one_replica_is_read_on_another(self): + """Сообщение, подтверждённое одной репликой, сразу видно при чтении через другую""" + alice = new_user('alice') + chat = new_chat(alice) + path = '/api/v1/chats/%s/messages' % chat['id'] + for i in range(6): + _, written_by, message = call('POST', path, {'text': 'm%d' % i}, alice['token']) + status, read_by, page = call('GET', path + '?limit=100', token=alice['token']) + self.assertEqual(status, 200) + self.assertIn(message['id'], [m['id'] for m in page['items']], + 'запись через %s не видна через %s' % (written_by['X-Instance-Id'], read_by['X-Instance-Id'])) + + def test_every_api_response_has_instance_id(self): + """X-Instance-Id есть и в успешных ответах, и в ошибках API, и в health""" + alice = new_user('alice') + for method, path, token, expected in ( + ('GET', '/health/live', None, 200), ('GET', '/health/ready', None, 200), + ('GET', '/api/v1/users/me', None, 401), ('GET', '/api/v1/users/me', alice['token'], 200), + ('GET', '/api/v1/chats/' + str(uuid.uuid4()), alice['token'], 404), ('GET', '/nope', None, 404)): + with self.subTest(path=path, expected=expected): + status, headers, _ = call(method, path, token=token) + self.assertEqual(status, expected) + self.assertTrue(headers['X-Instance-Id']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_storage.py b/student-tests/test_storage.py new file mode 100644 index 00000000..c51278c5 --- /dev/null +++ b/student-tests/test_storage.py @@ -0,0 +1,81 @@ +import subprocess +import unittest + +from helpers import compose, new_chat, new_user, sql +from imagegen import SAMPLE_PNG, noise_png +from test_attachments import upload + +LIMITED_CLIENT = ''' +import json +from botocore.exceptions import ClientError +from config import settings +from s3 import client +results = {} +for name, action in ( + ("put_get_delete_own_bucket", lambda: (client.put_object(Bucket=settings.s3_bucket, Key="probe/x", Body=b"x"), + client.get_object(Bucket=settings.s3_bucket, Key="probe/x"), + client.delete_object(Bucket=settings.s3_bucket, Key="probe/x"))), + ("list_bucket_objects", lambda: client.list_objects_v2(Bucket=settings.s3_bucket)), + ("create_bucket", lambda: client.create_bucket(Bucket="intruder")), + ("delete_bucket", lambda: client.delete_bucket(Bucket=settings.s3_bucket)), + ("list_all_buckets", lambda: client.list_buckets()["Buckets"])): + try: + value = action() + results[name] = "allowed" if name != "list_all_buckets" else "allowed:%d" % len(value) + except ClientError as error: + results[name] = error.response["Error"]["Code"] +print(json.dumps(results)) +''' + + +class StorageTest(unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.chat = new_chat(self.alice) + + def object_key(self, attachment_id): + return sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment_id) + + def test_object_key_is_generated_by_server(self): + """Ключ объекта = attachments/, filename в нём не участвует""" + att = upload(self.alice, self.chat, SAMPLE_PNG, filename='мой-файл.png')[2] + self.assertEqual(self.object_key(att['id']), 'attachments/' + att['id']) + + def test_bucket_is_private_outside_api(self): + """Анонимный GET объекта и листинг bucket напрямую в S3 (из сети backend) → 403""" + att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + for path in ('/messenger-attachments/' + self.object_key(att['id']), '/messenger-attachments'): + with self.subTest(path=path): + result = compose('exec', '-T', 'lb', 'wget', '-S', '-O', '/dev/null', 'http://s3:8333' + path, check=False) + self.assertNotIn('200 OK', result) + output = compose('exec', '-T', 'lb', 'sh', '-c', 'wget -S -O /dev/null http://s3:8333%s 2>&1; true' % path) + self.assertIn('403', output) + + def test_api_credentials_are_limited(self): + """Ключи API: свои объекты читать/писать/удалять можно; листинг, создание и удаление bucket — AccessDenied""" + import json + results = json.loads(compose('exec', '-T', 'api', 'python', '-c', 'import sys; sys.path.insert(0, "src")\n' + LIMITED_CLIENT)) + self.assertEqual(results['put_get_delete_own_bucket'], 'allowed') + for action in ('list_bucket_objects', 'create_bucket', 'delete_bucket'): + self.assertEqual(results[action], 'AccessDenied', results) + self.assertNotEqual(results['list_all_buckets'], 'allowed:1', 'API sees the bucket list') + + def test_bootstrap_is_idempotent(self): + """Повторный s3-init: код 0, bucket уже есть, приватность подтверждена""" + output = compose('run', '--rm', '--no-deps', 's3-init') + self.assertIn('already exists, nothing to do', output) + self.assertIn('is private', output) + + def test_no_original_bytes_in_database_or_api_container(self): + """В attachments единственный bytea — sha256 (байтов файла в БД нет); после загрузки 2 МБ в /tmp реплик API пусто""" + binary = sql("SELECT string_agg(column_name, ',') FROM information_schema.columns " + "WHERE table_name = 'attachments' AND data_type = 'bytea'") + self.assertEqual(binary, 'sha256') + self.assertEqual(upload(self.alice, self.chat, noise_png(2 * 1024 * 1024))[0], 201) + for container in compose('ps', '-q', 'api').split(): + files = subprocess.run(['docker', 'exec', container, 'sh', '-c', 'find /tmp -type f | wc -l'], + capture_output=True, text=True).stdout.strip() + self.assertEqual(files, '0', container[:12]) + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..c5a0e8cb --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,118 @@ +"""Usage (стенд запущен; на время эксперимента лучше остановить другие стенды): + python student-tests/upload_memory.py --output evidence/upload-memory.json + +N одновременных загрузок PNG ~10 MiB через LB при двух настройках буфера тела запроса +(UPLOAD_MEMORY_BYTES = 1 MiB в RAM, дальше временный файл — рабочая настройка; и 16 MiB — весь файл в RAM). +Перед каждым прогоном реплики перезапускаются (memory.peak cgroup считается с момента старта контейнера); +во время прогона каждые 0,1 с снимаются memory.current и anon/file из memory.stat каждой реплики. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import http.client +import json +import os +import subprocess +import threading +import time +import urllib.parse + +from helpers import BASE_URL, api_containers, compose, new_chat, new_user, wait_in_rotation, wait_ready +from imagegen import noise_png + +MIB = 1024 * 1024 +SAMPLER = ('while true; do echo "current $(cat /sys/fs/cgroup/memory.current)"; ' + 'grep -E "^(anon|file) " /sys/fs/cgroup/memory.stat; sleep 0.1; done') + + +class Sampler: + def __init__(self, container): + self.container, self.peaks = container, {'current': 0, 'anon': 0, 'file': 0} + self.process = subprocess.Popen(['docker', 'exec', container, 'sh', '-c', SAMPLER], + stdout=subprocess.PIPE, text=True) + self.thread = threading.Thread(target=self.read, daemon=True) + self.thread.start() + + def read(self): + for line in self.process.stdout: + name, _, value = line.partition(' ') + if name in self.peaks and value.strip().isdigit(): + self.peaks[name] = max(self.peaks[name], int(value)) + + def stop(self): + self.process.terminate() + self.process.wait() + return {name: round(value / MIB, 1) for name, value in self.peaks.items()} + + +def cgroup(container, name): + return int(subprocess.run(['docker', 'exec', container, 'cat', '/sys/fs/cgroup/' + name], + capture_output=True, text=True).stdout.strip()) + + +def post(token, chat_id, data): + target = urllib.parse.urlsplit(BASE_URL) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=120) + started = time.monotonic() + try: + connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat_id, body=data, + headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'image/png'}) + status = connection.getresponse().status + except OSError as error: + status = type(error).__name__ + connection.close() + return status, time.monotonic() - started + + +def restart_api(memory_bytes): + env = dict(os.environ, UPLOAD_MEMORY_BYTES=str(memory_bytes)) + compose('up', '-d', '--no-deps', '--no-build', '--force-recreate', 'api', env=env) + time.sleep(3) + wait_ready() + for container in api_containers(): + wait_in_rotation(container) + + +def run(label, memory_bytes, parallel, data): + restart_api(memory_bytes) + alice = new_user('memory') + chat = new_chat(alice) + replicas = api_containers() + idle = {c[:12]: round(cgroup(c, 'memory.current') / MIB, 1) for c in replicas} + samplers = [Sampler(c) for c in replicas] + started = time.monotonic() + with ThreadPoolExecutor(max_workers=parallel) as pool: + results = list(pool.map(lambda _: post(alice['token'], chat['id'], data), range(parallel))) + wall = time.monotonic() - started + time.sleep(0.5) + sampled = {c[:12]: s.stop() for c, s in zip(replicas, samplers)} + peak = {c[:12]: round(cgroup(c, 'memory.peak') / MIB, 1) for c in replicas} + limit = round(cgroup(replicas[0], 'memory.max') / MIB) + statuses = [r[0] for r in results] + summary = {'label': label, 'upload_memory_bytes': memory_bytes, 'parallel': parallel, + 'file_mib': round(len(data) / MIB, 2), 'ok': statuses.count(201), + 'errors': sorted({str(s) for s in statuses if s != 201}), 'wall_s': round(wall, 2), + 'max_request_s': round(max(r[1] for r in results), 2), 'limit_mib': limit, + 'idle_current_mib': idle, 'peak_since_start_mib': peak, 'sampled_peak_mib': sampled} + worst_anon = max(v['anon'] for v in sampled.values()) + print('%-28s ok=%2d/%d wall=%5.1fs peak(memory.peak)=%s anon max=%.1f MiB' + % (label, summary['ok'], parallel, wall, list(peak.values()), worst_anon)) + return summary + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--parallel', default='10,20') + args = parser.parse_args() + data = noise_png(10 * MIB - 50000) + runs = [run('spool 1 MiB, %s parallel' % n, MIB, int(n), data) for n in args.parallel.split(',')] + runs.append(run('all in RAM, %s parallel' % args.parallel.split(',')[-1], 16 * MIB, + int(args.parallel.split(',')[-1]), data)) + restart_api(MIB) + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + json.dump({'runs': runs}, file, ensure_ascii=False, indent=2) + + +if __name__ == '__main__': + main() -- 2.54.0 From bd7f6fc43acd97029f527726115183a793120dea Mon Sep 17 00:00:00 2001 From: Pavel Grigorev Date: Fri, 13 Nov 2026 16:22:54 +0300 Subject: [PATCH 3/3] =?UTF-8?q?=D0=97=D0=B0=D0=BF=D0=BE=D0=BB=D0=BD=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D0=BE=D1=82=D1=87=D1=91=D1=82=20=D0=B8=20=D0=B4?= =?UTF-8?q?=D0=BE=D0=BA=D0=B0=D0=B7=D0=B0=D1=82=D0=B5=D0=BB=D1=8C=D1=81?= =?UTF-8?q?=D1=82=D0=B2=D0=B0=20lab4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 193 +++++++- evidence/README.md | 20 + evidence/consistency.txt | 28 ++ evidence/expiry.txt | 10 + evidence/failover-series.json | 162 +++++++ evidence/failover-series.txt | 5 + evidence/failover.txt | 20 + evidence/failure.txt | 40 ++ evidence/load.json | 17 + evidence/persistence.txt | 29 ++ evidence/rolling-update.json | 59 +++ evidence/rolling-update.txt | 9 + evidence/sigterm.txt | 16 + evidence/smoke.txt | 4 + evidence/student-tests.txt | 146 ++++++ evidence/topology.txt | 32 ++ evidence/track2/cost.json | 45 ++ evidence/track2/race.json | 109 +++++ evidence/track2/results.md | 39 ++ evidence/track2/scenario.json | 866 ++++++++++++++++++++++++++++++++++ evidence/upload-memory.json | 97 ++++ 21 files changed, 1929 insertions(+), 17 deletions(-) create mode 100644 evidence/consistency.txt create mode 100644 evidence/expiry.txt create mode 100644 evidence/failover-series.json create mode 100644 evidence/failover-series.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/failure.txt create mode 100644 evidence/load.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/rolling-update.json create mode 100644 evidence/rolling-update.txt create mode 100644 evidence/sigterm.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/topology.txt create mode 100644 evidence/track2/cost.json create mode 100644 evidence/track2/race.json create mode 100644 evidence/track2/results.md create mode 100644 evidence/track2/scenario.json create mode 100644 evidence/upload-memory.json diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..1809d148 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,196 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Григорьев Павел, БПИ2302 +- Целевая оценка; выбранный трек 5, если нужен: 5, трек 2 «Согласованность двух хранилищ» +- Commit/PR: ветка `solution/bpi2302-grigoriev`; ссылка на PR — после публикации +- Использованные источники, библиотеки и помощь ИИ: FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.3.6 + psycopg-pool 3.3.3, argon2-cffi 25.1.0, redis-py 8.1.0, boto3/botocore 1.44.1, Pillow 13.0.0 (полный список — `requirements.txt`); SeaweedFS 4.48, HAProxy 3.0.30, PostgreSQL 17.11, Valkey 8.1.11; документация AWS S3 (PutObject, ListObjectsV2, SigV4), SeaweedFS (S3 API, identities), PNG specification (chunk CRC, IDAT/zlib, filter types, Adam7), Pillow (draft, DecompressionBomb). При разработке использовался ИИ-ассистент. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker Engine 27.x с Compose v2 (проверено: Docker Desktop, Engine 27.2.0, WSL2 на Windows 10); Python 3.9+ для тестов. Порты хоста по умолчанию: `127.0.0.1:8080` (LB) и `127.0.0.1:8404` (статистика HAProxy); меняются переменными `LB_PORT`/`STATS_PORT` в `.env` (на машине автора lab4 запускался на 8090/8405 рядом со стендом lab3). Запуск из копии только Git-файлов на портах по умолчанию проверен двумя холодными стартами с пустыми volumes: `up --wait` — exit 0 за 29 и 40 с, smoke PASS. + +Образы закреплены тегом и digest: `python:3.12.15-slim-trixie@sha256:a6e34c59…` (API), `chrislusf/seaweedfs:4.48@sha256:4e61d15f…`, `haproxy:3.0.30-alpine@sha256:d2b3567d…`, `postgres:17.11-alpine@sha256:266d3c25…`, `valkey/valkey:8.1.11-alpine@sha256:2602ace2…`. + +```sh +git clone mtusi2026lab4 && cd mtusi2026lab4 +git switch solution/bpi2302-grigoriev +cp .env.example .env +python scripts/init_secrets.py # пароль БД, S3-ключи admin/api/reconciler, secrets/s3_config.json (вне Git) +docker compose config --quiet +docker compose up -d --build --wait # postgres, valkey, s3 → migrate + s3-init (exit 0) → api ×2, reconciler → lb + +python tests/smoke.py --base-url http://127.0.0.1:8080 +python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 +docker compose start api # вернуть остановленную реплику +curl -s -o /dev/null http://127.0.0.1:8080/health/live # один лишний запрос: сдвинуть round robin +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 # теперь — вторая реплика +docker compose start api +# на Windows — scripts/restart.cmd и scripts/stop-one.cmd +python -m unittest discover -s student-tests -v # BASE_URL=... если LB не на 8080 +python student-tests/consistency_check.py +docker compose logs reconciler # периодический сверщик: проход сразу после старта и раз в 10 мин +docker compose run --rm reconcile # разовый запуск: dry-run отчёт +docker compose run --rm reconcile --apply # разовый запуск: исправить безопасные расхождения +python research/track2.py all # трек 2 +``` + +**Bootstrap.** Миграции — одноразовый `migrate` (0001–0005; `0005_attachments` только добавляет таблицы и ослабляет CHECK длины текста — совместима с работающей версией). S3 — одноразовый `s3-init` с admin-ключами: создаёт bucket, только если его нет (иначе `already exists, nothing to do`), делает пробную запись (SeaweedFS выделяет тома при первой записи — без этого первая пользовательская загрузка ждала 11 с), проверяет, что анонимный листинг и GET запрещены. Повтор `docker compose run --rm --no-deps s3-init` безопасен. + +**Healthcheck образа API**: `--interval=10s --start-period=60s --start-interval=1s`. В период старта проверка идёт раз в секунду, поэтому `--wait` не ждёт лишнего; неудачи первые 60 с не делают контейнер unhealthy. Прежний `--start-period=5s` на перегруженной VM (сборка + одновременный старт migrate, s3-init и двух API) однажды привёл к unhealthy через ~35 с и падению `up --wait`. + +**Порядок failover-проверки обеих реплик.** Тест останавливает реплику, обслужившую его первый запрос. Round robin детерминирован, поэтому без вмешательства оба прогона попадают в одну и ту же реплику. Перед вторым прогоном делается один лишний `GET /health/live`: очередь сдвигается, и целью становится вторая реплика (`evidence/failover.txt`). + +Остановка без потери данных: `docker compose stop`. Намеренное удаление учебных данных: `docker compose down -v` (удаляет `pgdata` и `s3data`). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент ──► lb (HAProxy) ──► api ×2 ──┬──► postgres : метаданные вложений (+ всё из lab3) + ├──► s3 (SeaweedFS, приватный bucket) : байты вложений + └──► valkey : кэш сессий +s3-init (one-shot, admin-ключи) ──► s3 reconciler (раз в 10 мин, ключи сверщика) ──► s3 + postgres +``` + +| Роль | Service | Версия/digest | Health | Volume | Сеть/порт | Лимиты | Credentials | +| --- | --- | --- | --- | --- | --- | --- | --- | +| load_balancer | `lb` | haproxy 3.0.30 | `/lb/health` | — | frontend+backend; `127.0.0.1:${LB_PORT}`, `${STATS_PORT}` | 0.5 CPU, 128M | — | +| api ×2 | `api` | сборка из `Dockerfile` | `/health/ready` | — | backend, порт не публикуется | 0.5 CPU, 320M | `database_url`; S3-ключи **api** | +| s3 | `s3` | seaweedfs 4.48@sha256:4e61… | `GET :8333/healthz` | `s3data` | backend; наружу из контейнера только S3-порт 8333, мастер/filer/тома привязаны к 127.0.0.1 внутри; ничего не опубликовано на хост | 1 CPU, 512M | `s3_config.json` (identities) | +| bootstrap | `s3-init` | образ API | exit 0 | — | backend | 0.5 CPU, 128M | S3-ключи **admin** | +| сверщик, периодический | `reconciler` | образ API | файл-пульс обновлён не позже интервал + 2 мин | — | backend | 0.25 CPU, 192M | `database_url`; S3-ключи **reconciler** | +| сверщик, разовый запуск | `reconcile` (profile `tools`) | образ API | — | — | backend | 0.5 CPU, 256M | то же | +| postgres, valkey, migrate | как в lab3 | | | `pgdata` | backend | | | + +**Почему SeaweedFS.** План был MinIO, но его образы недоступны: на Docker Hub репозиторий `minio/minio` удалён («repository does not exist»), на quay.io — `unauthorized`. Из разрешённых заданием вариантов выбран SeaweedFS (стабильный релиз 4.48 месячной давности). Флаги: `-s3.iam=false` (без встроенного IAM API), `-master.telemetry=false`, бюджеты памяти `-s3.concurrentUploadLimitMB=64`, `-filer.concurrentUploadLimitMB=64`, кэши 16/32 МБ, `GOMEMLIMIT=300MiB` (без них процесс Go держал ~430 МиБ и под нагрузкой упирался в лимит контейнера). + +**Права S3** (файл identities, генерирует `init_secrets.py`): + +| Ключ | Действия | Кому | +| --- | --- | --- | +| admin | Admin, Read, List, Tagging, Write | только `s3-init` | +| api | `Read:messenger-attachments`, `Write:messenger-attachments` | реплики API | +| reconciler | `Read`, `List`, `Write` на bucket | сверщик | +Ключ API: чтение и запись объектов своего bucket; листинг объектов, создание и удаление bucket, запись в другой bucket — AccessDenied; `ListBuckets` SeaweedFS выполняет, но возвращает пустой список (показываются только bucket'ы с правом List). Анонимной identity нет — анонимные запросы получают 403. Административной консоли наружу нет (UI мастера/filer слушают только localhost внутри контейнера, порты не опубликованы). + +**Схема** (`migrations/0005_attachments.sql`): `attachments(id, chat_id → chats, owner_id → users, filename text 1–255, content_type ∈ {png, jpeg}, size_bytes 1..10485760, sha256 bytea(32), object_key UNIQUE, status ∈ {uploading, ready}, created_at, updated_at)`, индексы `(chat_id)`, `(status, created_at)`; `message_attachments(message_id, attachment_id, position)`, PK `(message_id, attachment_id)`, индекс `(attachment_id)`. Байтов файлов в PostgreSQL нет. Ключ объекта — `attachments/`, filename в нём не участвует. + +**Загрузка** (`POST /chats/{id}/attachments?filename=`, сырые байты): +1. Сессия (401) → filename 1–255 без `/`, `\`, NUL, управляющих (400) → `Content-Type` только image/png|image/jpeg (415) → членство в чате (404) → `Content-Length` > 10 MiB → **413 до чтения тела**. +2. Тело читается потоком с жёстким счётчиком (413 и без Content-Length), SHA-256 на лету, буфер `SpooledTemporaryFile`: до 1 МиБ в RAM, дальше временный файл (удаляется после запроса). +3. Не больше 4 одновременных обработок на реплику (`UPLOAD_CONCURRENCY`, ожидание ≤ 30 с, иначе 503). +4. Сигнатура содержимого должна совпасть с заявленным типом (иначе **415**); Pillow читает только заголовок — ширина × высота > 40 Мп → **413 без распаковки**; целостность: PNG — собственная потоковая проверка за один проход кусками по 64 КиБ: CRC каждого чанка; IDAT распаковывается `zlib.decompressobj` с `max_length` (вывод не хранится); объём пиксельных данных должен совпасть с вычисленным по IHDR (с учётом Adam7); байт фильтра каждой строки ≤ 4; zlib-поток завершён. Pillow `Image.verify()` не подошёл: читает чанк целиком в память и пиксели не распаковывает. JPEG — декодирование в масштабе 1/8. Битый файл → **415** (в том числе PNG с верными CRC, но мусором или нехваткой данных в IDAT). +5. Строка-заготовка `uploading` → S3 PUT → строка `ready` → **201**. S3 недоступен → заготовка удаляется (компенсация), **503**. + +**Согласованность БД и S3.** Порядок «намерение → объект → ready» гарантирует, что у любого объекта, положенного API, строка появилась раньше объекта. Сбой между шагами оставляет заготовку `uploading` (с объектом или без) либо, если потерян ответ PUT после компенсации, объект без строки. Ни то, ни другое не видно пользователям (не ready → 404), и всё это убирает сверщик. + +**Сверщик** `src/reconcile.py` (сервис `reconcile`): листинг объектов по префиксу `attachments/` → снимок таблицы → категории `ok`, `orphan` / `fresh_orphan`, `stale_upload` / `fresh_upload`, `missing`, `size_mismatch`. По умолчанию **dry-run** (JSON-отчёт). `--apply` удаляет только старых сирот (с повторной проверкой, что строки нет) и зависшие заготовки старше **grace period** (по умолчанию 900 с; строка удаляется только если всё ещё `uploading` и не привязана к сообщению, затем её объект). `missing` и `size_mismatch` — только отчёт: восстановить байты неоткуда, а удаление метаданных молча сломало бы сообщения. Повторный запуск идемпотентен. **Автоматический запуск** — постоянный сервис `reconciler`: `reconcile.py --apply --interval 600 --grace-seconds 900` (`RECONCILE_INTERVAL_SECONDS`, `RECONCILE_GRACE_SECONDS`). Первый проход — сразу после старта, дальше раз в 10 мин. Ошибка прохода (S3 или БД недоступны) пишется в лог, следующий проход повторит попытку. По SIGTERM сервис завершается за ~1 с. Healthcheck проверяет файл-пульс, который обновляется после каждого прохода. Так мусор живёт не дольше grace + интервал ≈ 25 мин. Разовые запуски (dry-run по умолчанию) — `docker compose run --rm reconcile`; на время экспериментов трека 2 периодический сервис останавливается, чтобы не вмешаться в состаренные синтетические данные. + +**Скачивание**: проверка членства (404) → поток из S3 кусками по 64 КиБ с `Content-Type` и `Content-Length` оригинала, без redirect; S3 недоступен → 503; метаданные (`GET /attachments/{id}`) отдаются из PostgreSQL и при недоступном S3. + +**Сообщения**: `attachment_ids` 0–10 уникальных UUID (иначе 400); `text=""` допустим при ≥ 1 вложении; все вложения — этого чата и `ready`, иначе **404**, сообщение не создаётся (одна транзакция); любой участник может прикрепить вложение чата; идемпотентный хеш тела включает `attachment_ids`. + +**Readiness** не зависит от S3: при его отказе текстовый чат и метаданные работают, вложения отвечают 503 — исключать реплики из LB из-за отказа файлового хранилища бессмысленно. + +**Перенесено из lab3:** всё (LB, 2 реплики, идемпотентность, drain, отзыв сессий, rolling update, тесты). **Изменено/добавлено:** S3, вложения, сверщик (разовый и периодический); `s3` в restart-хуках; порты LB настраиваются; HAProxy `timeout server 60s` (загрузка 10 МиБ на нагруженном стенде дольше прежних 10 с; у API свои сроки на БД и S3); бюджет ожидания БД перезапускается после чтения тела и ожидания очереди (иначе его съедала передача 10 МиБ). + +Переменные `.env.example` (новые): `LB_PORT`, `STATS_PORT` — хостовые порты; `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET`; `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` — ключи API из secrets; `S3_TIMEOUT_SECONDS=15`; `MAX_UPLOAD_BYTES=10485760`; `MAX_IMAGE_PIXELS=40000000`; `UPLOAD_MEMORY_BYTES=1048576` — порог перехода буфера на диск; `UPLOAD_CONCURRENCY=4`; `RECONCILE_INTERVAL_SECONDS=600`, `RECONCILE_GRACE_SECONDS=900` — периодический сверщик. `FAULTS` — только для тестов внедрения сбоев (`after_s3_put`, срабатывает лишь для filename `fault-*`), в рабочей конфигурации пусто. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Файлы — `evidence/`; основной прогон 2026-11-12 со свежих volumes (LB на 8090, стенд lab3 остановлен). После исправлений по итогам проверки 2026-11-13 на том же стенде повторены smoke, student-tests, consistency, persistence, failover, rolling update, upload-memory и track2 scenario; запуск из чистой копии проверен отдельно. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Публичный smoke (upload, message с вложением, download вторым участником, SHA-256, 404 постороннему) | `tests/smoke.py` | PASS | `smoke.txt` | +| 3: S3 в стенде, метаданные в PostgreSQL, байты в приватном bucket | `docker compose ps`; `test_storage.py` | s3 healthy, `s3data`; в `attachments` единственный bytea — `sha256`; анонимный GET/листинг напрямую в S3 → 403 | `topology.txt`, `student-tests.txt` | +| 3: 201 ready после сохранения, побайтное скачивание, attachment_ids | `test_attachments.py` | PNG и JPEG: 201, ready, sha/size совпадают, скачивание = исходные байты через обе реплики | `student-tests.txt` | +| 3: PNG/JPEG ≤ 10 MiB и ≤ 40 Мп, MIME по содержимому, filename, лимиты до чтения/декодирования | `test_attachments.py` | 10 MiB+1 → 413 (по Content-Length и chunked); PNG ~10 MiB → 201; бомба 50 Мп (≈ 50 КБ) → 413; подмена MIME → 415; `/`, `\`, NUL, управляющие, > 255 → 400 | `student-tests.txt` | +| 3: upload → остановка обслужившей реплики → download через другую; restart сохраняет | `tests/resilience.py` failover (обе реплики) и persistence (+ s3) | PASS; в логах `Start Seaweed S3 API Server` после рестарта, volumes на месте | `failover.txt`, `persistence.txt` | +| 4: большой файл, повреждённое изображение, подмена MIME, чужое attachment_id, файл другого чата; без 500 | `test_attachments.py` | 413 / 415 / 415 / 404 / 404; обрезанный PNG, битый CRC, мусор в IDAT и нехватка пикселей при верных CRC, обрезанный JPEG → 415 | `student-tests.txt` | +| 4: обрыв загрузки и сбой между S3 и БД не оставляют вечного мусора; механизм с grace period | `consistency_check.py` | обрыв посреди тела → ни строки, ни объекта; S3 down → 503, заготовка удалена компенсацией; сбой после PUT (`FAULTS`) → `uploading` + объект; сверщик: dry-run grace 900 с — не трогает, apply после grace — удаляет оба; сирота: свежая не тронута, старая удалена; повтор — no-op; периодический сервис `reconciler` сам удалил заготовку возрастом 1 ч | `consistency.txt` | +| 4: лимиты/потоковая обработка; ≥ 10 одновременных загрузок, пик памяти | `upload_memory.py` | см. «Эксперименты»: 10 и 20 загрузок по 10 MiB — пик памяти процесса реплики ≤ 100 МиБ (не растёт с числом загрузок) | `upload-memory.json` | +| 4: bootstrap идемпотентен; отдельные ограниченные S3-ключи; консоль не открыта; приватность вне API | `test_storage.py` | повторный `s3-init` → `already exists`; ключи API: объекты своего bucket — можно; листинг объектов, создание/удаление bucket, запись в чужой — AccessDenied (`ListBuckets` возвращает пустой список); анонимный доступ → 403 | `student-tests.txt` | +| недоступность S3 | `consistency_check.py` (2) | upload/download 503 за 0,1 с, метаданные 200, текст 201, ready 200; после старта S3 — 201 | `consistency.txt` | +| 5, трек 2 | `research/track2.py` | см. ниже | `track2/` | +| сохранено из lab3 | `failover_series.py`, `rolling_update_check.py`, `sigterm_check.py`, `failure_check.py`, `expiry_check.py` | PASS: stop каждой реплики — 0 ошибок, kill — 1–2 × 502 без потерь/дублей; rolling update со сменой образа — 0 ошибок; SIGTERM — 182 запроса после сигнала, все 200 | `failover-series.json`, `rolling-update.json`, `sigterm.txt`, `failure.txt`, `expiry.txt` | + +Всего собственных тестов: 67 (`python -m unittest discover -s student-tests`), все OK. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Пик памяти при одновременных загрузках (уровень 4) -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +`student-tests/upload_memory.py`: PNG ≈ 10 MiB (шум, почти не сжимается) N раз одновременно через LB; перед каждым прогоном реплики пересоздаются; каждые 0,1 с снимаются `memory.current` и `anon`/`file` из `memory.stat` (cgroup v2). `anon` — память процесса, `file` — вытесняемый страничный кэш (в т.ч. временные файлы буфера). + +| Конфигурация | Загрузок OK | Время | Пик `anon` на реплику | Пик `memory.current` | `memory.peak` | +| --- | --- | --- | --- | --- | --- | +| буфер 1 МиБ, 10 одновременно | 10/10 | 3,0 с | 93–100 МиБ | 151–159 МиБ | 152–160 МиБ | +| буфер 1 МиБ, 20 одновременно | 20/20 | 5,3 с | 94–96 МиБ | 189–209 МиБ | 189–211 МиБ | +| весь файл в RAM (16 МиБ), 20 одновременно | 20/20 | 5,2 с | 221–250 МиБ | 248–261 МиБ | 249–262 МиБ | + +Замер 2026-11-13, после добавления распаковки IDAT в проверку PNG: память процесса не изменилась (распаковка идёт кусками по 64 КиБ). Первый замер 2026-11-12 (без неё): 93–100 / 94–107 / 232–236 МиБ. + +Память процесса при рабочей настройке **не растёт** с числом одновременных загрузок (очередь из 4 обработок + тела на диске); рост `memory.current` — это вытесняемый кэш временных файлов. При хранении тел в RAM память процесса растёт на ~10 МиБ на загрузку и при лимите 320 МиБ около 30 одновременных загрузок привели бы к OOM. Простой реплики — ~100 МиБ. + +По ходу эксперимента найдено и исправлено: (1) проверка PNG через Pillow `verify()` читала чанк целиком — заменена собственной потоковой проверкой; (2) без ограничения одновременных обработок 20 загрузок упирались в 0,5 CPU реплики и обрывались LB по `timeout server 10s` (504); (3) SeaweedFS без бюджетов памяти упирался в свой лимит и рвал загрузки; (4) бюджет ожидания БД съедала передача тела. Отдельно: когда одновременно работали стенды lab3 и lab4, VM Docker (1,9 ГиБ) ушла в swap (свободно 47 МБ, давление IO 30–40%) — тяжёлые эксперименты выполнены при остановленном lab3. + +### Трек 2. Согласованность PostgreSQL и S3 + +**Гипотезы.** +- H1 (точность): сверщик находит каждое внедрённое расхождение с правильной категорией и без ложных срабатываний; dry-run ничего не меняет; `--apply` исправляет только безопасные категории; повторный запуск — no-op. +- H2 (безопасность): свежие (моложе grace) и используемые в сообщениях вложения не удаляются — в том числе когда сверщик с `--apply` работает во время непрерывных загрузок. +- H3 (стоимость): время обхода линейно по числу объектов, память сверщика ограничена. + +**Методика** (`research/track2.py`, `research/s3ops.py`; повторяемо: `python research/track2.py all`). Синтетические строки — в отдельном чате, объекты кладутся/удаляются ключами сверщика из контейнера; grace 30 с. +- scenario: 200 ready-вложений (20 привязаны к сообщению) + внедрение: 25 старых сирот; 25 зависших заготовок с объектом и 25 без (возраст 1 ч); 10 пропавших объектов + 5 пропавших у **привязанных** вложений; 10 объектов неверного размера; после ожидания grace — 10 свежих сирот и 10 свежих заготовок. Затем dry-run ×2 → apply → apply → dry-run; плюс настоящее вложение, загруженное через API и привязанное к сообщению. +- race: 40 с, 4 потока загрузок через API, сверщик `--apply --grace-seconds 30` в цикле; затем скачивание и сверка каждого подтверждённого вложения. +- cost: 3,7 тыс. → 5 тыс. → 10 тыс. → 20 тыс. объектов; dry-run × 3, медиана. + +**Результаты** (`evidence/track2/results.md`, `scenario.json`, `race.json`, `cost.json`): + +| категория | внедрено | найдено dry-run | apply | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| orphan (старше grace) | 25 | 25 | удалено 25 объектов | +| fresh_orphan | 10 | 10 | не тронуто | +| stale_upload (с объектом / без) | 25 + 25 | 50 | удалено 50 строк и 25 объектов | +| fresh_upload | 10 | 10 | не тронуто | +| missing (в т.ч. 5 привязанных) | 15 | 15 | только отчёт, строки на месте | +| size_mismatch | 10 | 10 | только отчёт | + +Все проверки True: найдено = внедрено; dry-run ничего не меняет (два прогона — одинаковые счётчики и число объектов); apply удалил ровно старых сирот и зависшие заготовки (`skipped: 0`); повторный apply — 0 действий; свежие сироты и заготовки на месте; 20 привязанных строк на месте; настоящее вложение скачивается с исходным SHA-256. **H1 подтверждена.** + +Race: 3 691 загрузка → все 3 691 скачаны без изменений; 5 запусков сверщика с apply ничего не удалили, увидели 5 загрузок «в полёте» и пропустили их как свежие. Первый прогон ошибочно засчитал FAIL: сверщик удалил 20 синтетических «свежих» объектов, оставшихся от scenario и состарившихся за время теста; в исправленной методике состояние перед нагрузкой приводится к чистому. **H2 подтверждена.** + +| объектов | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB | +| --- | --- | --- | --- | --- | --- | --- | +| 3 719 | 4 | 0,62 | 0,014 | 0,68 | 183 | 90 | +| 4 985 | 5 | 0,82 | 0,017 | 0,90 | 180 | 91 | +| 9 985 | 10 | 1,54 | 0,072 | 1,71 | 172 | 97 | +| 19 985 | 20 | 3,29 | 0,201 | 3,69 | 185 | 111 | + +Время линейно (~180 мкс/объект), ~90 % — листинг S3 (страницы по 1000 ключей); скан БД — доли секунды; память растёт медленно (словари ключей в памяти, ~1 МиБ на 1 000 объектов). **H3 подтверждена.** + +**Ограничения.** Сверщик держит список ключей и строк в памяти (на 20 тыс. — 111 МиБ; для миллионов нужен постраничный обход с сортированным слиянием). Grace защищает только при условии, что загрузка длится меньше grace (у нас ≤ 15 мин против секунд). Источник времени для «свежести» объектов — часы SeaweedFS, для строк — часы PostgreSQL (один хост; в распределённой системе нужен запас на рассинхронизацию). `missing` не исправляется автоматически — это сигнал к восстановлению из бэкапа; загрузка, завершившаяся между листингом S3 и чтением таблицы, может один раз попасть в `missing` (ложное срабатывание только в отчёте, удалений не вызывает; следующий прогон — `ok`). Замеры на одном ноутбуке (Docker Desktop, WSL2); мс — ориентир, категории и счётчики воспроизводятся точно. + +### Отказы и нагрузка + +| Воздействие | Наблюдали | +| --- | --- | +| S3 остановлен | upload/download 503 за 0,1 с, метаданные 200, текст 201, ready 200; после старта всё работает | +| обрыв загрузки клиентом | ни строки, ни объекта | +| сбой после S3 PUT до ready | ответ 500 (внедрённый сбой), заготовка + объект → сверщик удаляет после grace | +| restart стенда (+ s3) | файл и токен на месте | +| остановка реплики, загрузившей файл | файл скачивается через другую | +| `docker stop` / `docker kill` каждой реплики под нагрузкой | 0 ошибок / 1–2 × 502, потерь и дублей нет, возврат ~8 с | +| rolling update под нагрузкой со сменой образа (`bb7d6d901b97` → `8ed3c1001fea`) | 27,7 с, 0 ошибок, потерь и дублей нет, RPS 207 → 173 → 173 | + +Нагрузка public-скриптом (`tests/load.py --duration 30 --concurrency 4`): 5 158 успешных, 0 ошибок, **171 RPS**, p50 12 / p95 70 / p99 94 мс (стенд lab3 остановлен; при параллельно работающем lab3 в lab3 намерено 65 RPS — VM в swap). ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один узел SeaweedFS без репликации томов, одна БД, один хост — единые точки отказа; бэкап `s3data` не реализован (для PostgreSQL — см. lab2). +- Периодический сверщик — один экземпляр; два одновременных прохода безопасны (удаление с повторной проверкой), но запускать несколько экземпляров незачем. +- Неиспользуемые ready-вложения (загружены, но не прикреплены) не удаляются — это учтённые данные; политика их хранения не определена контрактом. +- Подписанные прямые загрузки (трек 1) не реализованы. +- На машине автора VM Docker 1,9 ГиБ: одновременная работа стендов lab3 и lab4 приводит к swap; для экспериментов lab3 останавливался. diff --git a/evidence/README.md b/evidence/README.md index 8df70015..c7614e81 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -3,3 +3,23 @@ Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Файлы этого решения (2026-11-12, свежие volumes, Windows 10 + Docker Desktop 27.2.0, VM 12 CPU / 1,9 GiB, api 2 × 0,5 CPU / 320M, s3 1 CPU / 512M, LB на 127.0.0.1:8090 через `LB_PORT`, стенд lab3 остановлен). После исправлений по итогам проверки 2026-11-13 на том же стенде (не свежие volumes) перегенерированы `topology`, `smoke`, `student-tests`, `consistency`, `persistence`, `failover`, `rolling-update`, `upload-memory`, `track2/scenario.json` и `track2/results.md` + +| Файл | Команда | Что показывает | +| --- | --- | --- | +| `topology.txt` | `docker compose ps`, `/health/ready` через LB, `docker stats` | две реплики + s3 + периодический `reconciler` + one-shot `migrate`/`s3-init`, порт только у LB, лимиты, лог сверщика | +| `smoke.txt` | `python tests/smoke.py` | публичный сценарий lab4: upload, сообщение с вложением, download вторым участником, 404 постороннему | +| `persistence.txt` | `tests/resilience.py --mode persistence` + `scripts/restart.cmd` | файл и токен после рестарта postgres/valkey/s3 | +| `failover.txt` | `tests/resilience.py --mode failover` + `scripts/stop-one.cmd`, для каждой реплики | файл, загруженный через остановленную реплику, скачивается через другую; перед вторым прогоном один лишний GET, чтобы цель сменилась | +| `student-tests.txt` | `python -m unittest discover -s student-tests -v` | 67 тестов: вложения (лимиты, MIME, бомба, повреждения, доступ), приватность bucket и права ключей, всё из lab3 | +| `consistency.txt` | `python student-tests/consistency_check.py` | обрыв загрузки, отказ S3 (503 без мусора), сбой после S3 PUT + сверщик с grace, сирота, периодический сервис сам удаляет зависшую заготовку | +| `upload-memory.json` | `python student-tests/upload_memory.py` | 10 и 20 одновременных загрузок по 10 MiB: пик памяти реплик (cgroup), буфер на диске vs в RAM | +| `failure.txt`, `expiry.txt`, `sigterm.txt` | `failure_check.py`, `expiry_check.py`, `sigterm_check.py` | отказы Valkey/PostgreSQL, истечение сессии, мягкая остановка (из lab3) | +| `failover-series.json`, `.txt` | `python student-tests/failover_series.py` | stop и kill каждой реплики под нагрузкой | +| `rolling-update.json`, `.txt` | `python student-tests/rolling_update_check.py` | замена реплик под нагрузкой на новый образ, 0 ошибок | +| `load.json` | `python tests/load.py --duration 30 --concurrency 4` | базовая нагрузка через LB | +| `track2/scenario.json` | `python research/track2.py scenario` | H1: внедрённые расхождения, dry-run/apply/повтор, проверки | +| `track2/race.json` | `python research/track2.py race` | H2: сверщик с `--apply` во время непрерывных загрузок | +| `track2/cost.json` | `python research/track2.py cost` | H3: время и память сверщика на 3,7–20 тыс. объектов | +| `track2/results.md` | `python research/track2.py report` | сводка трека 2 | diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 00000000..b96e4e85 --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1,28 @@ +# python student-tests/consistency_check.py +# date: 2026-11-13T11:33:26Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090 + +1. Обрыв загрузки посреди тела + строк не прибавилось (ready/uploading) OK 109/0 -> 109/0 +2. S3 остановлен: загрузка, скачивание, остальной API + upload при остановленном S3 → 503 OK 503 за 0.0 с + заготовка удалена компенсацией OK 110/0 -> 110/0 + download при остановленном S3 → 503 OK 503 за 0.1 с + метаданные вложения доступны (они в PostgreSQL) OK + текстовое сообщение → 201 OK + /health/ready → 200 (S3 не критичен для текстового API) OK + после старта S3 upload снова 201 OK +3. Сбой после S3 PUT до фиксации ready (FAULTS=after_s3_put) + клиент получил ошибку (не 201) OK 500 + осталась строка uploading и объект в S3 OK 519f7e1c-a7a9-492e-a92a-a1d44f90281e attachments/519f7e1c-a7a9-492e-a92a-a1d44f90281e + dry-run, grace 900 с: заготовка свежая, не трогается OK {'ok': 111, 'orphan': 0, 'fresh_orphan': 0, 'stale_upload': 0, 'fresh_upload': 1, 'missing': 0, 'size_mismatch': 0} + apply, grace 0: строка и объект удалены OK {'orphan_objects_deleted': 0, 'stale_rows_deleted': 1, 'stale_objects_deleted': 1, 'skipped': 0} + объекта больше нет OK +4. Объект без строки в БД (orphan) + dry-run, grace 900 с: свежая сирота не трогается OK {'ok': 111, 'orphan': 0, 'fresh_orphan': 1, 'stale_upload': 0, 'fresh_upload': 0, 'missing': 0, 'size_mismatch': 0} + apply, grace 0: сирота удалена OK {'orphan_objects_deleted': 1, 'stale_rows_deleted': 0, 'stale_objects_deleted': 0, 'skipped': 0} + повторный запуск ничего не делает OK {'orphan_objects_deleted': 0, 'stale_rows_deleted': 0, 'stale_objects_deleted': 0, 'skipped': 0} + рабочие вложения не пострадали (missing = 0) OK {'ok': 111, 'orphan': 0, 'fresh_orphan': 0, 'stale_upload': 0, 'fresh_upload': 0, 'missing': 0, 'size_mismatch': 0} +5. Периодический сервис reconciler убирает зависшую заготовку сам + сервис reconciler запущен (docker compose ps) OK + заготовка старше grace удалена без ручного запуска OK +PASS: сбои загрузки не оставляют неучтённых данных diff --git a/evidence/expiry.txt b/evidence/expiry.txt new file mode 100644 index 00000000..e7a43f6d --- /dev/null +++ b/evidence/expiry.txt @@ -0,0 +1,10 @@ +# python student-tests/expiry_check.py --ttl 2 +# date: 2026-11-12T18:31:48Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090 + +пересоздаю api с SESSION_TTL_SECONDS=2 +сразу после входа: /users/me -> 200 +TTL ключа этой сессии в Valkey: 1 с (-2 = ключа нет) +через 3.5 с: /users/me -> 401, WWW-Authenticate: Bearer realm="messenger" +logout истёкшим токеном -> 401 +PASS: сессия истекла по SESSION_TTL_SECONDS=2, кэш не продлил её +возвращаю api с TTL из .env diff --git a/evidence/failover-series.json b/evidence/failover-series.json new file mode 100644 index 00000000..0ce5bede --- /dev/null +++ b/evidence/failover-series.json @@ -0,0 +1,162 @@ +{ + "replicas": [ + "f2d964e843f1", + "4f6e551ada27" + ], + "warmup_s": 5, + "observe_s": 10, + "workers": 4, + "lb_check": "GET /health/ready every 1s, fall 2, rise 2", + "results": [ + { + "replica": "f2d964e843f1", + "mode": "stop", + "action_seconds": 3.87, + "target_served_until_s": 1.6, + "first_success_on_other_s": 0.02, + "before": { + "seconds": 5.0, + "ok": 777, + "errors": 0, + "rps": 155.4, + "p50_ms": 14.6, + "p95_ms": 79.8, + "p99_ms": 104.1, + "error_kinds": [] + }, + "during_10s": { + "seconds": 10.0, + "ok": 1090, + "errors": 0, + "rps": 109.0, + "p50_ms": 20.2, + "p95_ms": 91.7, + "p99_ms": 106.1, + "error_kinds": [] + }, + "back_in_rotation_s": 7.87, + "data": { + "confirmed": 2252, + "stored": 2252, + "lost": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "uncertain_at_stop": 0 + } + }, + { + "replica": "4f6e551ada27", + "mode": "stop", + "action_seconds": 4.8, + "target_served_until_s": 1.63, + "first_success_on_other_s": 0.02, + "before": { + "seconds": 5.0, + "ok": 643, + "errors": 0, + "rps": 128.6, + "p50_ms": 18.0, + "p95_ms": 85.4, + "p99_ms": 167.4, + "error_kinds": [] + }, + "during_10s": { + "seconds": 10.0, + "ok": 499, + "errors": 0, + "rps": 49.9, + "p50_ms": 62.4, + "p95_ms": 223.6, + "p99_ms": 282.9, + "error_kinds": [] + }, + "back_in_rotation_s": 8.0, + "data": { + "confirmed": 1430, + "stored": 1430, + "lost": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "uncertain_at_stop": 0 + } + }, + { + "replica": "f2d964e843f1", + "mode": "kill", + "action_seconds": 0.32, + "target_served_until_s": 0.0, + "first_success_on_other_s": 0.14, + "before": { + "seconds": 5.0, + "ok": 695, + "errors": 3, + "rps": 139.0, + "p50_ms": 18.6, + "p95_ms": 86.7, + "p99_ms": 150.6, + "error_kinds": [ + "502" + ] + }, + "during_10s": { + "seconds": 10.0, + "ok": 636, + "errors": 1, + "rps": 63.6, + "p50_ms": 31.1, + "p95_ms": 104.7, + "p99_ms": 1014.3, + "error_kinds": [ + "502" + ] + }, + "back_in_rotation_s": 8.41, + "data": { + "confirmed": 1367, + "stored": 1367, + "lost": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "uncertain_at_stop": 0 + } + }, + { + "replica": "4f6e551ada27", + "mode": "kill", + "action_seconds": 0.38, + "target_served_until_s": 0.06, + "first_success_on_other_s": 0.02, + "before": { + "seconds": 5.0, + "ok": 889, + "errors": 0, + "rps": 177.8, + "p50_ms": 15.4, + "p95_ms": 68.3, + "p99_ms": 101.3, + "error_kinds": [] + }, + "during_10s": { + "seconds": 10.0, + "ok": 745, + "errors": 2, + "rps": 74.5, + "p50_ms": 28.4, + "p95_ms": 95.4, + "p99_ms": 1010.7, + "error_kinds": [ + "502" + ] + }, + "back_in_rotation_s": 8.59, + "data": { + "confirmed": 1675, + "stored": 1675, + "lost": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "uncertain_at_stop": 0 + } + } + ] +} \ No newline at end of file diff --git a/evidence/failover-series.txt b/evidence/failover-series.txt new file mode 100644 index 00000000..861c62c2 --- /dev/null +++ b/evidence/failover-series.txt @@ -0,0 +1,5 @@ +f2d964e843f1 stop errors=0 rps 155.4 -> 109.0 target last answer +1.60s restore 7.9s lost=0 dup=0 +4f6e551ada27 stop errors=0 rps 128.6 -> 49.9 target last answer +1.63s restore 8.0s lost=0 dup=0 +f2d964e843f1 kill errors=1 rps 139.0 -> 63.6 target last answer +0.00s restore 8.4s lost=0 dup=0 +4f6e551ada27 kill errors=2 rps 177.8 -> 74.5 target last answer +0.06s restore 8.6s lost=0 dup=0 +PASS: every replica stopped once per mode; no loss, no duplicates diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..d1c51e88 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,20 @@ +# failover with attachment: tests/resilience.py --mode failover --action-script scripts/stop-one.cmd, for each replica +# date: 2026-11-13T11:35:32Z; before run 2 one extra GET /health/live shifts round robin so that the test targets the other replica + +## run 1 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 4.134} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +grigoriev-lab4-api-6 a040ba43e8d1 Up 31 seconds (healthy) +grigoriev-lab4-api-5 d42206b660ea Exited (0) 11 seconds ago +stopped replica: d42206b660ea +restored d42206b660ea + +## run 2 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 4.171} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +grigoriev-lab4-api-6 a040ba43e8d1 Exited (0) 11 seconds ago +grigoriev-lab4-api-5 d42206b660ea Up 28 seconds (healthy) +stopped replica: a040ba43e8d1 +restored a040ba43e8d1 diff --git a/evidence/failure.txt b/evidence/failure.txt new file mode 100644 index 00000000..7c8ed380 --- /dev/null +++ b/evidence/failure.txt @@ -0,0 +1,40 @@ +# python student-tests/failure_check.py +# date: 2026-11-12T18:30:53Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090 + +1. Valkey остановлен: кэш обходится, всё работает через PostgreSQL + первый запрос: обнаружение отказа кэша -> 200 за 0.01 с OK + GET /users/me действующим токеном -> 200 за 0.01 с OK + POST сообщение -> 201 за 0.01 с OK + неизвестный токен -> 401 за 0.01 с OK + без Authorization -> 401 за 0.00 с OK + /health/ready (Valkey некритичен) -> 200 за 0.01 с OK + регистрация и вход без кэша -> 200 за 0.06 с OK + logout без кэша -> 204 за 0.01 с OK + отозванный токен -> 401 за 0.00 с OK +1б. Valkey на паузе: logout, затем unpause — отозванный токен не оживает из кэша + logout при паузе Valkey -> 204 за 0.42 с OK + через 6 с после unpause: /users/me -> 401 за 0.03 с OK + через 6 с после unpause: сообщение -> 401 за 0.01 с OK +1в. Valkey на паузе: logout, рестарт API, unpause — отзыв переживает рестарт процесса + logout при паузе Valkey -> 204 за 0.41 с OK + после рестарта API и unpause: /users/me -> 401 за 0.02 с OK + после рестарта API и unpause: сообщение -> 401 за 0.01 с OK +2. PostgreSQL остановлен: ограниченная ошибка 503, анонимный запрос не проходит + GET история -> 503 за 2.01 с OK + POST сообщение -> 503 за 2.00 с OK + неизвестный токен (не 200!) -> 503 за 2.01 с OK + без Authorization -> 401 за 0.01 с OK + вход по Basic верным паролем -> 503 за 2.00 с OK + /health/live -> 200 за 0.00 с OK + /health/ready -> 503 за 1.00 с OK +3. PostgreSQL запущен: готовность через 15.4 с после остановки + прежний токен -> 200; история: ['до отказа', 'без кэша'] +4. PostgreSQL заморожен (pause): запросы получают 503 в пределах общего срока, а не зависают + GET история -> 503 за 2.01 с OK + POST сообщение -> 503 за 2.00 с OK + неизвестный токен (не 200!) -> 503 за 2.01 с OK + вход по Basic -> 503 за 2.00 с OK + /health/ready -> 503 за 1.00 с OK + /health/live -> 200 за 0.00 с OK + готовность через 0.2 с после unpause; история: ['до отказа', 'без кэша'] +PASS: отказы Valkey и PostgreSQL обработаны diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..ef26580c --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.114, + "concurrency": 4, + "successful_requests": 5158, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 171.285, + "successful_latency_ms": { + "p50": 11.787, + "p95": 69.894, + "p99": 94.394 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..6ec9b151 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,29 @@ +# persistence: python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8090 +# date: 2026-11-13T11:35:32Z; stack after QA fixes (not fresh volumes); host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; api 2 x (0.5 CPU, 320M), s3 1 CPU / 512M + +$ docker compose ps -a (before) +NAME STATUS +grigoriev-lab4-api-5 Up About a minute (healthy) +grigoriev-lab4-api-6 Up About a minute (healthy) +grigoriev-lab4-lb-1 Up 3 minutes (healthy) +grigoriev-lab4-migrate-1 Exited (0) 3 minutes ago +grigoriev-lab4-postgres-1 Up 12 minutes (healthy) +grigoriev-lab4-reconciler-1 Up 31 seconds (healthy) +grigoriev-lab4-s3-1 Up About a minute (healthy) +grigoriev-lab4-s3-init-1 Exited (0) 3 minutes ago +grigoriev-lab4-valkey-1 Up 12 minutes (healthy) + +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +$ docker compose ps -a (after) +NAME STATUS +grigoriev-lab4-api-5 Up 5 seconds (healthy) +grigoriev-lab4-api-6 Up 5 seconds (healthy) +grigoriev-lab4-lb-1 Up 4 minutes (healthy) +grigoriev-lab4-migrate-1 Exited (0) 7 seconds ago +grigoriev-lab4-postgres-1 Up 49 seconds (healthy) +grigoriev-lab4-reconciler-1 Up About a minute (healthy) +grigoriev-lab4-s3-1 Up 34 seconds (healthy) +grigoriev-lab4-s3-init-1 Exited (0) 5 seconds ago +grigoriev-lab4-valkey-1 Up 49 seconds (healthy) diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 00000000..cc91ee0c --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,59 @@ +{ + "replicas_before": [ + "546a297b3e8d", + "f00b37e093a8" + ], + "replicas_after": [ + "b667024b618d", + "cfd28971fda4" + ], + "update_seconds": 27.7, + "update_exit_code": 0, + "before_5s": { + "seconds": 5.0, + "ok": 1035, + "errors": 0, + "rps": 207.0, + "p50_ms": 10.6, + "p95_ms": 60.6, + "p99_ms": 68.7, + "error_kinds": [] + }, + "during_update": { + "seconds": 27.71, + "ok": 4804, + "errors": 0, + "rps": 173.4, + "p50_ms": 12.6, + "p95_ms": 68.0, + "p99_ms": 93.0, + "error_kinds": [] + }, + "after_5s": { + "seconds": 5.0, + "ok": 864, + "errors": 0, + "rps": 172.8, + "p50_ms": 11.0, + "p95_ms": 71.2, + "p99_ms": 96.2, + "error_kinds": [] + }, + "data": { + "confirmed": 6708, + "stored": 6708, + "lost": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "uncertain_at_stop": 0 + }, + "log": [ + "old replicas: 546a297b3e8d(image bb7d6d901b97), f00b37e093a8(image bb7d6d901b97)", + "migrations applied once", + "+ b667024b618d (image 8ed3c1001fea) in rotation after 7.2s", + "- 546a297b3e8d stopped and removed after 4.3s", + "+ cfd28971fda4 (image 8ed3c1001fea) in rotation after 6.1s", + "- f00b37e093a8 stopped and removed after 4.4s", + "new replicas: b667024b618d(image 8ed3c1001fea), cfd28971fda4(image 8ed3c1001fea)" + ] +} \ No newline at end of file diff --git a/evidence/rolling-update.txt b/evidence/rolling-update.txt new file mode 100644 index 00000000..7a88ccdd --- /dev/null +++ b/evidence/rolling-update.txt @@ -0,0 +1,9 @@ +old replicas: 546a297b3e8d(image bb7d6d901b97), f00b37e093a8(image bb7d6d901b97) +migrations applied once ++ b667024b618d (image 8ed3c1001fea) in rotation after 7.2s +- 546a297b3e8d stopped and removed after 4.3s ++ cfd28971fda4 (image 8ed3c1001fea) in rotation after 6.1s +- f00b37e093a8 stopped and removed after 4.4s +new replicas: b667024b618d(image 8ed3c1001fea), cfd28971fda4(image 8ed3c1001fea) +rolling update 27.7s: errors=0, rps 207.0 -> 173.4 -> 172.8, lost=0, duplicates=0, all replaced=True +PASS diff --git a/evidence/sigterm.txt b/evidence/sigterm.txt new file mode 100644 index 00000000..f3433768 --- /dev/null +++ b/evidence/sigterm.txt @@ -0,0 +1,16 @@ +# python student-tests/sigterm_check.py + +сообщение с маркером отправлено: HTTP 201; маркер в логах: False +docker stop занял: 3.75 с (лимит 10 с) +код выхода контейнера: 0 (0 — сам завершился по SIGTERM; 137 — убит SIGKILL) +запросы до SIGTERM: 40, все 200: True +запросы после SIGTERM: 182, результаты: ['200'] +ответов 5xx за всё время: 0 + +логи остановки: + 2026-11-12 18:32:18,839 INFO messenger: signal 15: readiness off, draining for 3.0s + 2026-11-12 18:32:21,858 INFO uvicorn.error: Shutting down + 2026-11-12 18:32:21,958 INFO uvicorn.error: Waiting for application shutdown. + 2026-11-12 18:32:21,958 INFO messenger: application stopping + 2026-11-12 18:32:21,960 INFO uvicorn.error: Application shutdown complete. + 2026-11-12 18:32:21,960 INFO uvicorn.error: Finished server process [1] diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..eef3c2a2 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,4 @@ +# python tests/smoke.py --base-url http://127.0.0.1:8090 +# date: 2026-11-13T11:32:48Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090 + +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..31447f77 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,146 @@ +# python -m unittest discover -s student-tests -v +# date: 2026-11-13T11:32:48Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090 + +test_duplicate_username (test_api.DuplicateUsernameTest.test_duplicate_username) +Повторная регистрация того же username -> 409 conflict ... ok +test_empty_text (test_api.EmptyMessageTest.test_empty_text) +Пустой текст -> 400 ... ok +test_missing_text (test_api.EmptyMessageTest.test_missing_text) +Тело без поля text -> 400 ... ok +test_text_length_limit (test_api.EmptyMessageTest.test_text_length_limit) +4000 символов можно (эмодзи = 1 символ), 4001 — 400 ... ok +test_whitespace_text (test_api.EmptyMessageTest.test_whitespace_text) +Текст только из пробельных символов -> 400 ... ok +test_member_has_access (test_api.ForeignChatTest.test_member_has_access) +Участник видит чат (контроль: 404 выше не из-за ошибки в тесте) ... ok +test_outsider_cannot_read_chat (test_api.ForeignChatTest.test_outsider_cannot_read_chat) +Посторонний читает чат -> 404, как будто чата нет ... ok +test_outsider_cannot_write_or_read_history (test_api.ForeignChatTest.test_outsider_cannot_write_or_read_history) +Посторонний пишет/читает историю -> 404 ... ok +test_outsider_does_not_see_chat_in_list (test_api.ForeignChatTest.test_outsider_does_not_see_chat_in_list) +Чужой чат не попадает в список чатов ... ok +test_lone_surrogate_is_rejected (test_api.IdentityTest.test_lone_surrogate_is_rejected) +Одиночный суррогат не кодируется в UTF-8 -> 400, а не 500 ... ok +test_nul_character_is_preserved (test_api.IdentityTest.test_nul_character_is_preserved) +U+0000 — допустимый code point: сообщение, display_name и title сохраняются и читаются без изменений ... ok +test_sender_cannot_be_forged (test_api.IdentityTest.test_sender_cannot_be_forged) +sender_id в теле запрещён: отправитель только из сессии ... ok +test_extra_field (test_api.InvalidJsonTest.test_extra_field) +Лишнее поле в теле -> 400 ... ok +test_json_not_object (test_api.InvalidJsonTest.test_json_not_object) +JSON-массив вместо объекта -> 400 ... ok +test_malformed_json (test_api.InvalidJsonTest.test_malformed_json) +Битый JSON -> 400 invalid_request ... ok +test_wrong_type (test_api.InvalidJsonTest.test_wrong_type) +Число вместо строки -> 400 (без тихого приведения типов) ... ok +test_cursor_from_other_chat (test_api.PaginationTest.test_cursor_from_other_chat) +cursor от другого чата -> 400 ... ok +test_invalid_cursor_and_limit (test_api.PaginationTest.test_invalid_cursor_and_limit) +Мусорный cursor и limit вне 1..100 -> 400 ... ok +test_walk_several_pages (test_api.PaginationTest.test_walk_several_pages) +7 сообщений по 3: три страницы, без потерь, повторов и перестановок ... ok +test_foreign_or_unknown_attachment (test_attachments.MessageAttachmentsTest.test_foreign_or_unknown_attachment) +Вложение другого чата или несуществующее → 404, сообщение не создаётся ... ok +test_idempotency_includes_attachments (test_attachments.MessageAttachmentsTest.test_idempotency_includes_attachments) +Тот же ключ: те же вложения → тот же Message; другие вложения → 409; нет поля = [] для текста ... ok +test_invalid_attachment_lists (test_attachments.MessageAttachmentsTest.test_invalid_attachment_lists) +Повтор id, 11 вложений, не-UUID, пустой текст без вложений, пробелы без вложений → 400 ... ok +test_message_with_attachment_only (test_attachments.MessageAttachmentsTest.test_message_with_attachment_only) +text="" + одно вложение → 201; attachment_ids видны в истории; обычное сообщение получает [] ... ok +test_other_member_can_reference_attachment (test_attachments.MessageAttachmentsTest.test_other_member_can_reference_attachment) +Боб прикрепляет вложение, загруженное Алисой в этот же чат ... ok +test_access_rules (test_attachments.UploadTest.test_access_rules) +Без токена 401; посторонний не может загрузить в чужой чат и не видит метаданные/байты (404) ... ok +test_corrupted_images (test_attachments.UploadTest.test_corrupted_images) +Обрезанный PNG, битый CRC, мусор или нехватка пикселей в IDAT при верных CRC, обрезанный JPEG → 415 ... ok +test_decompression_bomb (test_attachments.UploadTest.test_decompression_bomb) +PNG 10000×5000 (50 Мп) весом в десятки КБ → 413 без распаковки ... ok +test_download_through_every_replica (test_attachments.UploadTest.test_download_through_every_replica) +Файл скачивается через любую API-реплику (байты не лежат локально у загрузившей) ... ok +test_filename_is_display_only_and_validated (test_attachments.UploadTest.test_filename_is_display_only_and_validated) +Unicode-имя сохраняется как есть; нет имени, пустое, /, \, NUL, управляющий символ, >255 → 400 ... ok +test_mime_spoofing (test_attachments.UploadTest.test_mime_spoofing) +Подмена MIME: PNG как jpeg, JPEG как png, текст и GIF под видом png → 415 ... ok +test_png_and_jpeg_are_stored_and_downloaded_byte_exact (test_attachments.UploadTest.test_png_and_jpeg_are_stored_and_downloaded_byte_exact) +PNG и JPEG → 201 ready; второй участник скачивает те же байты с верным Content-Type ... ok +test_too_large_file (test_attachments.UploadTest.test_too_large_file) +10 MiB + 1 байт → 413 (и по Content-Length, и chunked без длины); файл ~10 MiB принимается ... ok +test_wrong_declared_type (test_attachments.UploadTest.test_wrong_declared_type) +Content-Type не image/png|jpeg (octet-stream, multipart, gif) → 415 ... ok +test_authentication_before_validation (test_auth.BearerTest.test_authentication_before_validation) +Аноним с битым JSON на защищённом endpoint получает 401, а не 400; с сессией — 400 ... ok +test_only_bearer_is_accepted (test_auth.BearerTest.test_only_bearer_is_accepted) +Нет заголовка, Basic, X-User-Id, мусорный токен -> 401 Bearer ... ok +test_scheme_is_case_insensitive (test_auth.BearerTest.test_scheme_is_case_insensitive) +Схема аутентификации нечувствительна к регистру (RFC 9110) ... ok +test_x_user_id_does_not_override_session (test_auth.BearerTest.test_x_user_id_does_not_override_session) +X-User-Id игнорируется: владелец определяется только токеном ... ok +test_login_rejects_body (test_auth.LoginTest.test_login_rejects_body) +POST /auth/sessions принимает только заголовок, JSON body -> 400 ... ok +test_login_returns_opaque_session (test_auth.LoginTest.test_login_returns_opaque_session) +Вход: 201, token_type=Bearer, токен случайный и не содержит username/UUID ... ok +test_malformed_basic (test_auth.LoginTest.test_malformed_basic) +Нет заголовка, не base64, нет двоеточия, Bearer вместо Basic -> 401 Basic ... ok +test_password_with_colon_and_unicode (test_auth.LoginTest.test_password_with_colon_and_unicode) +Двоеточие и не-ASCII в пароле допустимы: делим только по первому ':' ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.LoginTest.test_wrong_password_and_unknown_user_look_the_same) +Неверный пароль и несуществующий пользователь -> одинаковый 401 с Basic challenge ... ok +test_logout_revokes_only_current_session (test_auth.LogoutTest.test_logout_revokes_only_current_session) +Logout -> 204 без тела; этот токен -> 401, вторая сессия того же пользователя работает ... ok +test_password_is_not_returned (test_auth.RegistrationTest.test_password_is_not_returned) +Ответ регистрации и /users/me не содержат password/hash ... ok +test_password_length_limits (test_auth.RegistrationTest.test_password_length_limits) +Пароль 12..128 символов; 11 и 129 -> 400; эмодзи считается одним символом ... ok +test_password_required_and_not_blank (test_auth.RegistrationTest.test_password_required_and_not_blank) +Без password или из одних пробелов -> 400 ... ok +test_100_messages_from_10_clients (test_concurrency.ConcurrencyTest.test_100_messages_from_10_clients) +100 параллельных сообщений от 10 клиентов -> ровно 100 уникальных в истории ... ok +test_parallel_registration_of_same_username (test_concurrency.ConcurrencyTest.test_parallel_registration_of_same_username) +20 одновременных регистраций одного username -> ровно один 201, остальные 409; вход только с паролем победителя ... ok +test_chat_creation_is_rolled_back (test_db.DatabaseTest.test_chat_creation_is_rolled_back) +Неизвестный участник: INSERT chats выполнен, INSERT chat_members упал по FK, всё откатано ... ok +test_history_query_uses_index (test_db.DatabaseTest.test_history_query_uses_index) +Запрос страницы истории идёт по индексу (chat_id, created_at, id), без Sort ... ok +test_migrations_are_idempotent (test_db.DatabaseTest.test_migrations_are_idempotent) +Повторный запуск миграций ничего не меняет и завершается успешно ... ok +test_secrets_are_not_stored_in_plaintext (test_db.DatabaseTest.test_secrets_are_not_stored_in_plaintext) +В users лежит argon2id-хеш, в sessions — sha256 токена, не сам токен ... ok +test_json_formatting_does_not_matter (test_idempotency.IdempotencyTest.test_json_formatting_does_not_matter) +Порядок/пробелы в JSON семантически равны: повтор с другим форматированием -> тот же Message ... ok +test_key_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_key_scope_is_user_and_chat) +Одинаковый ключ у разных пользователей и в разных чатах — независимые сообщения ... ok +test_missing_or_invalid_key (test_idempotency.IdempotencyTest.test_missing_or_invalid_key) +Нет ключа, пустой, недопустимые символы, длиннее 128 -> 400 ... ok +test_other_body_with_same_key_conflicts (test_idempotency.IdempotencyTest.test_other_body_with_same_key_conflicts) +Тот же ключ с другим телом -> 409, второе сообщение не создаётся ... ok +test_parallel_retries_through_replicas_create_one_message (test_idempotency.IdempotencyTest.test_parallel_retries_through_replicas_create_one_message) +20 одновременных POST с одним ключом через LB (разные реплики) -> ровно одно сообщение, все 201 с одним id ... ok +test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTest.test_replay_rechecks_session_and_access) +Повтор снова проверяет сессию и доступ: после logout -> 401, посторонний с тем же ключом -> 404 ... ok +test_replay_returns_same_message (test_idempotency.IdempotencyTest.test_replay_returns_same_message) +Повтор того же запроса с тем же ключом -> тот же 201 и тот же Message, дубля нет ... ok +test_every_api_response_has_instance_id (test_replicas.ReplicasTest.test_every_api_response_has_instance_id) +X-Instance-Id есть и в успешных ответах, и в ошибках API, и в health ... ok +test_one_session_is_served_by_several_replicas (test_replicas.ReplicasTest.test_one_session_is_served_by_several_replicas) +Один токен без повторного входа обслуживается разными репликами (X-Instance-Id), все ответы 200 ... ok +test_write_on_one_replica_is_read_on_another (test_replicas.ReplicasTest.test_write_on_one_replica_is_read_on_another) +Сообщение, подтверждённое одной репликой, сразу видно при чтении через другую ... ok +test_api_credentials_are_limited (test_storage.StorageTest.test_api_credentials_are_limited) +Ключи API: свои объекты читать/писать/удалять можно; листинг, создание и удаление bucket — AccessDenied ... ok +test_bootstrap_is_idempotent (test_storage.StorageTest.test_bootstrap_is_idempotent) +Повторный s3-init: код 0, bucket уже есть, приватность подтверждена ... ok +test_bucket_is_private_outside_api (test_storage.StorageTest.test_bucket_is_private_outside_api) +Анонимный GET объекта и листинг bucket напрямую в S3 (из сети backend) → 403 ... ok +test_no_original_bytes_in_database_or_api_container (test_storage.StorageTest.test_no_original_bytes_in_database_or_api_container) +В attachments единственный bytea — sha256 (байтов файла в БД нет); после загрузки 2 МБ в /tmp реплик API пусто ... ok +test_object_key_is_generated_by_server (test_storage.StorageTest.test_object_key_is_generated_by_server) +Ключ объекта = attachments/, filename в нём не участвует ... ok + +---------------------------------------------------------------------- +Ran 67 tests in 28.928s + +OK + + отправлено 201: 100, уникальных id: 100, прочитано: 100 за 4 страниц(ы) + + 20 повторов: реплики ['6f817671b243', '967308a0d498'], сообщений в истории: 1 diff --git a/evidence/topology.txt b/evidence/topology.txt new file mode 100644 index 00000000..0831c8b9 --- /dev/null +++ b/evidence/topology.txt @@ -0,0 +1,32 @@ +# topology +# date: 2026-11-13T11:47:08Z; stack after QA fixes; LB on 127.0.0.1:8090 (LB_PORT); host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; api 2 x (0.5 CPU, 320M), s3 1 CPU / 512M, reconciler 0.25 CPU / 192M + +SERVICE NAME STATUS PORTS +api grigoriev-lab4-api-5 Up About a minute (healthy) +api grigoriev-lab4-api-6 Up About a minute (healthy) +lb grigoriev-lab4-lb-1 Up About a minute (healthy) 127.0.0.1:8090->8080/tcp, 127.0.0.1:8405->8404/tcp +migrate grigoriev-lab4-migrate-1 Exited (0) About a minute ago +postgres grigoriev-lab4-postgres-1 Up 2 minutes (healthy) +reconciler grigoriev-lab4-reconciler-1 Up About a minute (healthy) +s3 grigoriev-lab4-s3-1 Up 2 minutes (healthy) +s3-init grigoriev-lab4-s3-init-1 Exited (0) About a minute ago +valkey grigoriev-lab4-valkey-1 Up 2 minutes (healthy) + +$ docker stats --no-stream +NAME CPU % MEM USAGE / LIMIT +grigoriev-lab4-api-5 0.53% 103MiB / 320MiB +grigoriev-lab4-api-6 0.51% 96.47MiB / 320MiB +grigoriev-lab4-lb-1 0.17% 17.33MiB / 128MiB +grigoriev-lab4-postgres-1 0.07% 32.48MiB / 384MiB +grigoriev-lab4-reconciler-1 0.00% 67.75MiB / 192MiB +grigoriev-lab4-s3-1 0.33% 161.1MiB / 512MiB +grigoriev-lab4-valkey-1 0.68% 13.7MiB / 96MiB + +$ docker compose logs s3-init +2026-11-13 11:45:26,442 s3-bootstrap: bucket messenger-attachments already exists, nothing to do +2026-11-13 11:45:26,451 s3-bootstrap: warm-up write done in 0.0s (storage volumes allocated before first upload) +2026-11-13 11:45:26,469 s3-bootstrap: anonymous list/get denied: bucket messenger-attachments is private + +$ docker compose logs reconciler +2026-11-13 11:45:32,048 reconcile: periodic mode: every 600s, apply, grace 900s +2026-11-13 11:45:32,366 reconcile: run: objects 165, rows 165, counts {'ok': 165}, actions {'orphan_objects_deleted': 0, 'stale_rows_deleted': 0, 'stale_objects_deleted': 0, 'skipped': 0}, 0.32s diff --git a/evidence/track2/cost.json b/evidence/track2/cost.json new file mode 100644 index 00000000..c490271e --- /dev/null +++ b/evidence/track2/cost.json @@ -0,0 +1,45 @@ +{ + "repeats": 3, + "rows": [ + { + "objects": 3719, + "rows": 3734, + "list_pages": 4, + "list_s3_s": 0.625, + "scan_db_s": 0.014, + "total_s": 0.679, + "max_rss_mib": 89.7, + "us_per_object": 182.6 + }, + { + "objects": 4985, + "rows": 5000, + "list_pages": 5, + "list_s3_s": 0.818, + "scan_db_s": 0.017, + "total_s": 0.896, + "max_rss_mib": 91.4, + "us_per_object": 179.7 + }, + { + "objects": 9985, + "rows": 10000, + "list_pages": 10, + "list_s3_s": 1.539, + "scan_db_s": 0.072, + "total_s": 1.712, + "max_rss_mib": 97.0, + "us_per_object": 171.5 + }, + { + "objects": 19985, + "rows": 20000, + "list_pages": 20, + "list_s3_s": 3.289, + "scan_db_s": 0.201, + "total_s": 3.692, + "max_rss_mib": 111.4, + "us_per_object": 184.7 + } + ] +} diff --git a/evidence/track2/race.json b/evidence/track2/race.json new file mode 100644 index 00000000..f8bcd668 --- /dev/null +++ b/evidence/track2/race.json @@ -0,0 +1,109 @@ +{ + "seconds": 40, + "grace_seconds": 30, + "preclean_before_load": { + "orphan_objects_deleted": 0, + "stale_rows_deleted": 0, + "stale_objects_deleted": 0, + "skipped": 0 + }, + "reconcile_runs": 5, + "uploads_201": 3691, + "upload_errors": [], + "downloaded_intact": 3691, + "deleted_by_reconciler": 0, + "fresh_uploads_seen_in_flight": 5, + "runs": [ + { + "counts": { + "ok": 4101, + "orphan": 0, + "fresh_orphan": 0, + "stale_upload": 0, + "fresh_upload": 3, + "missing": 248, + "size_mismatch": 16276 + }, + "actions": { + "orphan_objects_deleted": 0, + "stale_rows_deleted": 0, + "stale_objects_deleted": 0, + "skipped": 0 + }, + "total_s": 5.115 + }, + { + "counts": { + "ok": 4865, + "orphan": 0, + "fresh_orphan": 0, + "stale_upload": 0, + "fresh_upload": 1, + "missing": 245, + "size_mismatch": 16276 + }, + "actions": { + "orphan_objects_deleted": 0, + "stale_rows_deleted": 0, + "stale_objects_deleted": 0, + "skipped": 0 + }, + "total_s": 5.269 + }, + { + "counts": { + "ok": 5659, + "orphan": 0, + "fresh_orphan": 0, + "stale_upload": 0, + "fresh_upload": 1, + "missing": 235, + "size_mismatch": 16276 + }, + "actions": { + "orphan_objects_deleted": 0, + "stale_rows_deleted": 0, + "stale_objects_deleted": 0, + "skipped": 0 + }, + "total_s": 5.72 + }, + { + "counts": { + "ok": 6359, + "orphan": 0, + "fresh_orphan": 0, + "stale_upload": 0, + "fresh_upload": 0, + "missing": 188, + "size_mismatch": 16276 + }, + "actions": { + "orphan_objects_deleted": 0, + "stale_rows_deleted": 0, + "stale_objects_deleted": 0, + "skipped": 0 + }, + "total_s": 5.422 + }, + { + "counts": { + "ok": 7077, + "orphan": 0, + "fresh_orphan": 0, + "stale_upload": 0, + "fresh_upload": 0, + "missing": 220, + "size_mismatch": 16276 + }, + "actions": { + "orphan_objects_deleted": 0, + "stale_rows_deleted": 0, + "stale_objects_deleted": 0, + "skipped": 0 + }, + "total_s": 5.409 + } + ], + "passed": true +} diff --git a/evidence/track2/results.md b/evidence/track2/results.md new file mode 100644 index 00000000..9255744a --- /dev/null +++ b/evidence/track2/results.md @@ -0,0 +1,39 @@ +# Трек 2: результаты + +## H1/H2. Внедрённые сбои (grace 30 с) + +| категория | внедрено | найдено dry-run | действие apply | +|---|---|---|---| +| orphan | 25 | 25 | удалено объектов: 25 | +| fresh_orphan | 10 | 10 | не тронуто | +| stale_upload | 50 | 50 | удалено строк: 50, объектов: 25 | +| fresh_upload | 10 | 10 | не тронуто | +| missing | 15 | 15 | только отчёт | +| size_mismatch | 10 | 10 | только отчёт | + +| проверка | результат | +|---|---| +| detected_equals_injected | True | +| dry_run_changes_nothing | True | +| apply_deleted_exactly_old_orphans_and_stale | True | +| rerun_is_noop | True | +| fresh_orphans_kept | True | +| fresh_uploads_kept | True | +| old_orphans_gone | True | +| stale_objects_gone | True | +| referenced_rows_kept | True | +| missing_still_reported_not_deleted | True | +| real_attachment_intact | True | + +## H2. Сверщик под нагрузкой загрузок + +40 с, 4 потока загрузок, apply с grace 30 с в цикле: запусков сверщика 5, загрузок 201 — 3691, скачано без изменений — 3691, удалено сверщиком — 0, «свежих» загрузок в полёте увидено — 5. + +## H3. Стоимость обхода (медиана из 3) + +| объектов | строк | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB | +|---|---|---|---|---|---|---|---| +| 3719 | 3734 | 4 | 0.62 | 0.014 | 0.68 | 183 | 90 | +| 4985 | 5000 | 5 | 0.82 | 0.017 | 0.90 | 180 | 91 | +| 9985 | 10000 | 10 | 1.54 | 0.072 | 1.71 | 172 | 97 | +| 19985 | 20000 | 20 | 3.29 | 0.201 | 3.69 | 185 | 111 | diff --git a/evidence/track2/scenario.json b/evidence/track2/scenario.json new file mode 100644 index 00000000..b1ed5c2f --- /dev/null +++ b/evidence/track2/scenario.json @@ -0,0 +1,866 @@ +{ + "grace_seconds": 30, + "injected": { + "orphan": 25, + "fresh_orphan": 10, + "stale_upload": 50, + "fresh_upload": 10, + "missing": 15, + "size_mismatch": 10 + }, + "baseline_counts": { + "ok": 165, + "orphan": 0, + "fresh_orphan": 0, + "stale_upload": 0, + "fresh_upload": 0, + "missing": 0, + "size_mismatch": 0 + }, + "dry_run": { + "mode": "dry-run", + "grace_seconds": 30.0, + "objects_listed": 420, + "list_pages": 1, + "rows_scanned": 425, + "counts": { + "ok": 340, + "orphan": 25, + "fresh_orphan": 10, + "stale_upload": 50, + "fresh_upload": 10, + "missing": 15, + "size_mismatch": 10 + }, + "actions": null, + "seconds": { + "list_s3": 0.174, + "scan_db": 0.006, + "total": 0.181 + }, + "max_rss_mib": 86.5, + "missing": [ + { + "id": "5f1c0d1b-d923-441a-a546-4434f77dde61", + "object_key": "attachments/5f1c0d1b-d923-441a-a546-4434f77dde61", + "age_s": 67, + "referenced": true, + "object_exists": false + }, + { + "id": "8f3b6fd1-b291-47db-8765-de71de23284e", + "object_key": "attachments/8f3b6fd1-b291-47db-8765-de71de23284e", + "age_s": 67, + "referenced": true, + "object_exists": false + }, + { + "id": "b5166ba2-32b2-430d-be01-ae9c0b5d018c", + "object_key": "attachments/b5166ba2-32b2-430d-be01-ae9c0b5d018c", + "age_s": 67, + "referenced": true, + "object_exists": false + }, + { + "id": "f4726273-cab8-4ee7-b043-571080b4e5a4", + "object_key": "attachments/f4726273-cab8-4ee7-b043-571080b4e5a4", + "age_s": 67, + "referenced": true, + "object_exists": false + }, + { + "id": "6f1158a3-ad63-4ac1-8147-df4c27f346cb", + "object_key": "attachments/6f1158a3-ad63-4ac1-8147-df4c27f346cb", + "age_s": 67, + "referenced": true, + "object_exists": false + }, + { + "id": "45a0b2de-7270-4998-a448-578d9b93e75b", + "object_key": "attachments/45a0b2de-7270-4998-a448-578d9b93e75b", + "age_s": 67, + "referenced": false, + "object_exists": false + }, + { + "id": "a0ca6b51-1540-41d9-9bd2-d44191420b6e", + "object_key": "attachments/a0ca6b51-1540-41d9-9bd2-d44191420b6e", + "age_s": 67, + "referenced": false, + "object_exists": false + }, + { + "id": "0923b50f-a2df-46f3-a32a-83b67ac5bc6b", + "object_key": "attachments/0923b50f-a2df-46f3-a32a-83b67ac5bc6b", + "age_s": 67, + "referenced": false, + "object_exists": false + }, + { + "id": "e7e0f57a-79a9-41b2-a515-deaddbbdd32a", + "object_key": "attachments/e7e0f57a-79a9-41b2-a515-deaddbbdd32a", + "age_s": 67, + "referenced": false, + "object_exists": false + }, + { + "id": "f17c8321-bc41-443a-b77a-e81b2b3aaf57", + "object_key": "attachments/f17c8321-bc41-443a-b77a-e81b2b3aaf57", + "age_s": 67, + "referenced": false, + "object_exists": false + }, + { + "id": "176e6dec-27b7-48cc-925f-26f5439bf2b8", + "object_key": "attachments/176e6dec-27b7-48cc-925f-26f5439bf2b8", + "age_s": 67, + "referenced": false, + "object_exists": false + }, + { + "id": "c79d7776-1852-4504-ab18-014604a41a36", + "object_key": "attachments/c79d7776-1852-4504-ab18-014604a41a36", + "age_s": 67, + "referenced": false, + "object_exists": false + }, + { + "id": "264e2e76-d3b6-479e-8ed1-472467b9eed4", + "object_key": "attachments/264e2e76-d3b6-479e-8ed1-472467b9eed4", + "age_s": 67, + "referenced": false, + "object_exists": false + }, + { + "id": "fc71784e-0d0a-4283-bb9e-0cce558416e2", + "object_key": "attachments/fc71784e-0d0a-4283-bb9e-0cce558416e2", + "age_s": 67, + "referenced": false, + "object_exists": false + }, + { + "id": "11ff3a85-8391-45f0-97ef-7644072cc3e8", + "object_key": "attachments/11ff3a85-8391-45f0-97ef-7644072cc3e8", + "age_s": 67, + "referenced": false, + "object_exists": false + } + ], + "size_mismatch": [ + { + "id": "2c1a8bd6-839a-41da-9ad1-d81561aef6ab", + "object_key": "attachments/2c1a8bd6-839a-41da-9ad1-d81561aef6ab", + "age_s": 67, + "referenced": false, + "object_exists": true + }, + { + "id": "092c24f5-2cc1-4af0-9724-c83e0c4f60b2", + "object_key": "attachments/092c24f5-2cc1-4af0-9724-c83e0c4f60b2", + "age_s": 67, + "referenced": false, + "object_exists": true + }, + { + "id": "d3a1b67d-0eed-4658-a358-74cca5ebd991", + "object_key": "attachments/d3a1b67d-0eed-4658-a358-74cca5ebd991", + "age_s": 67, + "referenced": false, + "object_exists": true + }, + { + "id": "8da2384a-087e-43be-8ab7-ad515655e7a1", + "object_key": "attachments/8da2384a-087e-43be-8ab7-ad515655e7a1", + "age_s": 67, + "referenced": false, + "object_exists": true + }, + { + "id": "4a66083e-d775-4085-9870-ffd89cdd89f0", + "object_key": "attachments/4a66083e-d775-4085-9870-ffd89cdd89f0", + "age_s": 67, + "referenced": false, + "object_exists": true + }, + { + "id": "1f24119f-719e-4c96-b6c2-a920944c3eec", + "object_key": "attachments/1f24119f-719e-4c96-b6c2-a920944c3eec", + "age_s": 67, + "referenced": false, + "object_exists": true + }, + { + "id": "94c09a04-b40a-417b-b953-26a194c2a551", + "object_key": "attachments/94c09a04-b40a-417b-b953-26a194c2a551", + "age_s": 67, + "referenced": false, + "object_exists": true + }, + { + "id": "187049b9-f44f-43fe-9601-c57c1e93c821", + "object_key": "attachments/187049b9-f44f-43fe-9601-c57c1e93c821", + "age_s": 67, + "referenced": false, + "object_exists": true + }, + { + "id": "0291536d-4e75-4547-8562-7ca0cacb178f", + "object_key": "attachments/0291536d-4e75-4547-8562-7ca0cacb178f", + "age_s": 67, + "referenced": false, + "object_exists": true + }, + { + "id": "c2e14c84-6ab8-4a9f-b26d-301fc3817f31", + "object_key": "attachments/c2e14c84-6ab8-4a9f-b26d-301fc3817f31", + "age_s": 67, + "referenced": false, + "object_exists": true + } + ] + }, + "apply": { + "mode": "apply", + "grace_seconds": 30.0, + "objects_listed": 420, + "list_pages": 1, + "rows_scanned": 425, + "counts": { + "ok": 340, + "orphan": 25, + "fresh_orphan": 10, + "stale_upload": 50, + "fresh_upload": 10, + "missing": 15, + "size_mismatch": 10 + }, + "actions": { + "orphan_objects_deleted": 25, + "stale_rows_deleted": 50, + "stale_objects_deleted": 25, + "skipped": 0 + }, + "seconds": { + "list_s3": 0.136, + "scan_db": 0.005, + "total": 0.512 + }, + "max_rss_mib": 85.5, + "missing": [ + { + "id": "5f1c0d1b-d923-441a-a546-4434f77dde61", + "object_key": "attachments/5f1c0d1b-d923-441a-a546-4434f77dde61", + "age_s": 75, + "referenced": true, + "object_exists": false + }, + { + "id": "8f3b6fd1-b291-47db-8765-de71de23284e", + "object_key": "attachments/8f3b6fd1-b291-47db-8765-de71de23284e", + "age_s": 75, + "referenced": true, + "object_exists": false + }, + { + "id": "b5166ba2-32b2-430d-be01-ae9c0b5d018c", + "object_key": "attachments/b5166ba2-32b2-430d-be01-ae9c0b5d018c", + "age_s": 75, + "referenced": true, + "object_exists": false + }, + { + "id": "f4726273-cab8-4ee7-b043-571080b4e5a4", + "object_key": "attachments/f4726273-cab8-4ee7-b043-571080b4e5a4", + "age_s": 75, + "referenced": true, + "object_exists": false + }, + { + "id": "6f1158a3-ad63-4ac1-8147-df4c27f346cb", + "object_key": "attachments/6f1158a3-ad63-4ac1-8147-df4c27f346cb", + "age_s": 75, + "referenced": true, + "object_exists": false + }, + { + "id": "45a0b2de-7270-4998-a448-578d9b93e75b", + "object_key": "attachments/45a0b2de-7270-4998-a448-578d9b93e75b", + "age_s": 75, + "referenced": false, + "object_exists": false + }, + { + "id": "a0ca6b51-1540-41d9-9bd2-d44191420b6e", + "object_key": "attachments/a0ca6b51-1540-41d9-9bd2-d44191420b6e", + "age_s": 75, + "referenced": false, + "object_exists": false + }, + { + "id": "0923b50f-a2df-46f3-a32a-83b67ac5bc6b", + "object_key": "attachments/0923b50f-a2df-46f3-a32a-83b67ac5bc6b", + "age_s": 75, + "referenced": false, + "object_exists": false + }, + { + "id": "e7e0f57a-79a9-41b2-a515-deaddbbdd32a", + "object_key": "attachments/e7e0f57a-79a9-41b2-a515-deaddbbdd32a", + "age_s": 75, + "referenced": false, + "object_exists": false + }, + { + "id": "f17c8321-bc41-443a-b77a-e81b2b3aaf57", + "object_key": "attachments/f17c8321-bc41-443a-b77a-e81b2b3aaf57", + "age_s": 75, + "referenced": false, + "object_exists": false + }, + { + "id": "176e6dec-27b7-48cc-925f-26f5439bf2b8", + "object_key": "attachments/176e6dec-27b7-48cc-925f-26f5439bf2b8", + "age_s": 75, + "referenced": false, + "object_exists": false + }, + { + "id": "c79d7776-1852-4504-ab18-014604a41a36", + "object_key": "attachments/c79d7776-1852-4504-ab18-014604a41a36", + "age_s": 75, + "referenced": false, + "object_exists": false + }, + { + "id": "264e2e76-d3b6-479e-8ed1-472467b9eed4", + "object_key": "attachments/264e2e76-d3b6-479e-8ed1-472467b9eed4", + "age_s": 75, + "referenced": false, + "object_exists": false + }, + { + "id": "fc71784e-0d0a-4283-bb9e-0cce558416e2", + "object_key": "attachments/fc71784e-0d0a-4283-bb9e-0cce558416e2", + "age_s": 75, + "referenced": false, + "object_exists": false + }, + { + "id": "11ff3a85-8391-45f0-97ef-7644072cc3e8", + "object_key": "attachments/11ff3a85-8391-45f0-97ef-7644072cc3e8", + "age_s": 75, + "referenced": false, + "object_exists": false + } + ], + "size_mismatch": [ + { + "id": "2c1a8bd6-839a-41da-9ad1-d81561aef6ab", + "object_key": "attachments/2c1a8bd6-839a-41da-9ad1-d81561aef6ab", + "age_s": 75, + "referenced": false, + "object_exists": true + }, + { + "id": "092c24f5-2cc1-4af0-9724-c83e0c4f60b2", + "object_key": "attachments/092c24f5-2cc1-4af0-9724-c83e0c4f60b2", + "age_s": 75, + "referenced": false, + "object_exists": true + }, + { + "id": "d3a1b67d-0eed-4658-a358-74cca5ebd991", + "object_key": "attachments/d3a1b67d-0eed-4658-a358-74cca5ebd991", + "age_s": 75, + "referenced": false, + "object_exists": true + }, + { + "id": "8da2384a-087e-43be-8ab7-ad515655e7a1", + "object_key": "attachments/8da2384a-087e-43be-8ab7-ad515655e7a1", + "age_s": 75, + "referenced": false, + "object_exists": true + }, + { + "id": "4a66083e-d775-4085-9870-ffd89cdd89f0", + "object_key": "attachments/4a66083e-d775-4085-9870-ffd89cdd89f0", + "age_s": 75, + "referenced": false, + "object_exists": true + }, + { + "id": "1f24119f-719e-4c96-b6c2-a920944c3eec", + "object_key": "attachments/1f24119f-719e-4c96-b6c2-a920944c3eec", + "age_s": 75, + "referenced": false, + "object_exists": true + }, + { + "id": "94c09a04-b40a-417b-b953-26a194c2a551", + "object_key": "attachments/94c09a04-b40a-417b-b953-26a194c2a551", + "age_s": 75, + "referenced": false, + "object_exists": true + }, + { + "id": "187049b9-f44f-43fe-9601-c57c1e93c821", + "object_key": "attachments/187049b9-f44f-43fe-9601-c57c1e93c821", + "age_s": 75, + "referenced": false, + "object_exists": true + }, + { + "id": "0291536d-4e75-4547-8562-7ca0cacb178f", + "object_key": "attachments/0291536d-4e75-4547-8562-7ca0cacb178f", + "age_s": 75, + "referenced": false, + "object_exists": true + }, + { + "id": "c2e14c84-6ab8-4a9f-b26d-301fc3817f31", + "object_key": "attachments/c2e14c84-6ab8-4a9f-b26d-301fc3817f31", + "age_s": 75, + "referenced": false, + "object_exists": true + } + ] + }, + "rerun": { + "mode": "apply", + "grace_seconds": 30.0, + "objects_listed": 370, + "list_pages": 1, + "rows_scanned": 375, + "counts": { + "ok": 340, + "orphan": 0, + "fresh_orphan": 10, + "stale_upload": 0, + "fresh_upload": 10, + "missing": 15, + "size_mismatch": 10 + }, + "actions": { + "orphan_objects_deleted": 0, + "stale_rows_deleted": 0, + "stale_objects_deleted": 0, + "skipped": 0 + }, + "seconds": { + "list_s3": 0.167, + "scan_db": 0.006, + "total": 0.174 + }, + "max_rss_mib": 85.7, + "missing": [ + { + "id": "5f1c0d1b-d923-441a-a546-4434f77dde61", + "object_key": "attachments/5f1c0d1b-d923-441a-a546-4434f77dde61", + "age_s": 80, + "referenced": true, + "object_exists": false + }, + { + "id": "8f3b6fd1-b291-47db-8765-de71de23284e", + "object_key": "attachments/8f3b6fd1-b291-47db-8765-de71de23284e", + "age_s": 80, + "referenced": true, + "object_exists": false + }, + { + "id": "b5166ba2-32b2-430d-be01-ae9c0b5d018c", + "object_key": "attachments/b5166ba2-32b2-430d-be01-ae9c0b5d018c", + "age_s": 80, + "referenced": true, + "object_exists": false + }, + { + "id": "f4726273-cab8-4ee7-b043-571080b4e5a4", + "object_key": "attachments/f4726273-cab8-4ee7-b043-571080b4e5a4", + "age_s": 80, + "referenced": true, + "object_exists": false + }, + { + "id": "6f1158a3-ad63-4ac1-8147-df4c27f346cb", + "object_key": "attachments/6f1158a3-ad63-4ac1-8147-df4c27f346cb", + "age_s": 80, + "referenced": true, + "object_exists": false + }, + { + "id": "45a0b2de-7270-4998-a448-578d9b93e75b", + "object_key": "attachments/45a0b2de-7270-4998-a448-578d9b93e75b", + "age_s": 80, + "referenced": false, + "object_exists": false + }, + { + "id": "a0ca6b51-1540-41d9-9bd2-d44191420b6e", + "object_key": "attachments/a0ca6b51-1540-41d9-9bd2-d44191420b6e", + "age_s": 80, + "referenced": false, + "object_exists": false + }, + { + "id": "0923b50f-a2df-46f3-a32a-83b67ac5bc6b", + "object_key": "attachments/0923b50f-a2df-46f3-a32a-83b67ac5bc6b", + "age_s": 80, + "referenced": false, + "object_exists": false + }, + { + "id": "e7e0f57a-79a9-41b2-a515-deaddbbdd32a", + "object_key": "attachments/e7e0f57a-79a9-41b2-a515-deaddbbdd32a", + "age_s": 80, + "referenced": false, + "object_exists": false + }, + { + "id": "f17c8321-bc41-443a-b77a-e81b2b3aaf57", + "object_key": "attachments/f17c8321-bc41-443a-b77a-e81b2b3aaf57", + "age_s": 80, + "referenced": false, + "object_exists": false + }, + { + "id": "176e6dec-27b7-48cc-925f-26f5439bf2b8", + "object_key": "attachments/176e6dec-27b7-48cc-925f-26f5439bf2b8", + "age_s": 80, + "referenced": false, + "object_exists": false + }, + { + "id": "c79d7776-1852-4504-ab18-014604a41a36", + "object_key": "attachments/c79d7776-1852-4504-ab18-014604a41a36", + "age_s": 80, + "referenced": false, + "object_exists": false + }, + { + "id": "264e2e76-d3b6-479e-8ed1-472467b9eed4", + "object_key": "attachments/264e2e76-d3b6-479e-8ed1-472467b9eed4", + "age_s": 80, + "referenced": false, + "object_exists": false + }, + { + "id": "fc71784e-0d0a-4283-bb9e-0cce558416e2", + "object_key": "attachments/fc71784e-0d0a-4283-bb9e-0cce558416e2", + "age_s": 80, + "referenced": false, + "object_exists": false + }, + { + "id": "11ff3a85-8391-45f0-97ef-7644072cc3e8", + "object_key": "attachments/11ff3a85-8391-45f0-97ef-7644072cc3e8", + "age_s": 80, + "referenced": false, + "object_exists": false + } + ], + "size_mismatch": [ + { + "id": "2c1a8bd6-839a-41da-9ad1-d81561aef6ab", + "object_key": "attachments/2c1a8bd6-839a-41da-9ad1-d81561aef6ab", + "age_s": 80, + "referenced": false, + "object_exists": true + }, + { + "id": "092c24f5-2cc1-4af0-9724-c83e0c4f60b2", + "object_key": "attachments/092c24f5-2cc1-4af0-9724-c83e0c4f60b2", + "age_s": 80, + "referenced": false, + "object_exists": true + }, + { + "id": "d3a1b67d-0eed-4658-a358-74cca5ebd991", + "object_key": "attachments/d3a1b67d-0eed-4658-a358-74cca5ebd991", + "age_s": 80, + "referenced": false, + "object_exists": true + }, + { + "id": "8da2384a-087e-43be-8ab7-ad515655e7a1", + "object_key": "attachments/8da2384a-087e-43be-8ab7-ad515655e7a1", + "age_s": 80, + "referenced": false, + "object_exists": true + }, + { + "id": "4a66083e-d775-4085-9870-ffd89cdd89f0", + "object_key": "attachments/4a66083e-d775-4085-9870-ffd89cdd89f0", + "age_s": 80, + "referenced": false, + "object_exists": true + }, + { + "id": "1f24119f-719e-4c96-b6c2-a920944c3eec", + "object_key": "attachments/1f24119f-719e-4c96-b6c2-a920944c3eec", + "age_s": 80, + "referenced": false, + "object_exists": true + }, + { + "id": "94c09a04-b40a-417b-b953-26a194c2a551", + "object_key": "attachments/94c09a04-b40a-417b-b953-26a194c2a551", + "age_s": 80, + "referenced": false, + "object_exists": true + }, + { + "id": "187049b9-f44f-43fe-9601-c57c1e93c821", + "object_key": "attachments/187049b9-f44f-43fe-9601-c57c1e93c821", + "age_s": 80, + "referenced": false, + "object_exists": true + }, + { + "id": "0291536d-4e75-4547-8562-7ca0cacb178f", + "object_key": "attachments/0291536d-4e75-4547-8562-7ca0cacb178f", + "age_s": 80, + "referenced": false, + "object_exists": true + }, + { + "id": "c2e14c84-6ab8-4a9f-b26d-301fc3817f31", + "object_key": "attachments/c2e14c84-6ab8-4a9f-b26d-301fc3817f31", + "age_s": 80, + "referenced": false, + "object_exists": true + } + ] + }, + "after": { + "mode": "dry-run", + "grace_seconds": 30.0, + "objects_listed": 370, + "list_pages": 1, + "rows_scanned": 375, + "counts": { + "ok": 340, + "orphan": 0, + "fresh_orphan": 10, + "stale_upload": 0, + "fresh_upload": 10, + "missing": 15, + "size_mismatch": 10 + }, + "actions": null, + "seconds": { + "list_s3": 0.129, + "scan_db": 0.005, + "total": 0.135 + }, + "max_rss_mib": 84.9, + "missing": [ + { + "id": "5f1c0d1b-d923-441a-a546-4434f77dde61", + "object_key": "attachments/5f1c0d1b-d923-441a-a546-4434f77dde61", + "age_s": 84, + "referenced": true, + "object_exists": false + }, + { + "id": "8f3b6fd1-b291-47db-8765-de71de23284e", + "object_key": "attachments/8f3b6fd1-b291-47db-8765-de71de23284e", + "age_s": 84, + "referenced": true, + "object_exists": false + }, + { + "id": "b5166ba2-32b2-430d-be01-ae9c0b5d018c", + "object_key": "attachments/b5166ba2-32b2-430d-be01-ae9c0b5d018c", + "age_s": 84, + "referenced": true, + "object_exists": false + }, + { + "id": "f4726273-cab8-4ee7-b043-571080b4e5a4", + "object_key": "attachments/f4726273-cab8-4ee7-b043-571080b4e5a4", + "age_s": 84, + "referenced": true, + "object_exists": false + }, + { + "id": "6f1158a3-ad63-4ac1-8147-df4c27f346cb", + "object_key": "attachments/6f1158a3-ad63-4ac1-8147-df4c27f346cb", + "age_s": 84, + "referenced": true, + "object_exists": false + }, + { + "id": "45a0b2de-7270-4998-a448-578d9b93e75b", + "object_key": "attachments/45a0b2de-7270-4998-a448-578d9b93e75b", + "age_s": 84, + "referenced": false, + "object_exists": false + }, + { + "id": "a0ca6b51-1540-41d9-9bd2-d44191420b6e", + "object_key": "attachments/a0ca6b51-1540-41d9-9bd2-d44191420b6e", + "age_s": 84, + "referenced": false, + "object_exists": false + }, + { + "id": "0923b50f-a2df-46f3-a32a-83b67ac5bc6b", + "object_key": "attachments/0923b50f-a2df-46f3-a32a-83b67ac5bc6b", + "age_s": 84, + "referenced": false, + "object_exists": false + }, + { + "id": "e7e0f57a-79a9-41b2-a515-deaddbbdd32a", + "object_key": "attachments/e7e0f57a-79a9-41b2-a515-deaddbbdd32a", + "age_s": 84, + "referenced": false, + "object_exists": false + }, + { + "id": "f17c8321-bc41-443a-b77a-e81b2b3aaf57", + "object_key": "attachments/f17c8321-bc41-443a-b77a-e81b2b3aaf57", + "age_s": 84, + "referenced": false, + "object_exists": false + }, + { + "id": "176e6dec-27b7-48cc-925f-26f5439bf2b8", + "object_key": "attachments/176e6dec-27b7-48cc-925f-26f5439bf2b8", + "age_s": 84, + "referenced": false, + "object_exists": false + }, + { + "id": "c79d7776-1852-4504-ab18-014604a41a36", + "object_key": "attachments/c79d7776-1852-4504-ab18-014604a41a36", + "age_s": 84, + "referenced": false, + "object_exists": false + }, + { + "id": "264e2e76-d3b6-479e-8ed1-472467b9eed4", + "object_key": "attachments/264e2e76-d3b6-479e-8ed1-472467b9eed4", + "age_s": 84, + "referenced": false, + "object_exists": false + }, + { + "id": "fc71784e-0d0a-4283-bb9e-0cce558416e2", + "object_key": "attachments/fc71784e-0d0a-4283-bb9e-0cce558416e2", + "age_s": 84, + "referenced": false, + "object_exists": false + }, + { + "id": "11ff3a85-8391-45f0-97ef-7644072cc3e8", + "object_key": "attachments/11ff3a85-8391-45f0-97ef-7644072cc3e8", + "age_s": 84, + "referenced": false, + "object_exists": false + } + ], + "size_mismatch": [ + { + "id": "2c1a8bd6-839a-41da-9ad1-d81561aef6ab", + "object_key": "attachments/2c1a8bd6-839a-41da-9ad1-d81561aef6ab", + "age_s": 84, + "referenced": false, + "object_exists": true + }, + { + "id": "092c24f5-2cc1-4af0-9724-c83e0c4f60b2", + "object_key": "attachments/092c24f5-2cc1-4af0-9724-c83e0c4f60b2", + "age_s": 84, + "referenced": false, + "object_exists": true + }, + { + "id": "d3a1b67d-0eed-4658-a358-74cca5ebd991", + "object_key": "attachments/d3a1b67d-0eed-4658-a358-74cca5ebd991", + "age_s": 84, + "referenced": false, + "object_exists": true + }, + { + "id": "8da2384a-087e-43be-8ab7-ad515655e7a1", + "object_key": "attachments/8da2384a-087e-43be-8ab7-ad515655e7a1", + "age_s": 84, + "referenced": false, + "object_exists": true + }, + { + "id": "4a66083e-d775-4085-9870-ffd89cdd89f0", + "object_key": "attachments/4a66083e-d775-4085-9870-ffd89cdd89f0", + "age_s": 84, + "referenced": false, + "object_exists": true + }, + { + "id": "1f24119f-719e-4c96-b6c2-a920944c3eec", + "object_key": "attachments/1f24119f-719e-4c96-b6c2-a920944c3eec", + "age_s": 84, + "referenced": false, + "object_exists": true + }, + { + "id": "94c09a04-b40a-417b-b953-26a194c2a551", + "object_key": "attachments/94c09a04-b40a-417b-b953-26a194c2a551", + "age_s": 84, + "referenced": false, + "object_exists": true + }, + { + "id": "187049b9-f44f-43fe-9601-c57c1e93c821", + "object_key": "attachments/187049b9-f44f-43fe-9601-c57c1e93c821", + "age_s": 84, + "referenced": false, + "object_exists": true + }, + { + "id": "0291536d-4e75-4547-8562-7ca0cacb178f", + "object_key": "attachments/0291536d-4e75-4547-8562-7ca0cacb178f", + "age_s": 84, + "referenced": false, + "object_exists": true + }, + { + "id": "c2e14c84-6ab8-4a9f-b26d-301fc3817f31", + "object_key": "attachments/c2e14c84-6ab8-4a9f-b26d-301fc3817f31", + "age_s": 84, + "referenced": false, + "object_exists": true + } + ] + }, + "detected": { + "orphan": 25, + "fresh_orphan": 10, + "stale_upload": 50, + "fresh_upload": 10, + "missing": 15, + "size_mismatch": 10 + }, + "checks": { + "detected_equals_injected": true, + "dry_run_changes_nothing": true, + "apply_actions": { + "orphan_objects_deleted": 25, + "stale_rows_deleted": 50, + "stale_objects_deleted": 25, + "skipped": 0 + }, + "apply_deleted_exactly_old_orphans_and_stale": true, + "rerun_is_noop": true, + "fresh_orphans_kept": true, + "fresh_uploads_kept": true, + "old_orphans_gone": true, + "stale_objects_gone": true, + "referenced_rows_kept": true, + "missing_still_reported_not_deleted": true, + "real_attachment_intact": true + }, + "passed": true +} diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..71a29ada --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,97 @@ +{ + "runs": [ + { + "label": "spool 1 MiB, 10 parallel", + "upload_memory_bytes": 1048576, + "parallel": 10, + "file_mib": 9.95, + "ok": 10, + "errors": [], + "wall_s": 3.01, + "max_request_s": 3.0, + "limit_mib": 320, + "idle_current_mib": { + "32c51c16fad0": 101.2, + "b455a3d794e7": 98.2 + }, + "peak_since_start_mib": { + "32c51c16fad0": 159.8, + "b455a3d794e7": 151.8 + }, + "sampled_peak_mib": { + "32c51c16fad0": { + "current": 159.0, + "anon": 99.5, + "file": 51.7 + }, + "b455a3d794e7": { + "current": 151.0, + "anon": 93.4, + "file": 50.0 + } + } + }, + { + "label": "spool 1 MiB, 20 parallel", + "upload_memory_bytes": 1048576, + "parallel": 20, + "file_mib": 9.95, + "ok": 20, + "errors": [], + "wall_s": 5.26, + "max_request_s": 5.25, + "limit_mib": 320, + "idle_current_mib": { + "a0248524555b": 115.5, + "2ce74c7f9f9b": 96.3 + }, + "peak_since_start_mib": { + "a0248524555b": 211.2, + "2ce74c7f9f9b": 189.4 + }, + "sampled_peak_mib": { + "a0248524555b": { + "current": 209.4, + "anon": 93.9, + "file": 106.7 + }, + "2ce74c7f9f9b": { + "current": 189.2, + "anon": 95.6, + "file": 86.3 + } + } + }, + { + "label": "all in RAM, 20 parallel", + "upload_memory_bytes": 16777216, + "parallel": 20, + "file_mib": 9.95, + "ok": 20, + "errors": [], + "wall_s": 5.21, + "max_request_s": 5.19, + "limit_mib": 320, + "idle_current_mib": { + "1b0ebeafeae4": 111.6, + "5cf20821aa82": 97.8 + }, + "peak_since_start_mib": { + "1b0ebeafeae4": 248.5, + "5cf20821aa82": 261.9 + }, + "sampled_peak_mib": { + "1b0ebeafeae4": { + "current": 247.8, + "anon": 221.2, + "file": 17.9 + }, + "5cf20821aa82": { + "current": 261.1, + "anon": 250.1, + "file": 5.5 + } + } + } + ] +} \ No newline at end of file -- 2.54.0