diff --git a/.dockerignore b/.dockerignore index bee7419..5a62abf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,3 +42,5 @@ compose*.yaml lab.json requirements-dev.txt .claude + +.verification/ diff --git a/.env.example b/.env.example index 72acc0a..731464e 100644 --- a/.env.example +++ b/.env.example @@ -32,8 +32,8 @@ OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab # LB queue per replica, drain window before closing the listener on SIGTERM, PostgreSQL # limits, load generator limits (profile bench), cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone: # only the load balancer is published. -LB_HTTP_PORT=8080 -LB_STATS_PORT=8404 +LB_HTTP_PORT=18086 +LB_STATS_PORT=26085 API_CPUS=1 API_MEMORY=512m WEB_CONCURRENCY=2 @@ -63,7 +63,7 @@ MALLOC_MMAP_THRESHOLD_=131072 # Solution-specific for lab5: RabbitMQ management UI on 127.0.0.1, backoff base of a transient # retry (doubles per attempt up to 60 s), relay poll and sweep intervals, worker limits and the # PostgreSQL pool of relay and worker. FAULT_POINT is set only by infra/compose.faults.yaml. -BROKER_MANAGEMENT_PORT=15672 +BROKER_MANAGEMENT_PORT=26086 JOB_BACKOFF_BASE_SECONDS=2 OUTBOX_POLL_SECONDS=1 SWEEP_INTERVAL_SECONDS=5 @@ -76,3 +76,31 @@ PIPELINE_DB_POOL_MAX_SIZE=4 RENDER_TIMEOUT_SECONDS=30 RENDER_MEMORY_MB=448 RENDER_TASKS_PER_CHILD=100 +# Solution-specific for lab6: host ports of Prometheus, Loki, Tempo and Grafana (127.0.0.1 only), +# Prometheus retention, OTLP export timeout in seconds, metric export interval in ms, SDK queue +# sizes, the bounded flush on shutdown; OTEL_SDK_DISABLED=true turns the SDK and instrumentations off. +# relay and worker get OTEL_SERVICE_NAME messenger-relay and messenger-worker in compose.yaml. +PROMETHEUS_PORT=26090 +LOKI_PORT=26310 +TEMPO_PORT=26320 +GRAFANA_PORT=26300 +PROMETHEUS_RETENTION=7d +OTEL_SDK_DISABLED=false +OTEL_EXPORTER_OTLP_TIMEOUT=2 +OTEL_METRIC_EXPORT_INTERVAL=10000 +OTEL_BSP_MAX_QUEUE_SIZE=2048 +OTEL_BLRP_MAX_QUEUE_SIZE=2048 +TELEMETRY_SHUTDOWN_SECONDS=2 +# Minimal client transfer rate, bytes/s: every body byte moved in either direction extends the +# request deadline by 1/rate, so slow but steady uploads and downloads outlive REQUEST_TIMEOUT_SECONDS. +MIN_TRANSFER_RATE_BYTES=131072 +# Solution-specific for lab4 track 2: limits of the one-off reconciler (profile reconcile), its S3 +# read timeout for listing pages, optional CPU pinning of the measured services (empty = no pinning) +# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for the 100k point). +RECONCILE_CPUS=1 +RECONCILE_MEMORY=256m +RECONCILE_S3_READ_TIMEOUT_SECONDS=10 +RECONCILE_CPUSET= +S3_CPUSET= +PG_CPUSET= +S3_CPUS=1 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml index 2c61282..e942d00 100644 --- a/.github/workflows/application.yml +++ b/.github/workflows/application.yml @@ -59,6 +59,9 @@ jobs: - name: Несколько реплик на одном токене и гонка одного ключа run: make lb-check + - name: Повтор неподтверждённой записи после потерянного ответа и убитой реплики + run: make retry-check + - name: Остановка реплики под запросами, затем возврат run: | make test-failover ACTION_SCRIPT=scripts/stop-one.sh @@ -84,7 +87,17 @@ jobs: - name: Процессы работают не от root run: | - docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}' + for id in $(docker compose ps -q); do + docker top "$id" -eo pid,uid,comm | awk 'NR>1 && $2==0 {print; bad=1} END {exit bad}' || exit 1 + done + + - name: Одноразовые сервисы не от root + run: | + for service in migrate s3-init; do + user=$(docker inspect -f '{{.Config.User}}' "$(docker compose ps -aq "$service")") + echo "$service: $user" + case "${user%%:*}" in ""|0|root) exit 1 ;; esac + done - name: Корректная остановка relay и worker по SIGTERM run: | diff --git a/.gitignore b/.gitignore index 5b7c398..bbdec97 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ coverage/ # Локальные настройки агента .claude/settings.local.json + +.verification/ diff --git a/Dockerfile b/Dockerfile index 1f4e6fe..ada3258 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ ENV PYTHONUNBUFFERED=1 \ WEB_CONCURRENCY=2 \ REQUEST_TIMEOUT_SECONDS=8 \ DRAIN_SECONDS=2 \ - SHUTDOWN_TIMEOUT_SECONDS=5 + SHUTDOWN_TIMEOUT_SECONDS=6 RUN groupadd --system --gid 10001 messenger \ && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger @@ -36,6 +36,9 @@ COPY --from=builder --chown=root:root /opt/venv /opt/venv WORKDIR /app +COPY --chown=root:root alembic.ini ./ +COPY --chown=root:root alembic ./alembic + USER 10001:10001 EXPOSE 8080 diff --git a/Makefile b/Makefile index ccde1de..2afa8bb 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://$(shell docker compose port lb 8080 2>/dev/null || echo localhost:8080) CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) @@ -8,13 +8,20 @@ VENV ?= .venv UV ?= uv COMPOSE ?= docker compose RATE ?= 100 -API_IMAGE ?= mtusi-messenger:lab6 FROM_REV ?= -SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ - scripts/upload_memory.py scripts/make_images.py scripts/pipeline_suite.py \ - scripts/pipeline_evidence.py +SHIFT ?= 0 +SCRIPTS_PY = scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ + scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \ + scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py SCENARIO ?= failover-upload PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q +OBS_CHECK = $(PYTHON) scripts/obs_check.py +APP_METRICS = {job=~"messenger-.*"} + +TRACK2_OUTPUT ?= evidence/track2.json +TRACK2_SCENARIOS = smoke_pipeline,duplicate_after_ready,crash_commit_publish,crash_publish_confirm,crash_after_s3_before_commit,crash_after_commit_before_ack,lease_expiry_fencing,crash_before_side_effect,worker_kill_mid_task,duplicate_in_flight,reorder_attempts,reorder_versions,stale_after_new_version +PIPELINE_VERSION ?= 2 +PYTEST_ARGS ?= .PHONY: help check validate test load test-persistence test-failover \ install lint format typecheck student-test \ @@ -23,7 +30,12 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q limits detection failover-each rolling-update lab4-level4 \ scenario s3-check attachments-evidence upload-faults upload-memory \ images queues topology worker-stop worker-start broker-restart suite-preflight level3 \ - pipeline-evidence level4 replay failed dlq + pipeline-evidence level4 replay failed dlq \ + obs-validate telemetry-check \ + faults-evidence retry-check reconcile reconcile-demo reconcile-cost load-evidence \ + clean-clone-evidence overload-leftovers + +.NOTPARALLEL: help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -36,10 +48,25 @@ help: @echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/" @echo "lab5: queues, topology, worker-stop/worker-start, broker-restart, images, suite-preflight, level3, level4" @echo "lab5 оператор: failed (реестр), replay ID=, dlq (просмотр DLQ без извлечения)" + @echo "lab6: obs-validate (конфиги Collector, Prometheus, Loki, Tempo, dashboards); telemetry-check → evidence/*.json" + + @echo "retry-check: потерянный ответ на живой и на убитой реплике, повтор тем же ключом" + @echo "lab4-level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps" + @echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence" + @echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда" + @echo "upload-memory ARGS='--out evidence/upload-memory/runN': пик памяти реплик при 10/20/40 загрузках 1 и 10 MiB" + @echo "load-evidence: tests/load.py ×3 клиентом-контейнером в сети edge стенда → evidence/load.txt, evidence/load/" + @echo "overload-leftovers: после upload-memory — намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt" + @echo "clean-clone-evidence: блок «Запуск из чистого clone» в свежем клоне рядом → evidence/clean-clone.txt" + @echo "reconcile ARGS='--apply --deep': сверка PostgreSQL ↔ S3 (без ARGS — dry-run); трек 2:" + @echo "reconcile-demo: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/" + @echo "reconcile-cost: стоимость обхода 1k/10k/30k/100k объектов × 3 прогона на чистом стенде → evidence/reconcile/cost/" + @echo "BASE_URL=$(BASE_URL) (порт lb из docker compose); failover-evidence SHIFT=1 — сдвиг round robin на вторую реплику" check: $(PYTHON) scripts/check_contract.py +validate: PYTHON := $(if $(wildcard $(VENV)/bin/python),$(VENV)/bin/python,$(PYTHON)) validate: $(PYTHON) scripts/validate_openapi.py @@ -63,32 +90,34 @@ install: lint: $(VENV)/bin/ruff check src student-tests $(SCRIPTS_PY) $(VENV)/bin/ruff format --check src student-tests $(SCRIPTS_PY) - $(VENV)/bin/ty check src student-tests + $(VENV)/bin/ty check src student-tests alembic format: $(VENV)/bin/ruff check --fix src student-tests $(SCRIPTS_PY) $(VENV)/bin/ruff format src student-tests $(SCRIPTS_PY) typecheck: - $(VENV)/bin/ty check src student-tests + $(VENV)/bin/ty check src student-tests alembic student-test: - $(VENV)/bin/pytest + $(VENV)/bin/pytest $(PYTEST_ARGS) secrets: sh scripts/secrets.sh sh scripts/broker-secrets.sh + sh scripts/secrets-lab6.sh up: secrets $(COMPOSE) build $(COMPOSE) up -d --wait + scripts/lb-wait.sh down: - $(COMPOSE) down + $(COMPOSE) --profile bench --profile tools --profile reconcile down reset-data: @echo "Удаляются volumes с учебными данными стенда" - $(COMPOSE) down -v --remove-orphans + $(COMPOSE) --profile bench --profile tools --profile reconcile down -v --remove-orphans migrate: $(COMPOSE) run --rm migrate @@ -105,19 +134,23 @@ restore: lb-check: $(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json +retry-check: + $(PYTHON) scripts/probe.py retry --out evidence/retry.json + limits: $(PYTHON) scripts/limits.py --out evidence/limits.txt detection: - sh scripts/detection.sh evidence/detection + rm -rf evidence/detection + for run in 1 2 3; do sh scripts/detection.sh evidence/detection/run$$run || exit 1; done failover-each: RATE=$(RATE) sh scripts/failover-each.sh evidence/failover-each rolling-update: ifneq ($(FROM_REV),) - git archive "$(FROM_REV)" | docker build -q -t "$(API_IMAGE)" - - $(COMPOSE) up -d --no-deps --force-recreate --wait api + git archive "$(FROM_REV)" | docker build -q -t "mtusi-messenger:$(FROM_REV)" - + API_IMAGE="mtusi-messenger:$(FROM_REV)" $(COMPOSE) up -d --no-deps --force-recreate --wait api scripts/lb-wait.sh endif $(COMPOSE) build -q api @@ -144,10 +177,28 @@ upload-faults: status=$$?; cat evidence/upload-faults.txt; exit $$status upload-memory: - $(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)" + $(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)" $(ARGS) + +load-evidence: + sh scripts/load-evidence.sh + +overload-leftovers: + sh scripts/overload-leftovers.sh + +clean-clone-evidence: + sh scripts/clean-clone.sh + +reconcile: + $(COMPOSE) run --rm --no-deps reconciler $(ARGS) + +reconcile-demo: + $(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)" $(ARGS) + +reconcile-cost: + $(PYTHON) scripts/reconcile_track.py cost $(ARGS) failover-evidence: - sh scripts/failover-evidence.sh "$(BASE_URL)" + sh scripts/failover-evidence.sh "$(BASE_URL)" "$(SHIFT)" stack-evidence: sh scripts/evidence-stack.sh @@ -166,6 +217,12 @@ cache-evidence: test-faults: BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack +faults-evidence: + { echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; echo "\$$ make test-faults"; \ + $(MAKE) --no-print-directory test-faults 2>&1; } | sed 's#$(CURDIR)##g' > evidence/faults.txt + cat evidence/faults.txt + grep -qE '^=+ [0-9]+ passed in' evidence/faults.txt + seed: $(PSQL) -f - < scripts/sql/seed-history.sql @@ -226,3 +283,26 @@ dlq: pipeline-evidence: $(PYTHON) scripts/pipeline_evidence.py --base-url "$(BASE_URL)" > evidence/pipeline-stand.txt; \ status=$$?; cat evidence/pipeline-stand.txt; exit $$status + +obs-validate: + sh scripts/obs-validate.sh + +telemetry-check: + $(OBS_CHECK) grafana > evidence/grafana.json + $(OBS_CHECK) dashboards --strict > evidence/dashboards.json + $(OBS_CHECK) request --base-url "$(BASE_URL)" > evidence/trace-request.json + $(OBS_CHECK) job --base-url "$(BASE_URL)" > evidence/trace-job.json + $(OBS_CHECK) leaks --base-url "$(BASE_URL)" > evidence/leaks.json + $(OBS_CHECK) query 'count by (__name__, job) ($(APP_METRICS))' \ + 'count by (http_route, http_request_method) (http_server_request_duration_seconds_count)' \ + 'count by (job_type, outcome) (messenger_jobs_processed_total)' > evidence/cardinality.json + @echo "telemetry-check: evidence/grafana.json dashboards.json trace-request.json trace-job.json leaks.json cardinality.json" + +.PHONY: track2 reprocess + +track2: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 5 --only "$(TRACK2_SCENARIOS)" --output "$(TRACK2_OUTPUT)" + +reprocess: + @test -n "$(ID)" || (echo "Set ID="; exit 2) + $(COMPOSE) run --rm --no-deps -T worker python -m messenger.admin reprocess --attachment-id "$(ID)" --pipeline-version "$(PIPELINE_VERSION)" diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..934bba6 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,4 @@ +[alembic] +script_location = %(here)s/alembic +prepend_sys_path = %(here)s/src %(here)s/alembic +path_separator = space diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..f0a94ae --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,102 @@ +import asyncio + +from alembic import context +from alembic.script import ScriptDirectory +from legacy import adopt_legacy +from sqlalchemy import Connection, func, make_url, pool, select +from sqlalchemy.ext.asyncio import create_async_engine + +from messenger.attachments.infrastructure.postgres import ( + AttachmentRow, + JobEventRow, + JobRow, + OutboxRow, + VariantRow, +) +from messenger.identity.infrastructure.postgres import SessionRow, UserRow +from messenger.messaging.infrastructure.postgres import ( + ChatMemberRow, + ChatRow, + IdempotencyKeyRow, + MessageAttachmentRow, + MessageRow, +) +from messenger.shared.config import PlatformSettings + +LOCK_KEY = 20_260_002 +MODELS = ( + UserRow, + SessionRow, + ChatRow, + ChatMemberRow, + MessageRow, + IdempotencyKeyRow, + AttachmentRow, + JobRow, + OutboxRow, + JobEventRow, + VariantRow, + MessageAttachmentRow, +) +target_metadata = UserRow.metadata +config = context.config + + +def include_object(_obj, name, _type, _reflected, _compare_to): + return name != "schema_migrations" + + +def run_migrations(connection: Connection) -> None: + with connection.engine.connect() as lock: + lock.execute(select(func.pg_advisory_lock(LOCK_KEY))) + lock.commit() + try: + with connection.begin(): + adopt_legacy(connection, ScriptDirectory.from_config(config)) + context.configure( + connection=connection, + target_metadata=target_metadata, + transaction_per_migration=True, + compare_type=True, + include_object=include_object, + ) + with context.begin_transaction(): + context.run_migrations() + finally: + lock.execute(select(func.pg_advisory_unlock(LOCK_KEY))) + lock.commit() + + +async def run_async_migrations() -> None: + dsn = config.attributes.get("database_url") + if dsn is None: + dsn = PlatformSettings().database_url.get_secret_value() + engine = create_async_engine( + make_url(dsn).set(drivername="postgresql+asyncpg"), + poolclass=pool.NullPool, + hide_parameters=True, + connect_args={ + "timeout": 10, + "server_settings": {"lock_timeout": "10000", "statement_timeout": "60000"}, + }, + ) + try: + async with engine.connect() as connection: + await connection.run_sync(run_migrations) + finally: + await engine.dispose() + + +if context.is_offline_mode(): + context.configure( + dialect_name="postgresql", + target_metadata=target_metadata, + literal_binds=True, + transaction_per_migration=True, + ) + with context.begin_transaction(): + context.run_migrations() +elif config.attributes.get("connection") is not None: + run_migrations(config.attributes["connection"]) +else: + asyncio.run(run_async_migrations()) diff --git a/alembic/legacy.py b/alembic/legacy.py new file mode 100644 index 0000000..3652afa --- /dev/null +++ b/alembic/legacy.py @@ -0,0 +1,94 @@ +from uuid import uuid4 + +from alembic.migration import MigrationContext +from alembic.operations import Operations +from alembic.script import ScriptDirectory +from sqlalchemy import Connection, MetaData, Table, func, inspect, select, text +from sqlalchemy.schema import CreateSchema + +LEGACY = { + "0001_init": "9a13618f078ae9056431391e2e5197a31b981ae1fff047a9b3421ac750d3fd0d", + "0002_idempotency_keys": "d9159a35553249a1e51af5f2e4e7c39b014e97593c49748e13f3799839063dd0", + "0003_attachments": "c4417f560538a083efba1c67f660147dc94532fcbb2fa91f3f3e8258a0f01f2e", + "0004_jobs": "c3887fa7a886ae8153a216aa43fd4c3c6190a2074e8ce20c3e77f29bfb01550e", + "0005_failed_registry": "02b80061a08a6929feb092148c453ad970e3d001901c651b5a73b90b1e40f145", + "0006_attachment_quarantine": ( + "541f5399d87fac3584c4102b4e0bf92c23cd372cc20a7d56dad03e771773202c" + ), +} +SNAPSHOT = text(""" + WITH relations AS ( + SELECT c.* FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = :schema AND c.relkind IN ('r', 'p') + AND c.relname NOT IN ('schema_migrations', 'alembic_version') + ) + SELECT 'table' AS kind, c.relname AS name, + concat_ws('|', c.relkind, c.relpersistence, + c.relrowsecurity, c.relforcerowsecurity) AS definition + FROM relations c + UNION ALL + SELECT 'column', c.relname || '.' || a.attname, + concat_ws('|', a.attnum, format_type(a.atttypid, a.atttypmod), a.attnotnull, + a.attidentity, a.attgenerated, a.attcollation::regcollation, + pg_get_expr(d.adbin, d.adrelid)) + FROM relations c JOIN pg_attribute a ON a.attrelid = c.oid + LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum + WHERE a.attnum > 0 AND NOT a.attisdropped + UNION ALL + SELECT 'constraint', c.relname || '.' || k.conname, + concat_ws('|', pg_get_constraintdef(k.oid), k.convalidated) + FROM relations c JOIN pg_constraint k ON k.conrelid = c.oid + UNION ALL + SELECT 'index', c.relname || '.' || x.relname, + concat_ws('|', pg_get_indexdef(i.indexrelid), i.indisvalid, i.indisready) + FROM relations c JOIN pg_index i ON i.indrelid = c.oid + JOIN pg_class x ON x.oid = i.indexrelid + UNION ALL + SELECT 'trigger', c.relname || '.' || t.tgname, + concat_ws('|', pg_get_triggerdef(t.oid), t.tgenabled) + FROM relations c JOIN pg_trigger t ON t.tgrelid = c.oid WHERE NOT t.tgisinternal + ORDER BY kind, name +""") + + +def snapshot(connection: Connection, schema: str) -> list[tuple[str, str, str]]: + return [ + (kind, name, definition.replace(f"{schema}.", "")) + for kind, name, definition in connection.execute(SNAPSHOT, {"schema": schema}) + ] + + +def adopt_legacy(connection: Connection, scripts: ScriptDirectory) -> None: + migration = MigrationContext.configure(connection) + if migration.get_current_heads(): + return + tables = set(inspect(connection).get_table_names(schema="public")) + if tables <= {"alembic_version"}: + return + if "schema_migrations" not in tables: + raise RuntimeError("Existing schema has no legacy migration journal") + journal = Table("schema_migrations", MetaData(), schema="public", autoload_with=connection) + applied = dict(connection.execute(select(journal.c.version, journal.c.checksum)).tuples().all()) + expected = dict(list(LEGACY.items())[: len(applied)]) + if applied != expected: + raise RuntimeError("Unknown or changed legacy migrations") + if not applied: + if tables - {"schema_migrations", "alembic_version"}: + raise RuntimeError("Existing schema has an empty migration journal") + return + revision = list(expected)[-1] + actual = snapshot(connection, "public") + reference = f"alembic_verify_{uuid4().hex}" + savepoint = connection.begin_nested() + try: + connection.execute(CreateSchema(reference)) + connection.execute(select(func.set_config("search_path", reference, True))) + with Operations.context(MigrationContext.configure(connection)): + for script in reversed(list(scripts.walk_revisions(base="base", head=revision))): + script.module.upgrade() + wanted = snapshot(connection, reference) + finally: + savepoint.rollback() + if actual != wanted: + raise RuntimeError("Legacy schema differs from the recorded migrations") + migration.stamp(scripts, revision) diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..fceee8a --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,16 @@ +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/0001_init.py b/alembic/versions/0001_init.py new file mode 100644 index 0000000..21c7bb2 --- /dev/null +++ b/alembic/versions/0001_init.py @@ -0,0 +1,87 @@ +import sqlalchemy as sa +from alembic import op + +revision = "0001_init" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "users", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("username", sa.Text(), nullable=False), + sa.Column("display_name", sa.Text(), nullable=False), + sa.Column("password_hash", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("password_hash LIKE '$argon2id$%'", name="users_password_hash_check"), + sa.CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="users_username_format"), + sa.CheckConstraint( + "char_length(display_name) BETWEEN 1 AND 100", name="users_display_name_check" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("username", name="users_username_key"), + ) + op.create_table( + "sessions", + sa.Column("token_hash", sa.LargeBinary(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("expires_at > created_at", name="sessions_check"), + sa.CheckConstraint("octet_length(token_hash) = 32", name="sessions_token_hash_check"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("token_hash"), + ) + op.create_index("sessions_user_idx", "sessions", ["user_id"], unique=False) + op.create_table( + "chats", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("title", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_message_at", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="chats_title_check"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "chat_members", + sa.Column("chat_id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("position", sa.SmallInteger(), nullable=False), + sa.Column("chat_created_at", sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("position BETWEEN 1 AND 100", name="chat_members_position_check"), + sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + ), + sa.PrimaryKeyConstraint("chat_id", "user_id"), + ) + op.create_index( + "chat_members_user_idx", + "chat_members", + ["user_id", "chat_created_at", "chat_id"], + unique=False, + ) + op.create_table( + "messages", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("chat_id", sa.Uuid(), nullable=False), + sa.Column("sender_id", sa.Uuid(), nullable=False), + sa.Column("text", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("char_length(text) BETWEEN 1 AND 4000", name="messages_text_check"), + sa.ForeignKeyConstraint( + ["chat_id", "sender_id"], + ["chat_members.chat_id", "chat_members.user_id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "messages_history_idx", "messages", ["chat_id", "created_at", "id"], unique=False + ) + + +def downgrade() -> None: + raise RuntimeError("Downgrade would delete application data") diff --git a/alembic/versions/0002_idempotency_keys.py b/alembic/versions/0002_idempotency_keys.py new file mode 100644 index 0000000..ccaa253 --- /dev/null +++ b/alembic/versions/0002_idempotency_keys.py @@ -0,0 +1,33 @@ +import sqlalchemy as sa +from alembic import op + +revision = "0002_idempotency_keys" +down_revision = "0001_init" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "idempotency_keys", + sa.Column("chat_id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("key", sa.Text(), nullable=False), + sa.Column("fingerprint", sa.LargeBinary(), nullable=False), + sa.Column("message_id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="idempotency_keys_key_check"), + sa.CheckConstraint( + "octet_length(fingerprint) = 32", name="idempotency_keys_fingerprint_check" + ), + sa.ForeignKeyConstraint( + ["chat_id", "user_id"], + ["chat_members.chat_id", "chat_members.user_id"], + ), + sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("chat_id", "user_id", "key"), + ) + + +def downgrade() -> None: + raise RuntimeError("Downgrade would delete application data") diff --git a/alembic/versions/0003_attachments.py b/alembic/versions/0003_attachments.py new file mode 100644 index 0000000..f0395d0 --- /dev/null +++ b/alembic/versions/0003_attachments.py @@ -0,0 +1,77 @@ +import sqlalchemy as sa +from alembic import op + +revision = "0003_attachments" +down_revision = "0002_idempotency_keys" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_constraint("messages_text_check", "messages", type_="check") + op.create_check_constraint("messages_text_check", "messages", "char_length(text) <= 4000") + op.create_table( + "attachments", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("chat_id", sa.Uuid(), nullable=False), + sa.Column("owner_id", sa.Uuid(), nullable=False), + sa.Column("filename", sa.Text(), nullable=False), + sa.Column("content_type", sa.Text(), nullable=False), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("sha256", sa.Text(), nullable=False), + sa.Column("width", sa.Integer(), nullable=False), + sa.Column("height", sa.Integer(), nullable=False), + sa.Column("object_key", sa.Text(collation="C"), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("object_key", name="attachments_object_key_key"), + sa.UniqueConstraint("id", "chat_id", name="attachments_chat_key"), + sa.ForeignKeyConstraint( + ["chat_id", "owner_id"], + ["chat_members.chat_id", "chat_members.user_id"], + name="attachments_owner_fkey", + ), + sa.CheckConstraint( + "char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_check" + ), + sa.CheckConstraint( + "content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_check" + ), + sa.CheckConstraint( + "size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_check" + ), + sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachments_sha256_check"), + sa.CheckConstraint("width > 0", name="attachments_width_check"), + sa.CheckConstraint("height > 0", name="attachments_height_check"), + sa.CheckConstraint("status IN ('pending', 'ready')", name="attachments_status_check"), + ) + op.create_index( + "attachments_pending_idx", + "attachments", + ["created_at"], + postgresql_where=sa.text("status = 'pending'"), + ) + op.create_table( + "message_attachments", + sa.Column("message_id", sa.Uuid(), nullable=False), + sa.Column("position", sa.SmallInteger(), nullable=False), + sa.Column("chat_id", sa.Uuid(), nullable=False), + sa.Column("attachment_id", sa.Uuid(), nullable=False), + sa.PrimaryKeyConstraint("message_id", "position"), + sa.UniqueConstraint("message_id", "attachment_id"), + sa.CheckConstraint("position BETWEEN 0 AND 9", name="message_attachments_position_check"), + sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["attachment_id", "chat_id"], + ["attachments.id", "attachments.chat_id"], + name="message_attachments_attachment_fkey", + ondelete="RESTRICT", + ), + ) + op.create_index("message_attachments_attachment_idx", "message_attachments", ["attachment_id"]) + + +def downgrade() -> None: + raise RuntimeError("Downgrade would delete application data") diff --git a/alembic/versions/0004_jobs.py b/alembic/versions/0004_jobs.py new file mode 100644 index 0000000..231ab99 --- /dev/null +++ b/alembic/versions/0004_jobs.py @@ -0,0 +1,146 @@ +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "0004_jobs" +down_revision = "0003_attachments" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_constraint("attachments_status_check", "attachments", type_="check") + op.create_check_constraint( + "attachments_status_check", + "attachments", + "status IN ('pending', 'queued', 'processing', 'ready', 'failed')", + ) + for name in ("staging_key", "original_key"): + op.add_column("attachments", sa.Column(name, sa.Text(collation="C"), nullable=True)) + op.create_unique_constraint(f"attachments_{name}_key", "attachments", [name]) + op.add_column( + "attachments", + sa.Column("published_version", sa.Integer(), nullable=False, server_default="0"), + ) + op.create_check_constraint( + "attachments_published_version_check", "attachments", "published_version >= 0" + ) + op.add_column("attachments", sa.Column("error_code", sa.Text(), nullable=True)) + op.create_check_constraint( + "attachments_error_code_check", + "attachments", + "error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')", + ) + op.alter_column("attachments", "object_key", nullable=True) + attachments = sa.table( + "attachments", + sa.column("status"), + sa.column("object_key"), + sa.column("original_key"), + sa.column("staging_key"), + sa.column("published_version"), + ) + op.execute( + attachments.update() + .where(attachments.c.status == "ready") + .values(original_key=attachments.c.object_key, published_version=1) + ) + op.execute( + attachments.update() + .where(attachments.c.status == "pending") + .values(staging_key=attachments.c.object_key) + ) + op.create_index( + "attachments_staged_idx", + "attachments", + ["updated_at"], + postgresql_where=sa.column("staging_key").is_not(None), + ) + op.create_table( + "attachment_variants", + sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False), + sa.Column("pipeline_version", sa.Integer(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("object_key", sa.Text(collation="C"), nullable=False), + sa.Column("content_type", sa.Text(), nullable=False), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("width", sa.Integer(), nullable=False), + sa.Column("height", sa.Integer(), nullable=False), + sa.Column("sha256", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("attachment_id", "pipeline_version", "name"), + sa.UniqueConstraint("object_key", name="attachment_variants_object_key_key"), + sa.CheckConstraint( + "pipeline_version >= 1", name="attachment_variants_pipeline_version_check" + ), + sa.CheckConstraint( + "name IN ('thumbnail', 'crop', 'optimized')", name="attachment_variants_name_check" + ), + sa.CheckConstraint( + "content_type IN ('image/jpeg', 'image/png', 'image/webp')", + name="attachment_variants_content_type_check", + ), + sa.CheckConstraint("size_bytes >= 1", name="attachment_variants_size_bytes_check"), + sa.CheckConstraint("width >= 1", name="attachment_variants_width_check"), + sa.CheckConstraint("height >= 1", name="attachment_variants_height_check"), + sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachment_variants_sha256_check"), + ) + op.create_table( + "jobs", + sa.Column("job_id", sa.Uuid(), primary_key=True), + sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False), + sa.Column("pipeline_version", sa.Integer(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("attempt", sa.Integer(), nullable=False), + sa.Column("claims", sa.Integer(), nullable=False, server_default="0"), + sa.Column("lease_token", sa.Uuid(), nullable=True), + sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True), + sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint("attachment_id", "pipeline_version", name="jobs_effect_key"), + sa.CheckConstraint("pipeline_version >= 1", name="jobs_pipeline_version_check"), + sa.CheckConstraint( + "status IN ('queued', 'processing', 'done', 'failed', 'superseded')", + name="jobs_status_check", + ), + sa.CheckConstraint("attempt >= 1", name="jobs_attempt_check"), + sa.CheckConstraint("claims >= 0", name="jobs_claims_check"), + ) + op.create_index( + "jobs_lease_idx", + "jobs", + ["lease_until"], + postgresql_where=sa.column("status") == "processing", + ) + op.create_table( + "outbox", + sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True), + sa.Column("job_id", sa.Uuid(), sa.ForeignKey("jobs.job_id"), nullable=False), + sa.Column("payload", postgresql.JSONB(), nullable=False), + sa.Column("available_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("published_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "outbox_pending_idx", + "outbox", + ["available_at", "id"], + postgresql_where=sa.column("published_at").is_(None), + ) + op.create_index("outbox_job_idx", "outbox", ["job_id"]) + op.create_table( + "job_events", + sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True), + sa.Column("job_id", sa.Uuid(), nullable=False), + sa.Column("attachment_id", sa.Uuid(), nullable=False), + sa.Column("kind", sa.Text(), nullable=False), + sa.Column("attempt", sa.Integer(), nullable=True), + sa.Column("detail", postgresql.JSONB(), nullable=False, server_default="{}"), + sa.Column("at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("job_events_attachment_idx", "job_events", ["attachment_id", "id"]) + + +def downgrade() -> None: + raise RuntimeError("Downgrade would delete pipeline data") diff --git a/alembic/versions/0005_failed_registry.py b/alembic/versions/0005_failed_registry.py new file mode 100644 index 0000000..79a5466 --- /dev/null +++ b/alembic/versions/0005_failed_registry.py @@ -0,0 +1,20 @@ +import sqlalchemy as sa +from alembic import op + +revision = "0005_failed_registry" +down_revision = "0004_jobs" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index( + "jobs_failed_idx", + "jobs", + [sa.column("updated_at").desc()], + postgresql_where=sa.column("status") == "failed", + ) + + +def downgrade() -> None: + op.drop_index("jobs_failed_idx", table_name="jobs") diff --git a/alembic/versions/0006_attachment_quarantine.py b/alembic/versions/0006_attachment_quarantine.py new file mode 100644 index 0000000..6e34315 --- /dev/null +++ b/alembic/versions/0006_attachment_quarantine.py @@ -0,0 +1,28 @@ +import sqlalchemy as sa +from alembic import op + +revision = "0006_attachment_quarantine" +down_revision = "0005_failed_registry" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("attachments", sa.Column("quarantine_reason", sa.Text(), nullable=True)) + op.create_check_constraint( + "attachments_quarantine_reason_check", + "attachments", + "quarantine_reason IN ('missing', 'size_mismatch', 'sha_mismatch')", + ) + op.add_column( + "attachments", sa.Column("quarantined_at", sa.DateTime(timezone=True), nullable=True) + ) + op.create_check_constraint( + "attachments_quarantine_check", + "attachments", + "(quarantine_reason IS NULL) = (quarantined_at IS NULL)", + ) + + +def downgrade() -> None: + raise RuntimeError("Downgrade would delete quarantine data") diff --git a/compose.yaml b/compose.yaml index f464e1e..35e1fe5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,5 @@ x-app: &app - image: ${API_IMAGE:-mtusi-messenger:lab6} + image: ${API_IMAGE:-mtusi-messenger:${COMPOSE_PROJECT_NAME}} networks: [backend] x-s3-env: &s3-env @@ -19,7 +19,7 @@ x-s3-env: &s3-env UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100} x-pipeline: &pipeline - image: ${API_IMAGE:-mtusi-messenger:lab6} + image: ${API_IMAGE:-mtusi-messenger:${COMPOSE_PROJECT_NAME}} pull_policy: never restart: unless-stopped init: true @@ -44,8 +44,33 @@ x-pipeline: &pipeline broker: condition: service_healthy +x-otel-env: &otel-env + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-2} + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab} + OTEL_SEMCONV_STABILITY_OPT_IN: http + OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-10000} + OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048} + OTEL_BLRP_MAX_QUEUE_SIZE: ${OTEL_BLRP_MAX_QUEUE_SIZE:-2048} + TELEMETRY_SHUTDOWN_SECONDS: ${TELEMETRY_SHUTDOWN_SECONDS:-2} + +x-obs-logging: &obs-logging + driver: json-file + options: + max-size: 10m + max-file: "3" + +x-obs-hardening: &obs-hardening + restart: unless-stopped + stop_grace_period: 10s + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + logging: *obs-logging + x-pipeline-env: &pipeline-env - <<: *s3-env + <<: [*s3-env, *otel-env] DATABASE_URL_FILE: /run/secrets/database_url BROKER_URL_FILE: /run/secrets/broker_url DB_POOL_MAX_SIZE: ${PIPELINE_DB_POOL_MAX_SIZE:-4} @@ -64,10 +89,17 @@ x-s3-client: &s3-client S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} RC_CONFIG_DIR: /tmp/rc + user: "65534:65534" volumes: ["./infra/s3:/s3:ro"] secrets: - [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key, - s3_worker_access_key_id, s3_worker_secret_access_key] + - s3_root_user + - s3_root_password + - s3_access_key_id + - s3_secret_access_key + - s3_reconciler_access_key_id + - s3_reconciler_secret_access_key + - s3_worker_access_key_id + - s3_worker_secret_access_key cap_drop: [ALL] security_opt: ["no-new-privileges:true"] restart: "no" @@ -83,15 +115,16 @@ services: API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64} volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] ports: - - "127.0.0.1:${LB_HTTP_PORT:-8080}:8080" - - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" + - "127.0.0.1:${LB_HTTP_PORT:-18086}:8080" + - "127.0.0.1:${LB_STATS_PORT:-26085}:8404" networks: [edge, backend] healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"] interval: 2s timeout: 2s retries: 3 - start_period: 10s + start_period: 60s + start_interval: 1s sysctls: net.ipv4.ip_local_port_range: "10240 65535" net.ipv4.tcp_tw_reuse: "1" @@ -108,7 +141,8 @@ services: <<: *app build: . environment: - <<: *s3-env + <<: [*s3-env, *otel-env] + OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-messenger-api} DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL_FILE: /run/secrets/cache_url CURSOR_SECRET_FILE: /run/secrets/cursor_secret @@ -117,10 +151,11 @@ services: WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2} DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8} REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8} + MIN_TRANSFER_RATE_BYTES: ${MIN_TRANSFER_RATE_BYTES:-131072} DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1} DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2} DRAIN_SECONDS: ${DRAIN_SECONDS:-2} - SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-6} LOG_LEVEL: ${LOG_LEVEL:-INFO} MALLOC_MMAP_THRESHOLD_: ${MALLOC_MMAP_THRESHOLD_:-131072} secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key] @@ -130,7 +165,7 @@ services: timeout: 3s retries: 3 start_period: 30s - start_interval: 1s + start_interval: 200ms stop_grace_period: 8s restart: unless-stopped read_only: true @@ -153,7 +188,8 @@ services: migrate: <<: *app pull_policy: never - command: ["python", "-m", "messenger.migrate"] + init: true + command: ["alembic", "upgrade", "head"] environment: DATABASE_URL_FILE: /run/secrets/database_url LOG_LEVEL: ${LOG_LEVEL:-INFO} @@ -192,12 +228,14 @@ services: secrets: [postgres_password] volumes: [pgdata:/var/lib/postgresql/data] shm_size: 256m + cpuset: ${PG_CPUSET:-} healthcheck: test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] interval: 2s timeout: 3s retries: 30 start_period: 10s + start_interval: 200ms stop_grace_period: 10s restart: unless-stopped networks: [backend] @@ -245,6 +283,7 @@ services: RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn} secrets: [s3_root_user, s3_root_password] volumes: [s3data:/data] + cpuset: ${S3_CPUSET:-} read_only: true tmpfs: ["/tmp:size=64m"] cap_drop: [ALL] @@ -260,7 +299,36 @@ services: networks: [backend] deploy: resources: - limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"} + limits: {cpus: "${S3_CPUS:-1}", memory: "${S3_MEMORY:-512m}"} + + reconciler: + <<: *app + pull_policy: never + profiles: [reconcile] + init: true + entrypoint: ["python", "-m", "messenger.attachments.entrypoints.reconcile"] + command: ["--dry-run"] + environment: + <<: *s3-env + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key + S3_READ_TIMEOUT_SECONDS: ${RECONCILE_S3_READ_TIMEOUT_SECONDS:-10} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key] + healthcheck: + disable: true + restart: "no" + read_only: true + cpuset: ${RECONCILE_CPUSET:-} + depends_on: + postgres: + condition: service_healthy + s3: + condition: service_healthy + deploy: + resources: + limits: {cpus: "${RECONCILE_CPUS:-1}", memory: "${RECONCILE_MEMORY:-256m}"} s3-init: <<: *s3-client @@ -284,7 +352,7 @@ services: security_opt: ["no-new-privileges:true"] cap_drop: [ALL] ports: - - "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672" + - "127.0.0.1:${BROKER_MANAGEMENT_PORT:-26086}:15672" networks: [backend, edge] volumes: - broker-data:/var/lib/rabbitmq @@ -312,6 +380,7 @@ services: command: ["python", "-m", "messenger.relay"] environment: <<: *pipeline-env + OTEL_SERVICE_NAME: messenger-relay OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS:-1} SWEEP_INTERVAL_SECONDS: ${SWEEP_INTERVAL_SECONDS:-5} secrets: @@ -329,6 +398,7 @@ services: command: ["python", "-m", "messenger.worker"] environment: <<: *pipeline-env + OTEL_SERVICE_NAME: messenger-worker WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} RENDER_TIMEOUT_SECONDS: ${RENDER_TIMEOUT_SECONDS:-30} RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-448} @@ -356,6 +426,161 @@ services: resources: limits: {cpus: "${K6_CPUS:-2}", memory: "${K6_MEMORY:-512m}"} + healthtools: + image: busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 + command: ["cp", "/bin/busybox", "/hc/busybox"] + restart: "no" + network_mode: none + volumes: + - healthtools:/hc + logging: *obs-logging + + otel-collector: + <<: *obs-hardening + image: otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849 + command: ["--config=/etc/otelcol/collector.yaml"] + networks: [backend, edge] + environment: + GOMEMLIMIT: 200MiB + volumes: + - ./infra/otel:/etc/otelcol:ro + - healthtools:/hc:ro + depends_on: + healthtools: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "/hc/busybox", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:13133/"] + interval: 10s + timeout: 3s + retries: 6 + start_period: 10s + start_interval: 2s + deploy: + resources: + limits: {cpus: "0.50", memory: 256M} + + prometheus: + <<: *obs-hardening + image: prom/prometheus:v3.15.1@sha256:fa3302279710be2f21a8301ec17a8b4f2301fd829bd113839b3b6b02b6f191a2 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d} + - --storage.tsdb.retention.size=1GB + - --web.enable-otlp-receiver + - --enable-feature=exemplar-storage + networks: [backend, edge] + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - prometheus-data:/prometheus + ports: + - "127.0.0.1:${PROMETHEUS_PORT:-26090}:9090" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"] + interval: 10s + timeout: 3s + retries: 6 + start_period: 15s + start_interval: 2s + deploy: + resources: + limits: {cpus: "0.50", memory: 512M} + + loki: + <<: *obs-hardening + image: grafana/loki:3.7.9@sha256:0db216b630ae1f348a1584c67f3cead8b4c1fc839169a39501db664744158e28 + command: ["-config.file=/etc/loki/loki.yaml"] + networks: [edge] + environment: + GOMEMLIMIT: 300MiB + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki-data:/loki + - healthtools:/hc:ro + depends_on: + healthtools: + condition: service_completed_successfully + ports: + - "127.0.0.1:${LOKI_PORT:-26310}:3100" + healthcheck: + test: ["CMD", "/hc/busybox", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3100/ready"] + interval: 10s + timeout: 3s + retries: 10 + start_period: 30s + start_interval: 2s + deploy: + resources: + limits: {cpus: "0.50", memory: 384M} + + tempo: + <<: *obs-hardening + image: grafana/tempo:2.10.10@sha256:12b0820f13ffa10ce956b6e9acc3b03f3cffded99109be6cc09d106fe56e5d38 + command: ["-config.file=/etc/tempo/tempo.yaml"] + networks: [edge] + environment: + GOMEMLIMIT: 300MiB + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo-data:/var/tempo + - healthtools:/hc:ro + depends_on: + healthtools: + condition: service_completed_successfully + ports: + - "127.0.0.1:${TEMPO_PORT:-26320}:3200" + healthcheck: + test: ["CMD", "/hc/busybox", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3200/ready"] + interval: 10s + timeout: 3s + retries: 10 + start_period: 30s + start_interval: 2s + deploy: + resources: + limits: {cpus: "0.50", memory: 384M} + + grafana: + <<: *obs-hardening + image: grafana/grafana:12.4.14@sha256:ae509d9ae8c1021ca32ec370a9ca975d40165eda9fccd1e1102d317ff8787cd7 + networks: [edge] + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + GF_PLUGINS_PREINSTALL_DISABLED: "true" + GF_LOG_LEVEL: warn + secrets: [grafana_admin_password] + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/etc/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + ports: + - "127.0.0.1:${GRAFANA_PORT:-26300}:3000" + healthcheck: + test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:3000/api/health"] + interval: 10s + timeout: 3s + retries: 10 + start_period: 30s + start_interval: 2s + deploy: + resources: + limits: {cpus: "0.50", memory: 384M} + secrets: postgres_password: file: ./secrets/postgres_password @@ -387,12 +612,23 @@ secrets: file: ./secrets/broker_url_api broker_url_worker: file: ./secrets/broker_url_worker + grafana_admin_password: + file: ./secrets/grafana_admin_password + s3_reconciler_access_key_id: + file: ./secrets/s3_reconciler_access_key_id + s3_reconciler_secret_access_key: + file: ./secrets/s3_reconciler_secret_access_key volumes: pgdata: valkeydata: s3data: broker-data: + healthtools: + prometheus-data: + loki-data: + tempo-data: + grafana-data: networks: edge: diff --git a/infra/grafana/dashboards/messenger-api.json b/infra/grafana/dashboards/messenger-api.json new file mode 100644 index 0000000..51cf452 --- /dev/null +++ b/infra/grafana/dashboards/messenger-api.json @@ -0,0 +1,863 @@ +{ + "uid": "messenger-api", + "title": "Messenger · API RED", + "description": "RED по endpoint, SLI/SLO, логи и трассы API.", + "tags": [ + "messenger", + "lab6", + "red" + ], + "timezone": "browser", + "editable": false, + "graphTooltip": 1, + "refresh": "10s", + "schemaVersion": 41, + "version": 1, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "fiscalYearStartMonth": 0, + "liveNow": false, + "links": [ + { + "title": "Messenger", + "type": "dashboards", + "tags": [ + "messenger" + ], + "asDropdown": true, + "includeVars": false, + "keepTime": true + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "iconColor": "red", + "name": "Firing alerts", + "expr": "ALERTS{alertstate=\"firing\"}", + "step": "15s", + "titleFormat": "{{alertname}}", + "textFormat": "{{alertname}} firing", + "useValueForTime": false + } + ] + }, + "templating": { + "list": [ + { + "name": "route", + "label": "Маршрут", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, http_route)", + "refId": "PrometheusVariableQueryEditor-VariableQuery", + "qryType": 1 + }, + "definition": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, http_route)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "sort": 1 + }, + { + "name": "instance", + "label": "Реплика", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery", + "qryType": 1 + }, + "definition": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, instance)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "sort": 1 + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "RPS бизнес-запросов", + "description": "Все маршруты, кроме /health/*; сумма по репликам.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + } + }, + { + "type": "stat", + "title": "Доля 5xx (2m)", + "description": "SLI доступности: 5xx / все бизнес-запросы без /health/*. Неверный пароль (401) не считается отказом.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "messenger:http_errors:ratio_rate2m", + "legendFormat": "__auto", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "id": 2, + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + } + }, + { + "type": "stat", + "title": "p95 latency (5m)", + "description": "histogram_quantile по сумме buckets всех реплик, не среднее p95.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "messenger:http_request_duration_seconds:p95_5m", + "legendFormat": "__auto", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "id": 3, + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + } + }, + { + "type": "stat", + "title": "Доля не-5xx (окно 1h)", + "description": "Доля не-5xx бизнес-запросов за 1h; цель 99%.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "1 - messenger:http_errors:ratio_rate1h", + "legendFormat": "__auto", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "decimals": 3 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "id": 4, + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + } + }, + { + "type": "row", + "title": "Rate / Errors / Duration по endpoint", + "collapsed": false, + "panels": [], + "id": 5, + "gridPos": { + "x": 0, + "y": 4, + "w": 24, + "h": 1 + } + }, + { + "type": "timeseries", + "title": "RPS по маршруту и классу статуса", + "description": "Recording rule messenger:http_requests:rate2m; status class выводится из http.response.status_code.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (http_route, http_request_method, status_class) (messenger:http_requests:rate2m{http_route=~\"$route\"})", + "legendFormat": "{{http_request_method}} {{http_route}} {{status_class}}", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 6, + "gridPos": { + "x": 0, + "y": 5, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Доля ошибок", + "description": "Числитель SLI — только 5xx; 4xx показаны отдельно и в SLO не входят.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\", http_response_status_code=~\"5..\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "5xx / все (SLI)", + "range": true, + "exemplar": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "B", + "expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\", http_response_status_code=~\"4..\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "4xx / все (клиент)", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7, + "gridPos": { + "x": 12, + "y": 5, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Latency p50 / p95 / p99", + "description": "Из histogram buckets (секунды) всех выбранных реплик; exemplars ведут в Tempo.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p95", + "range": true, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8, + "gridPos": { + "x": 0, + "y": 13, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "p95 по маршруту", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "{{http_route}}", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 9, + "gridPos": { + "x": 12, + "y": 13, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "RPS по реплике", + "description": "instance = service.instance.id (реплика + pid процесса gunicorn).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (instance) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{instance}}", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10, + "gridPos": { + "x": 0, + "y": 21, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Активные запросы", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (instance) (http_server_active_requests{job=\"messenger-api\"})", + "legendFormat": "{{instance}}", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 11, + "gridPos": { + "x": 12, + "y": 21, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "Логи и трассы API", + "collapsed": false, + "panels": [], + "id": 12, + "gridPos": { + "x": 0, + "y": 29, + "w": 24, + "h": 1 + } + }, + { + "type": "logs", + "title": "Логи API (ошибки и предупреждения)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "expr": "{service_name=\"messenger-api\"} | severity_text=~\"(?i)error|warn.*\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true, + "dedupStrategy": "none", + "prettifyLogMessage": false + }, + "id": 13, + "gridPos": { + "x": 0, + "y": 30, + "w": 12, + "h": 10 + } + }, + { + "type": "table", + "title": "Трассы API с ошибкой", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "refId": "A", + "queryType": "traceql", + "query": "{ resource.service.name = \"messenger-api\" && status = error }", + "limit": 20, + "tableType": "traces" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "id": 14, + "gridPos": { + "x": 12, + "y": 30, + "w": 12, + "h": 10 + } + } + ] +} diff --git a/infra/grafana/dashboards/messenger-pipeline.json b/infra/grafana/dashboards/messenger-pipeline.json new file mode 100644 index 0000000..c23dc39 --- /dev/null +++ b/infra/grafana/dashboards/messenger-pipeline.json @@ -0,0 +1,307 @@ +{ + "uid": "messenger-pipeline", + "title": "Messenger · Worker", + "description": "Задания worker: исходы, длительность обработки, логи и трассы.", + "tags": [ + "messenger", + "lab6", + "worker" + ], + "timezone": "browser", + "editable": false, + "graphTooltip": 1, + "refresh": "10s", + "schemaVersion": 41, + "version": 1, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "fiscalYearStartMonth": 0, + "liveNow": false, + "links": [ + { + "title": "Messenger", + "type": "dashboards", + "tags": [ + "messenger" + ], + "asDropdown": true, + "includeVars": false, + "keepTime": true + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "iconColor": "red", + "name": "Firing alerts", + "expr": "ALERTS{alertstate=\"firing\"}", + "step": "15s", + "titleFormat": "{{alertname}}", + "textFormat": "{{alertname}} firing", + "useValueForTime": false + } + ] + }, + "templating": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Worker", + "collapsed": false, + "panels": [], + "id": 8, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + } + }, + { + "type": "timeseries", + "title": "Jobs по исходу", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (job_type, outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))", + "legendFormat": "{{job_type}} {{outcome}}", + "range": true, + "exemplar": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 9, + "gridPos": { + "x": 0, + "y": 1, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Длительность обработки job", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket{}[$__rate_interval])))", + "legendFormat": "p50 {{job_type}}", + "range": true, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket{}[$__rate_interval])))", + "legendFormat": "p95 {{job_type}}", + "range": true, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket{}[$__rate_interval])))", + "legendFormat": "p99 {{job_type}}", + "range": true, + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10, + "gridPos": { + "x": 12, + "y": 1, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "Логи и трассы worker", + "collapsed": false, + "panels": [], + "id": 13, + "gridPos": { + "x": 0, + "y": 9, + "w": 24, + "h": 1 + } + }, + { + "type": "logs", + "title": "Логи worker", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "expr": "{service_name=\"messenger-worker\"}", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true, + "dedupStrategy": "none", + "prettifyLogMessage": false + }, + "id": 14, + "gridPos": { + "x": 0, + "y": 10, + "w": 12, + "h": 10 + } + }, + { + "type": "table", + "title": "Трассы worker", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "refId": "A", + "queryType": "traceql", + "query": "{ resource.service.name = \"messenger-worker\" }", + "limit": 20, + "tableType": "traces" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "id": 15, + "gridPos": { + "x": 12, + "y": 10, + "w": 12, + "h": 10 + } + } + ] +} diff --git a/infra/grafana/provisioning/alerting/alerting.yaml b/infra/grafana/provisioning/alerting/alerting.yaml new file mode 100644 index 0000000..55439cb --- /dev/null +++ b/infra/grafana/provisioning/alerting/alerting.yaml @@ -0,0 +1 @@ +apiVersion: 1 diff --git a/infra/grafana/provisioning/dashboards/messenger.yaml b/infra/grafana/provisioning/dashboards/messenger.yaml new file mode 100644 index 0000000..d8336a8 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/messenger.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 + +providers: + - name: messenger + orgId: 1 + folder: Messenger + folderUid: messenger + type: file + disableDeletion: true + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..937be30 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,79 @@ +apiVersion: 1 + +prune: true + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + prometheusType: Prometheus + prometheusVersion: 3.15.1 + timeInterval: 15s + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + urlDisplayLabel: Trace + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 1000 + derivedFields: + - name: TraceID + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: "$${__value.raw}" + urlDisplayLabel: Trace + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + nodeGraph: + enabled: true + search: + hide: false + traceQuery: + timeShiftEnabled: true + spanStartTimeShift: -30m + spanEndTimeShift: 30m + streamingEnabled: + search: true + metrics: false + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: true + filterBySpanID: false + tags: + - key: service.name + value: service_name + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: -5m + spanEndTimeShift: 5m + tags: + - key: service.name + value: job + queries: + - name: RPS маршрута + query: sum(rate(http_server_request_duration_seconds_count{$$__tags}[1m])) + - name: p95 маршрута + query: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{$$__tags}[5m]))) + lokiSearch: + datasourceUid: loki diff --git a/infra/grafana/provisioning/plugins/plugins.yaml b/infra/grafana/provisioning/plugins/plugins.yaml new file mode 100644 index 0000000..4064289 --- /dev/null +++ b/infra/grafana/provisioning/plugins/plugins.yaml @@ -0,0 +1,3 @@ +apiVersion: 1 + +apps: [] diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..8415033 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,77 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2026-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ingester: + chunk_idle_period: 5m + max_chunk_age: 1h + +limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 168h + reject_old_samples: true + reject_old_samples_max_age: 24h + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + max_query_lookback: 168h + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: + - service.name + - deployment.environment.name + - action: structured_metadata + attributes: + - service.instance.id + - service.version + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + compaction_interval: 10m + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 64 + +pattern_ingester: + enabled: false + +ruler: + enable_api: false + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..70109ed --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,105 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + max_request_body_size: 8388608 + +processors: + memory_limiter: + check_interval: 1s + limit_percentage: 75 + spike_limit_percentage: 20 + redaction: + allow_all_keys: true + summary: silent + blocked_key_patterns: + - "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$" + - "^http\\.(request|response)\\.header\\..+" + - "^url\\.(query|full)$" + - "^http\\.(url|target)$" + - "^(messenger\\.)?(message\\.)?(text|body)$" + - "^(file\\.)?filename$" + blocked_values: + - "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}" + - "(?i)basic\\s+[A-Za-z0-9+/=]{8,}" + - "(?i)x-amz-(signature|credential)=[^&\\s]+" + - "-----BEGIN [A-Z ]*PRIVATE KEY-----" + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + +exporters: + otlp_http/tempo: + traces_endpoint: http://tempo:4318/v1/traces + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + sizer: items + queue_size: 20000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + otlp_http/prometheus: + metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + sizer: items + queue_size: 50000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + otlp_http/loki: + logs_endpoint: http://loki:3100/otlp/v1/logs + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + sizer: items + queue_size: 20000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + metrics: + level: normal + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + without_type_suffix: false + without_units: false + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, redaction, batch] + exporters: [otlp_http/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, redaction, batch] + exporters: [otlp_http/prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, redaction, batch] + exporters: [otlp_http/loki] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..f854543 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,41 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/rules/*.yml + +otlp: + translation_strategy: UnderscoreEscapingWithSuffixes + promote_resource_attributes: + - service.name + - service.version + - deployment.environment.name + - container.name + - compose.project + - compose.service + keep_identifying_resource_attributes: true + +storage: + tsdb: + out_of_order_time_window: 30m + exemplars: + max_exemplars: 100000 + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + - job_name: loki + static_configs: + - targets: ["loki:3100"] + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] + - job_name: grafana + static_configs: + - targets: ["grafana:3000"] diff --git a/infra/prometheus/rules/messenger.recording.yml b/infra/prometheus/rules/messenger.recording.yml new file mode 100644 index 0000000..d85c820 --- /dev/null +++ b/infra/prometheus/rules/messenger.recording.yml @@ -0,0 +1,32 @@ +groups: + - name: messenger-http + interval: 15s + rules: + - record: messenger:http_requests:rate2m + expr: | + sum by (http_route, http_request_method, status_class) ( + label_replace( + rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[2m]), + "status_class", "${1}xx", "http_response_status_code", "([1-5])[0-9][0-9]" + ) + ) + - record: messenger:http_errors:ratio_rate2m + expr: | + (sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_code=~"5.."}[2m])) or vector(0)) + / + sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[2m])) + - record: messenger:http_errors:ratio_rate1h + expr: | + (sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_code=~"5.."}[1h])) or vector(0)) + / + sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1h])) + - record: messenger:http_request_duration_seconds:p50_5m + expr: histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m]))) + - record: messenger:http_request_duration_seconds:p95_5m + expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m]))) + - record: messenger:http_request_duration_seconds:p99_5m + expr: histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m]))) + - record: messenger:jobs:rate5m + expr: sum by (job_type, outcome) (rate(messenger_jobs_processed_total[5m])) + - record: messenger:job_duration_seconds:p95_5m + expr: histogram_quantile(0.95, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket[5m]))) \ No newline at end of file diff --git a/infra/s3/api-policy.json b/infra/s3/api-policy.json index 1e46aa4..90635a6 100644 --- a/infra/s3/api-policy.json +++ b/infra/s3/api-policy.json @@ -4,20 +4,8 @@ { "Sid": "MessengerApiObjects", "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject", - "s3:AbortMultipartUpload", - "s3:ListMultipartUploadParts" - ], + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], "Resource": ["arn:aws:s3:::messenger-attachments/*"] - }, - { - "Sid": "MessengerApiBucket", - "Effect": "Allow", - "Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"], - "Resource": ["arn:aws:s3:::messenger-attachments"] } ] } diff --git a/infra/s3/cli.sh b/infra/s3/cli.sh index 9e0937f..54ff852 100755 --- a/infra/s3/cli.sh +++ b/infra/s3/cli.sh @@ -4,7 +4,8 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; } endpoint="${S3_ENDPOINT:-http://s3:9000}" RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}" -export RC_HOST_root RC_HOST_api +RC_HOST_reconciler="${endpoint%%://*}://$(secret s3_reconciler_access_key_id):$(secret s3_reconciler_secret_access_key)@${endpoint#*://}" +export RC_HOST_root RC_HOST_api RC_HOST_reconciler rc alias set anon "$endpoint" --anonymous >/dev/null case "${1:-}" in sh|wget) exec "$@" ;; diff --git a/infra/s3/init.sh b/infra/s3/init.sh index c9bd299..67c9c4f 100755 --- a/infra/s3/init.sh +++ b/infra/s3/init.sh @@ -4,23 +4,22 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; } bucket="${S3_BUCKET:-messenger-attachments}" endpoint="${S3_ENDPOINT:-http://s3:9000}" policy="${S3_API_POLICY:-messenger-api}" +reconciler="${S3_RECONCILER_POLICY:-messenger-reconciler}" RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" export RC_HOST_root -api_key=$(secret s3_access_key_id) +grant() { + sed "s/messenger-attachments/$bucket/g" "/s3/$2" > "/tmp/$2" + rc admin policy create root "$1" "/tmp/$2" >/dev/null + rc admin user add root "$(secret "$3")" "$(secret "$4")" >/dev/null + rc admin policy attach root "$1" --user "$(secret "$3")" >/dev/null + rc admin user info root "$(secret "$3")" --json | grep -q "\"$1\"" +} rc ready root --timeout 10 >/dev/null rc bucket create --ignore-existing "root/$bucket" >/dev/null rc bucket anonymous set private "root/$bucket" >/dev/null -sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json -rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null -rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null -rc admin policy attach root "$policy" --user "$api_key" >/dev/null -worker_key=$(secret s3_worker_access_key_id) -sed "s/messenger-attachments/$bucket/g" /s3/worker-policy.json > /tmp/worker-policy.json -rc admin policy create root messenger-worker /tmp/worker-policy.json >/dev/null -rc admin user add root "$worker_key" "$(secret s3_worker_secret_access_key)" >/dev/null -rc admin policy attach root messenger-worker --user "$worker_key" >/dev/null +grant "$policy" api-policy.json s3_access_key_id s3_secret_access_key +grant "$reconciler" reconciler-policy.json s3_reconciler_access_key_id s3_reconciler_secret_access_key +grant messenger-worker worker-policy.json s3_worker_access_key_id s3_worker_secret_access_key rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"' -rc admin user info root "$api_key" --json | grep -q "\"$policy\"" -rc admin user info root "$worker_key" --json | grep -q '"messenger-worker"' -echo "s3-init: bucket=$bucket anonymous=private policy=$policy worker=messenger-worker lifecycle=abort-incomplete-uploads" +echo "s3-init: bucket=$bucket anonymous=private policies=$policy,$reconciler lifecycle=abort-incomplete-uploads" diff --git a/infra/s3/reconciler-policy.json b/infra/s3/reconciler-policy.json new file mode 100644 index 0000000..a28fa83 --- /dev/null +++ b/infra/s3/reconciler-policy.json @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "MessengerReconcilerObjects", + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + }, + { + "Sid": "MessengerReconcilerListing", + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..01e1117 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,48 @@ +stream_over_http_enabled: true + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3200 + grpc_listen_port: 9095 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + trace_idle_period: 10s + +compactor: + compaction: + block_retention: 168h + +query_frontend: + search: + duration_slo: 5s + trace_by_id: + duration_slo: 5s + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +overrides: + defaults: + ingestion: + rate_limit_bytes: 15000000 + burst_size_bytes: 20000000 + max_traces_per_user: 50000 + global: + max_bytes_per_trace: 5000000 + +usage_report: + reporting_enabled: false diff --git a/openspec/changes/forward-lab5-sqlalchemy-foundation/findings.json b/openspec/changes/forward-lab5-sqlalchemy-foundation/findings.json new file mode 100644 index 0000000..671e10f --- /dev/null +++ b/openspec/changes/forward-lab5-sqlalchemy-foundation/findings.json @@ -0,0 +1,7 @@ +[ + { + "severity": "minor", + "file": "scripts/limits.py:33", + "problem": "Унаследованный POLICY и проверка опубликованных портов в .github/workflows/application.yml:85 ещё не учитывают существующий OTel WIP. Вызов declared() с текущей Compose-конфигурацией помечает все шесть OTel/helper-сервисов как service is not covered by the policy; CI разрешает порты только lb/broker. Это ограничение вспомогательного аудита; smoke и проверки переноса проходят. Полный CI в этой фазе не запускался." + } +] diff --git a/openspec/changes/forward-lab5-sqlalchemy-foundation/tasks.md b/openspec/changes/forward-lab5-sqlalchemy-foundation/tasks.md index 593953d..9e4543b 100644 --- a/openspec/changes/forward-lab5-sqlalchemy-foundation/tasks.md +++ b/openspec/changes/forward-lab5-sqlalchemy-foundation/tasks.md @@ -1,16 +1,16 @@ ## 1. Основа до грейдов ЛР6 -- [ ] 1.1 Зафиксировать источник, WIP и контрольные суммы защищённых файлов. -- [ ] 1.2 Перенести зависимости, платформу SQLAlchemy и Alembic с legacy adoption. -- [ ] 1.3 Перенести identity и собственные проверки зоны. -- [ ] 1.4 Перенести messaging и собственные проверки зоны. -- [ ] 1.5 Перенести attachments, reconciler, pipeline и собственные проверки зоны. -- [ ] 1.6 Объединить точки сборки и OTel-порты, сохранив ack после COMMIT. -- [ ] 1.7 Объединить Dockerfile, Compose, Makefile, свои скрипты/CI; сохранить OTel WIP и порты ЛР6. -- [ ] 1.8 Выполнить make check, make lint, make student-test и проверить отсутствие raw asyncpg адаптеров. +- [x] 1.1 Зафиксировать источник, WIP и контрольные суммы защищённых файлов. +- [x] 1.2 Перенести зависимости, платформу SQLAlchemy и Alembic с legacy adoption. +- [x] 1.3 Перенести identity и собственные проверки зоны. +- [x] 1.4 Перенести messaging и собственные проверки зоны. +- [x] 1.5 Перенести attachments, reconciler, pipeline и собственные проверки зоны. +- [x] 1.6 Объединить точки сборки и OTel-порты, сохранив ack после COMMIT. +- [x] 1.7 Объединить Dockerfile, Compose, Makefile, свои скрипты/CI; сохранить OTel WIP и порты ЛР6. +- [x] 1.8 Выполнить make check, make lint, make student-test и проверить отсутствие raw asyncpg адаптеров. ## 2. Проверка и завершение переноса -- [ ] 2.1 Поднять свой стенд, выполнить публичный smoke и повторный migrate; записать Docker-фон. -- [ ] 2.2 Остановить свои стенды без удаления постоянных volumes, сверить защищённые файлы. +- [x] 2.1 Поднять свой стенд, выполнить публичный smoke и повторный migrate; записать Docker-фон. +- [x] 2.2 Остановить свои стенды без удаления постоянных volumes, сверить защищённые файлы. - [ ] 2.3 Записать реальные результаты и ограничения в verification.md; синхронизировать specs и архивировать change. diff --git a/openspec/changes/forward-lab5-sqlalchemy-foundation/verification.md b/openspec/changes/forward-lab5-sqlalchemy-foundation/verification.md new file mode 100644 index 0000000..340eb9a --- /dev/null +++ b/openspec/changes/forward-lab5-sqlalchemy-foundation/verification.md @@ -0,0 +1,36 @@ +# Проверка переноса основы ЛР5 → ЛР6 + +Источник: immutable snapshot `lab5-f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071`, SHA `f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071`. Перенесён разрешённый diff от `64d5c67`. Исходный HEAD ЛР6: `a8836ce`; proposal: `dc9256b`. Дата: 2026-11-06. + +Перенесены SQLAlchemy 2.0.45/Alembic 1.16.5, поздние исправления identity/messaging, versioned pipeline и reconciler, собственные тесты, Dockerfile/Compose, зависимости, скрипты и CI. Существующие OTel-порты, SDK и инфраструктура сохранены. Версии всех общих зависимостей совпадают с snapshot, исключая намеренную версию проекта 1.6.0 вместо 1.5.0. Raw asyncpg адаптеров больше нет; asyncpg остаётся драйвером SQLAlchemy и точкой инструментации. Legacy SQL только в тестовых fixtures. + +## Команды и результаты + +- `make check` — PASS, контракт 1.6.0 и синтаксис Python 3.9 скриптов. +- `make validate` — PASS, OpenAPI 3.1 и JSON Schema. +- `make lint` — ruff check, format check, ty: PASS. +- Первый `make student-test` — 513 passed, 1 skipped, 19 deselected, 2 errors за 213.00 с. Обе ошибки — ready-probe RustFS через системный HTTP-прокси macOS. Они не объявляются успехом. +- `NO_PROXY=localhost,127.0.0.1,::1 make student-test PYTEST_ARGS=student-tests/adapters/test_s3_storage.py` — 3 passed за 6.91 с: обе ошибки устранены настройкой окружения, тесты и assertions не менялись. +- Итоговый `NO_PROXY=localhost,127.0.0.1,::1 make student-test` — **515 passed, 1 skipped, 19 deselected за 154.75 с**, exit 0. Включает 36 проверок миграций, наследуемые сценарии pipeline и сохранённые OTel-тесты. +- `docker compose config --quiet`, `make secrets`, `docker compose build`, `docker compose up -d --wait` — exit 0; все долгоживущие сервисы healthy, migrate/s3-init/healthtools завершены. Использованы переменные окружения ниже. +- Первый `make test BASE_URL=http://127.0.0.1:18086` — FAIL RemoteDisconnected из-за системного proxy. Прямой curl без proxy — 200. +- `NO_PROXY=localhost,127.0.0.1,::1 make test BASE_URL=http://127.0.0.1:18086` — `PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.` +- `make migrate` — повторный Alembic upgrade head, exit 0. +- `make down` — exit 0; project mtusi2026lab6 остановлен, девять постоянных volumes сохранены. +- SHA-256 сравнение 22 защищённых файлов и 14 исходных OTel WIP файлов — без изменений; исходный snapshot остался чистым. `git diff --check` — PASS. + +## Условия Docker + +Все Compose-команды выполнялись из основного каталога с `COMPOSE_PROJECT_NAME=mtusi2026lab6`, `LB_HTTP_PORT=18086`, `LB_STATS_PORT=26085`, `BROKER_MANAGEMENT_PORT=26086`, `PROMETHEUS_PORT=26090`, `LOKI_PORT=26310`, `TEMPO_PORT=26320`, `GRAFANA_PORT=26300`. При up задано `OTEL_SDK_DISABLED=false`; это подтверждено для обеих реплик API, relay и worker. Порты тестовых контейнеров — 26000–26999. + +При первой проверке docker ps был пуст. К 09:20:06 UTC появился чужой project `hlpipe-lab4-forward-reconcile-20261106`: API ×2, PostgreSQL, Valkey, S3 и LB. Он не останавливался и не изменялся. Одновременно работали собственные одноразовые контейнеры тестов. Фоновая нагрузка неизвестна; нагрузочных замеров в этой фазе нет. Эти результаты подтверждают функциональность, не производительность. Повтор затронутого suite выполнен с NO_PROXY; позднее полный suite повторён с тем же окружением для единого результата. + +Локальные исходные логи сохранены в gitignored `.verification/forward/`: student-test*.txt, smoke*.txt, build.txt, up.txt, migrate-rerun.txt, compose-ps.txt, telemetry-env.json, docker-background.json, down.txt, limits-wip.json. Они не содержат дампов и не читают значения файлов secrets. + +## Ограничения для отдельного отчётного этапа + +Один тест RLIMIT_AS пропускается на macOS и должен выполняться на Linux; он не объявлен пройденным. Девятнадцать stack-тестов исключены штатным маркером; вместо полного отказного suite в этой фазе выполнен заданный публичный smoke. Грейды 3/4/5 ЛР6, нагрузка, telemetry-check, публикация и Telegram не выполнялись. Активный change `add-lab6-telemetry-pipeline` не закрывается этим переносом. + +Minor из findings.json: вспомогательный POLICY в scripts/limits.py и CI-allowlist портов ещё не учитывают OTel WIP. Это сохранено для раздела ограничений будущего REPORT; текущие REPORT.md, AGENTS.md и evidence не изменены по прямой инструкции пользователя. Замечание не является дефектом переноса бизнес-API и не требует fix в этой фазе. + +Полный повтор завершён после остановки стенда ЛР6; одноразовые контейнеры тестов удалены fixture-finalizers. Конфигурации наблюдаемости сохранены как исходный WIP, а не как завершённый грейд ЛР6. diff --git a/openspec/config.yaml b/openspec/config.yaml index f09c24b..90441c1 100644 --- a/openspec/config.yaml +++ b/openspec/config.yaml @@ -41,7 +41,7 @@ context: | Python 3.14 (аннотации вычисляются лениво по PEP 649, `TYPE_CHECKING` и `from __future__ import annotations` не используются), FastAPI, gunicorn с несколькими `UvicornWorker` (пакет `uvicorn-worker`, `preload_app = False`), dishka (DI, интеграции FastAPI - и FastStream), pydantic v2 + pydantic-settings, structlog, asyncpg, redis-py для Valkey, + и FastStream), pydantic v2 + pydantic-settings, structlog, SQLAlchemy 2.x async (asyncpg), Alembic, redis-py для Valkey, argon2-cffi, aiobotocore, Pillow, FastStream 0.7 с `faststream[rabbit,otel]` поверх aio-pika, OpenTelemetry Python (API/SDK, OTLP/HTTP protobuf exporter без gRPC, инструментации FastAPI и asyncpg, версии в `uv.lock`), pytest + pytest-asyncio + httpx. Зависимости — uv и `uv.lock`; @@ -68,7 +68,7 @@ context: | main.py сборка HTTP-приложения: вклады зон, связи зон портами, телеметрия процесса relay.py сборка процесса relay: outbox → брокер, возврат просроченных аренд worker.py сборка процесса worker: FastStream-подписчик очереди заданий - migrate.py одноразовая точка входа: применяет SQL-миграции под advisory lock + alembic/ async миграции с legacy adoption и advisory lock identity/ пользователи, пароли и сессии messaging/ чаты, история и идемпотентная отправка, ссылки сообщения на вложения attachments/ вложения: приём и staging, задания и outbox, обработка и варианты, порты diff --git a/openspec/specs/lab2-persistent-storage/spec.md b/openspec/specs/lab2-persistent-storage/spec.md index 9884db4..1c2492d 100644 --- a/openspec/specs/lab2-persistent-storage/spec.md +++ b/openspec/specs/lab2-persistent-storage/spec.md @@ -19,27 +19,32 @@ PostgreSQL SHALL хранить пользователей, сессии, чат ### Requirement: Версионируемые миграции -Схема SHALL создаваться только миграциями: упорядоченными SQL-файлами, каждый из которых применяется в своей транзакции и записывается в журнал применённых версий. Миграции SHALL применяться одним одноразовым процессом до старта реплик API, а не каждой репликой; одновременный запуск двух процессов SHALL сериализоваться. Повторный запуск SHALL NOT менять схему и данные. Изменение текста уже применённой миграции SHALL останавливать запуск с ошибкой. Новая миграция SHALL быть аддитивной: работающая предыдущая версия API SHALL продолжать работать на схеме после неё. +Схема SHALL создаваться версионируемыми миграциями, каждая в своей транзакции. Одноразовый процесс `migrate` SHALL завершаться до старта API; одновременные запуски SHALL сериализоваться. Повторный запуск SHALL NOT менять данные. При переходе со старого SQL-журнала процесс SHALL проверить контрольные суммы и фактическую схему до принятия версии. Неизвестная или изменённая история SHALL останавливать запуск без изменения пользовательских данных. Аддитивные изменения SHALL сохранять совместимость с предыдущей версией приложения. Основание: обязательное постоянное хранение и сохранение lab1–lab4 в README. #### Scenario: Миграции с пустого volume -- **WHEN** стенд поднимается на пустом volume PostgreSQL -- **THEN** процесс миграций применяет все версии по порядку и завершается кодом 0, после чего стартуют реплики API +- **WHEN** стенд запускается с пустой PostgreSQL +- **THEN** все миграции применяются по порядку, migrate завершается кодом 0 и запускается API #### Scenario: Повторный запуск -- **WHEN** процесс миграций запускается повторно на той же БД -- **THEN** он завершается кодом 0, журнал версий и число строк во всех таблицах не меняются +- **WHEN** migrate повторно запускается на той же БД +- **THEN** он завершается кодом 0 без изменения пользовательских строк + +#### Scenario: Переход старого стенда ЛР5 + +- **WHEN** БД содержит проверенный SQL-журнал ЛР5, вложения, задания, outbox и варианты +- **THEN** переход сохраняет строки, ограничения, состояния и ссылки на объекты #### Scenario: Изменённая миграция -- **WHEN** текст применённой миграции изменён и процесс миграций запущен -- **THEN** процесс завершается ненулевым кодом и схему не трогает +- **WHEN** контрольная сумма SQL-журнала или фактическая схема не совпадает с ожидаемой +- **THEN** принятие истории завершается ошибкой без изменения данных и отметки версии #### Scenario: Реплики не мигрируют схему -- **WHEN** стартует или перезапускается реплика API -- **THEN** она не применяет миграции; журнал версий меняет только процесс `migrate` +- **WHEN** стартует реплика API, relay или worker +- **THEN** только отдельный процесс migrate изменяет журнал миграций ### Requirement: Атомарное создание чата diff --git a/openspec/specs/lab4-storage-reconciler/spec.md b/openspec/specs/lab4-storage-reconciler/spec.md new file mode 100644 index 0000000..6f50966 --- /dev/null +++ b/openspec/specs/lab4-storage-reconciler/spec.md @@ -0,0 +1,39 @@ +# lab4-storage-reconciler Specification + +## Purpose + +Сохраняет совместимость перенесённой из ЛР4 сверки PostgreSQL и S3 с ранее записанными объектами и ограничивает её область, чтобы не вмешиваться в асинхронную обработку ЛР5. Основание — сохранение предыдущего решения при переносе; не является заявкой на грейд ЛР5. + +## Requirements + +### Requirement: Ручная сверка legacy-объектов + +Одноразовый reconciler SHALL по умолчанию работать в dry-run над прежним префиксом `attachments/`, выдавая JSON со счётчиками расхождений. Apply SHALL сохранять свежие объекты и повторно проверять ссылку в БД перед удалением старого orphan. Одновременный второй проход SHALL завершаться с признаком skipped. + +#### Scenario: Dry-run без изменения данных + +- **WHEN** оператор запускает reconciler без apply для legacy-объектов +- **THEN** он возвращает JSON и не меняет строки или байты + +#### Scenario: Повторная проверка защищает объект + +- **WHEN** после листинга старого orphan появляется строка с его ключом +- **THEN** apply сохраняет объект + +### Requirement: Карантин сохраняет данные + +При подтверждённом расхождении готового legacy-вложения apply SHALL поместить метаданные в карантин без удаления строки и объекта. GET `/api/v1/attachments/{id}/content` SHALL возвращать 404 с ошибкой not_found для карантина, метаданные остаются доступны участнику. Снятие карантина SHALL требовать совпадения SHA-256 даже без deep. + +#### Scenario: Возвращены другие байты той же длины + +- **WHEN** объект в карантине заменён байтами прежнего размера, но другого SHA-256 +- **THEN** карантин остаётся и скачивание не разрешается + +### Requirement: Разделение со staging и pipeline + +Перенесённая сверка SHALL NOT обходить или удалять объекты из `staging/`, `originals/` и `variants/`. Этими префиксами управляет pipeline ЛР5 и его существующая уборка staging. + +#### Scenario: Готовое вложение ЛР5 вне legacy-префикса + +- **WHEN** у нового вложения оригинал и варианты опубликованы pipeline +- **THEN** legacy-сверка не считает их orphan и не изменяет их diff --git a/openspec/specs/lab5-versioned-pipeline/spec.md b/openspec/specs/lab5-versioned-pipeline/spec.md new file mode 100644 index 0000000..8d9c272 --- /dev/null +++ b/openspec/specs/lab5-versioned-pipeline/spec.md @@ -0,0 +1,51 @@ +# lab5-versioned-pipeline Specification + +## Purpose + +Определяет наблюдаемые гарантии versioned pipeline и воспроизводимое доказательство идемпотентности для трека 2 оценки «5» README лабораторной 5. + +## Requirements + +### Requirement: Явная постановка версии + +Оператор SHALL иметь команду reprocess с attachment-id и pipeline-version 1 или 2. Повтор той же версии SHALL возвращать noop без новой job или outbox. Для неизвестного вложения возвращается not_found, для pending/failed — conflict без изменения состояния; failed восстанавливается существующим replay. Успех queued SHALL означать durable job и outbox. Требование трассируется на README, трек 2. + +#### Scenario: Одновременные операторские запросы +- **WHEN** несколько команд запрашивают версию 2 одного вложения +- **THEN** ровно одна получает queued, остальные noop; создаётся одна job версии 2 и одно намерение доставки + +#### Scenario: Ошибочный запрос +- **WHEN** указана неподдерживаемая версия, неизвестный id или failed-вложение +- **THEN** новая работа не создаётся, оператор получает явную ошибку/результат отказа + +### Requirement: Монотонная публикация различимых версий + +Версии 1 и 2 SHALL давать детерминированные варианты с различимыми recipe при сохранении форматов/размеров контракта. API SHALL отдавать только опубликованную версию, published_version SHALL возрастать, а ready SHALL NOT откатываться. Работа или ошибка старой версии SHALL NOT перезаписывать новую. Ошибка одной версии SHALL NOT завершать всё вложение в failed, пока остаётся активная версия. Требование трассируется на README, трек 2 и contracts/attachment-states.md. + +#### Scenario: Новая версия завершилась первой +- **WHEN** версия 2 достигла ready до доставки версии 1 +- **THEN** последующая версия 1 не меняет опубликованные варианты; оригинал и байты версии 2 сохраняются + +#### Scenario: Старый держатель завершился позже +- **WHEN** worker версии 1 задержан после записи объектов, а версия 2 опубликована +- **THEN** завершение версии 1 не перезаписывает опубликованный результат версии 2 + +#### Scenario: Уборка при активной версии +- **WHEN** новая версия уже готова, а старый worker задержан до или после side effect и запускается очистка staging +- **THEN** исходник активного worker сохраняется до его завершения; последующая обработка ready-вложения использует устойчивый опубликованный оригинал и не зависит от незавершённой уборки + +#### Scenario: Ошибка при нескольких версиях +- **WHEN** одна версия завершилась ошибкой, другая ещё обрабатывается +- **THEN** вложение не становится failed до завершения оставшейся работы, а уже ready продолжает скачиваться + +### Requirement: Воспроизводимые инварианты при отказах + +Одна команда make track2 SHALL запускать контроль без отказа и автоматические duplicate, reorder, crash до/после эффекта, SIGKILL и stale job после версии 2. Результат SHALL включать UTC/SHA, фактические контейнеры, отсутствие потери подтверждённых 202, лишних эффектов и отката ready/версии, соответствие байтов метаданным и объектов собственных вложений ожидаемым ключам. Исследование SHALL содержать три последовательных повтора, сравнение, ограничения и объяснение at-least-once доставки. Требование трассируется на README, трек 2; унаследованные негативные сценарии остаются в спецификациях уровней 3/4. + +#### Scenario: Повтор после эффекта +- **WHEN** worker падает после PUT либо после COMMIT до ack и получает ту же job снова +- **THEN** итоговое вложение доступно, лишнего набора вариантов нет, принятая job не потеряна + +#### Scenario: Проверяемая серия +- **WHEN** три раза последовательно выполняется make track2 с разными output paths +- **THEN** каждый отчёт проверяет собственные принятые IDs, содержит реальные результаты и не удаляет данные предыдущих прогонов diff --git a/pyproject.toml b/pyproject.toml index e4bb50e..635e61f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,8 @@ dependencies = [ "gunicorn>=26.2.0", "orjson>=3.12.0", "asyncpg>=0.32.0", + "sqlalchemy[asyncio]==2.0.45", + "alembic==1.16.5", "argon2-cffi>=25.1.0", "uvicorn-worker>=0.4.0", "redis>=8.1.0", @@ -34,6 +36,7 @@ dev = [ "httpx>=0.28.0", "ruff>=0.14.0", "ty>=0.0.1a21", + "openapi-spec-validator==0.7.2", ] [build-system] @@ -87,6 +90,7 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "ARG002", "T20", "TID251"] +"tools/*.py" = ["T20"] "scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"] "scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"] "scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"] @@ -96,10 +100,10 @@ ignore = [ "src/messenger/*/entrypoints/**" = ["TID251", "N818", "PLR0917", "S105"] "src/messenger/shared/config.py" = ["ARG002", "ARG003"] "src/messenger/main.py" = ["TID251"] -"src/messenger/migrate.py" = ["TID251"] "src/messenger/relay.py" = ["TID251"] "src/messenger/worker.py" = ["TID251"] "src/messenger/admin.py" = ["TID251"] +"alembic/**" = ["TID251"] "src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"] [tool.ruff.lint.flake8-tidy-imports.banned-api] @@ -112,7 +116,10 @@ ignore = [ known-first-party = ["messenger"] [tool.ty.src] -include = ["src", "student-tests"] +include = ["src", "student-tests", "alembic"] + +[tool.ty.environment] +extra-paths = ["alembic"] [tool.ty.rules] unresolved-import = "error" @@ -123,7 +130,7 @@ possibly-missing-attribute = "warn" testpaths = ["student-tests"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" -addopts = "-q --strict-markers -m 'not stack'" +addopts = "--strict-markers -m 'not stack'" markers = [ "stack: black-box проверки против поднятого compose-стенда (make test-faults)", ] diff --git a/scripts/capture_evidence.py b/scripts/capture_evidence.py new file mode 100644 index 0000000..6f9ff85 --- /dev/null +++ b/scripts/capture_evidence.py @@ -0,0 +1,73 @@ +import argparse +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SETTINGS = ( + "COMPOSE_PROJECT_NAME", + "COMPOSE_FILE", + "LB_HTTP_PORT", + "LB_STATS_PORT", + "BASE_URL", + "EVIDENCE_DIR", + "RECONCILE_EVIDENCE_DIR", + "NO_PROXY", + "S3_CPUS", + "S3_MEMORY", + "UPLOAD_GRACE_SECONDS", + "API_IMAGE", + "API_REPLICAS", + "WEB_CONCURRENCY", + "MALLOC_MMAP_THRESHOLD_", +) + + +def main(): + parser = argparse.ArgumentParser(description="Capture a command without overwriting evidence") + parser.add_argument("output", type=Path) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("command is required") + args.output.parent.mkdir(parents=True, exist_ok=True) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + started = time.monotonic() + with args.output.open("x") as output: + + def write(line): + line = line.replace(str(ROOT), "") + output.write(line) + output.flush() + sys.stdout.write(line) + sys.stdout.flush() + + write(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit: {commit}\n") + write( + "environment: " + + json.dumps({k: os.environ[k] for k in SETTINGS if k in os.environ}) + + "\n" + ) + write("$ " + shlex.join(command) + "\n") + with subprocess.Popen( + command, + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + ) as process: + for line in process.stdout: + write(line) + code = process.wait() + write(f"exit_code: {code} elapsed_seconds: {time.monotonic() - started:.3f}\n") + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/clean-clone.sh b/scripts/clean-clone.sh new file mode 100755 index 0000000..b4348ce --- /dev/null +++ b/scripts/clean-clone.sh @@ -0,0 +1,54 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +repo=$PWD +project="hlpipe-lab6-clean-$(date -u +%Y%m%d%H%M%S)" +dir="$repo/.verification/$project" +out="$repo/${EVIDENCE_DIR:-evidence}/clean-clone.txt" +sha=$(git rev-parse HEAD) +[ ! -e "$out" ] || { echo "Choose a new EVIDENCE_DIR" >&2; exit 2; } +mkdir -p "$(dirname "$dir")" "$(dirname "$out")" +exec 3>&1 +exec > "$out" 2>&1 +cleanup() { + status=$? + trap - EXIT HUP INT TERM + if [ -d "$dir/.git" ]; then + cd "$dir" + docker compose --profile bench --profile tools --profile reconcile down || status=1 + docker volume ls --filter "label=com.docker.compose.project=$project" + fi + echo "exit_code: $status; volumes retained" + sed "s#$repo##g" "$out" >&3 + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' HUP INT TERM +step() { echo "\$ $*"; "$@"; } +export COMPOSE_PROJECT_NAME="$project" COMPOSE_FILE=compose.yaml +export LB_HTTP_PORT=18086 LB_STATS_PORT=26085 BROKER_MANAGEMENT_PORT=26086 BASE_URL=http://127.0.0.1:18086 +export PROMETHEUS_PORT=26090 LOKI_PORT=26310 TEMPO_PORT=26320 GRAFANA_PORT=26300 +export NO_PROXY=localhost,127.0.0.1,::1 +unset API_IMAGE +echo "date: $(date -u +%FT%TZ) commit: $sha project: $project" +echo "HTTP=$LB_HTTP_PORT stats=$LB_STATS_PORT" +step git clone --quiet --no-hardlinks "$repo" "$dir" +cd "$dir" +step git checkout --quiet --detach "$sha" +step make secrets +step docker compose --env-file .env config --quiet +step make up +step docker compose ps -a --format '{{.Service}} {{.State}} {{.Status}} {{.Ports}}' +step make test +step make test-persistence ACTION_SCRIPT=scripts/restart.sh +step make test-failover ACTION_SCRIPT=scripts/stop-one.sh +step docker compose ps -a --format '{{.Service}} {{.Name}} {{.State}}' api +step make restore +step make scenario SCENARIO=failover-upload +step make restore +step make scenario SCENARIO=s3-outage +step sh scripts/s3-check.sh +step make reconcile +step make migrate +step make down +step docker volume ls --filter "label=com.docker.compose.project=$project" diff --git a/scripts/detection.sh b/scripts/detection.sh index 5ef0d35..5282bae 100755 --- a/scripts/detection.sh +++ b/scripts/detection.sh @@ -8,7 +8,7 @@ replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; } detect() { python3 scripts/probe.py detect "$@"; } failed=0 detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \ - -- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1 + -- docker stop -t "${STOP_TIMEOUT:-7}" "$(replica)" || failed=1 detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \ -- docker kill "$(replica)" || failed=1 hung=$(replica) diff --git a/scripts/docker-stats.sh b/scripts/docker-stats.sh new file mode 100755 index 0000000..48ea851 --- /dev/null +++ b/scripts/docker-stats.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +out=${1:?usage: docker-stats.sh } +vm=$(dirname "$out")/vm_stat.csv +cd "$(dirname "$0")/.." +project=$(docker compose config | sed -n 's/^name: //p') +echo 'ts,name,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids' > "$out" +echo 'ts,busy,total,cpus' > "$vm" +while :; do + ts=$(date +%s) + docker compose exec -T valkey cat /proc/stat 2>/dev/null | awk -v ts="$ts" '/^cpu /{b=$2+$3+$4+$7+$8+$9; t=b+$5+$6} /^cpu[0-9]/{n++} END{if (t) print ts "," b "," t "," n}' >> "$vm" || true + ids=$(docker ps -q --filter "label=com.docker.compose.project=$project") + [ -z "$ids" ] || docker stats --no-stream --format "$ts,{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true + sleep "${INTERVAL:-1}" +done diff --git a/scripts/evidence-attachments.sh b/scripts/evidence-attachments.sh index 87c241a..5161b56 100755 --- a/scripts/evidence-attachments.sh +++ b/scripts/evidence-attachments.sh @@ -1,12 +1,14 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." +out=${EVIDENCE_DIR:-evidence} +mkdir -p "$out" psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; } value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; } cli() { docker compose run --rm --no-deps -T s3-cli "$@"; } bucket=${S3_BUCKET:-messenger-attachments} { - echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" + echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)" echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'" docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}' for table in attachments message_attachments; do @@ -36,5 +38,5 @@ bucket=${S3_BUCKET:-messenger-attachments} docker compose config --format json | python3 -c 'import json,sys for name, service in sorted(json.load(sys.stdin)["services"].items()): print(name, sorted(s["source"] for s in service.get("secrets", [])))' -} > evidence/storage-layout.txt -cat evidence/storage-layout.txt +} > "$out"/storage-layout.txt +cat "$out"/storage-layout.txt diff --git a/scripts/evidence-cache.sh b/scripts/evidence-cache.sh index ad01762..f116557 100755 --- a/scripts/evidence-cache.sh +++ b/scripts/evidence-cache.sh @@ -11,14 +11,25 @@ cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status' echo "\$ valkey-cli dbsize; info stats | grep keyspace" cli dbsize cli "info stats" | grep -E '^keyspace_(hits|misses)' -key=$(cli "--scan --pattern session:* --count 1000" | head -1) -hash=${key#session:} -echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)" -echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")" -echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")" -echo "\$ количество отметок отзыва и закэшированных сессий" -revoked=0; cached=0 +echo "\$ valkey-cli --scan --pattern 'session:*' и вид значения (хеш сокращён, значения не печатаются)" +revoked=0; cached=0; key=""; stored="" for k in $(cli "--scan --pattern session:* --count 1000"); do - if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi + if [ "$(cli "get $k")" = "-" ]; then + revoked=$((revoked + 1)); printf '%.16s… отметка отзыва "-"\n' "$k"; continue + fi + cached=$((cached + 1)); printf '%.16s… сессия\n' "$k" + if [ -z "$key" ]; then + stored=$(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('${k#session:}', 'hex')") + if [ -n "$stored" ]; then key=$k; fi + fi done echo "tombstones: $revoked, cached sessions: $cached" +if [ "$revoked" -eq 0 ] || [ -z "$key" ]; then + echo "нужны закэшированная сессия из БД и отметка отзыва: сначала make test" >&2 + exit 1 +fi +expires=$(cli "pexpiretime $key") +echo "\$ срок ключа кэша против срока сессии для $(printf '%.16s' "$key")…" +echo "valkey pexpiretime, ms: $expires" +echo "postgres expires_at, ms: $stored" +if [ "$expires" -le "$stored" ]; then echo "ключ не переживает сессию: да"; else echo "ключ не переживает сессию: НЕТ"; exit 1; fi diff --git a/scripts/evidence-stack.sh b/scripts/evidence-stack.sh index 7a4bc96..cbd5266 100755 --- a/scripts/evidence-stack.sh +++ b/scripts/evidence-stack.sh @@ -1,9 +1,11 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." +out=${EVIDENCE_DIR:-evidence} +mkdir -p "$out" base=${1:-http://$(docker compose port lb 8080)} stats=http://$(docker compose port lb 8404)/stats\;csv -header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; } +header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; } { header echo "\$ docker compose --env-file .env config --quiet && echo valid" @@ -25,7 +27,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()): print(r["svname"], r["status"], r["check_status"], r["check_code"])' echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id" for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done -} > evidence/stack.txt +} > "$out"/stack.txt { header echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)" @@ -38,7 +40,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()): echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)" echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)" echo "packages (uv.lock):" - for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \ + for name in fastapi uvicorn uvicorn-worker gunicorn sqlalchemy alembic asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \ faststream aio-pika aiormq dishka-faststream; do printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)" done @@ -47,5 +49,5 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()): service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id") docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}' done | sort -u -} > evidence/environment.txt -cat evidence/stack.txt evidence/environment.txt +} > "$out"/environment.txt +cat "$out"/stack.txt "$out"/environment.txt diff --git a/scripts/evidence-storage.sh b/scripts/evidence-storage.sh index b913aff..57bf665 100755 --- a/scripts/evidence-storage.sh +++ b/scripts/evidence-storage.sh @@ -1,20 +1,22 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." +out=${EVIDENCE_DIR:-evidence} +mkdir -p "$out" psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; } -header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; } -counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions" -kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1" +header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; } +counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM alembic_version) versions" +kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, username = 'seed_author' AS seed_stub, count(*) AS users, min(length(password_hash)) AS min_length, max(length(password_hash)) AS max_length FROM users GROUP BY 1, 2 ORDER BY 2" tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1" { header - for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do + for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments alembic_version; do echo "\$ psql -c '\\d+ $table'" psql "\\d+ $table" done - echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'" - psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations" -} > evidence/schema.txt + echo "\$ psql -c 'SELECT version_num FROM alembic_version'" + psql "SELECT version_num FROM alembic_version" +} > "$out"/schema.txt { header echo "\$ psql -c \"$counts\"" @@ -23,14 +25,14 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR docker compose run --rm migrate 2>/dev/null echo "\$ psql -c \"$counts\"" psql "$counts" -} > evidence/migrations.txt +} > "$out"/migrations.txt { header echo "\$ psql -c \"$kdf\"" psql "$kdf" echo "\$ psql -c \"$tokens\"" psql "$tokens" -} > evidence/kdf.txt +} > "$out"/kdf.txt { header echo "\$ git ls-files secrets .env backups volumes" @@ -38,7 +40,8 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR echo "(пусто: секретов, .env, дампов и volumes в Git нет)" echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/" git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/ - echo "\$ docker volume ls --filter name=mtusi2026lab6" - docker volume ls --filter name=mtusi2026lab6 -} > evidence/secrets.txt -echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены" + project=$(docker compose config --format json | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])') + echo "\$ docker volume ls --filter label=com.docker.compose.project=$project" + docker volume ls --filter "label=com.docker.compose.project=$project" +} > "$out"/secrets.txt +echo ""$out"/{schema,migrations,kdf,secrets}.txt обновлены" diff --git a/scripts/failover-each.sh b/scripts/failover-each.sh index 64050ef..02f1b7c 100755 --- a/scripts/failover-each.sh +++ b/scripts/failover-each.sh @@ -10,7 +10,7 @@ failed=0 set -- for host in $hosts; do TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \ - -- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1 + -- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1 scripts/restore.sh set -- "$@" "$out/$host/result.json" done diff --git a/scripts/failover-evidence.sh b/scripts/failover-evidence.sh index fb30bd3..74e15c9 100755 --- a/scripts/failover-evidence.sh +++ b/scripts/failover-evidence.sh @@ -1,28 +1,33 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." +out=${EVIDENCE_DIR:-evidence} +mkdir -p "$out" replicas() { docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}") } -tmp=evidence/failover.tmp +url=${1:-http://localhost:8080} +shift=0 +while [ "$shift" -lt "${2:-0}" ]; do curl -fsS -o /dev/null "$url/health/live"; shift=$((shift + 1)); done +tmp="$out"/failover.tmp since=$(date -u +%FT%TZ) { - echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" + echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)" echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt" replicas echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh" - make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}" + make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$url" echo "== after: api replicas" replicas } > "$tmp" 2>&1 stopped=$(replicas | awk '$2 == "exited" {print $1}') { echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit" - docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16 + docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|service.draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16 echo "\$ scripts/restore.sh" scripts/restore.sh echo "== after restore" replicas } >> "$tmp" 2>&1 -mv "$tmp" "evidence/failover-$stopped.txt" -cat "evidence/failover-$stopped.txt" +mv "$tmp" "$out/failover-$stopped.txt" +cat "$out/failover-$stopped.txt" diff --git a/scripts/lab4_scenarios.py b/scripts/lab4_scenarios.py index 9ac01c7..cd48098 100755 --- a/scripts/lab4_scenarios.py +++ b/scripts/lab4_scenarios.py @@ -23,6 +23,19 @@ def compose(*args, stdin=None): return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False) +def containers(service): + ids = compose("ps", "-aq", service).stdout.split() + fields = ("hostname", "status", "exit_code", "started_at", "finished_at") + template = ( + "{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}" + " {{.State.StartedAt}} {{.State.FinishedAt}}" + ) + out = subprocess.run( + ["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False + ) + return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()] + + def upload(c, actor, chat, expected=202): path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png" schema = "Attachment" if expected == 202 else None @@ -90,6 +103,10 @@ def failover_upload(c, args): recovered = time.monotonic() - started served = got["headers"].get("X-Instance-Id") check(served != target, "download served by the stopped replica") + during = containers(args.service) + states = {item["hostname"]: item["status"] for item in during} + check(states.get(target) == "exited", f"target {target} is not exited: {states}") + check(states.get(served) == "running", f"neighbour {served} is not running: {states}") digest = hashlib.sha256(got["bytes"]).hexdigest() check( digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(), @@ -104,7 +121,9 @@ def failover_upload(c, args): "sha256_equal": True, "hook_seconds": round(hook_seconds, 3), "download_ok_after_seconds": round(recovered, 3), + "replicas_at_download": during, "restored": args.restore, + "replicas_after_restore": containers(args.service) if args.restore else None, } @@ -124,6 +143,7 @@ def s3_outage(c, args): "text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")), "health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])), } + stopped = containers(args.s3_service) finally: started = time.monotonic() compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service) @@ -131,8 +151,11 @@ def s3_outage(c, args): got = wait(lambda: content(c, bob, att), 30) check(got["bytes"] == FIXTURE, "attachment changed across S3 outage") expected = {"upload": 503, "download": 503, "text_message": 201} + check(all(during[k]["status"] == v for k, v in expected.items()), f"S3 outage: {during}") + check(all(during[k]["seconds"] < 6 for k in ("upload", "download")), "S3 timeout exceeded") return { "scenario": "s3-outage", + "s3_container_during_outage": stopped, "during_outage": during, "expected": expected, "as_expected": all(during[k]["status"] == v for k, v in expected.items()), @@ -180,6 +203,8 @@ def abort_upload(c, args): ) rows = out.stdout.split() counts = dict(zip(rows[::2], (int(x) for x in rows[1::2]))) + check(out.returncode == 0, "cannot inspect aborted upload") + check(counts.get("ready", 0) == 0, "aborted upload became ready") alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"] return { "scenario": "abort-upload", @@ -216,10 +241,17 @@ def main(): "s3-outage": s3_outage, "abort-upload": abort_upload, }[args.scenario] + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, check=False + ) + stamp = { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "commit": commit.stdout.decode().strip(), + } try: - report = {"ok": True, **run(c, args)} + report = {"ok": True, **stamp, **run(c, args)} except Failure as exc: - report = {"scenario": args.scenario, "ok": False, "error": str(exc)} + report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)} text = json.dumps(report, ensure_ascii=False, indent=2) if args.output: Path(args.output).write_text(text + "\n") diff --git a/scripts/lb-stats.sh b/scripts/lb-stats.sh new file mode 100755 index 0000000..13754a5 --- /dev/null +++ b/scripts/lb-stats.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +out=${1:?usage: lb-stats.sh } +cd "$(dirname "$0")/.." +url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv} +curl -fsS "$url" | sed -n '1s/^# /ts,/p' > "$out" +while :; do + ts=$(date +%s) + curl -fsS "$url" | grep -E '^(public|api),' | sed "s/^/$ts,/" >> "$out" || true + sleep "${INTERVAL:-1}" +done diff --git a/scripts/limits.py b/scripts/limits.py index c1da642..2c97c9b 100644 --- a/scripts/limits.py +++ b/scripts/limits.py @@ -9,12 +9,20 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] S3_CLIENT_SECRETS = [ "s3_access_key_id", + "s3_reconciler_access_key_id", + "s3_reconciler_secret_access_key", + "s3_worker_access_key_id", + "s3_worker_secret_access_key", "s3_root_password", "s3_root_user", "s3_secret_access_key", - "s3_worker_access_key_id", - "s3_worker_secret_access_key", ] +RECONCILER_SECRETS = [ + "database_url", + "s3_reconciler_access_key_id", + "s3_reconciler_secret_access_key", +] +OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli") POLICY = { "lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True}, "api": { @@ -29,6 +37,11 @@ POLICY = { "networks": ["backend"], }, "migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]}, + "reconciler": { + "secrets": RECONCILER_SECRETS, + "volumes": [], + "networks": ["backend"], + }, "postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]}, "valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]}, "k6": {"secrets": [], "volumes": [], "networks": ["edge"]}, @@ -94,6 +107,9 @@ def declared(name, service, networks, problems): problems.append(f"{name}: publishes {ports} beyond loopback") if name == "api" and not service.get("read_only"): problems.append("api: root filesystem is writable") + user = str(service.get("user", "")) + if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"): + problems.append(f"{name}: runs as root") internal = [n for n in nets if networks.get(n, {}).get("internal")] return [ name, @@ -103,6 +119,7 @@ def declared(name, service, networks, problems): ",".join(volumes) or "-", ",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets), str(service.get("read_only", False)).lower(), + user or "image", ] @@ -149,6 +166,8 @@ def main(): "bench", "--profile", "tools", + "--profile", + "reconcile", "config", "--format", "json", @@ -162,9 +181,14 @@ def main(): if not any(row[0] == "api" for row in containers): problems.append("no running api replicas to inspect") lines = [ - f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}", - "$ docker compose --profile bench --profile tools config --format json (declared)", - *table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services), + f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} " + f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}", + "$ docker compose --profile bench --profile tools --profile reconcile config --format json" + " (declared)", + *table( + ["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"], + services, + ), "", "$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)", *table(["service", "hostname", "cpus", "memory", "published"], containers), diff --git a/scripts/load-evidence.sh b/scripts/load-evidence.sh new file mode 100755 index 0000000..4b9953b --- /dev/null +++ b/scripts/load-evidence.sh @@ -0,0 +1,62 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +runs="${RUNS:-3}" +pause="${PAUSE:-15}" +project=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$(docker compose ps -q lb)") +network="${project}_edge" +image=$(sed -n 's/^FROM \(python:[^ ]*\) AS runtime$/\1/p' Dockerfile) +name="$project-load-client" +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +out=${EVIDENCE_DIR:-evidence} +mkdir -p "$out/load" +[ -z "$(ls -A "$out/load")" ] || { echo "Choose an empty EVIDENCE_DIR/load" >&2; exit 2; } +client() { + docker run --rm --name "$name" --network "$network" --cpus 2 --memory 512m \ + --sysctl net.ipv4.tcp_tw_reuse=1 -v "$PWD/tests:/w/tests:ro" -v "$PWD/contracts:/w/contracts:ro" \ + -v "$PWD/lab.json:/w/lab.json:ro" -v "$PWD/$out/load:/out" "$image" python /w/tests/load.py "$@" +} +handled() { + docker compose logs --no-log-prefix --since "$1" api | python3 -c 'import collections, json, sys +seen = collections.Counter() +for line in sys.stdin: + if line.startswith("{"): + event = json.loads(line) + if event.get("event") == "request.handled": + seen[(event["status"], event["method"], event["route"])] += 1 +for (status, method, route), count in sorted(seen.items()): + print(status, method, route, count)' +} +{ + echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) network: $network" + echo "# make load-evidence (scripts/load-evidence.sh): закрытый контур tests/load.py, 64 потока, 8 с, все записи" + echo "# прогона в один новый чат, $runs повтора с паузой $pause с; клиент — одноразовый контейнер в сети <проект>_edge" + echo "# стенда (2 CPU, 512 MiB, базовый образ API по digest, tests/, contracts/ и lab.json только для чтения," + echo "# tcp_tw_reuse=1), адрес http://lb:8080. На 4-й секунде — docker stats; после прогона — request.handled реплик." + echo "\$ docker ps --format '{{.Names}}' | grep -v '^$project-' # чужие контейнеры на VM во время серии" + docker ps --format '{{.Names}}' | grep -v "^$project-" || true + echo "\$ client() { docker run --rm --name $name --network $network --cpus 2 --memory 512m --sysctl net.ipv4.tcp_tw_reuse=1 -v \"\$PWD/tests:/w/tests:ro\" -v \"\$PWD/contracts:/w/contracts:ro\" -v \"\$PWD/lab.json:/w/lab.json:ro\" -v \"\$PWD/$out/load:/out\" $image python /w/tests/load.py \"\$@\"; }" + run=1 + while [ "$run" -le "$runs" ]; do + since=$(date -u +%FT%TZ) + echo "\$ client --base-url http://lb:8080 --duration 8 --concurrency 64 --output /out/run$run.json" + client --base-url http://lb:8080 --duration 8 --concurrency 64 --output "/out/run$run.json" > "$work/out" 2>&1 & + pid=$! + sleep 4 + stats=$(docker stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}}' $(docker compose ps -q) "$name" | sed "s/^$project-//" | sort) + if wait "$pid"; then code=0; else code=$?; fi + echo "exit $code" + cat "$work/out" + echo "\$ docker stats --no-stream # 4-я секунда прогона: контейнер CPU% память" + echo "$stats" + echo "\$ docker compose logs --since $since api | request.handled: статус метод маршрут число" + handled "$since" + [ "$code" -eq 0 ] || { exit "$code"; } + [ "$run" -lt "$runs" ] && sleep "$pause" + run=$((run + 1)) + done + echo "\$ docker compose exec -T postgres psql -c 'SELECT count(*) FROM messages'" + docker compose exec -T postgres psql -U messenger -d messenger -qAt -c 'SELECT count(*) FROM messages' +} > "$out/load.txt" +cat "$out/load.txt" diff --git a/scripts/obs-validate.sh b/scripts/obs-validate.sh new file mode 100755 index 0000000..37c5a53 --- /dev/null +++ b/scripts/obs-validate.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +otel=otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849 +prom=prom/prometheus:v3.15.1@sha256:fa3302279710be2f21a8301ec17a8b4f2301fd829bd113839b3b6b02b6f191a2 +loki=grafana/loki:3.7.9@sha256:0db216b630ae1f348a1584c67f3cead8b4c1fc839169a39501db664744158e28 +tempo=grafana/tempo:2.10.10@sha256:12b0820f13ffa10ce956b6e9acc3b03f3cffded99109be6cc09d106fe56e5d38 +docker run --rm --network none -v "$PWD/infra/otel:/etc/otelcol:ro" "$otel" validate --config=/etc/otelcol/collector.yaml +echo "otelcol-contrib validate collector.yaml: OK" +docker run --rm --network none --entrypoint promtool -v "$PWD/infra/prometheus:/etc/prometheus:ro" "$prom" check config /etc/prometheus/prometheus.yml +docker run --rm --network none -v "$PWD/infra/loki/loki.yaml:/etc/loki/loki.yaml:ro" "$loki" -config.file=/etc/loki/loki.yaml -verify-config +echo "loki -verify-config: OK" +docker run --rm --network none -v "$PWD/infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro" "$tempo" -config.file=/etc/tempo/tempo.yaml -config.verify=true +echo "tempo -config.verify: OK" +for board in infra/grafana/dashboards/*.json; do + python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d["uid"] and d["panels"]' "$board" +done +echo "grafana dashboards: valid JSON with uid and panels" diff --git a/scripts/obs_check.py b/scripts/obs_check.py new file mode 100755 index 0000000..a75843d --- /dev/null +++ b/scripts/obs_check.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python3 +import argparse +import base64 +import json +import os +import re +import secrets +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PROMQL_WORDS = { + "sum", "min", "max", "avg", "count", "by", "without", "on", "ignoring", "group_left", "group_right", "and", "or", + "unless", "offset", "bool", "rate", "irate", "increase", "delta", "deriv", "histogram_quantile", "label_replace", + "label_join", "absent", "absent_over_time", "vector", "scalar", "clamp_min", "clamp_max", "topk", "bottomk", + "max_over_time", "min_over_time", "avg_over_time", "sum_over_time", "count_over_time", "quantile_over_time", + "time", "timestamp", "sort", "sort_desc", "round", "abs", "le", "inf", "nan", "predict_linear", "changes", "resets", +} + + +class Failure(Exception): + pass + + +def now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def http(method, url, body=None, headers=None, timeout=10, expect=None): + data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode() + head = {"Accept": "application/json"} + if body is not None and not isinstance(body, bytes): + head["Content-Type"] = "application/json" + head.update(headers or {}) + request = urllib.request.Request(url, data=data, headers=head, method=method) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + status, raw, got = response.status, response.read(), dict(response.headers) + except urllib.error.HTTPError as error: + status, raw, got = error.code, error.read(), dict(error.headers) + if expect is not None and status not in expect: + raise Failure(f"{method} {url.split('?')[0]} -> {status}: {raw[:200]!r}") + try: + parsed = json.loads(raw) if raw else None + except ValueError: + parsed = None + return status, parsed, got, raw + + +def get_json(url, headers=None, params=None): + if params: + url += ("&" if "?" in url else "?") + urllib.parse.urlencode(params) + return http("GET", url, headers=headers, expect=(200,))[1] + + +def wait_for(what, check, timeout, interval=2.0): + deadline = time.monotonic() + timeout + while True: + try: + result = check() + if result: + return result + except Failure: + pass + if time.monotonic() > deadline: + raise Failure(f"timeout waiting for {what}") + time.sleep(interval) + + +class Backends: + def __init__(self, args): + self.prometheus = args.prometheus.rstrip("/") + self.loki = args.loki.rstrip("/") + self.tempo = args.tempo.rstrip("/") + self.grafana = args.grafana.rstrip("/") + self.grafana_password_file = args.grafana_password_file + + def grafana_headers(self): + password = Path(self.grafana_password_file).read_text().strip() + token = base64.b64encode(f"admin:{password}".encode()).decode() + return {"Authorization": "Basic " + token} + + def promql(self, query, at=None, base=None): + params = {"query": query} + if at is not None: + params["time"] = f"{at:.3f}" + result = get_json((base or self.prometheus) + "/api/v1/query", params=params) + if result.get("status") != "success": + raise Failure(f"PromQL failed: {query}") + return result["data"]["result"] + + def logs(self, query, start, end, limit=100, base=None): + params = {"query": query, "start": str(int(start * 1e9)), "end": str(int(end * 1e9)), "limit": str(limit), + "direction": "backward"} + result = get_json((base or self.loki) + "/loki/api/v1/query_range", params=params) + return [(stream["stream"], line) for stream in result["data"]["result"] for _, line in stream["values"]] + + def traceql(self, query, start, end, limit=20, base=None): + params = {"q": query, "start": str(int(start)), "end": str(int(end) + 1), "limit": str(limit)} + return get_json((base or self.tempo) + "/api/search", params=params).get("traces") or [] + + def trace(self, trace_id, base=None): + status, body, _, raw = http("GET", (base or self.tempo) + f"/api/v2/traces/{trace_id}", expect=(200, 404)) + return (body, raw.decode("utf-8", "replace")) if status == 200 else (None, "") + + +def otlp_attrs(values): + return [{"key": k, "value": {"intValue": str(v)} if isinstance(v, int) else {"stringValue": str(v)}} + for k, v in values.items()] + + +def otlp_resource(service, instance): + return {"attributes": otlp_attrs({"service.name": service, "service.version": "probe", + "service.instance.id": instance, "deployment.environment.name": "lab"})} + + +def tempo_spans(body): + spans = [] + for batch in (body or {}).get("trace", {}).get("resourceSpans", []): + service = next((a["value"].get("stringValue") for a in batch["resource"].get("attributes", []) + if a["key"] == "service.name"), "?") + for scope in batch.get("scopeSpans", []): + for span in scope.get("spans", []): + values = {a["key"]: next(iter(a["value"].values()), None) for a in span.get("attributes", [])} + duration = (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6 + spans.append({"service": service, "name": span["name"], "kind": span.get("kind"), + "status": span.get("status", {}).get("code", "STATUS_CODE_UNSET"), + "duration_ms": round(duration, 3), "links": len(span.get("links", [])), + "attributes": values}) + return spans + + +def probe(args, b): + leak = "probe-secret-" + secrets.token_hex(8) + rid = uuid.uuid4().hex + trace_id, server, producer, consumer = secrets.token_hex(16), secrets.token_hex(8), secrets.token_hex(8), secrets.token_hex(8) + t0 = time.time() + ns = time.time_ns() + api, worker = args.service_name, args.service_name + "-worker" + scope = {"name": "obs-check", "version": "1"} + + def span(span_id, parent, name, kind, start_ms, end_ms, values, links=()): + return {"traceId": trace_id, "spanId": span_id, "parentSpanId": parent, "name": name, "kind": kind, + "startTimeUnixNano": str(ns + start_ms * 1_000_000), "endTimeUnixNano": str(ns + end_ms * 1_000_000), + "attributes": otlp_attrs(values), "status": {"code": 1}, + "links": [{"traceId": trace_id, "spanId": s} for s in links]} + + traces = {"resourceSpans": [ + {"resource": otlp_resource(api, api + "-1"), "scopeSpans": [{"scope": scope, "spans": [ + span(server, "", "POST /api/v1/probe", 2, 0, 40, { + "http.request.method": "POST", "http.route": "/api/v1/probe", "http.response.status_code": 202, + "messenger.request_id": rid, "http.request.header.authorization": "Bearer " + leak, + "url.query": "filename=" + leak, "user.password": leak, "messenger.session.token": leak}), + span(producer, server, "probe.jobs publish", 4, 30, 35, {"messaging.operation.type": "send"})]}]}, + {"resource": otlp_resource(worker, worker + "-1"), "scopeSpans": [{"scope": scope, "spans": [ + span(consumer, producer, "probe.jobs process", 5, 50, 90, {"messaging.operation.type": "process"}, [producer])]}]}, + ]} + logs = {"resourceLogs": [{"resource": otlp_resource(api, api + "-1"), "scopeLogs": [{"scope": scope, "logRecords": [{ + "timeUnixNano": str(ns), "observedTimeUnixNano": str(ns), "severityNumber": 9, "severityText": "INFO", + "body": {"stringValue": "probe.request.completed"}, "traceId": trace_id, "spanId": server, + "attributes": otlp_attrs({"event": "probe.request.completed", "request_id": rid, "authorization": "Bearer " + leak})}]}]}]} + point = {"attributes": otlp_attrs({"probe.id": rid}), "startTimeUnixNano": str(ns - 10_000_000_000), "timeUnixNano": str(ns)} + metrics = {"resourceMetrics": [{"resource": otlp_resource(api, api + "-1"), "scopeMetrics": [{"scope": scope, "metrics": [ + {"name": "obs.probe.heartbeat", "unit": "{probe}", "sum": {"aggregationTemporality": 2, "isMonotonic": True, + "dataPoints": [dict(point, asInt="1")]}}, + {"name": "obs.probe.latency", "unit": "s", "histogram": {"aggregationTemporality": 2, "dataPoints": [dict( + point, count="1", sum=0.04, bucketCounts=["0", "1", "0"], explicitBounds=[0.01, 0.1])]}}, + ]}]}]} + otlp = args.otlp.rstrip("/") + for path, payload in (("/v1/traces", traces), ("/v1/logs", logs), ("/v1/metrics", metrics)): + http("POST", otlp + path, payload, expect=(200,)) + report = {"probe_id": rid, "trace_id": trace_id, "service": api, "sent_at": now()} + + def tempo_ok(base=None): + body, raw = b.trace(trace_id, base) + spans = tempo_spans(body) + return {"spans": len(spans), "services": sorted({s["service"] for s in spans}), "raw": raw} if len(spans) == 3 else None + + tempo = wait_for("trace in Tempo", tempo_ok, args.timeout) + found = wait_for("TraceQL search", lambda: b.traceql(f'{{ span.messenger.request_id = "{rid}" }}', t0 - 60, time.time() + 60), args.timeout) + loki_query = f'{{service_name="{api}"}} | request_id="{rid}"' + lines = wait_for("log in Loki", lambda: b.logs(loki_query, t0 - 60, time.time() + 60), args.timeout) + series = wait_for("metrics in Prometheus", lambda: (lambda r: r if len(r) == 2 else None)(b.promql( + f'{{__name__=~"obs_probe_heartbeat_total|obs_probe_latency_seconds_count", probe_id="{rid}"}}')), args.timeout) + labels = lines[0][0] + raw_logs = json.dumps(lines) + report.update({ + "tempo": {"spans": tempo["spans"], "services": tempo["services"], "traceql_hits": len(found)}, + "loki": {"lines": len(lines), "trace_id_matches": labels.get("trace_id") == trace_id, + "indexed_labels": sorted(k for k in labels if k in {"service_name", "deployment_environment_name"})}, + "prometheus": {"series": sorted(s["metric"].get("__name__", "") for s in series), + "job": series[0]["metric"].get("job")}, + "redaction": {"secret_in_tempo": leak in tempo["raw"], "secret_in_loki": leak in raw_logs, + "masked_in_tempo": "****" in tempo["raw"]}, + }) + if not args.no_grafana: + g = b.grafana + "/api/datasources/proxy/uid/" + headers = b.grafana_headers() + report["grafana"] = { + "prometheus": len(get_json(g + "prometheus/api/v1/query", headers, {"query": f'obs_probe_heartbeat_total{{probe_id="{rid}"}}'})["data"]["result"]), + "loki": len(get_json(g + "loki/loki/api/v1/query_range", headers, {"query": loki_query, "start": str(int((t0 - 60) * 1e9)), "end": str(int((time.time() + 60) * 1e9))})["data"]["result"]), + "tempo": http("GET", g + f"tempo/api/v2/traces/{trace_id}", headers=headers)[0], + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + ok = (report["tempo"]["spans"] == 3 and report["loki"]["trace_id_matches"] and len(series) == 2 + and not report["redaction"]["secret_in_tempo"] and not report["redaction"]["secret_in_loki"] + and ("grafana" not in report or (report["grafana"]["prometheus"] and report["grafana"]["loki"] and report["grafana"]["tempo"] == 200))) + if not ok: + raise Failure("probe verification failed") + + +class Api: + def __init__(self, base): + self.base = base.rstrip("/") + + def call(self, method, path, body=None, headers=None, expect=(200,)): + status, parsed, got, _ = http(method, self.base + path, body, headers, expect=expect) + return parsed, got.get("X-Request-Id") or got.get("x-request-id") + + def register(self): + username = "u_obs_" + secrets.token_hex(6) + password = "T3st-" + secrets.token_hex(16) + user, rid = self.call("POST", "/api/v1/users", {"username": username, "display_name": "obs check", "password": password}, expect=(201,)) + return user, username, password, rid + + def login(self, username, password): + basic = base64.b64encode(f"{username}:{password}".encode()).decode() + session, _ = self.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, expect=(201,)) + return {"Authorization": "Bearer " + session["token"]}, session["token"] + + def chat(self, auth, user): + chat, _ = self.call("POST", "/api/v1/chats", {"title": "obs", "member_ids": [user["id"]]}, auth, expect=(201,)) + return chat + + def logout(self, auth): + self.call("DELETE", "/api/v1/auth/sessions/current", headers=auth, expect=(204,)) + + +def request_check(args, b): + api = Api(args.base_url) + t0 = time.time() + _, _, _, rid = api.register() + query = f'{{service_name="{args.api_service}"}} | {args.log_request_id_key}="{rid}"' + lines = wait_for("API log by request_id", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout) + labels, line = lines[0] + trace_id = labels.get("trace_id") + if not trace_id: + raise Failure("log record has no trace_id") + spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout) + print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id, + "span_id": labels.get("span_id"), + "service_name": labels.get("service_name"), + "severity": labels.get("severity_text") or labels.get("detected_level")}, + "trace": [{k: s[k] for k in ("service", "name", "kind", "status", "duration_ms")} for s in spans]}, + ensure_ascii=False, indent=2)) + + +def job_check(args, b): + api = Api(args.base_url) + t0 = time.time() + user, username, password, _ = api.register() + auth, _ = api.login(username, password) + chat = api.chat(auth, user) + image = Path(args.image).read_bytes() + headers = dict(auth, **{"Content-Type": "image/png"}) + attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=obs.png", image, headers, expect=(202,)) + ready = wait_for("attachment ready", lambda: (lambda a: a if a["status"] in ("ready", "failed") else None)( + api.call("GET", f"/api/v1/attachments/{attachment['id']}", headers=auth)[0]), 60, 0.5) + key = args.attachment_attr + traces = wait_for("traces for attachment", lambda: b.traceql(f'{{ span.{key} = "{attachment["id"]}" }}', t0 - 30, time.time() + 30), args.timeout) + chain = [] + for item in traces: + for span in tempo_spans(b.trace(item["traceID"])[0]): + chain.append(dict({k: span[k] for k in ("service", "name", "kind", "status", "duration_ms", "links")}, + trace_id=item["traceID"], attempt=span["attributes"].get(args.attempt_attr))) + query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"' + worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout) + api.logout(auth) + print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"], + "traces": sorted({c["trace_id"] for c in chain}), "spans": chain, + "worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")} + for labels, line in worker_logs[:10]]}, ensure_ascii=False, indent=2)) + + +def leaks_check(args, b): + api = Api(args.base_url) + t0 = time.time() + user, username, password, _ = api.register() + auth, token = api.login(username, password) + chat = api.chat(auth, user) + text = "obs-leak-text-" + secrets.token_hex(8) + filename = "obs-leak-file-" + secrets.token_hex(6) + ".png" + api.call("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": text}, dict(auth, **{"Idempotency-Key": uuid.uuid4().hex}), expect=(201,)) + if Path(args.image).is_file(): + api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=" + urllib.parse.quote(filename), + Path(args.image).read_bytes(), dict(auth, **{"Content-Type": "image/png"}), expect=(202,)) + api.logout(auth) + time.sleep(args.flush) + needles = {"password": password, "token": token, "message_text": text, "filename": filename} + end = time.time() + 5 + hits = {} + for name, needle in needles.items(): + lines = b.logs(f'{{service_name=~"{args.service_regex}"}} |= `{needle}`', t0 - 30, end, limit=50) + hits[name] = {"loki": len(lines)} + traces = b.traceql(f'{{ resource.service.name =~ "{args.service_regex}" }}', t0 - 30, end, limit=500) + raws = [b.trace(t["traceID"])[1] for t in traces] + for name, needle in needles.items(): + hits[name]["tempo"] = sum(needle in raw for raw in raws) + report = {"checked_at": now(), "window_seconds": round(end - t0), "traces_scanned": len(raws), "hits": hits} + print(json.dumps(report, indent=2)) + if any(v["loki"] or v["tempo"] for v in hits.values()): + raise Failure("secret or message body found in telemetry") + + +def uploads_burst(args, b): + api = Api(args.base_url) + user, username, password, _ = api.register() + auth, _ = api.login(username, password) + chat = api.chat(auth, user) + image = Path(args.image).read_bytes() + accepted = [] + for index in range(args.count): + attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=burst-{index}.png", image, + dict(auth, **{"Content-Type": "image/png"}), expect=(202,)) + accepted.append({"attachment_id": attachment["id"], "request_id": rid, "status": attachment["status"]}) + time.sleep(args.interval) + api.logout(auth) + print(json.dumps({"ts": now(), "uploaded": accepted}, ensure_ascii=False)) + + +def query(args, b): + results = {expr: [{"labels": {k: v for k, v in r["metric"].items() if k != "__name__"}, "value": r["value"][1]} + for r in b.promql(expr)] for expr in args.expr} + print(json.dumps({"ts": now(), "results": results}, ensure_ascii=False)) + + +def expand(expr): + expr = re.sub(r"\$\{?__(rate_interval|interval|range)\}?", "1m", expr) + return re.sub(r"\$\{?(\w+)(?::\w+)?\}?", ".*", expr) + + +def metric_names(expr): + stripped = re.sub(r'"[^"]*"|`[^`]*`|\{[^}]*\}|\b(by|without|on|ignoring|group_left|group_right)\s*\([^)]*\)', " ", expr) + stripped = re.sub(r"\[[^\]]*\]", " ", stripped) + return {t for t in re.findall(r"[a-zA-Z_:][a-zA-Z0-9_:]*", stripped) if ("_" in t or ":" in t) and t not in PROMQL_WORDS} + + +def walk_panels(dashboard): + for panel in dashboard.get("panels", []): + yield panel + yield from panel.get("panels", []) + + +def dashboards_check(args, b): + known = set(get_json(b.prometheus + "/api/v1/label/__name__/values")["data"]) + problems, report = [], [] + for path in sorted(Path(args.dir).glob("*.json")): + dashboard = json.loads(path.read_text()) + for panel in walk_panels(dashboard): + for target in panel.get("targets", []): + uid = (target.get("datasource") or panel.get("datasource") or {}).get("uid") + entry = {"dashboard": path.name, "panel": panel.get("title"), "datasource": uid} + try: + if uid == "prometheus": + expr = expand(target["expr"]) + result = b.promql(expr) + missing = sorted(metric_names(expr) - known) + entry.update(series=len(result), missing_metrics=missing) + if missing and args.strict: + problems.append(f"{path.name}: {panel.get('title')}: missing {missing}") + elif uid == "loki": + entry["lines"] = len(b.logs(expand(target["expr"]), time.time() - 3600, time.time(), limit=5)) + elif uid == "tempo": + entry["traces"] = len(b.traceql(expand(target["query"]), time.time() - 3600, time.time(), limit=5)) + except Failure as error: + problems.append(f"{path.name}: {panel.get('title')}: {error}") + report.append(entry) + print(json.dumps({"checked_at": now(), "targets": report, "problems": problems}, ensure_ascii=False, indent=2)) + if problems: + raise Failure(f"{len(problems)} dashboard queries failed") + + +def grafana_check(args, b): + headers = b.grafana_headers() + health = get_json(b.grafana + "/api/health") + sources = {} + for uid in ("prometheus", "loki", "tempo"): + status, body, _, _ = http("GET", b.grafana + f"/api/datasources/uid/{uid}/health", headers=headers) + sources[uid] = {"http": status, "status": (body or {}).get("status"), "message": (body or {}).get("message")} + found = get_json(b.grafana + "/api/search", headers, {"type": "dash-db"}) + boards = {} + for item in found: + status, body, _, _ = http("GET", b.grafana + f"/api/dashboards/uid/{item['uid']}", headers=headers) + meta = (body or {}).get("meta", {}) + boards[item["uid"]] = {"title": item["title"], "http": status, "provisioned": meta.get("provisioned"), + "folder": meta.get("folderTitle"), "panels": len(list(walk_panels((body or {}).get("dashboard", {}))))} + expected = {json.loads(p.read_text())["uid"] for p in Path(args.dir).glob("*.json")} + print(json.dumps({"checked_at": now(), "grafana": health.get("version"), "datasources": sources, "dashboards": boards, + "missing_dashboards": sorted(expected - set(boards))}, ensure_ascii=False, indent=2)) + if expected - set(boards) or any(s["http"] != 200 or s["status"] != "OK" for s in sources.values()): + raise Failure("grafana provisioning incomplete") + + +def alerts_watch(args, b): + want_state, _, wanted = args.until.partition(":") if args.until else ("", "", "") + names = set(filter(None, wanted.split("|"))) + only = {n for value in args.only for n in value.split("|")} + deadline = time.monotonic() + args.timeout + last = None + output = open(args.output, "a") if args.output else None + while True: + alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"] + state = sorted((a["labels"]["alertname"], a["state"], a.get("activeAt", "")) for a in alerts + if not only or a["labels"]["alertname"] in only) + if state != last: + line = json.dumps({"ts": now(), "alerts": [{"alertname": n, "state": s, "active_at": at} for n, s, at in state]}, ensure_ascii=False) + print(line, flush=True) + if output: + output.write(line + "\n") + output.flush() + last = state + current = {n: s for n, s, _ in state} + firing = {n for n in names if current.get(n) == "firing"} + if names and ((want_state == "firing" and firing) or (want_state == "resolved" and not firing)): + return + if time.monotonic() > deadline: + raise Failure(f"timeout waiting for {args.until}") + time.sleep(args.interval) + + +def main(): + parser = argparse.ArgumentParser(description="Telemetry checks through public HTTP APIs of the messenger and of the observability stack") + parser.add_argument("--prometheus", default=os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090")) + parser.add_argument("--loki", default=os.environ.get("LOKI_URL", "http://127.0.0.1:3100")) + parser.add_argument("--tempo", default=os.environ.get("TEMPO_URL", "http://127.0.0.1:3200")) + parser.add_argument("--grafana", default=os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000")) + parser.add_argument("--grafana-password-file", default=os.environ.get("GRAFANA_PASSWORD_FILE", str(ROOT / "secrets/grafana_admin_password"))) + parser.add_argument("--timeout", type=float, default=60) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("probe", help="send a synthetic OTLP trace, log and metric and find them in every backend and in Grafana") + p.add_argument("--otlp", default=os.environ.get("OTLP_URL", "http://127.0.0.1:4318")) + p.add_argument("--service-name", default="obs-probe") + p.add_argument("--no-grafana", action="store_true") + for name, fn in (("request", request_check), ("job", job_check), ("leaks", leaks_check), ("uploads", uploads_burst)): + q = sub.add_parser(name) + q.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + q.add_argument("--api-service", default="messenger-api") + q.add_argument("--worker-service", default="messenger-worker") + q.add_argument("--service-regex", default="messenger-.*") + q.add_argument("--log-request-id-key", default="request_id") + q.add_argument("--log-attachment-key", default="attachment_id") + q.add_argument("--attachment-attr", default="messenger.attachment_id") + q.add_argument("--attempt-attr", default="messenger.job.attempt") + q.add_argument("--image", default=str(ROOT / "tests/fixtures/sample.png")) + q.add_argument("--flush", type=float, default=15) + q.add_argument("--count", type=int, default=10) + q.add_argument("--interval", type=float, default=1) + q.set_defaults(fn=fn) + q = sub.add_parser("query", help="print instant PromQL results with a timestamp") + q.add_argument("expr", nargs="+") + q.set_defaults(fn=query) + for name, fn in (("dashboards", dashboards_check), ("grafana", grafana_check)): + q = sub.add_parser(name) + q.add_argument("--dir", default=str(ROOT / "infra/grafana/dashboards")) + q.add_argument("--strict", action="store_true", help="fail when a referenced metric does not exist yet") + q.set_defaults(fn=fn) + w = sub.add_parser("alerts", help="record alert state changes until firing: or resolved:") + w.add_argument("--until", default="") + w.add_argument("--only", nargs="*", default=[]) + w.add_argument("--interval", type=float, default=5) + w.add_argument("--output", default="") + p.set_defaults(fn=probe) + w.set_defaults(fn=alerts_watch) + args = parser.parse_args() + try: + args.fn(args, Backends(args)) + except (Failure, OSError, KeyError, ValueError) as error: + print(f"FAIL: {error}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/overload-leftovers.sh b/scripts/overload-leftovers.sh new file mode 100755 index 0000000..923cdc5 --- /dev/null +++ b/scripts/overload-leftovers.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +limit="${WAIT_SECONDS:-1500}" +out=${EVIDENCE_DIR:-evidence} +mkdir -p "$out" +sql() { docker compose exec -T postgres psql -U messenger -d messenger -X "$@"; } +pending() { sql -tAc "SELECT count(*) FROM attachments WHERE status = 'pending'"; } +statuses="SELECT status, count(*), min(created_at), max(created_at) FROM attachments GROUP BY status ORDER BY status" +since=$(date -u +%FT%TZ) +{ + echo "date: $since commit: $(git rev-parse HEAD)" + echo "# make overload-leftovers (scripts/overload-leftovers.sh) после серии памяти: невидимые намерения от 503 по" + echo "# таймауту видны сверке свежими, уборка API удаляет их после UPLOAD_GRACE_SECONDS; стенд тот же" + echo "\$ make reconcile # dry-run" + docker compose run --rm --no-deps -T reconciler 2>/dev/null + echo "\$ psql -c \"$statuses\"" + sql -c "$statuses" + echo "\$ until [ \"\$(psql -tAc \"SELECT count(*) FROM attachments WHERE status = 'pending'\")\" = 0 ]; do sleep 10; done" + started=$(date +%s) + until [ "$(pending)" = 0 ] || [ $(( $(date +%s) - started )) -gt "$limit" ]; do sleep 10; done + echo "pending = $(pending) в $(date -u +%FT%TZ)" + [ "$(pending)" = 0 ] || { echo "FAIL: pending cleanup timed out"; exit 1; } + echo "\$ docker compose logs --since $since api | audit.attachment_discarded и attachments.swept" + docker compose logs --no-log-prefix --since "$since" api | python3 -c 'import collections, json, sys +reasons, swept, times = collections.Counter(), 0, [] +for line in sys.stdin: + if not line.startswith("{"): + continue + event = json.loads(line) + if event.get("event") == "audit.attachment_discarded": + reasons[event["reason"]] += 1 + times.append(event["ts"][:19] + "Z") + elif event.get("event") == "attachments.swept": + swept += event["count"] +print(dict(reasons), "swept", swept, "first", min(times, default="-"), "last", max(times, default="-"))' + echo "\$ psql -c \"$statuses\"" + sql -c "$statuses" + echo "\$ make reconcile # dry-run после уборки" + docker compose run --rm --no-deps -T reconciler 2>/dev/null +} > "$out/overload-leftovers.txt" +cat "$out/overload-leftovers.txt" diff --git a/scripts/persistence-evidence.sh b/scripts/persistence-evidence.sh index f10ce0c..582ffac 100755 --- a/scripts/persistence-evidence.sh +++ b/scripts/persistence-evidence.sh @@ -6,7 +6,7 @@ started() { docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id" done | sort } -echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" +echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)" echo "== before: service hostname StartedAt status" started make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}" diff --git a/scripts/pg-stats.sh b/scripts/pg-stats.sh new file mode 100755 index 0000000..736a5d5 --- /dev/null +++ b/scripts/pg-stats.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu +mode=${1:?usage: pg-stats.sh activity | statements | size | reset} +cd "$(dirname "$0")/.." +psql() { + docker compose exec -T postgres sh -c 'exec psql -X -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "${POSTGRES_DB:-$POSTGRES_USER}" "$@"' psql "$@" +} +case $mode in +reset) + psql -c 'set client_min_messages = warning' -c 'create extension if not exists pg_stat_statements' -c 'select pg_stat_statements_reset()' >/dev/null + ;; +size) + psql -At -F, -c "select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)" + ;; +statements) + psql -c "copy (select calls, round(total_exec_time::numeric, 3) as total_ms, round(mean_exec_time::numeric, 3) as mean_ms, rows, shared_blks_hit, shared_blks_read, left(regexp_replace(query, '\s+', ' ', 'g'), 200) as query from pg_stat_statements where dbid = (select oid from pg_database where datname = current_database()) and query not like '%pg_stat%' order by total_exec_time desc limit ${TOP:-20}) to stdout with csv header" > "${2:?csv}" + ;; +activity) + echo 'ts,state,wait_event_type,wait_event,backends,max_connections' > "${2:?csv}" + while :; do + ts=$(date +%s) + psql -At -F, -c "select coalesce(state, ''), coalesce(wait_event_type, ''), coalesce(wait_event, ''), count(*), current_setting('max_connections') from pg_stat_activity where backend_type = 'client backend' and pid <> pg_backend_pid() group by 1, 2, 3" | sed "s/^/$ts,/" >> "$2" || true + sleep "${INTERVAL:-1}" + done + ;; +*) + echo "unknown mode $mode" >&2 + exit 2 + ;; +esac diff --git a/scripts/pipeline_evidence.py b/scripts/pipeline_evidence.py index 5ed7b9c..edff2ce 100644 --- a/scripts/pipeline_evidence.py +++ b/scripts/pipeline_evidence.py @@ -53,6 +53,11 @@ def stop(service): check=True, ).stdout.strip() say(f"$ docker compose stop -t 10 {service} -> {seconds:.1f} s, exit code {code}") + say( + compose( + "ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", service + ).rstrip() + ) check(seconds <= 10 and code == "0", f"{service} did not stop cleanly on SIGTERM") diff --git a/scripts/pipeline_suite.py b/scripts/pipeline_suite.py index 1388f7f..33b4d56 100644 --- a/scripts/pipeline_suite.py +++ b/scripts/pipeline_suite.py @@ -107,9 +107,11 @@ staging_left: WHERE id = ANY(:'ids'::uuid[]) AND status = 'ready' AND staging_key IS NOT NULL object_keys: - SELECT object_key AS key FROM attachment_variants - UNION ALL SELECT original_key FROM attachments WHERE original_key IS NOT NULL - UNION ALL SELECT staging_key FROM attachments WHERE staging_key IS NOT NULL + SELECT object_key AS key FROM attachment_variants WHERE attachment_id = ANY(:'ids'::uuid[]) + UNION ALL SELECT original_key FROM attachments + WHERE id = ANY(:'ids'::uuid[]) AND original_key IS NOT NULL + UNION ALL SELECT staging_key FROM attachments + WHERE id = ANY(:'ids'::uuid[]) AND staging_key IS NOT NULL columns: SELECT table_name, column_name FROM information_schema.columns @@ -252,6 +254,35 @@ class Stack: out = self.rabbitmqadmin(*command, json.dumps(properties), stdin=json.dumps(payload)) check("routed successfully" in out, "broker did not route the injected job") + def s3_json(self, *argv, missing=False): + done = subprocess.run( + [*self.base, "run", "--rm", "--no-deps", "-T", "s3-cli", *argv, "--json"], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + body = json.loads(done.stdout or done.stderr) + if missing and done.returncode == 5 and body.get("details", {}).get("type") == "not_found": + return None + check(done.returncode == 0, "S3 inventory command failed: exit " + str(done.returncode)) + return body + + def objects(self, ids): + bucket = self.dc("exec", "-T", self.args.relay, "printenv", "S3_BUCKET").strip() + keys = set() + for ident in ids: + for prefix in ("staging", "originals"): + body = self.s3_json( + "object", "stat", f"root/{bucket}/{prefix}/{ident}", missing=True + ) + if body is not None: + keys.add(body["name"]) + body = self.s3_json("object", "list", "--recursive", f"root/{bucket}/variants/{ident}/") + check(not body["truncated"], "S3 variant inventory is truncated") + keys.update(item["key"] for item in body["items"]) + return keys + def admin(self, *argv): command = shlex.split(self.args.admin) + list(argv) out = self.dc("run", "--rm", "--no-deps", "-T", self.args.worker, *command) @@ -396,6 +427,9 @@ class Suite: code = self.stack.wait_exit(service) check(code == 137, f"{service} exited with {code}, expected 137 at {point}") seen = observe(ident) + seen["containers"] = self.stack.dc( + "ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", service + ).splitlines() self.stack.launch(service) ready = self.api.wait(ident, timeout=self.args.recovery_timeout) poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox") @@ -424,7 +458,12 @@ class Suite: steps.append({"at": now(), "step": step, **extra}) self.stack.stop(self.args.worker) - mark("worker stopped") + mark( + "worker stopped", + containers=self.stack.dc( + "ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", self.args.worker + ).splitlines(), + ) att = self.api.fixture() mark("upload 202 queued", attachment_id=att["id"]) self.api.content(att["id"], expected=409) @@ -594,15 +633,39 @@ class Suite: self.api.final[ident] = {"failed"} kept = self.record(ident)["staging_key"] check(kept is not None, "failed staging was removed before the retention period") - try: - self.stack.launch(self.args.relay, FAULT_STAGING_RETENTION_SECONDS="1") - poll(lambda: self.record(ident)["staging_key"] is None, 30, "failed staging cleanup") - finally: - self.stack.launch(self.args.relay) + retention = int( + self.stack.dc("exec", "-T", self.args.relay, "printenv", "STAGING_RETENTION_SECONDS") + ) + aged = self.stack.dc( + "exec", + "-T", + self.args.postgres, + "sh", + "-c", + PSQL, + "sh", + "-v", + "id=" + ident, + "-v", + "chat=" + self.api.chat["id"], + "-v", + "age=" + str(retention + 1), + stdin="UPDATE attachments SET created_at = created_at - :age * interval '1 second', " + "updated_at = updated_at - :age * interval '1 second' " + "WHERE id = :'id' AND chat_id = :'chat' AND status = 'failed' RETURNING id;\n", + ) + check(aged.strip() == ident, "did not age exactly the suite's failed attachment") + poll(lambda: self.record(ident)["staging_key"] is None, 30, "failed staging cleanup") replay = self.stack.admin("replay", "--attachment-id", ident) check(replay["result"] == "expired", f"replay after retention gave {replay}") check(self.api.get(ident)["status"] == "failed", "expired replay changed the status") - return {"attachment_id": ident, "staging_before": "kept", "replay": replay} + return { + "attachment_id": ident, + "staging_before": "kept", + "retention_seconds": retention, + "aged_only": ident, + "replay": replay, + } def image_matrix(self): folder = Path(self.args.images) @@ -696,11 +759,19 @@ class Suite: poll(lambda: self.record(ident)["status"] == "processing", 30, "processing") self.stack.kill(self.args.worker) code = self.stack.wait_exit(self.args.worker) + containers = self.stack.dc( + "ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", self.args.worker + ).splitlines() self.stack.launch(self.args.worker) ready = self.api.wait(ident, timeout=self.args.recovery_timeout) self.drained() self.verify(ready) - return {"attachment_id": ident, "exit_code": code, "jobs": self.stack.sql("jobs", id=ident)} + return { + "attachment_id": ident, + "exit_code": code, + "containers": containers, + "jobs": self.stack.sql("jobs", id=ident), + } def duplicate_in_flight(self): self.stack.stop(self.args.worker) @@ -743,36 +814,60 @@ class Suite: def reorder_versions(self): self.stack.stop(self.args.worker, self.args.relay) ident = self.api.fixture()["id"] - self.reprocess(ident) + check(self.reprocess(ident)["result"] == "queued", "version 2 was not scheduled") + check(self.reprocess(ident)["result"] == "noop", "duplicate reprocess was not a noop") self.stack.publish(self.payload(ident, 2)) self.queued(1) - self.stack.launch(self.args.relay) - self.queued(3) self.stack.launch(self.args.worker) self.published(ident, 2) - self.api.wait(ident) + before = self.snapshot(ident) + self.stack.launch(self.args.relay) + poll( + lambda: ( + {j["pipeline_version"]: j["status"] for j in self.stack.sql("jobs", id=ident)} + == {1: "superseded", 2: "done"} + ), + self.args.recovery_timeout, + "old version superseded", + ) + poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox") self.drained() final = self.snapshot(ident) check(final["published"] == 2, f"published_version rolled back to {final['published']}") + check(before["public"] == final["public"], "late v1 changed the published v2 variants") served = {v["name"]: v["sha256"] for v in final["public"]} stored = {r["name"]: r["sha256"] for r in final["rows"] if r["pipeline_version"] == 2} check(served == stored, "API serves variants of a version that is not published") + jobs = self.stack.sql("jobs", id=ident) + check( + {j["pipeline_version"]: j["status"] for j in jobs} == {1: "superseded", 2: "done"}, + "v1 did not finish after v2", + ) self.verify(self.api.get(ident)) - return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)} + return {"attachment_id": ident, "jobs": jobs, "v2_before_v1": before, "after_v1": final} def stale_after_new_version(self): ident = self.api.fixture()["id"] self.api.wait(ident) - self.reprocess(ident) + first = self.snapshot(ident) + check(self.reprocess(ident)["result"] == "queued", "version 2 was not scheduled") + check(self.api.get(ident)["status"] == "ready", "reprocess rolled back ready") self.published(ident, 2) self.drained() before = self.snapshot(ident) + check(first["public"] != before["public"], "recipes did not produce distinct results") self.stack.publish(self.payload(ident, 1)) self.drained() time.sleep(1) after = self.snapshot(ident) check(before == after and after["published"] == 2, "stale v1 job changed the v2 result") - return {"attachment_id": ident, "published_version": after["published"]} + return { + "attachment_id": ident, + "published_version": after["published"], + "v1": first, + "v2_before_stale": before, + "v2_after_stale": after, + } def invariants(self, dead_before): ids = self.api.accepted @@ -819,13 +914,21 @@ class Suite: check(queues[QUEUE] == (0, 0), "messages left in " + QUEUE) expected = dead_before + self.dead_letters check(queues[DLQ][0] == expected, f"unexpected dead letters: {queues[DLQ]}") - if not self.args.s3_list: - return {"queues": queues, "s3": "SKIP: --s3-list is not set"} - keys = {r["key"] for r in self.stack.sql("object_keys")} - listing = run(shlex.split(self.args.s3_list)).splitlines() - stray = [k for k in (x.split()[-1] for x in listing if x.strip()) if k not in keys] - check(not stray, f"S3 objects without metadata: {stray[:5]}") - return {"queues": queues, "s3_objects": len(listing)} + keys = {r["key"] for r in self.stack.sql("object_keys", ids=array)} + if self.args.s3_list: + listing = { + x.split()[-1] + for x in run(shlex.split(self.args.s3_list)).splitlines() + if x.strip() + } + listing = {key for key in listing if any("/" + ident in key for ident in ids)} + else: + listing = self.stack.objects(ids) + check( + listing == keys, + f"S3 mismatch: extra={sorted(listing - keys)}, missing={sorted(keys - listing)}", + ) + return {"queues": queues, "s3_objects": len(listing), "scoped_attachments": len(ids)} def no_loss(): known = {r["id"] for r in self.stack.sql("known", ids=array)} @@ -904,6 +1007,7 @@ class Suite: report = { "suite": "lab5-pipeline-idempotency", "started_at": now(), + "source_sha": run(["git", "rev-parse", "HEAD"]).strip(), "config": self.config(), } if self.args.preflight: @@ -925,6 +1029,7 @@ class Suite: finally: self.stack.up(self.args.s3) self.stack.restore() + report["accepted_ids"] = self.api.accepted if self.api else [] report["scenarios"] = results return self.finish(report, results + report.get("invariants", [])) diff --git a/scripts/probe.py b/scripts/probe.py index a624bd0..71652a5 100755 --- a/scripts/probe.py +++ b/scripts/probe.py @@ -10,12 +10,16 @@ import sys import threading import time import urllib.error +import urllib.parse import urllib.request import uuid +from datetime import datetime from pathlib import Path ROOT = Path(__file__).resolve().parents[1] COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx") +PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"] +PSQL += ["-X", "-q", "-At"] @functools.cache @@ -207,16 +211,112 @@ def check(args): return 0 if passed else 1 +def output(*command): + done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False) + return done.stdout + done.stderr + + +def replicas(): + ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split() + form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}" + return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()] + + +def answered(container, request_id): + for line in output("docker", "logs", container).splitlines(): + if request_id in line and "request.handled" in line: + return json.loads(line).get("status") + return None + + +def lost(alice, bob, chat, kill, args): + key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex + request_id = "lost-" + key[:12] + lock = subprocess.Popen( + PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True + ) + lock.stdin.write( + f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n" + f"SELECT pg_sleep({args.hold});\nCOMMIT;\n" + ) + lock.stdin.flush() + lock.stdout.readline() + sent = time.time() + headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id}) + path = f"/api/v1/chats/{chat}/messages" + first = call("POST", path, {"text": text}, headers, args.timeout)[0] + sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'" + waiting = set(output(*PSQL, "-c", sql).split()) + holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None]) + if kill and holder[0]: + subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL) + lock.stdin.close() + lock.wait() + retried = time.time() + attempts, (status, instance, body) = 1, send(alice, chat, text, key) + while status != 201 and attempts < 20: + time.sleep(0.2) + attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key) + page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []} + copies = [m for m in page["items"] if m["text"] == text] + stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None + report = { + "variant": "kill" if kill else "timeout", + "lost_attempt": { + "status": first, + "client_timeout_s": args.timeout, + "chat_lock_held_s": args.hold, + "replica": holder[1], + "replica_killed": bool(kill and holder[0]), + "replica_answered": answered(holder[0], request_id) if holder[0] else None, + }, + "retry": { + "after_s": round(retried - sent, 3), + "status": status, + "attempts": attempts, + "replica": instance, + }, + "copies_in_history": len(copies), + "same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"), + "written_by": None + if stamp is None + else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"), + } + if kill and holder[0]: + subprocess.call( + [str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + report["pass"] = ( + first == 0 + and holder[0] is not None + and status == 201 + and report["copies_in_history"] == 1 + and report["same_message"] + and (kill or report["lost_attempt"]["replica_answered"] == 201) + ) + return report + + +def retry(args): + alice, bob = actor("alice"), actor("bob") + chat = chat_of(alice, bob) + variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)] + passed = all(v["pass"] for v in variants) + emit({"variants": variants, "pass": passed}, args.out) + return 0 if passed else 1 + + def record(stop, alice, chat): - probes, trace = [], [] + probes, trace, confirmed = [], [], [] def prober(): key = uuid.uuid4().hex while not stop.is_set(): started = time.time() - status, instance, _ = send(alice, chat, "probe " + key, key) + status, instance, body = send(alice, chat, "probe " + key, key) probes.append((started, status, round(1000 * (time.time() - started), 1), instance)) if status == 201: + confirmed.append(body) key = uuid.uuid4().hex time.sleep(0.1) @@ -228,7 +328,38 @@ def record(stop, alice, chat): threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)] for thread in threads: thread.start() - return probes, trace, threads + return probes, trace, threads, confirmed + + +def history_check(reader, chat, confirmed): + items, cursors = [], set() + cursor = None + for _ in range(len(confirmed) // 100 + 2): + path = f"/api/v1/chats/{chat}/messages?limit=100" + if cursor: + path += "&cursor=" + urllib.parse.quote(cursor, safe="") + status, _, page = call("GET", path, headers=reader["auth"]) + if status != 200 or not isinstance(page, dict): + break + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"]) + return { + "confirmed": len(confirmed), + "stored": len(items), + "exact_match": bool(confirmed) and same, + "unique": len({m["id"] for m in items}) == len(items), + } + if cursor in cursors: + break + cursors.add(cursor) + return { + "confirmed": len(confirmed), + "stored": len(items), + "exact_match": False, + "unique": False, + } def write_csv(path, header, rows): @@ -241,6 +372,10 @@ def summarize(args, probes, trace, action, expected): t0, end = action["start"], action["end"] label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action") failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0] + successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0] + recovery = ( + next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0 + ) down = [t for t, up in trace if t >= t0 and up is not None and up < expected] back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected] outage = [t for t in down if not back or t < back[0]] or [end] @@ -256,13 +391,15 @@ def summarize(args, probes, trace, action, expected): "action_finished_before_load_end": end < action["load_end"], "lb_up_before": expected, "lb_up_min": min((up for _, up in trace if up is not None), default=None), + "lb_samples_missing": sum(up is None for _, up in trace), "detection_seconds": round(down[0] - t0, 3) if down else None, "rejoin_seconds": round(back[0] - t0, 3) if back else None, "probe": { "attempts": len(probes), "failed_attempts": sum(1 for _, s, _, _ in probes if s != 201), "first_failure_s": round(failures[0], 3) if failures else None, - "recovery_s": round(failures[-1], 3) if failures else 0.0, + "first_success_s": round(successes[0][1], 3) if successes else None, + "recovery_s": round(recovery, 3) if recovery is not None else None, "instances_before": dict(before), "instances_during_outage": dict(during), "instances_after": dict(after), @@ -275,7 +412,13 @@ def verdict(args, result): checks = { "action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"], "load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio, - "write_recovery_ok": probe["recovery_s"] <= args.max_recovery, + "load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0, + "lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0, + "write_recovery_ok": all( + probe[k] is not None and probe[k] <= args.max_recovery + for k in ("first_success_s", "recovery_s") + ), + "history_preserved": result["history"]["exact_match"] and result["history"]["unique"], } if args.expect == "dip": checks["detected"] = result["detection_seconds"] is not None @@ -296,7 +439,8 @@ def load(args): expected = lb_up() or 0 alice, bob = actor("alice"), actor("bob") stop, action = threading.Event(), {} - probes, trace, threads = record(stop, alice, chat_of(alice, bob)) + chat = chat_of(alice, bob) + probes, trace, threads, confirmed = record(stop, alice, chat) def act(): time.sleep(args.warmup + args.lead) @@ -327,6 +471,7 @@ def load(args): ) write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace)) result = summarize(args, probes, trace, action, expected) + result["history"] = history_check(bob, chat, confirmed) result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()} result["k6"] = dict( k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate @@ -344,10 +489,13 @@ def detect(args): started = time.time() process = subprocess.Popen(args.command, cwd=str(ROOT)) detected = acted = None + missing = 0 while time.time() - started < args.timeout and (detected is None or acted is None): if acted is None and process.poll() is not None: acted = time.time() - started - if detected is None and (lb_up() or 0) <= until: + up = lb_up() + missing += up is None + if detected is None and up is not None and up <= until: detected = time.time() - started time.sleep(0.05) code = process.wait() @@ -362,6 +510,7 @@ def detect(args): restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None checks = { "action_ok": code == 0, + "lb_observed": before >= 2 and missing == 0, "detected_in_time": detected is not None and detected <= args.max + args.slack, "lb_answer_ok": status == args.lb_status, } @@ -374,6 +523,7 @@ def detect(args): "action_exit": code, "action_seconds": round(acted, 3), "lb_up_before": before, + "lb_samples_missing": missing, "lb_up_counted_as_detected": until, "detection_seconds": None if detected is None else round(detected, 3), "declared_max_seconds": args.max, @@ -413,6 +563,10 @@ def main(): under.add_argument("--max-failed-ratio", type=float, default=0.01) under.add_argument("--max-recovery", type=float, default=10.0) under.add_argument("command", nargs=argparse.REMAINDER) + again = sub.add_parser("retry", help="retry a write whose answer was lost, same key") + again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)") + again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s") + again.add_argument("--out", help="JSON file, written only after a complete report") watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation") watch.add_argument("--max", type=float, required=True, help="declared worst detection, s") watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s") @@ -426,6 +580,8 @@ def main(): args = parser.parse_args() if args.mode == "check": return check(args) + if args.mode == "retry": + return retry(args) args.command = [c for c in args.command if c != "--"] if not args.command: parser.error(f"{args.mode} needs an action command after --") diff --git a/scripts/reconcile-inject.sh b/scripts/reconcile-inject.sh new file mode 100755 index 0000000..fb18b47 --- /dev/null +++ b/scripts/reconcile-inject.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +kind="${1:?usage: reconcile-inject.sh orphan|missing|corrupt|bitflip|stale-pending|pending-object|multipart [count]}" +count="${2:-1}" +age="${INJECT_AGE_SECONDS:-7200}" +prefix="${S3_PREFIX:-attachments/}" +sql() { docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -qAt -F ' ' -U "${PGUSER:-messenger}" -d "${PGDATABASE:-messenger}" -c "$1"; } +s3() { docker compose run --rm --no-deps -T -e ROWS="$1" s3-cli sh -c "$2"; } +ready() { sql "SELECT id, object_key, size_bytes FROM attachments WHERE status = 'ready' ${INJECT_IDS:+AND id = ANY('{$INJECT_IDS}'::uuid[])} ORDER BY random() LIMIT $count"; } +pending() { + sql "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at) + SELECT g.id, s.chat_id, s.owner_id, 'injected.png', 'image/png', 2048, repeat('0', 64), 1, 1, '$prefix' || g.id, 'pending', + clock_timestamp() - make_interval(secs => $age), clock_timestamp() - make_interval(secs => $age) + FROM (SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1) s, + (SELECT gen_random_uuid() AS id FROM generate_series(1, $count)) g + RETURNING id, object_key" +} +fresh() { sql "SELECT g, '$prefix' || g FROM (SELECT gen_random_uuid() AS g FROM generate_series(1, $count)) s"; } +put_random='printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; head -c "${size:-2048}" /dev/urandom | rc pipe "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' +case "$kind" in + orphan) rows=$(fresh); s3 "$rows" "$put_random" | sed "s/^/orphan /" ;; + missing) rows=$(ready); s3 "$rows" 'printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; rc object remove --force "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' | sed "s/^/missing /" ;; + corrupt) rows=$(ready | awk '{print $1, $2, int($3 / 2) + 1}'); s3 "$rows" "$put_random" | sed "s/^/corrupt /" ;; + bitflip) rows=$(ready); s3 "$rows" "$put_random" | sed "s/^/bitflip /" ;; + stale-pending) pending | sed "s/^/stale-pending /" ;; + pending-object) rows=$(pending); s3 "$rows" "$put_random" | sed "s/^/pending-object /" ;; + multipart) + fresh | while read -r _ key; do + docker compose exec -T -e KEY="$key" -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" s3 sh -c 'u="http://127.0.0.1:9000/$S3_BUCKET/$KEY"; a="$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"; s="aws:amz:${RUSTFS_REGION:-us-east-1}:s3" + upload=$(curl -fsS --aws-sigv4 "$s" --user "$a" -X POST "$u?uploads" | sed -n "s:.*\(.*\).*:\1:p") + head -c 4096 /dev/urandom | curl -fsS --aws-sigv4 "$s" --user "$a" -X PUT --data-binary @- "$u?partNumber=1&uploadId=$upload" -o /dev/null' &2; exit 2 ;; +esac diff --git a/scripts/reconcile_check.py b/scripts/reconcile_check.py new file mode 100755 index 0000000..8712c5a --- /dev/null +++ b/scripts/reconcile_check.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +import argparse +import datetime as dt +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SQL = ( + "SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256," + " floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint," + " EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id)," + " (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL" + " FROM attachments a ORDER BY a.object_key" +) +HASH = ( + 'while read -r k; do printf "%s %s\\n" "$k" ' + '"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done' +) +CLASSES = ( + "orphan_stale", + "orphan_fresh", + "missing_referenced", + "missing_unreferenced", + "size_mismatch", + "sha_mismatch", + "pending_stale_with_object", + "pending_stale_without_object", + "pending_fresh", + "quarantined", + "incomplete_stale", + "incomplete_fresh", + "lost_ready", +) +TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"} +BROKEN = ( + "missing_referenced", + "missing_unreferenced", + "size_mismatch", + "sha_mismatch", + "quarantined", +) + + +def timed(cmd, stdin=None): + started = time.monotonic() + out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False) + if out.returncode: + raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}") + return out.stdout, round(time.monotonic() - started, 3) + + +def s3(*args, stdin=None): + return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin) + + +def age(stamp, now): + return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp() + + +def load(bucket, prefix, deep): + listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}") + uploads, uploads_seconds = s3( + "object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}" + ) + user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger") + psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"] + rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL]) + objects = {} + for item in json.loads(listing).get("items", []): + key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"] + objects[key] = item + rows = {} + for line in rows_raw.splitlines(): + rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t") + rows[key] = { + "id": rid, + "status": status, + "size": int(size), + "sha256": sha, + "age": int(row_age), + "referenced": referenced == "t", + "quarantined": quarantined == "t", + } + hashes, deep_seconds = {}, 0.0 + if deep: + present = "".join( + f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects + ) + out, deep_seconds = s3("sh", "-c", HASH, stdin=present) + hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line) + seconds = { + "s3_list": list_seconds, + "s3_incomplete": uploads_seconds, + "pg_query": pg_seconds, + "deep_hash": deep_seconds, + } + return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds + + +def row_class(key, row, objects, hashes, grace): + if row["quarantined"]: + return "quarantined" + if row["status"] == "pending": + stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object" + return "pending_fresh" if row["age"] <= grace else stale + if key not in objects: + return "missing_referenced" if row["referenced"] else "missing_unreferenced" + if objects[key]["size_bytes"] != row["size"]: + return "size_mismatch" + return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None + + +def classify(objects, rows, hashes, uploads, grace): + now = time.time() + classes = {name: [] for name in CLASSES} + for key, item in objects.items(): + if key not in rows: + classes[ + "orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh" + ].append(key) + for key, row in rows.items(): + name = row_class(key, row, objects, hashes, grace) + if name: + classes[name].append(key if name.startswith("orphan") else row["id"]) + for item in uploads: + classes[ + "incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh" + ].append(item["key"]) + return classes + + +def main(): + p = argparse.ArgumentParser( + description="Independent PG <-> S3 invariant check for lab4 attachments" + ) + p.add_argument( + "--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600")) + ) + p.add_argument( + "--deep", action="store_true", help="stream every present object and compare sha256" + ) + p.add_argument( + "--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments" + ) + p.add_argument( + "--expect-ready", + metavar="FILE", + help="fail if any attachment from FILE is no longer healthy", + ) + p.add_argument("--limit", type=int, default=20, help="max ids per class in the report") + p.add_argument("--output", metavar="FILE") + args = p.parse_args() + started = time.monotonic() + prefix = os.environ.get("S3_PREFIX", "attachments/") + objects, rows, hashes, uploads, seconds = load( + os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep + ) + classes = classify(objects, rows, hashes, uploads, args.grace) + bad = {i for c in BROKEN for i in classes[c]} + healthy = { + r["id"]: r["sha256"] + for k, r in rows.items() + if r["status"] == "ready" and k in objects and r["id"] not in bad + } + if args.expect_ready: + expected = json.loads(Path(args.expect_ready).read_text()) + classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha) + if args.save_ready: + Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True)) + violations = [c for c in CLASSES if classes[c] and c not in TOLERATED] + seconds["total"] = round(time.monotonic() - started, 3) + report = { + "checked_at": dt.datetime.now(dt.timezone.utc) + .isoformat(timespec="seconds") + .replace("+00:00", "Z"), + "grace_seconds": args.grace, + "deep": args.deep, + "objects": len(objects), + "rows": len(rows), + "ready_healthy": len(healthy), + "counts": {c: len(v) for c, v in classes.items()}, + "samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v}, + "seconds": seconds, + "violations": violations, + "ok": not violations, + } + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).write_text(text + "\n") + print(text) + sys.exit(0 if report["ok"] else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/reconcile_track.py b/scripts/reconcile_track.py new file mode 100644 index 0000000..b2b1abe --- /dev/null +++ b/scripts/reconcile_track.py @@ -0,0 +1,823 @@ +#!/usr/bin/env python3 +import argparse +import http.client +import json +import os +import re +import statistics +import struct +import subprocess +import sys +import threading +import time +import uuid +import zlib +from pathlib import Path +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +client = __import__("client") +FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() +GRACE = 60 +FLAGS = ("--grace", str(GRACE), "--skew", "0") +INTENT_AGE = 300 +LOCK_KEY = 20260004 +FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale") +USER_HZ = 100 +BUSY = (0, 1, 2, 5, 6, 7) + + +def run(cmd, env=None, stdin=None): + started = time.monotonic() + done = subprocess.run( + cmd, + cwd=ROOT, + input=stdin, + capture_output=True, + check=False, + env=dict(os.environ, **(env or {})), + ) + out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace") + return done.returncode, out, err, round(time.monotonic() - started, 3) + + +def must(cmd, env=None, stdin=None): + code, out, err, _ = run(cmd, env, stdin) + if code: + raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}") + return out + + +def compose(*args, env=None, stdin=None): + return must(["docker", "compose", *args], env, stdin) + + +def psql(sql): + return compose( + "exec", + "-T", + "postgres", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-qAt", + "-U", + "messenger", + "-d", + "messenger", + "-c", + sql, + ).strip() + + +def reconcile(*flags, env=None): + cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags] + code, out, err, wall = run(cmd, env) + if code not in (0, 1) or not out.strip(): + raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}") + report = json.loads(out) + audit = [json.loads(line) for line in err.splitlines() if '"audit.' in line] + report.update(exit_code=code, wall_seconds=wall, audit=audit) + return report + + +def oracle(*flags): + code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags]) + if not out.strip(): + raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}") + return json.loads(out) + + +def inject(kind, count, ids=(), age=None): + env = {"INJECT_IDS": ",".join(ids)} if ids else {} + if age is not None: + env["INJECT_AGE_SECONDS"] = str(age) + out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env) + return [line.split()[1] for line in out.splitlines() if line.strip()] + + +def head(): + return must(["git", "rev-parse", "HEAD"]).strip() + + +def evidence_root(): + return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile") + + +def save(name, data, *, stamp=True): + path = evidence_root() / name + path.parent.mkdir(parents=True, exist_ok=True) + stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data} + path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n") + return path + + +class Checks: + def __init__(self): + self.items = [] + + def __call__(self, name, ok, **detail): + self.items.append(dict(name=name, ok=bool(ok), **detail)) + sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n") + + @property + def ok(self): + return all(item["ok"] for item in self.items) + + +def chunk(tag, data): + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data)) + + +def tiny_png(): + raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8)) + header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(raw)) + + chunk(b"IEND", b"") + ) + + +def pick(found, names): + return {name: found.get(name, 0) for name in names} + + +def faults(c, check, pre): + alice, bob, _, chat = c.bootstrap() + ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)] + linked = ids[:6] + for part in (linked[:3], linked[3:]): + c.message(alice, chat, "", attachments=part) + targets = { + "missing": [linked[0], linked[1], ids[6]], + "corrupt": [linked[2], ids[7], ids[8]], + "bitflip": [linked[3], ids[9], ids[10]], + } + damaged = {i for group in targets.values() for i in group} + ready = evidence_root() / "ready-before.json" + oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready)) + untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged} + save("ready-untouched.json", untouched, stamp=False) + started = time.monotonic() + injected = { + "orphan_stale": inject("orphan", 3), + "missing": inject("missing", 3, targets["missing"]), + "size_mismatch": inject("corrupt", 3, targets["corrupt"]), + "sha_mismatch": inject("bitflip", 3, targets["bitflip"]), + "pending_stale": inject("stale-pending", 3, age=INTENT_AGE) + + inject("pending-object", 3, age=INTENT_AGE), + } + time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started))) + injected["orphan_fresh"] = inject("orphan", 2) + injected["pending_fresh"] = inject("stale-pending", 2, age=0) + expected = {name: len(keys) for name, keys in injected.items()} + expected.update(quarantined=pre["counts"]["quarantined"]) + before = oracle( + "--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json") + ) + dry = reconcile("--dry-run", "--deep", *FLAGS) + save("dry-run.json", dry) + after_dry = oracle("--grace", str(GRACE), "--deep") + named = {k: sorted(s.rsplit("/", 1)[-1] for s in dry["samples"].get(k, [])) for k in injected} + check( + "dry-run находит каждый внедрённый класс, и найдено ровно внедрённое", + pick(dry["found"], expected) == expected + and named == {name: sorted(ids) for name, ids in injected.items()}, + expected=expected, + found=dry["found"], + ) + check( + "dry-run ничего не меняет", + not any(dry["fixed"].values()) + and after_dry["counts"] == before["counts"] + and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]), + objects=[before["objects"], after_dry["objects"]], + rows=[before["rows"], after_dry["rows"]], + ) + first = reconcile("--apply", "--deep", *FLAGS) + save("apply-1.json", first) + discarded = sorted(e["attachment_id"] for e in first["audit"] if e["reason"] == "reconcile") + check( + "apply исправляет ровно найденное dry-run, удаление намерений — в журнале аудита", + pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE) + and not first["errors"] + and discarded == sorted(injected["pending_stale"]), + dry_run=pick(dry["found"], FIXABLE), + fixed=pick(first["fixed"], FIXABLE), + audit_discarded=len(discarded), + ) + second = reconcile("--apply", "--deep", *FLAGS) + save("apply-2.json", second) + check( + "повторный apply — 0 действий", + not any(second["fixed"].values()), + fixed=second["fixed"], + found=second["found"], + ) + after = oracle( + "--grace", + str(GRACE), + "--deep", + "--expect-ready", + str(evidence_root() / "ready-untouched.json"), + "--output", + str(evidence_root() / "check-after.json"), + ) + tolerated = ("orphan_fresh", "pending_fresh") + check( + "независимая проверка после apply: нарушений нет, готовые не потеряны", + after["ok"] + and not after["counts"]["lost_ready"] + and pick(after["counts"], tolerated) == pick(before["counts"], tolerated) + and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9, + counts=after["counts"], + ) + marks = dict( + line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split() + ) + links = psql("SELECT attachment_id FROM message_attachments").split() + kept = {i: marks.get(i) for i in sorted(damaged)} + check( + "строки с расхождениями и ссылки сообщений сохранены", + all(kept.values()) and sum(i in linked for i in links) == len(linked), + quarantine=kept, + message_links=sum(i in linked for i in links), + ) + meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment") + blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob) + intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob) + check( + "карантин: метаданные 200, /content 404; нетронутое скачивается", + meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE, + ) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + for attachment, data in ((linked[0], FIXTURE), (linked[1], os.urandom(len(FIXTURE)))): + compose( + "run", + "--rm", + "--no-deps", + "-T", + "s3-cli", + "pipe", + f"root/{bucket}/attachments/{attachment}", + stdin=data, + ) + healed = reconcile("--apply", *FLAGS) + save("healed.json", healed) + restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob) + foreign = c.call("GET", f"/api/v1/attachments/{linked[1]}/content", 404, actor=bob) + check( + "вернули те же байты — карантин снят; другие байты той же длины — остался", + healed["fixed"]["healed"] == 1 + and healed["samples"].get("healed") == [linked[0]] + and linked[1] in healed["samples"].get("quarantined", []) + and restored["bytes"] == FIXTURE + and foreign["status"] == 404, + fixed=healed["fixed"], + s3_get=healed["cost"]["requests"]["s3_get"], + ) + return {"injected": injected, "targets": targets, "linked": linked} + + +def single(check): + holder = subprocess.Popen( + [ + "docker", + "compose", + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + "messenger", + "-c", + f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)", + ], + cwd=ROOT, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + time.sleep(3) + busy = reconcile("--apply", *FLAGS) + finally: + holder.wait() + save("lock-busy.json", busy) + check( + "второй одновременный проход пропущен", + busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()), + ) + + +def healthy(service, seconds=90): + container = compose("ps", "-q", service).strip() + probe = ["docker", "inspect", "-f", "{{.State.Health.Status}}", container] + deadline = time.monotonic() + seconds + while must(probe).strip() != "healthy": + if time.monotonic() > deadline: + raise SystemExit(f"{service} is not healthy after {seconds} s") + time.sleep(1) + + +def interrupt(check): + name = f"hlpipe-lab6-reconcile-sigterm-{os.getpid()}" + compose("pause", "s3") + try: + cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "--name", name, "reconciler"] + runner = subprocess.Popen( + [*cmd, "--apply", *FLAGS], + cwd=ROOT, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + time.sleep(4) + state = must(["docker", "inspect", "-f", "{{.State.Status}} {{.HostConfig.Init}}", name]) + started = time.monotonic() + must(["docker", "stop", name]) + stopped = round(time.monotonic() - started, 2) + code = runner.wait(timeout=60) + finally: + compose("unpause", "s3") + healthy("s3") + after = reconcile("--dry-run", *FLAGS) + status, init = state.split() + report = { + "s3": "paused", + "state_before_stop": status, + "init": init, + "stop_seconds": stopped, + "exit_code": code, + "next_pass_skipped": after["skipped"], + } + save("sigterm.json", report) + check( + "SIGTERM: идущий проход останавливается за секунды, блокировка освобождена", + status == "running" and code == 143 and stopped < 5 and not after["skipped"], + **report, + ) + + +def outage(check): + before = oracle("--grace", str(GRACE)) + compose("stop", "s3") + try: + cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", "--apply"] + code, out, err, wall = run([*cmd, *FLAGS]) + finally: + compose("up", "-d", "--wait", "--no-deps", "s3") + after = oracle("--grace", str(GRACE)) + log = [line for line in err.splitlines() if "reconcile." in line] + same = all(after[k] == before[k] for k in ("objects", "rows", "counts")) + save("s3-down.json", {"exit_code": code, "wall_seconds": wall, "stdout": out, "log": log}) + check( + "S3 остановлен: код 1 за ограниченное время, состояние не изменилось", + code == 1 and wall < 30 and not out.strip() and same, + exit_code=code, + wall_seconds=wall, + ) + + +def uploader(base, token, chat, *, deadline, results, lock): + host = urlsplit(base).netloc + conn = http.client.HTTPConnection(host, timeout=30) + headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"} + sent = 0 + while time.monotonic() < deadline: + path = f"/api/v1/chats/{chat}/attachments?filename=load.png" + try: + conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"}) + response = conn.getresponse() + status, data = response.status, response.read() + if status == 201 and sent % 10 == 0: + body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]}) + extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex} + conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra}) + linked = conn.getresponse() + linked.read() + status = 201 if linked.status == 201 else -linked.status + except (OSError, http.client.HTTPException): + conn.close() + conn = http.client.HTTPConnection(host, timeout=30) + status, data = 0, b"" + with lock: + results.append((status, json.loads(data) if status == 201 else None)) + sent += 1 + time.sleep(0.05) + + +def under_load(c, check, base, seconds, threads): + alice, _, _, chat = c.bootstrap() + ready = evidence_root() / "ready-load.json" + left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"] + leftover = { + "orphan_stale": left["orphan_fresh"] + left["orphan_stale"], + "pending_stale": left["pending_fresh"] + + left["pending_stale_with_object"] + + left["pending_stale_without_object"], + } + orphans = inject("orphan", 3) + intents = inject("stale-pending", 3, age=INTENT_AGE) + results, lock = [], threading.Lock() + deadline = time.monotonic() + seconds + workers = [ + threading.Thread( + target=uploader, + args=(base, alice["token"], chat["id"]), + kwargs={"deadline": deadline, "results": results, "lock": lock}, + ) + for _ in range(threads) + ] + for worker in workers: + worker.start() + runs = [] + while time.monotonic() < deadline: + runs.append(reconcile("--apply", *FLAGS)) + for worker in workers: + worker.join() + runs.append(reconcile("--apply", *FLAGS)) + created = {att["id"]: att["sha256"] for status, att in results if att} + expected = dict(json.loads(ready.read_text()), **created) + save("ready-load.json", expected, stamp=False) + after = oracle( + "--grace", + str(GRACE), + "--deep", + "--expect-ready", + str(ready), + "--output", + str(evidence_root() / "check-under-load.json"), + ) + statuses = {} + for status, _ in results: + statuses[str(status)] = statuses.get(str(status), 0) + 1 + fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE} + rechecked = {} + for r in runs: + for name, count in r["rechecked"].items(): + rechecked[name] = rechecked.get(name, 0) + count + seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")} + summary = { + "seconds": seconds, + "upload_threads": threads, + "uploads": statuses, + "created": len(created), + "linked_every": 10, + "reconciler_runs": len(runs), + "run_seconds": [r["cost"]["seconds"] for r in runs], + "fixed_total": fixed, + "fresh_seen_total": seen, + "rechecked_total": rechecked, + "errors": sum(r["errors"] for r in runs), + "skipped": sum(1 for r in runs if r["skipped"]), + "injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)}, + "left_from_faults": leftover, + "check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")}, + } + save("under-load.json", summary) + check( + "под нагрузкой: исправлены только внедрённые, ошибок нет", + fixed + == { + **dict.fromkeys(FIXABLE, 0), + "orphan_stale": leftover["orphan_stale"] + len(orphans), + "pending_stale": leftover["pending_stale"] + len(intents), + } + and not summary["errors"] + and not summary["skipped"], + fixed=fixed, + runs=len(runs), + ) + check( + "под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0", + after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"}, + uploads=statuses, + lost_ready=after["counts"]["lost_ready"], + ) + + +def demo(args): + evidence_root().mkdir(parents=True, exist_ok=False) + c = client.Client(args.base_url, timeout=30) + check = Checks() + pre = oracle("--grace", str(GRACE)) + dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"} + if dirty: + raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply") + started = time.monotonic() + result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()} + result["faults"] = faults(c, check, pre) + single(check) + interrupt(check) + outage(check) + under_load(c, check, args.base_url, args.load_seconds, args.load_threads) + result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok) + save("demo.json", result) + print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False)) + sys.exit(0 if check.ok else 1) + + +def pins(args): + cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip()) + sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory} + if args.no_pin or cpus < 8: + return sized + return dict( + sized, + S3_CPUSET=f"{cpus - 5}-{cpus - 4}", + PG_CPUSET=f"{cpus - 3}-{cpus - 2}", + RECONCILE_CPUSET=str(cpus - 1), + ) + + +def cpu_set(env): + cpus = set() + for name in ("S3_CPUSET", "PG_CPUSET", "RECONCILE_CPUSET"): + for part in filter(None, env.get(name, "").split(",")): + low, _, high = part.partition("-") + cpus.update(range(int(low), int(high or low) + 1)) + return cpus + + +def limits(): + ids = compose("ps", "-q", "s3", "postgres").split() + fmt = ( + "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}" + " cpuset={{.HostConfig.CpusetCpus}}" + ) + return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines() + + +def host(title): + stats = must( + [ + "docker", + "stats", + "--no-stream", + "--all", + "--format", + "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}", + ] + ) + running = must(["docker", "ps", "--format", "{{.Names}} {{.Image}} {{.Status}}"]) + stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + load = " ".join(f"{v:.2f}" for v in os.getloadavg()) + return ( + f"== {title} date: {stamp} commit: {head()} host loadavg: {load}\n" + f"$ docker stats --no-stream --all\n{stats}\n$ docker ps\n{running}\n" + ) + + +def usage(text): + return int(re.search(r"^usage_usec (\d+)$", text, re.M).group(1)) / 1e6 + + +def sample(): + vm = compose( + "exec", "-T", "postgres", "cat", "/proc/uptime", "/proc/stat", "/sys/fs/cgroup/cpu.stat" + ) + s3 = compose("exec", "-T", "s3", "cat", "/sys/fs/cgroup/cpu.stat") + busy = {} + for line in vm.splitlines()[1:]: + name, *values = line.split() or [""] + if name.startswith("cpu") and name[3:].isdigit(): + busy[int(name[3:])] = sum(int(values[i]) for i in BUSY) / USER_HZ + return float(vm.split()[0]), busy, usage(vm) + usage(s3) + + +def screen(first, last, own, pinned, args, *, started=True): + window = max(last[0] - first[0], 0.01) + busy = {cpu: spent - first[1].get(cpu, 0.0) for cpu, spent in last[1].items()} + ours = last[2] - first[2] + own + other = sum(busy.values()) - ours - args.start_other * started + on_pinned = sum(busy.get(cpu, 0.0) for cpu in pinned) - ours - args.start_pinned * started + other, on_pinned = max(0.0, other / window), max(0.0, on_pinned / window) if pinned else 0.0 + return { + "window_seconds": round(window, 2), + "ours_cores": round(ours / window, 2), + "other_cores": round(other, 2), + "other_on_pinned_cores": round(on_pinned, 2), + "host_loadavg_1m": round(os.getloadavg()[0], 2), + "disturbed": other > args.max_other or on_pinned > args.max_pinned, + } + + +def wait_quiet(args, pinned): + deadline, calm = time.monotonic() + args.quiet_timeout, 0 + while calm < 3 and time.monotonic() < deadline: + first = sample() + time.sleep(5) + quiet = screen(first, sample(), 0.0, pinned, args, started=False) + calm = 0 if quiet["disturbed"] else calm + 1 + sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n") + return calm == 3 + + +def measure(flags, env, args, pinned): + rejected = [] + for attempt in range(args.retries + 1): + first = sample() + report = reconcile(*flags, "--samples", "3", env=env) + report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args) + if not report["screen"]["disturbed"]: + return report, rejected + rejected.append(report) + if attempt < args.retries and not wait_quiet(args, pinned): + break + return None, rejected + + +def seed(count, env): + if count <= 0: + return {"added": 0} + env = dict(env, API_CPUS="4", API_MEMORY="512m") + tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro" + out = compose( + "run", + "--rm", + "--no-deps", + "-T", + "-v", + tools, + "--entrypoint", + "python", + "api", + "/tools/seed_storage.py", + "--count", + str(count), + "--concurrency", + "4", + env=env, + ) + records = [json.loads(line) for line in out.strip().splitlines()] + return {**records[-1], "progress": records[:-1]} + + +def median(runs, path): + values = [] + for r in runs: + value = r + for key in path: + value = value[key] + values.append(value) + return statistics.median(values) + + +def summarize(results, rejected): + rows = [] + for (n, mode), runs in sorted(results.items()): + seconds = median(runs, ("cost", "seconds")) + rows.append( + { + "objects": n, + "mode": mode, + "runs": len(runs), + "seconds_median": seconds, + "seconds_all": [r["cost"]["seconds"] for r in runs], + "wall_seconds_median": median(runs, ("wall_seconds",)), + "us_per_object": round(seconds / n * 1e6, 1), + "cpu_seconds_median": median(runs, ("cost", "cpu_seconds")), + "s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")), + "sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")), + "requests": runs[0]["cost"]["requests"], + "max_rss_mib_median": median(runs, ("cost", "max_rss_mib")), + "other_cores_max": max(r["screen"]["other_cores"] for r in runs), + "other_on_pinned_cores_max": max( + r["screen"]["other_on_pinned_cores"] for r in runs + ), + "disturbed_kept": sum(r["screen"]["disturbed"] for r in runs), + "rejected": rejected.get((n, mode), 0), + "scanned": runs[0]["scanned"], + "found_nonzero": {k: v for k, v in runs[0]["found"].items() if v}, + } + ) + return rows + + +def table(rows): + head = ( + "objects mode runs t_med,s wall,s us/obj cpu,s list,s sql,s lists sqls" + " gets rss,MiB other pinned rejected" + ) + lines = [head] + for r in rows: + q = r["requests"] + lines.append( + f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} " + f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} " + f"{r['cpu_seconds_median']:>5.2f} {r['s3_list_wait_median']:>6.3f} " + f"{r['sql_wait_median']:>5.3f} {q['s3_list']:>5} {q['sql']:>4} {q['s3_get']:>5} " + f"{r['max_rss_mib_median']:>7.1f} {r['other_cores_max']:>5.2f} " + f"{r['other_on_pinned_cores_max']:>6.2f} {r['rejected']:>8}" + ) + return "\n".join(lines) + "\n" + + +def cost(args): + out = evidence_root() / "cost" + out.mkdir(parents=True, exist_ok=False) + (out / "disturbed").mkdir() + rows = int(psql("SELECT count(*) FROM attachments")) + if rows > args.sizes[0]: + raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project") + snapshot = out / "host-docker-stats.txt" + snapshot.write_text(host("before the series")) + env = pins(args) + pinned = cpu_set(env) + compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env) + stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []} + plan["screen"] = { + "max_other_cores": args.max_other, + "max_other_on_pinned_cores": args.max_pinned, + "start_allowance_core_seconds": {"vm": args.start_other, "pinned": args.start_pinned}, + "retries": args.retries, + } + results, rejected = {}, {} + try: + for n in args.sizes: + plan["seeds"].append(dict(seed(n - rows, env), target=n)) + rows = n + time.sleep(args.settle) + save("cost/plan.json", plan) + if not wait_quiet(args, pinned): + raise SystemExit("no quiet measurement window before timeout") + modes = [("shallow", ["--dry-run"])] + if n <= args.deep_max: + modes.append(("deep", ["--dry-run", "--deep"])) + if n == args.page_compare: + modes.append(("page100", ["--dry-run", "--page-size", "100"])) + for mode, flags in modes: + for attempt in range(1, args.repeats + 1): + report, dropped = measure(flags, env, args, pinned) + name = f"n{n}-{mode}-{attempt}" + for number, bad in enumerate(dropped, 1): + save(f"cost/disturbed/{name}-try{number}.json", bad) + rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped) + if report is None: + raise SystemExit(f"{name}: no clean run after retries; rejected preserved") + if ( + report["exit_code"] + or report["errors"] + or report["skipped"] + or report["scanned"] != {"objects": n, "rows": n} + or any(report["found"].values()) + ): + save(f"cost/invalid/{name}.json", report) + raise SystemExit(f"{name}: inconsistent dataset or failed pass") + save(f"cost/{name}.json", report) + results.setdefault((n, mode), []).append(report) + save("cost/summary.json", {**plan, "results": summarize(results, rejected)}) + sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n") + finally: + save("cost/summary.json", {**plan, "results": summarize(results, rejected)}) + compose("up", "-d", "--wait", "--no-deps", "s3", "postgres") + with snapshot.open("a") as handle: + handle.write("\n" + host("after the series")) + summary = summarize(results, rejected) + save("cost/summary.json", {**plan, "results": summary}) + (out / "summary.txt").write_text( + f"date: {stamp} commit: {plan['commit']} pins: {env}\n" + table(summary) + ) + sys.stdout.write(table(summary)) + + +def main(): + p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass") + sub = p.add_subparsers(dest="command", required=True) + d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load") + d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + d.add_argument("--load-seconds", type=int, default=75) + d.add_argument("--load-threads", type=int, default=3) + k = sub.add_parser("cost", help="cost of a pass for N objects, repeated") + k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 30000, 100000]) + k.add_argument("--repeats", type=int, default=3) + k.add_argument("--deep-max", type=int, default=100000) + k.add_argument("--page-compare", type=int, default=10000) + k.add_argument("--settle", type=float, default=10) + k.add_argument("--s3-memory", default="3g") + k.add_argument("--no-pin", action="store_true") + k.add_argument("--max-other", type=float, default=2.0, help="foreign CPU in the VM, cores") + k.add_argument("--max-pinned", type=float, default=0.25, help="foreign CPU on pinned cores") + k.add_argument("--start-other", type=float, default=1.5, help="container start, core-s") + k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores") + k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run") + k.add_argument("--quiet-timeout", type=float, default=900.0) + for command in (d, k): + command.add_argument( + "--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile") + ) + args = p.parse_args() + os.environ["RECONCILE_EVIDENCE_DIR"] = args.out + if args.command == "demo": + demo(args) + else: + cost(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index 8bbcb64..777c667 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -2,7 +2,8 @@ set -eu cd "$(dirname "$0")/.." service=${API_SERVICE:-api} -show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; } +show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; } +scripts/stand-usage.sh & echo "$(date -u +%T) migrate once before the first replacement" docker compose run --rm --no-deps -T migrate 2>&1 old=$(docker compose ps -q "$service") @@ -13,8 +14,9 @@ for id in $old; do scripts/lb-wait.sh $((count + 1)) new=$(docker compose ps -q "$service" | grep -v -F "$running") echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")" - docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null + docker stop -t "${STOP_TIMEOUT:-7}" "$id" >/dev/null docker rm "$id" >/dev/null done scripts/lb-wait.sh "$count" +wait echo "$(date -u +%T) done: $count replicas replaced" diff --git a/scripts/s3-check.sh b/scripts/s3-check.sh index fa1bdae..79402bc 100755 --- a/scripts/s3-check.sh +++ b/scripts/s3-check.sh @@ -2,17 +2,30 @@ set -eu cd "$(dirname "$0")/.." cli() { docker compose run --rm --no-deps -T s3-cli "$@"; } +undated() { + python3 -c 'import json, sys +def clean(value): + if isinstance(value, dict): + return {k: clean(v) for k, v in value.items() if not k.endswith("_date")} + return [clean(v) for v in value] if isinstance(value, list) else value +text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0 +while text[at:].strip(): + at += len(text[at:]) - len(text[at:].lstrip()) + value, at = decoder.raw_decode(text, at) + print(json.dumps(clean(value), indent=2, ensure_ascii=False))' +} snapshot() { - cli sh -s <<'EOF' | grep -v '_date' + cli sh -s <<'EOF' | undated ak=$(cat /run/secrets/s3_access_key_id) -for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do +rk=$(cat /run/secrets/s3_reconciler_access_key_id) +for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do rc $c --json -done | sed "s/$ak//g" +done | sed "s/$ak//g; s/$rk//g" EOF } failed=0 verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; } -echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks" +echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) == S3 privacy and bootstrap checks" verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)" verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)" before=$(snapshot) @@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same cli sh -s <<'EOF' || failed=1 b=$S3_BUCKET key="attachments/s3-check-$(date +%s)-$$" +spare="$key-reconciler" other="s3-check-$(date +%s)-$$" head -c 4096 /dev/urandom > /tmp/probe result=0 expect() { want=$1; name=$2; shift 2 - if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi + if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi } expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key" expect allow "api: HeadObject" rc object stat "api/$b/$key" expect allow "api: GetObject" rc object show "api/$b/$key" -expect allow "api: ListBucket" rc object list "api/$b/attachments/" +expect deny "api: ListBucket" rc object list "api/$b/attachments/" +rc put /tmp/probe "api/$b/$spare" >/dev/null +expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/" +expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key" +expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key" +expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put" +expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare" +expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other" +expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b" +expect deny "reconciler: admin ListUsers" rc admin user list reconciler expect deny "api: CreateBucket" rc bucket create "api/$other" expect deny "api: DeleteBucket" rc bucket remove "api/$b" expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b" -expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json +expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json expect deny "api: admin ListUsers" rc admin user list api rc bucket create --ignore-existing "root/$other" >/dev/null expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x" diff --git a/scripts/scaling.py b/scripts/scaling.py new file mode 100755 index 0000000..def0ba1 --- /dev/null +++ b/scripts/scaling.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +import argparse +import collections +import contextlib +import csv +import json +import math +import os +import re +import signal +import statistics +import subprocess +import sys +import time +from pathlib import Path + +from probe import ROOT, k6_metrics, lb_up, run_k6 + +SCRIPTS = ROOT / "scripts" +UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9} +PINNED = { + "PG_CPUSET": "0-1", + "LB_CPUSET": "2", + "VALKEY_CPUSET": "3", + "K6_CPUSET": "4-6", + "K6_CPUS": "3", + "K6_MEMORY": "1g", +} +API_FIRST_CPU = 7 +FIELDS = [ + "raw_dir", + "variant", + "config", + "replicas", + "cpus", + "memory_mb", + "workers", + "pool", + "maxconn", + "api_cpuset", + "rate", + "repeat", + "started", + "db_mb", + "messages", + "k6_exit", + "saturated", + "requests", + "rps", + "ok_rps", + "failed_ratio", + "dropped_iterations", + "p50_ms", + "p95_ms", + "p99_ms", + "p95_post_ms", + "p95_get_ms", + "cpu_api", + "cpu_postgres", + "cpu_lb", + "cpu_valkey", + "cpu_k6", + "mem_api_mb", + "mem_postgres_mb", + "mem_k6_mb", + "pg_backends_max", + "pg_active_avg", + "pg_active_max", + "pg_idle_avg", + "pg_idle_tx_avg", + "pg_lock_avg", + "pg_lock_max", + "pg_lwlock_avg", + "pg_io_avg", + "lb_scur_max", + "lb_qcur_max", + "lb_qtime_max_ms", + "lb_rtime_avg_ms", + "cpu_vm", + "cpu_other", + "gap_s", + "wall_s", + "disturbed", +] + + +def compose(*args, env=None): + command = ["docker", "compose", *args] + return subprocess.run( + command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True + ).stdout + + +def size(text): + match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip()) + return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0 + + +def rows(path, start, end): + with path.open() as handle: + return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end] + + +def mean(values): + return round(statistics.mean(values), 2) if values else 0.0 + + +def docker_usage(path, project, start, end): + ticks = collections.defaultdict(collections.Counter) + memory = collections.Counter() + for r in rows(path, start, end): + name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :]) + ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0) + memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20) + cpu = { + name: mean([t[name] for t in ticks.values() if name in t]) + for name in ("api", "postgres", "lb", "valkey", "k6") + } + usage = {f"cpu_{name}": value for name, value in cpu.items()} + usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")}) + return usage + + +def vm_usage(path, start, end): + samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)] + if len(samples) < 2 or samples[-1][1] == samples[0][1]: + return 0.0 + first, last = samples[0], samples[-1] + return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1) + + +def gap(path, start, end): + ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)}) + edges = [start, *[t for t in ticks if start <= t <= end], end] + return max(b - a for a, b in zip(edges, edges[1:])) + + +def vm_counters(): + lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines() + values = [int(v) for v in lines[0].split()[1:]] + cpus = sum(1 for line in lines if re.match(r"cpu\d", line)) + return sum(values) - values[3] - values[4], sum(values), cpus + + +def wait_quiet(args): + deadline, calm = time.monotonic() + args.quiet_timeout, 0 + while calm < 3 and time.monotonic() < deadline: + first = vm_counters() + time.sleep(5) + last = vm_counters() + busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1]) + calm = calm + 1 if busy < args.max_other else 0 + print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True) + + +def pg_usage(path, start, end): + ticks = collections.defaultdict(collections.Counter) + for r in rows(path, start, end): + count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"] + tick = ticks[r["ts"]] + tick["backends"] += count + tick["active"] += count * (state == "active") + tick["idle"] += count * (state == "idle") + tick["idle_tx"] += count * state.startswith("idle in transaction") + tick["lock"] += count * (wait == "Lock") + tick["lwlock"] += count * (wait == "LWLock") + tick["io"] += count * (wait == "IO") + series = list(ticks.values()) + return { + "pg_backends_max": max((t["backends"] for t in series), default=0), + "pg_active_avg": mean([t["active"] for t in series]), + "pg_active_max": max((t["active"] for t in series), default=0), + "pg_idle_avg": mean([t["idle"] for t in series]), + "pg_idle_tx_avg": mean([t["idle_tx"] for t in series]), + "pg_lock_avg": mean([t["lock"] for t in series]), + "pg_lock_max": max((t["lock"] for t in series), default=0), + "pg_lwlock_avg": mean([t["lwlock"] for t in series]), + "pg_io_avg": mean([t["io"] for t in series]), + } + + +def lb_usage(path, start, end): + backend = [ + r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND" + ] + + def numbers(field): + return [int(r[field] or 0) for r in backend] + + return { + "lb_scur_max": max(numbers("scur"), default=0), + "lb_qcur_max": max(numbers("qcur"), default=0), + "lb_qtime_max_ms": max(numbers("qtime"), default=0), + "lb_rtime_avg_ms": mean(numbers("rtime")), + } + + +def trend(path, name): + metrics = json.loads(Path(path).read_text())["metrics"] + found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {}) + return round(found.get("values", {}).get("p(95)", 0.0), 2) + + +def sampler(*command): + return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True) + + +def stop(process): + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + process.wait() + + +def wait_lb(expected, timeout=120): + deadline, stable = time.monotonic() + timeout, 0 + while stable < 4: + if time.monotonic() > deadline: + raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})") + stable = stable + 1 if lb_up() == expected else 0 + time.sleep(0.5) + + +def layout(config, args, variant): + name, _, extra = variant.partition(":") + replicas, cpus = (float(v) for v in config.split("x")) + replicas = int(replicas) + workers = args.workers or max(1, int(cpus)) + width = max(1, math.ceil(replicas * cpus)) + cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "") + memory = args.api_memory or args.api_memory_total // replicas + settings = { + "API_REPLICAS": str(replicas), + "API_CPUS": f"{cpus:g}", + "API_MEMORY": f"{memory}m", + "WEB_CONCURRENCY": str(workers), + "DB_POOL_MAX_SIZE": str(args.pool), + "API_SERVER_MAXCONN": str(args.conn_per_worker * workers), + "API_CPUSET": cpuset if args.pin else "", + } + settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))])) + base = { + "variant": name, + "config": config, + "replicas": replicas, + "cpus": cpus, + "memory_mb": memory, + "workers": int(settings["WEB_CONCURRENCY"]), + "pool": int(settings["DB_POOL_MAX_SIZE"]), + "maxconn": int(settings["API_SERVER_MAXCONN"]), + "api_cpuset": settings["API_CPUSET"], + } + return base, settings + + +def apply(base, settings, env, out): + env = dict(env, **settings) + for attempt in range(3): + try: + compose("up", "-d", "--wait", "--wait-timeout", "180", env=env) + break + except subprocess.CalledProcessError as error: + log = Path(out) / "compose-up-failed.txt" + with log.open("a") as handle: + stamp = time.strftime("%FT%TZ", time.gmtime()) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + handle.write(f"date: {stamp} commit: {commit}\n") + handle.write(f"== {stamp} {base} attempt {attempt + 1}\n") + handle.write(error.stdout + error.stderr + "\n") + print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)})) + if attempt == 2: + raise + time.sleep(30) + wait_lb(base["replicas"]) + return env + + +def deployment(env): + ids = compose("ps", "-q", env=env).split() + template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}" + result = subprocess.run( + ["docker", "inspect", "--format", template, *ids], + check=True, + capture_output=True, + text=True, + ) + containers = [] + for line in result.stdout.splitlines(): + name, image, config = line.split(" ", 2) + limits = json.loads(config) + containers.append( + dict( + name=json.loads(name), + image=json.loads(image), + **{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")}, + ) + ) + return containers + + +def provenance(folder, args, env): + stamp = time.strftime("%FT%TZ", time.gmtime()) + metadata = { + "date": stamp, + "commit": args.commit, + "project": args.project, + "command": sys.argv, + "api_image": env.get("API_IMAGE", "compose default"), + "warmup_seconds": args.warmup, + "duration_seconds": args.duration, + "containers": args.deployment, + } + (folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n") + for path in folder.rglob("*"): + if path.suffix in {".csv", ".txt"}: + path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text()) + elif path.name == "summary.json": + data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text())) + path.write_text(json.dumps(data) + "\n") + + +def attempt_folder(out, base, rate, repeat): + name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}" + attempt = 1 + while (Path(out) / f"{name}_attempt{attempt}").exists(): + attempt += 1 + folder = Path(out) / f"{name}_attempt{attempt}" + folder.mkdir(parents=True) + return folder + + +def measure(folder, args, rate, env): + subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True) + database = subprocess.run( + [str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True + ).stdout.split(",") + samplers = [ + sampler("docker-stats.sh", str(folder / "docker_stats.csv")), + sampler("lb-stats.sh", str(folder / "lb_stats.csv")), + sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")), + ] + load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration)) + began = time.time() + try: + code, started = run_k6(folder / "k6", load) + finally: + wall = time.time() - began + for process in samplers: + stop(process) + subprocess.run( + [str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True + ) + if started is None: + raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}") + return code, started + args.warmup, database, wall + + +def step(base, rate, repeat, args, env): + folder = attempt_folder(args.out, base, rate, repeat) + code, start, database, wall = measure(folder, args, rate, env) + end = start + args.duration + summary = folder / "k6/summary.json" + metrics = k6_metrics(summary, args.duration) + saturated = ( + metrics["failed_ratio"] > args.max_error_ratio + or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration + or metrics["p95_ms"] > args.slo_p95_ms + ) + row = dict( + base, + raw_dir=folder.name, + rate=rate, + repeat=repeat, + k6_exit=code, + saturated=saturated, + **metrics, + ) + row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start)) + row["db_mb"] = round(int(database[0]) / 2**20, 1) + row["messages"] = int(database[1]) + row["p95_post_ms"] = trend(summary, "post_message") + row["p95_get_ms"] = trend(summary, "get_history") + row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end)) + row.update(pg_usage(folder / "pg_activity.csv", start, end)) + row.update(lb_usage(folder / "lb_stats.csv", start, end)) + row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end) + own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6")) + row["cpu_other"] = round(row["cpu_vm"] - own, 1) + row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1) + row["wall_s"] = round(wall, 1) + row["disturbed"] = ( + row["cpu_other"] > args.max_other + or row["gap_s"] > args.max_gap + or wall > args.warmup + args.duration + args.max_overhead + or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2")) + or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6")) + or not rows(folder / "pg_activity.csv", start, end) + or not rows(folder / "lb_stats.csv", start, end) + ) + provenance(folder, args, env) + if code or not metrics["requests"]: + raise SystemExit(f"invalid k6 run: {folder}") + print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True) + return row + + +def rates(args, base): + if args.rates: + return [int(r) for r in args.rates.split(",")] + factor, top = (float(v) for v in args.search.split(":")) + value, values = args.start_per_cpu * base["replicas"] * base["cpus"], [] + while value <= top: + values.append(int(round(value, -1))) + value *= factor + return values + + +def arguments(): + parser = argparse.ArgumentParser(description="Track 1: replicas against throughput") + parser.add_argument("--out", required=True) + parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica") + parser.add_argument( + "--variant", + dest="variants", + action="append", + help="NAME[:KEY=VALUE,...]; several variants alternate per repeat", + ) + parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas") + parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica") + parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus") + parser.add_argument("--pool", type=int, default=8, help="DB pool per worker") + parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker") + parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE") + parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs") + parser.add_argument("--rates", help="comma separated offered rates, rps") + parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu") + parser.add_argument("--start-per-cpu", type=float, default=100.0) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat") + parser.add_argument("--warmup", type=int, default=10) + parser.add_argument("--duration", type=int, default=30) + parser.add_argument("--pause", type=float, default=5.0) + parser.add_argument("--slo-p95-ms", type=float, default=500.0) + parser.add_argument("--max-error-ratio", type=float, default=0.01) + parser.add_argument("--keep", action="store_true", help="leave the last layout running") + parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core") + parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s") + parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s") + parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step") + parser.add_argument("--quiet-timeout", type=float, default=1200.0) + args = parser.parse_args() + args.variants = args.variants or ["base"] + if len(args.variants) > 1 and not args.rates: + parser.error("several variants need explicit --rates") + return args + + +def block(args, output, config, variant, repeats): + base, settings = layout(config, args, variant) + current, streak = apply(base, settings, args.base_env, args.out), 0 + args.deployment = deployment(current) + for rate in rates(args, base): + results = [] + for repeat in repeats: + for _ in range(args.retries + 1): + row = step(base, rate, repeat, args, current) + time.sleep(args.pause) + if not row["disturbed"]: + break + args.rejected.writerow(row) + wait_quiet(args) + else: + raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}") + results.append(row) + output.writerows(results) + streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0 + if streak == 2: + break + + +def series(args, output): + every = range(args.first_repeat, args.first_repeat + args.repeats) + configs = args.configs.split(",") + if not args.rates: + for config in configs: + block(args, output, config, args.variants[0], every) + return + for repeat in every: + for config in configs: + for variant in args.variants: + block(args, output, config, variant, [repeat]) + + +@contextlib.contextmanager +def table(path): + fresh = not path.exists() + with path.open("a", newline="", buffering=1) as handle: + writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore") + if fresh: + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n") + writer.writeheader() + yield writer + + +def main(): + args = arguments() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + args.project = json.loads(compose("config", "--format", "json"))["name"] + args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + args.base_env = dict(os.environ, **(PINNED if args.pin else {})) + try: + with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected: + series(args, writer) + finally: + if not args.keep: + compose("up", "-d", "--wait", "--wait-timeout", "180") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/secrets-lab6.sh b/scripts/secrets-lab6.sh new file mode 100755 index 0000000..5986030 --- /dev/null +++ b/scripts/secrets-lab6.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets +[ -s secrets/grafana_admin_password ] || od -An -N24 -tx1 /dev/urandom | tr -d ' \n' > secrets/grafana_admin_password +chmod 644 secrets/grafana_admin_password +echo "secrets/grafana_admin_password is ready (value not printed)" diff --git a/scripts/secrets.sh b/scripts/secrets.sh index bdc67ec..6e133aa 100755 --- a/scripts/secrets.sh +++ b/scripts/secrets.sh @@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)" put s3_root_password "$(random)" put s3_access_key_id "api$(random | cut -c1-16)" put s3_secret_access_key "$(random)" +put s3_reconciler_access_key_id "rec$(random | cut -c1-16)" +put s3_reconciler_secret_access_key "$(random)" [ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; } diff --git a/scripts/stand-usage.sh b/scripts/stand-usage.sh new file mode 100755 index 0000000..97d469f --- /dev/null +++ b/scripts/stand-usage.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +size="select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)" +echo "$(date -u +%T) database bytes,messages: $(docker compose exec -T postgres psql -X -At -F, -U messenger -d messenger -c "$size")" +docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q) diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index 222cd88..c508503 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -6,4 +6,4 @@ ids=$(docker compose ps -q "${API_SERVICE:-api}") [ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; } target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}') [ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; } -docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null +docker stop -t "${STOP_TIMEOUT:-7}" "$target" >/dev/null diff --git a/scripts/upload_memory.py b/scripts/upload_memory.py index eb6febb..9925e71 100755 --- a/scripts/upload_memory.py +++ b/scripts/upload_memory.py @@ -77,7 +77,9 @@ def main(): args = p.parse_args() out = ROOT / args.out out.mkdir(parents=True, exist_ok=True) - commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip() + if any(out.iterdir()): + p.error("output directory must be empty; choose a new --out") + commit = seed.run(["git", "rev-parse", "HEAD"]).strip() threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072") lines = [ f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU," @@ -102,7 +104,12 @@ def main(): link=0, ids_out=None, ) - report = {"upload_concurrency": slots, **seed.over_http(run)} + report = { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "commit": commit, + "upload_concurrency": slots, + **seed.over_http(run), + } report["lb"] = balancer() name = f"slots{slots}-{size >> 20}mib-c{level}.json" (out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") diff --git a/src/messenger/admin.py b/src/messenger/admin.py index 4c6f3cb..acae9d0 100644 --- a/src/messenger/admin.py +++ b/src/messenger/admin.py @@ -3,7 +3,7 @@ import sys from dishka import make_async_container -from messenger.attachments.domain.jobs import ReplayResult +from messenger.attachments.domain.jobs import ReplayResult, ReprocessResult from messenger.attachments.entrypoints.admin import rendered, run from messenger.attachments.entrypoints.ioc import AdminProvider from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal @@ -26,7 +26,7 @@ async def serve(argv: list[str]) -> int: await bus.stop() await container.close() sys.stdout.write(rendered(report) + "\n") - return int(report.get("result") == ReplayResult.NOT_FOUND) + return int(report.get("result") in (ReplayResult.NOT_FOUND, ReprocessResult.CONFLICT)) def main() -> None: diff --git a/src/messenger/attachments/application/commands.py b/src/messenger/attachments/application/commands.py index af89a4b..18f41c3 100644 --- a/src/messenger/attachments/application/commands.py +++ b/src/messenger/attachments/application/commands.py @@ -47,3 +47,21 @@ class CleanStaging(Command): class ReplayAttachment(Command): attachment_id: UUID operator: str + + +@dataclass(frozen=True, slots=True) +class ReprocessAttachment(Command): + attachment_id: UUID + pipeline_version: int + operator: str + + +@dataclass(frozen=True, slots=True) +class ReconcileStorage(Command): + apply: bool + grace: timedelta + skew: timedelta + deep: bool = False + page_size: int = 1000 + deep_concurrency: int = 8 + samples: int = 20 diff --git a/src/messenger/attachments/application/pipeline.py b/src/messenger/attachments/application/pipeline.py index acbeebb..8d51e34 100644 --- a/src/messenger/attachments/application/pipeline.py +++ b/src/messenger/attachments/application/pipeline.py @@ -8,6 +8,7 @@ from messenger.attachments.application.commands import ( PublishOutbox, RecoverExpiredLeases, ReplayAttachment, + ReprocessAttachment, ) from messenger.attachments.application.ports import ( AttachmentsUnitOfWork, @@ -18,20 +19,26 @@ from messenger.attachments.application.ports import ( VariantRenderer, ) from messenger.attachments.domain.errors import LeaseLost, UnreadableImage -from messenger.attachments.domain.events import AttachmentReplayRequested, JobSettled +from messenger.attachments.domain.events import ( + AttachmentReplayRequested, + AttachmentReprocessRequested, + JobSettled, +) from messenger.attachments.domain.jobs import ( + PIPELINE_VERSIONS, Claim, FailureCode, Job, Lease, Outcome, ReplayResult, + ReprocessResult, RetryPolicy, ) from messenger.attachments.domain.models import Attachment, AttachmentStatus, original_key from messenger.attachments.domain.variants import Variant, variant_key from messenger.shared.application.errors import Unavailable -from messenger.shared.application.ports import IdGenerator +from messenger.shared.application.ports import Clock, IdGenerator TRANSIENT_ERROR = "transient_error" STALL_AFTER_CLAIM = "stall_after_claim" @@ -299,3 +306,39 @@ class ReplayAttachmentHandler: return ReplayResult.EXPIRED job = await self._uow.jobs.replay(attachment, operator) return ReplayResult.NOOP if job is None else ReplayResult.REPLAYED + + +class ReprocessAttachmentHandler: + __slots__ = ("_clock", "_ids", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, ids: IdGenerator, clock: Clock) -> None: + self._uow = uow + self._ids = ids + self._clock = clock + + async def __call__(self, command: ReprocessAttachment) -> ReprocessResult: + if command.pipeline_version not in PIPELINE_VERSIONS: + raise ValueError("неподдерживаемая версия pipeline") + async with self._uow.transaction(): + attachment = await self._uow.attachments.lock(command.attachment_id) + result = await self._schedule(attachment, command) + self._uow.record( + AttachmentReprocessRequested( + command.attachment_id, command.pipeline_version, command.operator, result.value + ) + ) + await self._uow.commit() + return result + + async def _schedule( + self, attachment: Attachment | None, command: ReprocessAttachment + ) -> ReprocessResult: + if attachment is None: + return ReprocessResult.NOT_FOUND + if attachment.status in (AttachmentStatus.PENDING, AttachmentStatus.FAILED): + return ReprocessResult.CONFLICT + if command.pipeline_version <= attachment.published_version: + return ReprocessResult.NOOP + job = Job(self._ids.new_id(), attachment.id, command.pipeline_version, 1, self._clock.now()) + added = await self._uow.jobs.reprocess(job, command.operator) + return ReprocessResult.QUEUED if added else ReprocessResult.NOOP diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index 32922e4..839401d 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -1,4 +1,5 @@ from collections.abc import AsyncIterator, Sequence +from contextlib import AbstractAsyncContextManager from dataclasses import dataclass from datetime import datetime, timedelta from typing import Protocol, runtime_checkable @@ -40,6 +41,13 @@ class StagedObject: key: str +@dataclass(frozen=True, slots=True) +class StoredObject: + key: str + size: int + modified: datetime + + @runtime_checkable class ObjectStorage(Protocol): async def put(self, key: str, data: bytes, content_type: str) -> None: ... @@ -52,6 +60,10 @@ class ObjectStorage(Protocol): async def delete(self, key: str) -> None: ... + def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]: ... + + async def stat(self, key: str) -> StoredObject | None: ... + @runtime_checkable class ImageInspector(Protocol): @@ -95,6 +107,14 @@ class AttachmentRepository(Protocol): async def remove(self, attachment: Attachment) -> None: ... + async def page_after(self, key: str, limit: int) -> Sequence[Attachment]: ... + + async def by_key(self, key: str) -> Attachment | None: ... + + async def claim(self, attachment_id: UUID) -> Attachment | None: ... + + async def mark(self, attachment: Attachment) -> None: ... + async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: ... async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ... @@ -104,6 +124,8 @@ class AttachmentRepository(Protocol): class JobRepository(Protocol): async def add(self, job: Job) -> None: ... + async def reprocess(self, job: Job, operator: str) -> bool: ... + async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: ... async def complete( @@ -158,3 +180,8 @@ class AttachmentDirectory(Protocol): @runtime_checkable class JobObserver(Protocol): def observe(self, job: Job, outcome: str, seconds: float) -> None: ... + + +@runtime_checkable +class SingleRun(Protocol): + def hold(self) -> AbstractAsyncContextManager[bool]: ... diff --git a/src/messenger/attachments/application/queries.py b/src/messenger/attachments/application/queries.py index 973d759..e792f3c 100644 --- a/src/messenger/attachments/application/queries.py +++ b/src/messenger/attachments/application/queries.py @@ -49,6 +49,9 @@ class OpenAttachment: if found is None: raise AttachmentNotFound("у вложения нет такого варианта") key, content_type = found.object_key, found.content_type + if attachment.quarantine is not None: + self._logger.warning("attachment.quarantined", attachment_id=str(attachment.id)) + raise AttachmentNotFound("вложение не найдено") try: content = await self._storage.open(str(key)) except ObjectMissingError as error: diff --git a/src/messenger/attachments/application/reconcile.py b/src/messenger/attachments/application/reconcile.py new file mode 100644 index 0000000..638be55 --- /dev/null +++ b/src/messenger/attachments/application/reconcile.py @@ -0,0 +1,318 @@ +import asyncio +from collections import Counter +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from dataclasses import dataclass, field +from datetime import datetime +from hashlib import sha256 +from time import monotonic +from uuid import UUID + +from messenger.attachments.application.commands import ReconcileStorage +from messenger.attachments.application.errors import ObjectMissingError +from messenger.attachments.application.handlers import drop +from messenger.attachments.application.ports import ( + AttachmentsUnitOfWork, + ObjectStorage, + SingleRun, + StoredObject, +) +from messenger.attachments.domain.models import ( + OBJECT_PREFIX, + Attachment, + AttachmentStatus, + Quarantine, +) +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Clock, EventPublisher, Logger + +type UnitOfWorkFactory = Callable[[], AttachmentsUnitOfWork] + + +@dataclass(slots=True) +class Reconciliation: + apply: bool + skipped: bool = False + objects: int = 0 + rows: int = 0 + found: Counter[str] = field(default_factory=Counter) + fixed: Counter[str] = field(default_factory=Counter) + rechecked: Counter[str] = field(default_factory=Counter) + requests: Counter[str] = field(default_factory=Counter) + waited: dict[str, float] = field(default_factory=dict) + samples: dict[str, list[str]] = field(default_factory=dict) + hashed_bytes: int = 0 + errors: int = 0 + seconds: float = 0.0 + + +class StorageReconciler: + __slots__ = ("_clock", "_logger", "_publisher", "_single", "_storage", "_uow") + + def __init__( + self, + uow: UnitOfWorkFactory, + storage: ObjectStorage, + single: SingleRun, + clock: Clock, + *, + logger: Logger, + publisher: EventPublisher, + ) -> None: + self._uow = uow + self._storage = storage + self._single = single + self._clock = clock + self._logger = logger + self._publisher = publisher + + async def __call__(self, command: ReconcileStorage) -> Reconciliation: + report = Reconciliation(apply=command.apply) + started = monotonic() + async with self._single.hold() as held: + report.skipped = not held + if held: + before = self._clock.now() - command.grace - command.skew + await Pass( + self._uow, + self._storage, + command, + before=before, + report=report, + logger=self._logger, + publisher=self._publisher, + ).run() + report.seconds = round(monotonic() - started, 3) + return report + + +class Pass: + __slots__ = ( + "_before", + "_command", + "_logger", + "_publisher", + "_report", + "_slots", + "_storage", + "_uow", + ) + + def __init__( + self, + uow: UnitOfWorkFactory, + storage: ObjectStorage, + command: ReconcileStorage, + *, + before: datetime, + report: Reconciliation, + logger: Logger, + publisher: EventPublisher, + ) -> None: + self._uow = uow + self._storage = storage + self._command = command + self._before = before + self._report = report + self._logger = logger + self._publisher = publisher + self._slots = asyncio.Semaphore(command.deep_concurrency) + + async def run(self) -> None: + objects = aiter(self._objects()) + item = await anext(objects, None) + try: + async with asyncio.TaskGroup() as checks: + async for row in self._rows(): + while item is not None and item.key < row.object_key: + await self._orphan(item) + item = await anext(objects, None) + matched = item if item is not None and item.key == row.object_key else None + await self._row(row, matched, checks) + if matched is not None: + item = await anext(objects, None) + while item is not None: + await self._orphan(item) + item = await anext(objects, None) + except* Unavailable as failed: + error = failed.exceptions[0] + raise error from error.__cause__ + + async def _timed[ResultT](self, name: str, step: Awaitable[ResultT]) -> ResultT: + started = monotonic() + try: + return await step + finally: + spent = self._report.waited.get(name, 0.0) + monotonic() - started + self._report.waited[name] = spent + + async def _objects(self) -> AsyncIterator[StoredObject]: + pages = aiter(self._storage.listing(OBJECT_PREFIX, self._command.page_size)) + while (page := await self._timed("s3_list", anext(pages, None))) is not None: + self._report.requests["s3_list"] += 1 + self._report.objects += len(page) + for item in page: + yield item + + async def _rows(self) -> AsyncIterator[Attachment]: + after, size = "", self._command.page_size + while True: + page = await self._timed("sql", self._page(after, size + 1)) + self._report.requests["sql"] += 1 + rows = page[:size] + self._report.rows += len(rows) + for row in rows: + yield row + if len(page) <= size: + return + after = rows[-1].object_key + + async def _page(self, after: str, size: int) -> Sequence[Attachment]: + uow = self._uow() + async with uow.transaction(): + return await uow.attachments.page_after(after, size) + + def _found(self, name: str, subject: str) -> None: + self._report.found[name] += 1 + sample = self._report.samples.setdefault(name, []) + if len(sample) < self._command.samples: + sample.append(subject) + + async def _safely(self, name: str, action: Awaitable[bool], subject: UUID | None) -> None: + attachment_id = None if subject is None else str(subject) + try: + done = await action + except Exception as error: + self._report.errors += 1 + self._logger.warning( + "reconcile.action_failed", + reason=name, + attachment_id=attachment_id, + error=type(error).__name__, + ) + return + (self._report.fixed if done else self._report.rechecked)[name] += 1 + event = "reconcile.fixed" if done else "reconcile.rechecked" + self._logger.info(event, reason=name, attachment_id=attachment_id) + + async def _orphan(self, item: StoredObject) -> None: + if item.modified >= self._before: + self._found("orphan_fresh", item.key) + return + self._found("orphan_stale", item.key) + if self._command.apply: + await self._safely("orphan_stale", self._delete_orphan(item), None) + + async def _delete_orphan(self, item: StoredObject) -> bool: + uow = self._uow() + async with uow.transaction(): + owner = await uow.attachments.by_key(item.key) + self._report.requests["sql"] += 1 + if owner is not None: + return False + self._report.requests["s3_delete"] += 1 + await self._storage.delete(item.key) + return True + + async def _row( + self, row: Attachment, item: StoredObject | None, checks: asyncio.TaskGroup + ) -> None: + if row.status is AttachmentStatus.PENDING: + await self._pending(row) + elif item is None: + await self._broken(row, Quarantine.MISSING) + elif item.size != row.size_bytes: + await self._broken(row, Quarantine.SIZE_MISMATCH) + elif self._command.deep or row.quarantine is not None: + await self._slots.acquire() + checks.create_task(self._verify(row)) + + async def _verify(self, row: Attachment) -> None: + try: + digest = await self._digest(row.object_key) + except Exception as error: + self._report.errors += 1 + self._logger.warning( + "reconcile.hash_failed", attachment_id=str(row.id), error=type(error).__name__ + ) + return + finally: + self._slots.release() + if digest is None: + await self._broken(row, Quarantine.MISSING) + elif digest != row.sha256: + await self._broken(row, Quarantine.SHA_MISMATCH) + else: + await self._healthy(row) + + async def _digest(self, key: str) -> str | None: + self._report.requests["s3_get"] += 1 + try: + content = await self._storage.open(key) + except ObjectMissingError: + return None + digest = sha256() + async for chunk in content.chunks: + digest.update(chunk) + self._report.hashed_bytes += len(chunk) + return digest.hexdigest() + + async def _broken(self, row: Attachment, reason: Quarantine) -> None: + if row.quarantine is not None: + self._found("quarantined", str(row.id)) + return + self._found(reason.value, str(row.id)) + if self._command.apply: + await self._safely(reason.value, self._isolate(row, reason), row.id) + + async def _isolate(self, row: Attachment, reason: Quarantine) -> bool: + if reason is not Quarantine.SHA_MISMATCH: + self._report.requests["s3_head"] += 1 + found = await self._storage.stat(row.object_key) + if found is not None and found.size == row.size_bytes: + return False + row.isolate(reason) + await self._mark(row) + return True + + async def _healthy(self, row: Attachment) -> None: + if row.quarantine is None: + return + self._found("healed", str(row.id)) + if self._command.apply: + await self._safely("healed", self._release(row), row.id) + + async def _release(self, row: Attachment) -> bool: + row.release() + await self._mark(row) + return True + + async def _mark(self, row: Attachment) -> None: + uow = self._uow() + async with uow.transaction(): + await uow.attachments.mark(row) + await uow.commit() + self._report.requests["sql"] += 1 + + async def _pending(self, row: Attachment) -> None: + if row.created_at >= self._before: + self._found("pending_fresh", str(row.id)) + return + self._found("pending_stale", str(row.id)) + if self._command.apply: + await self._safely("pending_stale", self._drop(row), row.id) + + async def _drop(self, row: Attachment) -> bool: + uow = self._uow() + async with uow.transaction(): + current = await uow.attachments.claim(row.id) + self._report.requests["sql"] += 1 + if ( + current is None + or current.status is not AttachmentStatus.PENDING + or current.created_at >= self._before + ): + return False + self._report.requests["s3_delete"] += 1 + await drop(uow, self._storage, current, "reconcile") + await self._publisher.publish(uow.collect_new_events()) + return True diff --git a/src/messenger/attachments/domain/events.py b/src/messenger/attachments/domain/events.py index 4c5eae1..1571be2 100644 --- a/src/messenger/attachments/domain/events.py +++ b/src/messenger/attachments/domain/events.py @@ -33,6 +33,15 @@ class JobSettled(DomainEvent): name: str = field(init=False, default="job.settled") +@dataclass(frozen=True, slots=True) +class AttachmentReprocessRequested(DomainEvent): + attachment_id: UUID + pipeline_version: int + operator: str + result: str + name: str = field(init=False, default="attachment.reprocess_requested") + + @dataclass(frozen=True, slots=True) class AttachmentReplayRequested(DomainEvent): attachment_id: UUID diff --git a/src/messenger/attachments/domain/jobs.py b/src/messenger/attachments/domain/jobs.py index 9c526e5..38d3a8e 100644 --- a/src/messenger/attachments/domain/jobs.py +++ b/src/messenger/attachments/domain/jobs.py @@ -6,6 +6,7 @@ from uuid import UUID JOB_TYPE = "attachment.process.v1" SCHEMA_VERSION = 1 +PIPELINE_VERSIONS = (1, 2) class FailureCode(StrEnum): @@ -21,6 +22,13 @@ class ReplayResult(StrEnum): NOT_FOUND = "not_found" +class ReprocessResult(StrEnum): + QUEUED = "queued" + NOOP = "noop" + CONFLICT = "conflict" + NOT_FOUND = "not_found" + + class Outcome(StrEnum): DONE = "done" DUPLICATE = "duplicate_noop" diff --git a/src/messenger/attachments/domain/models.py b/src/messenger/attachments/domain/models.py index cc91704..146166f 100644 --- a/src/messenger/attachments/domain/models.py +++ b/src/messenger/attachments/domain/models.py @@ -8,6 +8,7 @@ from messenger.attachments.domain.values import Filename, ImageInfo from messenger.attachments.domain.variants import Variant from messenger.shared.domain.events import DomainEvent +OBJECT_PREFIX = "attachments/" STAGING_PREFIX = "staging/" ORIGINALS_PREFIX = "originals/" @@ -24,6 +25,12 @@ def original_key(attachment_id: UUID) -> str: return f"{ORIGINALS_PREFIX}{attachment_id}" +class Quarantine(StrEnum): + MISSING = "missing" + SIZE_MISMATCH = "size_mismatch" + SHA_MISMATCH = "sha_mismatch" + + @dataclass(slots=True) class Attachment: id: UUID @@ -41,6 +48,7 @@ class Attachment: published_version: int = 0 error_code: str | None = None variants: tuple[Variant, ...] = () + quarantine: Quarantine | None = None _events: list[DomainEvent] = field(init=False, default_factory=list, repr=False, compare=False) @classmethod @@ -70,6 +78,10 @@ class Attachment: staging_key=f"{STAGING_PREFIX}{attachment_id}", ) + @property + def object_key(self) -> str: + return self.original_key or self.staging_key or f"{OBJECT_PREFIX}{self.id}" + @property def content_type(self) -> str: return self.image.kind.value @@ -98,6 +110,13 @@ class Attachment: AttachmentDiscarded(attachment_id=self.id, chat_id=self.chat_id, reason=reason) ) + def isolate(self, reason: Quarantine) -> None: + if self.status is AttachmentStatus.READY: + self.quarantine = reason + + def release(self) -> None: + self.quarantine = None + def collect_events(self) -> list[DomainEvent]: events, self._events = self._events, [] return events diff --git a/src/messenger/attachments/entrypoints/admin.py b/src/messenger/attachments/entrypoints/admin.py index ca87a9a..6101fd2 100644 --- a/src/messenger/attachments/entrypoints/admin.py +++ b/src/messenger/attachments/entrypoints/admin.py @@ -9,9 +9,9 @@ from uuid import UUID from dishka import AsyncContainer from messenger.attachments.application.bus import AttachmentsCommandBus -from messenger.attachments.application.commands import ReplayAttachment +from messenger.attachments.application.commands import ReplayAttachment, ReprocessAttachment from messenger.attachments.application.ports import AttachmentDirectory -from messenger.attachments.domain.jobs import ReplayResult +from messenger.attachments.domain.jobs import PIPELINE_VERSIONS, ReplayResult, ReprocessResult REGISTRY_LIMIT = 20 @@ -26,6 +26,10 @@ def parser() -> argparse.ArgumentParser: replay = commands.add_parser("replay", help="вернуть failed-вложение в очередь") replay.add_argument("--attachment-id", type=UUID, required=True) replay.add_argument("--operator", default=os.environ.get("OPERATOR", "cli")) + reprocess = commands.add_parser("reprocess", help="поставить новую версию pipeline") + reprocess.add_argument("--attachment-id", type=UUID, required=True) + reprocess.add_argument("--pipeline-version", type=int, choices=PIPELINE_VERSIONS, required=True) + reprocess.add_argument("--operator", default=os.environ.get("OPERATOR", "cli")) return root @@ -36,10 +40,20 @@ async def run(container: AsyncContainer, argv: list[str]) -> dict[str, Any]: directory = await scope.get(AttachmentDirectory) return {"failed": [asdict(job) for job in await directory.failed_jobs(args.limit)]} bus = await scope.get(AttachmentsCommandBus) - result: ReplayResult = await bus.execute( - ReplayAttachment(args.attachment_id, args.operator) + command = ( + ReprocessAttachment(args.attachment_id, args.pipeline_version, args.operator) + if args.command == "reprocess" + else ReplayAttachment(args.attachment_id, args.operator) ) - return {"result": result.value, "attachment_id": args.attachment_id, "operator": args.operator} + result: ReplayResult | ReprocessResult = await bus.execute(command) + report = { + "result": result.value, + "attachment_id": args.attachment_id, + "operator": args.operator, + } + if args.command == "reprocess": + report["pipeline_version"] = args.pipeline_version + return report def plain(value: object) -> str: diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index 0e60b98..d348019 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -12,6 +12,7 @@ from messenger.attachments.application.commands import ( PublishOutbox, RecoverExpiredLeases, ReplayAttachment, + ReprocessAttachment, SweepAbandonedUploads, UploadAttachment, ) @@ -26,6 +27,7 @@ from messenger.attachments.application.pipeline import ( PublishOutboxHandler, RecoverExpiredLeasesHandler, ReplayAttachmentHandler, + ReprocessAttachmentHandler, ) from messenger.attachments.application.ports import ( AttachmentDirectory, @@ -76,7 +78,7 @@ async def object_storage(settings: AttachmentsSettings) -> AsyncIterator[ObjectS yield S3ObjectStorage(client, settings.s3_bucket) -class AttachmentsProvider(Provider): +class AttachmentStorageProvider(Provider): def __init__(self, settings: AttachmentsSettings) -> None: super().__init__() self._settings = settings @@ -90,6 +92,8 @@ class AttachmentsProvider(Provider): async with object_storage(settings) as storage: yield storage + +class AttachmentsProvider(AttachmentStorageProvider): @provide(scope=Scope.APP) def images(self, settings: AttachmentsSettings) -> ImageInspector: return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency) @@ -266,9 +270,10 @@ class AdminProvider(Provider): @provide(scope=Scope.REQUEST) def command_bus( - self, uow: AttachmentsUnitOfWork, publisher: EventPublisher + self, uow: AttachmentsUnitOfWork, publisher: EventPublisher, ids: IdGenerator, clock: Clock ) -> AttachmentsCommandBus: handlers: dict[type[Command], CommandHandler] = { - ReplayAttachment: ReplayAttachmentHandler(uow) + ReplayAttachment: ReplayAttachmentHandler(uow), + ReprocessAttachment: ReprocessAttachmentHandler(uow, ids, clock), } return AttachmentsCommandBus(handlers, uow, publisher) diff --git a/src/messenger/attachments/entrypoints/journal.py b/src/messenger/attachments/entrypoints/journal.py index 014f23f..9a1f9d7 100644 --- a/src/messenger/attachments/entrypoints/journal.py +++ b/src/messenger/attachments/entrypoints/journal.py @@ -2,6 +2,7 @@ from messenger.attachments.domain.events import ( AttachmentDiscarded, AttachmentQueued, AttachmentReplayRequested, + AttachmentReprocessRequested, JobSettled, ) from messenger.shared.application.messagebus import EventBus @@ -69,6 +70,15 @@ class AttachmentsJournal: result=event.result, ) + async def on_reprocess_requested(self, event: AttachmentReprocessRequested) -> None: + self._logger.info( + "audit.attachment_reprocess", + attachment_id=str(event.attachment_id), + pipeline_version=event.pipeline_version, + operator=event.operator, + result=event.result, + ) + def subscribe_journal(bus: EventBus, logger: Logger) -> None: journal = AttachmentsJournal(logger) @@ -76,3 +86,4 @@ def subscribe_journal(bus: EventBus, logger: Logger) -> None: bus.subscribe(AttachmentDiscarded, journal.on_attachment_discarded) bus.subscribe(JobSettled, journal.on_job_settled) bus.subscribe(AttachmentReplayRequested, journal.on_replay_requested) + bus.subscribe(AttachmentReprocessRequested, journal.on_reprocess_requested) diff --git a/src/messenger/attachments/entrypoints/reconcile.py b/src/messenger/attachments/entrypoints/reconcile.py new file mode 100644 index 0000000..bdecddf --- /dev/null +++ b/src/messenger/attachments/entrypoints/reconcile.py @@ -0,0 +1,174 @@ +import argparse +import asyncio +import json +import resource +import sys +from collections.abc import Callable, Sequence +from datetime import timedelta +from typing import Any + +from dishka import Scope, make_async_container, provide + +from messenger.attachments.application.commands import ReconcileStorage +from messenger.attachments.application.ports import ObjectStorage +from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler +from messenger.attachments.config import AttachmentsSettings +from messenger.attachments.domain.events import AttachmentDiscarded +from messenger.attachments.entrypoints.ioc import AttachmentStorageProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS, AttachmentsJournal +from messenger.attachments.infrastructure.postgres import ( + PostgresAttachmentsUnitOfWork, + PostgresSingleRun, +) +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.messagebus import EventBus +from messenger.shared.application.ports import Clock, EventPublisher, Logger +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger +from messenger.shared.infrastructure.postgres import Database + +LOCK_KEY = 20_260_004 +CLASSES = ( + "orphan_stale", + "orphan_fresh", + "missing", + "size_mismatch", + "sha_mismatch", + "pending_stale", + "pending_fresh", + "quarantined", + "healed", +) +FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale", "healed") +REQUESTS = ("s3_list", "s3_head", "s3_get", "s3_delete", "sql") + + +def bounded(low: int, high: int | None = None) -> Callable[[str], int]: + def parse(text: str) -> int: + value = int(text) + if value < low or (high is not None and value > high): + raise argparse.ArgumentTypeError(f"ожидается {low}..{high or '∞'}") + return value + + return parse + + +def command_of(argv: Sequence[str] | None = None) -> ReconcileStorage: + parser = argparse.ArgumentParser( + prog="python -m messenger.attachments.entrypoints.reconcile", + description="Сверка метаданных вложений в PostgreSQL с объектами bucket S3", + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--dry-run", dest="apply", action="store_false", help="только отчёт") + mode.add_argument("--apply", dest="apply", action="store_true", help="исправить найденное") + parser.add_argument("--grace", type=bounded(60), default=3600, help="секунды, ≥ 60") + parser.add_argument("--skew", type=bounded(0), default=60, help="запас на часы, секунды") + parser.add_argument("--deep", action="store_true", help="сверить SHA-256 каждого объекта") + parser.add_argument("--page-size", type=bounded(1, 1000), default=1000) + parser.add_argument("--deep-concurrency", type=bounded(1, 48), default=8) + parser.add_argument("--samples", type=bounded(0, 1000), default=20) + args = parser.parse_args(argv) + return ReconcileStorage( + apply=args.apply, + grace=timedelta(seconds=args.grace), + skew=timedelta(seconds=args.skew), + deep=args.deep, + page_size=args.page_size, + deep_concurrency=args.deep_concurrency, + samples=args.samples, + ) + + +class ReconcileProvider(AttachmentStorageProvider): + @provide(scope=Scope.APP) + def reconciler( + self, + database: Database, + storage: ObjectStorage, + clock: Clock, + logger: Logger, + publisher: EventPublisher, + ) -> StorageReconciler: + return StorageReconciler( + lambda: PostgresAttachmentsUnitOfWork(database), + storage, + PostgresSingleRun(database, LOCK_KEY), + clock, + logger=logger, + publisher=publisher, + ) + + +async def reconcile( + command: ReconcileStorage, + platform: PlatformSettings, + settings: AttachmentsSettings, + logger: Logger, +) -> Reconciliation: + container = make_async_container( + PlatformProvider(platform, logger), ReconcileProvider(settings) + ) + bus = await container.get(EventBus) + bus.subscribe(AttachmentDiscarded, AttachmentsJournal(logger).on_attachment_discarded) + bus.start() + try: + reconciler = await container.get(StorageReconciler) + return await reconciler(command) + finally: + await bus.stop() + await container.close() + + +def failure(error: BaseException) -> str: + if isinstance(error, BaseExceptionGroup): + return failure(error.exceptions[0]) + if isinstance(error, Unavailable) and error.__cause__ is not None: + return type(error.__cause__).__name__ + return type(error).__name__ + + +def report_of(command: ReconcileStorage, result: Reconciliation) -> dict[str, Any]: + usage = resource.getrusage(resource.RUSAGE_SELF) + return { + "mode": "apply" if command.apply else "dry-run", + "skipped": result.skipped, + "grace_seconds": int(command.grace.total_seconds()), + "skew_seconds": int(command.skew.total_seconds()), + "deep": command.deep, + "page_size": command.page_size, + "scanned": {"objects": result.objects, "rows": result.rows}, + "found": {name: result.found[name] for name in CLASSES}, + "fixed": {name: result.fixed[name] for name in FIXABLE}, + "rechecked": dict(result.rechecked), + "errors": result.errors, + "samples": result.samples, + "cost": { + "seconds": result.seconds, + "requests": {name: result.requests[name] for name in REQUESTS}, + "waited_seconds": {name: round(spent, 3) for name, spent in result.waited.items()}, + "hashed_bytes": result.hashed_bytes, + "cpu_seconds": round(usage.ru_utime + usage.ru_stime, 3), + "max_rss_mib": round(usage.ru_maxrss / 1024, 1), + }, + } + + +def main(argv: Sequence[str] | None = None) -> None: + command = command_of(argv) + platform = PlatformSettings(db_pool_max_size=command.deep_concurrency + 2) + settings = AttachmentsSettings() + logger = StructlogLogger( + level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS, stream=sys.stderr + ) + try: + result = asyncio.run(reconcile(command, platform, settings, logger)) + except Exception as error: + logger.error("reconcile.failed", error=failure(error)) + raise SystemExit(1) from None + sys.stdout.write(json.dumps(report_of(command, result), ensure_ascii=False, indent=2) + "\n") + raise SystemExit(1 if result.errors else 0) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/attachments/entrypoints/worker.py b/src/messenger/attachments/entrypoints/worker.py index c5a317e..f6ecb44 100644 --- a/src/messenger/attachments/entrypoints/worker.py +++ b/src/messenger/attachments/entrypoints/worker.py @@ -40,7 +40,12 @@ async def handle_job( outcome = (await bus.execute(ProcessJob(job))).value except Exception as error: observed(job, "error", started, observer, logger) - logger.warning("job.requeued", job_id=str(job.id), error=type(error).__name__) + logger.warning( + "job.requeued", + job_id=str(job.id), + error=type(error).__name__, + error_code=getattr(getattr(error, "orig", error), "sqlstate", None), + ) await asyncio.sleep(REQUEUE_DELAY_SECONDS) await message.nack(requeue=True) return diff --git a/src/messenger/attachments/infrastructure/images.py b/src/messenger/attachments/infrastructure/images.py index b4d0205..91cb69d 100644 --- a/src/messenger/attachments/infrastructure/images.py +++ b/src/messenger/attachments/infrastructure/images.py @@ -34,8 +34,8 @@ BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning) JPEG_SCALE = 8 WIDE_MODES = frozenset({"I", "I;16", "I;16B", "I;16L", "I;16N"}) ALPHA_MODES = frozenset({"LA", "La", "PA", "RGBa"}) -JPEG = {"quality": 85, "optimize": True} -WEBP = {"quality": 80, "method": 4} +JPEG = {1: {"quality": 85, "optimize": True}, 2: {"quality": 90, "optimize": True}} +WEBP = {1: {"quality": 80, "method": 4}, 2: {"quality": 85, "method": 4}} ENCODINGS = {"JPEG": (ImageKind.JPEG.value, "jpg"), "WEBP": ("image/webp", "webp")} REDUCING_GAP = 3.0 @@ -128,15 +128,27 @@ def decoded(data: bytes, max_pixels: int) -> tuple[Image.Image, int, int]: raise UnreadableImage("изображение не декодируется") from error -def render_variants(data: bytes, limits: RenderLimits) -> list[Rendition]: +def render_variants( + data: bytes, limits: RenderLimits, pipeline_version: int = 1 +) -> list[Rendition]: if len(data) > limits.max_bytes: raise ImageTooDemanding("оригинал больше допустимого размера") upright, width, height = decoded(data, limits.max_pixels) square = ImageOps.fit(upright, (CROP_SIDE, CROP_SIDE), Image.Resampling.LANCZOS) return [ - encoded(THUMBNAIL, scaled(upright, fit(width, height, THUMBNAIL_BOX)), "JPEG", **JPEG), - encoded(CROP, square, "JPEG", **JPEG), - encoded(OPTIMIZED, scaled(upright, fit(width, height, OPTIMIZED_BOX)), "WEBP", **WEBP), + encoded( + THUMBNAIL, + scaled(upright, fit(width, height, THUMBNAIL_BOX)), + "JPEG", + **JPEG[pipeline_version], + ), + encoded(CROP, square, "JPEG", **JPEG[pipeline_version]), + encoded( + OPTIMIZED, + scaled(upright, fit(width, height, OPTIMIZED_BOX)), + "WEBP", + **WEBP[pipeline_version], + ), ] @@ -145,7 +157,7 @@ def limit_memory(limit_bytes: int) -> None: resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes)) -type Render = Callable[[bytes, RenderLimits], list[Rendition]] +type Render = Callable[[bytes, RenderLimits, int], list[Rendition]] class ProcessVariantRenderer: @@ -173,7 +185,9 @@ class ProcessVariantRenderer: async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: pool = await self._idle.get() try: - future = asyncio.wrap_future(pool.submit(self._render, data, self._limits)) + future = asyncio.wrap_future( + pool.submit(self._render, data, self._limits, pipeline_version) + ) return await asyncio.wait_for(future, self._timeout) except TimeoutError as error: pool = self._replace(pool) diff --git a/src/messenger/attachments/infrastructure/postgres.py b/src/messenger/attachments/infrastructure/postgres.py index 1af88b2..451995c 100644 --- a/src/messenger/attachments/infrastructure/postgres.py +++ b/src/messenger/attachments/infrastructure/postgres.py @@ -1,211 +1,210 @@ import json -from collections.abc import Sequence +from collections.abc import AsyncIterator, Sequence +from contextlib import asynccontextmanager from datetime import datetime, timedelta +from typing import Any, cast from uuid import UUID -from asyncpg import ForeignKeyViolationError, Record +from sqlalchemy import ( + BigInteger, + CheckConstraint, + DateTime, + ForeignKey, + ForeignKeyConstraint, + Identity, + Index, + Integer, + RowMapping, + Select, + Table, + Text, + UniqueConstraint, + delete, + func, + insert, + or_, + select, + update, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Mapped, mapped_column from messenger.attachments.application.ports import OutboxEntry, StagedObject from messenger.attachments.domain.errors import ChatNotFound from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome -from messenger.attachments.domain.models import Attachment, AttachmentStatus +from messenger.attachments.domain.models import Attachment, AttachmentStatus, Quarantine from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind from messenger.attachments.domain.variants import Variant from messenger.shared.application.errors import Unavailable from messenger.shared.infrastructure.postgres import ( - Connection, + Base, Database, + DatabaseSession, EventSource, PostgresUnitOfWork, ) -INSERT = """ -INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - width, height, staging_key, status, created_at, updated_at) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) -""" -MARK = "UPDATE attachments SET status = $2, updated_at = $3 WHERE id = $1 AND status = 'pending'" -COLUMNS = """ -SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width, - a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code, - a.created_at, a.updated_at -FROM attachments a -""" -VISIBLE = """ -SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width, - a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code, - a.created_at, a.updated_at, - (SELECT coalesce(jsonb_agg(jsonb_build_array(v.name, v.content_type, v.size_bytes, v.width, - v.height, v.sha256, v.object_key) - ORDER BY v.name), '[]') - FROM attachment_variants v - WHERE v.attachment_id = a.id AND v.pipeline_version = a.published_version) AS variants -FROM attachments a -WHERE a.id = $1 AND a.status <> 'pending' -""" -FAILED_AMONG = """ -SELECT EXISTS (SELECT 1 FROM attachments - WHERE id = ANY($1::uuid[]) AND chat_id = $2 AND status = 'failed') -""" -LOCK = COLUMNS + "WHERE a.id = $1 FOR UPDATE" -ABANDONED = ( - COLUMNS - + """WHERE a.status = 'pending' AND a.created_at < $1 -ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED""" -) -REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'" -STAGED = """ -SELECT id, staging_key FROM attachments -WHERE staging_key IS NOT NULL - AND (status = 'ready' OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2))) -ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED -""" -FAILED_JOBS = """ -SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, a.error_code, j.last_error, - j.updated_at AS failed_at -FROM jobs j JOIN attachments a ON a.id = j.attachment_id -WHERE j.status = 'failed' -ORDER BY j.updated_at DESC LIMIT $1 -""" -UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])" -ADD_JOB = """ -WITH job AS ( - INSERT INTO jobs (job_id, attachment_id, pipeline_version, status, attempt, - created_at, updated_at) - VALUES ($1, $2, $3, 'queued', $4, $5, $5) - RETURNING job_id -), box AS ( - INSERT INTO outbox (job_id, payload, available_at) - SELECT job_id, $6::jsonb, now() FROM job -) -INSERT INTO job_events (job_id, attachment_id, kind, attempt) -SELECT job_id, $2, 'enqueued', $4 FROM job -""" -CLAIM = """ -WITH claimed AS ( - UPDATE jobs - SET status = 'processing', lease_token = $4, claims = claims + 1, updated_at = now(), - lease_until = now() + make_interval(secs => $5) - WHERE job_id = $1 AND attachment_id = $2 AND pipeline_version = $3 - AND ((status = 'queued' AND coalesce(next_attempt_at, '-infinity') <= now()) - OR (status = 'processing' AND lease_until < now())) - RETURNING job_id, attachment_id, attempt, created_at -), moved AS ( - UPDATE attachments a SET status = 'processing', updated_at = greatest(now(), a.created_at) - FROM claimed c WHERE a.id = c.attachment_id AND a.status = 'queued' -), audit AS ( - INSERT INTO job_events (job_id, attachment_id, kind, attempt) - SELECT job_id, attachment_id, 'claimed', attempt FROM claimed -) -SELECT c.attempt, c.created_at, a.staging_key, a.original_key -FROM claimed c JOIN attachments a ON a.id = c.attachment_id -""" -FINISH = """ -UPDATE jobs SET status = 'done', lease_token = NULL, lease_until = NULL, next_attempt_at = NULL, - last_error = NULL, updated_at = now() -WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 -""" -ADD_VARIANTS = """ -INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, - size_bytes, width, height, sha256, created_at) -SELECT $1, $2, v.name, v.object_key, v.content_type, v.size_bytes, v.width, v.height, v.sha256, - now() -FROM unnest($3::text[], $4::text[], $5::text[], $6::bigint[], $7::int[], $8::int[], $9::text[]) - AS v (name, object_key, content_type, size_bytes, width, height, sha256) -ON CONFLICT DO NOTHING -""" -PUBLISH = """ -UPDATE attachments -SET status = 'ready', published_version = $2, error_code = NULL, original_key = $3, - updated_at = greatest(now(), created_at) -WHERE id = $1 AND published_version < $2 -""" -SUPERSEDE = "UPDATE jobs SET status = 'superseded', updated_at = now() WHERE job_id = $1" -RESCHEDULE = """ -WITH job AS ( - UPDATE jobs - SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_until = NULL, - next_attempt_at = now() + make_interval(secs => $3), last_error = $4, updated_at = now() - WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 - RETURNING job_id, attachment_id, attempt, next_attempt_at -), moved AS ( - UPDATE attachments a SET status = 'queued', updated_at = greatest(now(), a.created_at) - FROM job WHERE a.id = job.attachment_id AND a.status = 'processing' -), audit AS ( - INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) - SELECT job_id, attachment_id, $5, attempt, - jsonb_build_object('error', $4::text, 'next_attempt_at', next_attempt_at, - 'delay_seconds', $3::float8) - FROM job -) -SELECT attempt, next_attempt_at FROM job -""" -RENEW = """ -UPDATE jobs SET lease_until = now() + make_interval(secs => $3), updated_at = now() -WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' -""" -ENQUEUE = "INSERT INTO outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, $3)" -FAIL = """ -WITH job AS ( - UPDATE jobs SET status = 'failed', lease_token = NULL, lease_until = NULL, last_error = $4, - updated_at = now() - WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 - RETURNING job_id, attachment_id, attempt -), moved AS ( - UPDATE attachments a SET status = 'failed', error_code = $3, - updated_at = greatest(now(), a.created_at) - FROM job WHERE a.id = job.attachment_id AND a.status IN ('queued', 'processing') -), audit AS ( - INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) - SELECT job_id, attachment_id, 'failed', attempt, - jsonb_build_object('error_code', $3::text, 'error', $4::text) - FROM job -) -SELECT attempt FROM job -""" -NOTE = "INSERT INTO job_events (job_id, attachment_id, kind, attempt) VALUES ($1, $2, $3, $4)" -EXPIRED = """ -SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token FROM jobs -WHERE status = 'processing' AND lease_until < now() -ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED -""" -REPLAY = """ -WITH job AS ( - UPDATE jobs - SET status = 'queued', attempt = 1, lease_token = NULL, lease_until = NULL, - next_attempt_at = NULL, last_error = NULL, updated_at = now() - WHERE job_id = (SELECT job_id FROM jobs WHERE attachment_id = $1 AND status = 'failed' - ORDER BY pipeline_version DESC LIMIT 1) - RETURNING job_id, attachment_id, pipeline_version, created_at, updated_at -), moved AS ( - UPDATE attachments a SET status = 'queued', error_code = NULL, - updated_at = greatest(now(), a.created_at) - FROM job WHERE a.id = job.attachment_id AND a.status = 'failed' -), audit AS ( - INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) - SELECT job_id, attachment_id, 'replayed', 1, - jsonb_build_object('operator', $2::text, 'error_code', $3::text) - FROM job -) -SELECT job_id, pipeline_version, created_at, updated_at FROM job -""" -NOTE_REPLAY = """ -INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) -SELECT job_id, attachment_id, 'replay_noop', attempt, - jsonb_build_object('operator', $2::text, 'result', $3::text) -FROM jobs WHERE attachment_id = $1 -ORDER BY pipeline_version DESC LIMIT 1 -""" -PENDING = """ -SELECT id, job_id, payload::text AS body FROM outbox -WHERE published_at IS NULL AND available_at <= now() -ORDER BY available_at, id LIMIT $1 FOR UPDATE SKIP LOCKED -""" -PUBLISHED = "UPDATE outbox SET published_at = now() WHERE id = ANY($1::bigint[])" +class AttachmentRow(Base): + __tablename__ = "attachments" + __table_args__ = ( + CheckConstraint( + "char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_check" + ), + CheckConstraint( + "content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_check" + ), + CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_check"), + CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachments_sha256_check"), + CheckConstraint("width > 0", name="attachments_width_check"), + CheckConstraint("height > 0", name="attachments_height_check"), + CheckConstraint( + "status IN ('pending', 'queued', 'processing', 'ready', 'failed')", + name="attachments_status_check", + ), + CheckConstraint( + "quarantine_reason IN ('missing', 'size_mismatch', 'sha_mismatch')", + name="attachments_quarantine_reason_check", + ), + CheckConstraint( + "(quarantine_reason IS NULL) = (quarantined_at IS NULL)", + name="attachments_quarantine_check", + ), + UniqueConstraint("object_key", name="attachments_object_key_key"), + CheckConstraint("published_version >= 0", name="attachments_published_version_check"), + CheckConstraint( + "error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')", + name="attachments_error_code_check", + ), + UniqueConstraint("id", "chat_id", name="attachments_chat_key"), + ForeignKeyConstraint( + ["chat_id", "owner_id"], + ["chat_members.chat_id", "chat_members.user_id"], + name="attachments_owner_fkey", + ), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + chat_id: Mapped[UUID] + owner_id: Mapped[UUID] + filename: Mapped[str] + content_type: Mapped[str] + size_bytes: Mapped[int] = mapped_column(BigInteger) + sha256: Mapped[str] + width: Mapped[int] = mapped_column(Integer) + height: Mapped[int] = mapped_column(Integer) + object_key: Mapped[str | None] = mapped_column(Text(collation="C")) + status: Mapped[str] + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + staging_key: Mapped[str | None] = mapped_column(Text(collation="C"), unique=True) + original_key: Mapped[str | None] = mapped_column(Text(collation="C"), unique=True) + published_version: Mapped[int] = mapped_column(server_default="0") + error_code: Mapped[str | None] + quarantine_reason: Mapped[str | None] + quarantined_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) -def attachment_of(row: Record, variants: tuple[Variant, ...] = ()) -> Attachment: +Index( + "attachments_pending_idx", + AttachmentRow.created_at, + postgresql_where=AttachmentRow.status == "pending", +) + + +class VariantRow(Base): + __tablename__ = "attachment_variants" + __table_args__ = ( + CheckConstraint("pipeline_version >= 1"), + CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')"), + CheckConstraint("content_type IN ('image/jpeg', 'image/png', 'image/webp')"), + CheckConstraint("size_bytes >= 1"), + CheckConstraint("width >= 1"), + CheckConstraint("height >= 1"), + CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'"), + ) + + attachment_id: Mapped[UUID] = mapped_column(ForeignKey("attachments.id"), primary_key=True) + pipeline_version: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(primary_key=True) + object_key: Mapped[str] = mapped_column(Text(collation="C"), unique=True) + content_type: Mapped[str] + size_bytes: Mapped[int] = mapped_column(BigInteger) + width: Mapped[int] + height: Mapped[int] + sha256: Mapped[str] + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class JobRow(Base): + __tablename__ = "jobs" + __table_args__ = ( + CheckConstraint("pipeline_version >= 1"), + CheckConstraint("status IN ('queued', 'processing', 'done', 'failed', 'superseded')"), + CheckConstraint("attempt >= 1"), + CheckConstraint("claims >= 0"), + UniqueConstraint("attachment_id", "pipeline_version", name="jobs_effect_key"), + ) + + job_id: Mapped[UUID] = mapped_column(primary_key=True) + attachment_id: Mapped[UUID] = mapped_column(ForeignKey("attachments.id")) + pipeline_version: Mapped[int] + status: Mapped[str] + attempt: Mapped[int] + claims: Mapped[int] = mapped_column(server_default="0") + lease_token: Mapped[UUID | None] + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_error: Mapped[str | None] + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class OutboxRow(Base): + __tablename__ = "outbox" + + id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True) + job_id: Mapped[UUID] = mapped_column(ForeignKey("jobs.job_id")) + payload: Mapped[dict[str, Any]] = mapped_column(JSONB) + available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + +class JobEventRow(Base): + __tablename__ = "job_events" + + id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True) + job_id: Mapped[UUID] + attachment_id: Mapped[UUID] + kind: Mapped[str] + attempt: Mapped[int | None] + detail: Mapped[dict[str, Any]] = mapped_column(JSONB, server_default="{}") + at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + +Index("jobs_lease_idx", JobRow.lease_until, postgresql_where=JobRow.status == "processing") +Index("jobs_failed_idx", JobRow.updated_at.desc(), postgresql_where=JobRow.status == "failed") +Index( + "outbox_pending_idx", + OutboxRow.available_at, + OutboxRow.id, + postgresql_where=OutboxRow.published_at.is_(None), +) +Index("outbox_job_idx", OutboxRow.job_id) +Index("job_events_attachment_idx", JobEventRow.attachment_id, JobEventRow.id) +Index( + "attachments_staged_idx", + AttachmentRow.updated_at, + postgresql_where=AttachmentRow.staging_key.is_not(None), +) + + +COLUMNS = select(AttachmentRow.__table__) + + +def attachment_of(row: RowMapping, variants: tuple[Variant, ...] = ()) -> Attachment: return Attachment( id=row["id"], chat_id=row["chat_id"], @@ -214,211 +213,142 @@ def attachment_of(row: Record, variants: tuple[Variant, ...] = ()) -> Attachment image=ImageInfo(ImageKind(row["content_type"]), row["width"], row["height"]), size_bytes=row["size_bytes"], sha256=row["sha256"], - status=AttachmentStatus(row["status"]), - created_at=row["created_at"], - updated_at=row["updated_at"], - staging_key=row["staging_key"], - original_key=row["original_key"], + staging_key=row["staging_key"] + or (row["object_key"] if row["status"] == "pending" else None), + original_key=row["original_key"] + or (row["object_key"] if row["status"] == "ready" else None), published_version=row["published_version"], error_code=row["error_code"], variants=variants, + status=AttachmentStatus(row["status"]), + created_at=row["created_at"], + updated_at=row["updated_at"], + quarantine=Quarantine(row["quarantine_reason"]) if row["quarantine_reason"] else None, ) -def payload(job: Job) -> str: - return json.dumps(job.payload(), separators=(",", ":")) - - class PostgresAttachmentRepository: - __slots__ = ("_connection", "_seen") + __slots__ = ("_seen", "_session") - def __init__(self, connection: Connection, seen: list[EventSource]) -> None: - self._connection = connection + def __init__(self, session: DatabaseSession, seen: list[EventSource]) -> None: + self._session = session self._seen = seen async def add(self, attachment: Attachment) -> None: try: - await self._connection.execute( - INSERT, - attachment.id, - attachment.chat_id, - attachment.owner_id, - attachment.filename.value, - attachment.content_type, - attachment.size_bytes, - attachment.sha256, - attachment.image.width, - attachment.image.height, - attachment.staging_key, - attachment.status.value, - attachment.created_at, - attachment.updated_at, + await self._session.execute( + insert(AttachmentRow).values( + id=attachment.id, + chat_id=attachment.chat_id, + owner_id=attachment.owner_id, + filename=attachment.filename.value, + content_type=attachment.content_type, + size_bytes=attachment.size_bytes, + sha256=attachment.sha256, + width=attachment.image.width, + height=attachment.image.height, + staging_key=attachment.staging_key, + original_key=attachment.original_key, + published_version=attachment.published_version, + status=attachment.status.value, + created_at=attachment.created_at, + updated_at=attachment.updated_at, + ) ) - except ForeignKeyViolationError as error: + except IntegrityError as error: + if getattr(error.orig, "sqlstate", None) != "23503": + raise raise ChatNotFound("чат не найден") from error self._seen.append(attachment) async def save(self, attachment: Attachment) -> None: - done = await self._connection.execute( - MARK, attachment.id, attachment.status.value, attachment.updated_at + done = await self._session.scalar( + update(AttachmentRow) + .where(AttachmentRow.id == attachment.id, AttachmentRow.status == "pending") + .values(status=attachment.status.value, updated_at=attachment.updated_at) + .returning(AttachmentRow.id) ) - if done == "UPDATE 0": + if done is None: raise Unavailable("намерение загрузки уже снято уборкой") self._seen.append(attachment) - async def lock(self, attachment_id: UUID) -> Attachment | None: - row = await self._connection.fetchrow(LOCK, attachment_id) + async def _one(self, statement: Select) -> Attachment | None: + row = (await self._session.execute(statement)).mappings().first() return None if row is None else attachment_of(row) + async def lock(self, attachment_id: UUID) -> Attachment | None: + return await self._one(COLUMNS.where(AttachmentRow.id == attachment_id).with_for_update()) + async def next_abandoned(self, before: datetime) -> Attachment | None: - row = await self._connection.fetchrow(ABANDONED, before) - return None if row is None else attachment_of(row) + return await self._one( + COLUMNS.where(AttachmentRow.status == "pending", AttachmentRow.created_at < before) + .order_by(AttachmentRow.created_at) + .limit(1) + .with_for_update(skip_locked=True) + ) async def remove(self, attachment: Attachment) -> None: - await self._connection.execute(REMOVE, attachment.id) + await self._session.execute( + delete(AttachmentRow).where( + AttachmentRow.id == attachment.id, AttachmentRow.status == "pending" + ) + ) self._seen.append(attachment) async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: - rows = await self._connection.fetch(STAGED, limit, retention.total_seconds()) - return [StagedObject(row["id"], row["staging_key"]) for row in rows] + rows = await self._session.execute( + select(AttachmentRow.id, AttachmentRow.staging_key) + .where( + AttachmentRow.staging_key.is_not(None), + ~select(JobRow.job_id) + .where( + JobRow.attachment_id == AttachmentRow.id, + JobRow.status.in_(["queued", "processing"]), + ) + .exists(), + or_( + AttachmentRow.status == "ready", + (AttachmentRow.status == "failed") + & (AttachmentRow.updated_at < func.now() - retention), + ), + ) + .order_by(AttachmentRow.updated_at) + .limit(limit) + .with_for_update(skip_locked=True) + ) + return [StagedObject(row.id, row.staging_key) for row in rows] async def unstage(self, attachment_ids: Sequence[UUID]) -> None: if attachment_ids: - await self._connection.execute(UNSTAGE, list(attachment_ids)) - - -class PostgresJobRepository: - __slots__ = ("_connection",) - - def __init__(self, connection: Connection) -> None: - self._connection = connection - - async def add(self, job: Job) -> None: - await self._connection.execute( - ADD_JOB, - job.id, - job.attachment_id, - job.pipeline_version, - job.attempt, - job.created_at, - payload(job), - ) - - async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: - row = await self._connection.fetchrow( - CLAIM, - job.id, - job.attachment_id, - job.pipeline_version, - token, - lease.total_seconds(), - ) - if row is None: - return None - claimed = Job( - job.id, job.attachment_id, job.pipeline_version, row["attempt"], row["created_at"] - ) - staged = row["staging_key"] is not None - source = row["staging_key"] if staged else row["original_key"] - return Claim(Lease(claimed, token), source, staged) - - async def complete( - self, lease: Lease, variants: Sequence[Variant], original_key: str - ) -> Outcome: - job = lease.job - if await self._connection.execute(FINISH, job.id, lease.token) == "UPDATE 0": - await self.note(job, Outcome.FENCED) - return Outcome.FENCED - await self._connection.execute( - ADD_VARIANTS, - job.attachment_id, - job.pipeline_version, - [v.name for v in variants], - [v.object_key for v in variants], - [v.content_type for v in variants], - [v.size_bytes for v in variants], - [v.width for v in variants], - [v.height for v in variants], - [v.sha256 for v in variants], - ) - published = await self._connection.execute( - PUBLISH, job.attachment_id, job.pipeline_version, original_key - ) - outcome = Outcome.SUPERSEDED if published == "UPDATE 0" else Outcome.DONE - if outcome is Outcome.SUPERSEDED: - await self._connection.execute(SUPERSEDE, job.id) - await self.note(job, outcome) - return outcome - - async def reschedule( - self, lease: Lease, delay: timedelta, kind: Outcome, error: str - ) -> Job | None: - row = await self._connection.fetchrow( - RESCHEDULE, lease.job.id, lease.token, delay.total_seconds(), error, kind.value - ) - if row is None: - return None - job = lease.job.retried(row["attempt"]) - await self._connection.execute(ENQUEUE, job.id, payload(job), row["next_attempt_at"]) - return job - - async def renew(self, lease: Lease, duration: timedelta) -> bool: - done = await self._connection.execute( - RENEW, lease.job.id, lease.token, duration.total_seconds() - ) - return done == "UPDATE 1" - - async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: - attempt = await self._connection.fetchval( - FAIL, lease.job.id, lease.token, code.value, error - ) - return attempt is not None - - async def note(self, job: Job, kind: Outcome) -> None: - await self._connection.execute(NOTE, job.id, job.attachment_id, kind.value, job.attempt) - - async def expired(self, limit: int) -> list[Lease]: - rows = await self._connection.fetch(EXPIRED, limit) - return [ - Lease( - Job( - row["job_id"], - row["attachment_id"], - row["pipeline_version"], - row["attempt"], - row["created_at"], - ), - row["lease_token"], + await self._session.execute( + update(AttachmentRow) + .where(AttachmentRow.id.in_(attachment_ids)) + .values(staging_key=None) ) - for row in rows - ] - async def replay(self, attachment: Attachment, operator: str) -> Job | None: - row = await self._connection.fetchrow( - REPLAY, attachment.id, operator, attachment.error_code + async def page_after(self, key: str, limit: int) -> Sequence[Attachment]: + rows = await self._session.execute( + COLUMNS.where(AttachmentRow.object_key > key) + .order_by(AttachmentRow.object_key) + .limit(limit) ) - if row is None: - return None - job = Job(row["job_id"], attachment.id, row["pipeline_version"], 1, row["created_at"]) - await self._connection.execute(ENQUEUE, job.id, payload(job), row["updated_at"]) - return job + return [attachment_of(row) for row in rows.mappings()] - async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None: - await self._connection.execute(NOTE_REPLAY, attachment_id, operator, result) + async def by_key(self, key: str) -> Attachment | None: + return await self._one(COLUMNS.where(AttachmentRow.object_key == key)) + async def claim(self, attachment_id: UUID) -> Attachment | None: + return await self._one( + COLUMNS.where(AttachmentRow.id == attachment_id).with_for_update(skip_locked=True) + ) -class PostgresOutboxRepository: - __slots__ = ("_connection",) - - def __init__(self, connection: Connection) -> None: - self._connection = connection - - async def pending(self, limit: int) -> list[OutboxEntry]: - rows = await self._connection.fetch(PENDING, limit) - return [OutboxEntry(row["id"], row["job_id"], row["body"].encode()) for row in rows] - - async def published(self, entry_ids: Sequence[int]) -> None: - await self._connection.execute(PUBLISHED, list(entry_ids)) + async def mark(self, attachment: Attachment) -> None: + reason = None if attachment.quarantine is None else attachment.quarantine.value + await self._session.execute( + update(AttachmentRow) + .where(AttachmentRow.id == attachment.id, AttachmentRow.status == "ready") + .values(quarantine_reason=reason, quarantined_at=None if reason is None else func.now()) + ) class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork): @@ -426,19 +356,15 @@ class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork): @property def attachments(self) -> PostgresAttachmentRepository: - return PostgresAttachmentRepository(self.connection, self.seen) + return PostgresAttachmentRepository(self.session, self.seen) @property def jobs(self) -> PostgresJobRepository: - return PostgresJobRepository(self.connection) + return PostgresJobRepository(self.session) @property def outbox(self) -> PostgresOutboxRepository: - return PostgresOutboxRepository(self.connection) - - -def variants_of(raw: str) -> tuple[Variant, ...]: - return tuple(Variant(*item) for item in json.loads(raw)) + return PostgresOutboxRepository(self.session) class PostgresAttachmentDirectory: @@ -448,14 +374,459 @@ class PostgresAttachmentDirectory: self._database = database async def visible(self, attachment_id: UUID) -> Attachment | None: - async with self._database.connection() as connection: - row = await connection.fetchrow(VISIBLE, attachment_id) - return None if row is None else attachment_of(row, variants_of(row["variants"])) + async with self._database.session() as session: + row = ( + ( + await session.execute( + COLUMNS.where( + AttachmentRow.id == attachment_id, AttachmentRow.status != "pending" + ) + ) + ) + .mappings() + .first() + ) + if row is None: + return None + rows = ( + await session.execute( + select( + VariantRow.name, + VariantRow.content_type, + VariantRow.size_bytes, + VariantRow.width, + VariantRow.height, + VariantRow.sha256, + VariantRow.object_key, + ) + .where( + VariantRow.attachment_id == attachment_id, + VariantRow.pipeline_version == row["published_version"], + ) + .order_by(VariantRow.name) + ) + ).all() + return attachment_of(row, tuple(Variant(*item) for item in rows)) async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: - async with self._database.connection() as connection: - return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id)) + async with self._database.session() as session: + return bool( + await session.scalar( + select( + select(AttachmentRow.id) + .where( + AttachmentRow.id.in_(attachment_ids), + AttachmentRow.chat_id == chat_id, + AttachmentRow.status == "failed", + ) + .exists() + ) + ) + ) async def failed_jobs(self, limit: int) -> list[FailedJob]: - async with self._database.connection() as connection: - return [FailedJob(**row) for row in await connection.fetch(FAILED_JOBS, limit)] + async with self._database.session() as session: + rows = await session.execute( + select( + JobRow.job_id, + JobRow.attachment_id, + JobRow.pipeline_version, + JobRow.attempt, + AttachmentRow.error_code, + JobRow.last_error, + JobRow.updated_at.label("failed_at"), + ) + .join(AttachmentRow, AttachmentRow.id == JobRow.attachment_id) + .where(JobRow.status == "failed") + .order_by(JobRow.updated_at.desc()) + .limit(limit) + ) + return [FailedJob(**row) for row in rows.mappings()] + + +class PostgresSingleRun: + __slots__ = ("_database", "_key") + + def __init__(self, database: Database, key: int) -> None: + self._database = database + self._key = key + + @asynccontextmanager + async def hold(self) -> AsyncIterator[bool]: + async with self._database.session() as session: + connection = await session.connection( + execution_options={"isolation_level": "AUTOCOMMIT"} + ) + held = bool(await connection.scalar(select(func.pg_try_advisory_lock(self._key)))) + try: + yield held + finally: + if held: + await connection.execute(select(func.pg_advisory_unlock(self._key))) + + +class PostgresJobRepository: + __slots__ = ("_session",) + + def __init__(self, session: DatabaseSession) -> None: + self._session = session + + async def _enqueue(self, job: Job, available_at: datetime | None = None) -> None: + await self._session.execute( + insert(OutboxRow).values( + job_id=job.id, + payload=job.payload(), + available_at=func.now() if available_at is None else available_at, + ) + ) + + async def _note(self, job: Job, kind: str, detail: dict[str, Any] | None = None) -> None: + await self._session.execute( + insert(JobEventRow).values( + job_id=job.id, + attachment_id=job.attachment_id, + kind=kind, + attempt=job.attempt, + detail=detail or {}, + ) + ) + + async def _move( + self, attachment_id: UUID, previous: Sequence[str], status: str, **values: Any + ) -> None: + await self._session.execute( + update(AttachmentRow) + .where(AttachmentRow.id == attachment_id, AttachmentRow.status.in_(previous)) + .values( + status=status, + updated_at=func.greatest(func.now(), AttachmentRow.created_at), + **values, + ) + ) + + def _owned(self, lease: Lease): + return update(cast(Table, JobRow.__table__)).where( + JobRow.job_id == lease.job.id, + JobRow.status == "processing", + JobRow.lease_token == lease.token, + ) + + async def add(self, job: Job) -> None: + await self._session.execute( + insert(JobRow).values( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + status="queued", + attempt=job.attempt, + created_at=job.created_at, + updated_at=job.created_at, + ) + ) + await self._enqueue(job) + await self._note(job, "enqueued") + + async def reprocess(self, job: Job, operator: str) -> bool: + existing = await self._session.scalar( + select(JobRow.job_id).where( + JobRow.attachment_id == job.attachment_id, + JobRow.pipeline_version == job.pipeline_version, + ) + ) + if existing is not None: + return False + await self.add(job) + await self._note(job, "reprocess_requested", {"operator": operator}) + return True + + async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: + row = ( + await self._session.execute( + update(cast(Table, JobRow.__table__)) + .where( + JobRow.job_id == job.id, + JobRow.attachment_id == job.attachment_id, + JobRow.pipeline_version == job.pipeline_version, + or_( + (JobRow.status == "queued") + & or_( + JobRow.next_attempt_at.is_(None), JobRow.next_attempt_at <= func.now() + ), + (JobRow.status == "processing") & (JobRow.lease_until < func.now()), + ), + ) + .values( + status="processing", + lease_token=token, + claims=JobRow.claims + 1, + updated_at=func.now(), + lease_until=func.now() + lease, + ) + .returning(JobRow.attempt, JobRow.created_at) + ) + ).first() + if row is None: + return None + claimed = Job(job.id, job.attachment_id, job.pipeline_version, row.attempt, row.created_at) + await self._move(job.attachment_id, ["queued"], "processing") + await self._note(claimed, "claimed") + source = ( + await self._session.execute( + select( + AttachmentRow.status, AttachmentRow.staging_key, AttachmentRow.original_key + ).where(AttachmentRow.id == job.attachment_id) + ) + ).one() + staged = source.status != "ready" and source.staging_key is not None + return Claim( + Lease(claimed, token), source.staging_key if staged else source.original_key, staged + ) + + async def complete( + self, lease: Lease, variants: Sequence[Variant], original_key: str + ) -> Outcome: + job = lease.job + done = await self._session.scalar( + self._owned(lease) + .values( + status="done", + lease_token=None, + lease_until=None, + next_attempt_at=None, + last_error=None, + updated_at=func.now(), + ) + .returning(JobRow.job_id) + ) + if done is None: + await self.note(job, Outcome.FENCED) + return Outcome.FENCED + if variants: + await self._session.execute( + pg_insert(VariantRow) + .values( + [ + { + "attachment_id": job.attachment_id, + "pipeline_version": job.pipeline_version, + "name": v.name, + "object_key": v.object_key, + "content_type": v.content_type, + "size_bytes": v.size_bytes, + "width": v.width, + "height": v.height, + "sha256": v.sha256, + "created_at": func.now(), + } + for v in variants + ] + ) + .on_conflict_do_nothing() + ) + published = await self._session.scalar( + update(AttachmentRow) + .where( + AttachmentRow.id == job.attachment_id, + AttachmentRow.published_version < job.pipeline_version, + ) + .values( + status="ready", + published_version=job.pipeline_version, + error_code=None, + original_key=original_key, + updated_at=func.greatest(func.now(), AttachmentRow.created_at), + ) + .returning(AttachmentRow.id) + ) + outcome = Outcome.SUPERSEDED if published is None else Outcome.DONE + if outcome is Outcome.SUPERSEDED: + await self._session.execute( + update(cast(Table, JobRow.__table__)) + .where(JobRow.job_id == job.id) + .values(status="superseded", updated_at=func.now()) + ) + await self.note(job, outcome) + return outcome + + async def reschedule( + self, lease: Lease, delay: timedelta, kind: Outcome, error: str + ) -> Job | None: + row = ( + await self._session.execute( + self._owned(lease) + .values( + status="queued", + attempt=JobRow.attempt + 1, + lease_token=None, + lease_until=None, + next_attempt_at=func.now() + delay, + last_error=error, + updated_at=func.now(), + ) + .returning(JobRow.attempt, JobRow.next_attempt_at) + ) + ).first() + if row is None: + return None + job = lease.job.retried(row.attempt) + await self._move(job.attachment_id, ["processing"], "queued") + await self._note( + job, + kind.value, + { + "error": error, + "next_attempt_at": row.next_attempt_at.isoformat(), + "delay_seconds": delay.total_seconds(), + }, + ) + await self._enqueue(job, row.next_attempt_at) + return job + + async def renew(self, lease: Lease, duration: timedelta) -> bool: + return ( + await self._session.scalar( + self._owned(lease) + .values(lease_until=func.now() + duration, updated_at=func.now()) + .returning(JobRow.job_id) + ) + is not None + ) + + async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: + attempt = await self._session.scalar( + self._owned(lease) + .values( + status="failed", + lease_token=None, + lease_until=None, + last_error=error, + updated_at=func.now(), + ) + .returning(JobRow.attempt) + ) + if attempt is None: + return False + await self._session.execute( + select(AttachmentRow.id) + .where(AttachmentRow.id == lease.job.attachment_id) + .with_for_update() + ) + active = ( + select(JobRow.job_id) + .where( + JobRow.attachment_id == lease.job.attachment_id, + JobRow.status.in_(["queued", "processing"]), + ) + .exists() + ) + await self._session.execute( + update(AttachmentRow) + .where( + AttachmentRow.id == lease.job.attachment_id, + AttachmentRow.status.in_(["queued", "processing"]), + ~active, + ) + .values( + status="failed", + error_code=code.value, + updated_at=func.greatest(func.now(), AttachmentRow.created_at), + ) + ) + await self._note( + lease.job.retried(attempt), "failed", {"error_code": code.value, "error": error} + ) + return True + + async def note(self, job: Job, kind: Outcome) -> None: + await self._note(job, kind.value) + + async def expired(self, limit: int) -> list[Lease]: + rows = await self._session.execute( + select(JobRow.__table__) + .where(JobRow.status == "processing", JobRow.lease_until < func.now()) + .order_by(JobRow.lease_until) + .limit(limit) + .with_for_update(skip_locked=True) + ) + return [ + Lease( + Job( + row.job_id, row.attachment_id, row.pipeline_version, row.attempt, row.created_at + ), + row.lease_token, + ) + for row in rows + ] + + async def replay(self, attachment: Attachment, operator: str) -> Job | None: + job_id = ( + select(JobRow.job_id) + .where(JobRow.attachment_id == attachment.id, JobRow.status == "failed") + .order_by(JobRow.pipeline_version.desc()) + .limit(1) + .scalar_subquery() + ) + row = ( + await self._session.execute( + update(cast(Table, JobRow.__table__)) + .where(JobRow.job_id == job_id) + .values( + status="queued", + attempt=1, + lease_token=None, + lease_until=None, + next_attempt_at=None, + last_error=None, + updated_at=func.now(), + ) + .returning( + JobRow.job_id, JobRow.pipeline_version, JobRow.created_at, JobRow.updated_at + ) + ) + ).first() + if row is None: + return None + job = Job(row.job_id, attachment.id, row.pipeline_version, 1, row.created_at) + await self._move(attachment.id, ["failed"], "queued", error_code=None) + await self._note( + job, "replayed", {"operator": operator, "error_code": attachment.error_code} + ) + await self._enqueue(job, row.updated_at) + return job + + async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None: + row = ( + await self._session.execute( + select(JobRow.__table__) + .where(JobRow.attachment_id == attachment_id) + .order_by(JobRow.pipeline_version.desc()) + .limit(1) + ) + ).first() + if row is not None: + job = Job( + row.job_id, row.attachment_id, row.pipeline_version, row.attempt, row.created_at + ) + await self._note(job, "replay_noop", {"operator": operator, "result": result}) + + +class PostgresOutboxRepository: + __slots__ = ("_session",) + + def __init__(self, session: DatabaseSession) -> None: + self._session = session + + async def pending(self, limit: int) -> list[OutboxEntry]: + rows = await self._session.execute( + select(OutboxRow.__table__) + .where(OutboxRow.published_at.is_(None), OutboxRow.available_at <= func.now()) + .order_by(OutboxRow.available_at, OutboxRow.id) + .limit(limit) + .with_for_update(skip_locked=True) + ) + return [OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode()) for row in rows] + + async def published(self, entry_ids: Sequence[int]) -> None: + await self._session.execute( + update(OutboxRow).where(OutboxRow.id.in_(entry_ids)).values(published_at=func.now()) + ) diff --git a/src/messenger/attachments/infrastructure/s3.py b/src/messenger/attachments/infrastructure/s3.py index 3acea37..e71a82b 100644 --- a/src/messenger/attachments/infrastructure/s3.py +++ b/src/messenger/attachments/infrastructure/s3.py @@ -1,4 +1,4 @@ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Sequence from contextlib import asynccontextmanager from typing import Any @@ -8,7 +8,7 @@ from botocore.exceptions import BotoCoreError, ClientError from botocore.handlers import add_expect_header from messenger.attachments.application.errors import ObjectMissingError -from messenger.attachments.application.ports import ObjectContent +from messenger.attachments.application.ports import ObjectContent, StoredObject from messenger.shared.application.errors import Unavailable CHUNK_BYTES = 64 * 1024 @@ -98,6 +98,28 @@ class S3ObjectStorage: except FAILURES as error: raise Unavailable("S3 недоступно") from error + async def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]: + pages = self._client.get_paginator("list_objects_v2").paginate( + Bucket=self._bucket, Prefix=prefix, PaginationConfig={"PageSize": page_size} + ) + try: + async for page in pages: + yield [ + StoredObject(item["Key"], item["Size"], item["LastModified"]) + for item in page.get("Contents", ()) + ] + except FAILURES as error: + raise Unavailable("S3 недоступно") from error + + async def stat(self, key: str) -> StoredObject | None: + try: + head = await self._client.head_object(Bucket=self._bucket, Key=key) + except FAILURES as error: + if missing(error): + return None + raise Unavailable("S3 недоступно") from error + return StoredObject(key, head["ContentLength"], head["LastModified"]) + async def _chunks(self, body: Any) -> AsyncIterator[bytes]: async with body as stream: async for chunk in stream.iter_chunks(CHUNK_BYTES): diff --git a/src/messenger/gunicorn_conf.py b/src/messenger/gunicorn_conf.py index 4ccd498..140d588 100644 --- a/src/messenger/gunicorn_conf.py +++ b/src/messenger/gunicorn_conf.py @@ -1,5 +1,4 @@ import asyncio -import logging import os import sys from typing import Any @@ -10,12 +9,13 @@ from uvicorn.server import Server from uvicorn_worker import UvicornWorker from messenger.main import LOG_FIELDS -from messenger.shared.entrypoints.health import DRAINING -from messenger.shared.infrastructure.logging import configure +from messenger.shared.infrastructure.logging import StructlogLogger, configure +from messenger.shared.infrastructure.readiness import DRAINING LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") -SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")) +SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "6")) DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2")) +LOGGER = StructlogLogger(level=LOG_LEVEL, allowed_fields=LOG_FIELDS) class StructlogGunicornLogger(GunicornBaseLogger): @@ -31,7 +31,7 @@ class DrainingServer(Server): await super().main_loop() if not self.force_exit: DRAINING.set() - logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS) + LOGGER.info("service.draining", limit=DRAIN_SECONDS) await asyncio.sleep(DRAIN_SECONDS) @@ -40,7 +40,7 @@ class MessengerWorker(UvicornWorker): "access_log": False, "server_header": False, "date_header": False, - "timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)), + "timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS) - 1), } async def _serve(self) -> None: diff --git a/src/messenger/identity/application/handlers.py b/src/messenger/identity/application/handlers.py index b5811ff..28b6f45 100644 --- a/src/messenger/identity/application/handlers.py +++ b/src/messenger/identity/application/handlers.py @@ -4,7 +4,7 @@ from messenger.identity.application.commands import CloseSession, OpenSession, R from messenger.identity.application.ports import ( IdentityUnitOfWork, PasswordHasher, - SessionDirectory, + SessionRevocations, ) from messenger.identity.domain.events import SessionClosed, SessionOpened from messenger.identity.domain.models import Session, User @@ -62,15 +62,19 @@ class OpenSessionHandler: class CloseSessionHandler: - __slots__ = ("_sessions", "_uow") + __slots__ = ("_revocations", "_uow") - def __init__(self, uow: IdentityUnitOfWork, sessions: SessionDirectory) -> None: + def __init__(self, uow: IdentityUnitOfWork, revocations: SessionRevocations) -> None: self._uow = uow - self._sessions = sessions + self._revocations = revocations async def __call__(self, command: CloseSession) -> None: - await self._sessions.invalidate(command.session) - async with self._uow.transaction(): - await self._uow.sessions.remove(command.session.token_hash) - self._uow.record(SessionClosed(user_id=command.session.user_id)) - await self._uow.commit() + await self._revocations.revoke(command.session) + try: + async with self._uow.transaction(): + await self._uow.sessions.remove(command.session.token_hash) + self._uow.record(SessionClosed(user_id=command.session.user_id)) + await self._uow.commit() + except BaseException: + await self._revocations.restore(command.session) + raise diff --git a/src/messenger/identity/application/ports.py b/src/messenger/identity/application/ports.py index bba207e..d9354c8 100644 --- a/src/messenger/identity/application/ports.py +++ b/src/messenger/identity/application/ports.py @@ -38,7 +38,12 @@ class UserDirectory(Protocol): class SessionDirectory(Protocol): async def find(self, token_hash: bytes, now: datetime) -> Session | None: ... - async def invalidate(self, session: Session) -> None: ... + +@runtime_checkable +class SessionRevocations(Protocol): + async def revoke(self, session: Session) -> None: ... + + async def restore(self, session: Session) -> None: ... @runtime_checkable diff --git a/src/messenger/identity/entrypoints/ioc.py b/src/messenger/identity/entrypoints/ioc.py index 01ae21c..1616d95 100644 --- a/src/messenger/identity/entrypoints/ioc.py +++ b/src/messenger/identity/entrypoints/ioc.py @@ -17,6 +17,7 @@ from messenger.identity.application.ports import ( IdentityUnitOfWork, PasswordHasher, SessionDirectory, + SessionRevocations, UserDirectory, ) from messenger.identity.application.queries import Authenticate, GetCurrentUser, VerifyCredentials @@ -27,7 +28,7 @@ from messenger.identity.infrastructure.postgres import ( PostgresSessionDirectory, PostgresUserDirectory, ) -from messenger.identity.infrastructure.valkey import CachedSessionDirectory +from messenger.identity.infrastructure.valkey import CachedSessionDirectory, NoRevocations from messenger.shared.application.commands import Command from messenger.shared.application.messagebus import CommandHandler from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger @@ -76,6 +77,10 @@ class IdentityProvider(Provider): finally: await client.aclose() + @provide(scope=Scope.APP) + def revocations(self, sessions: SessionDirectory) -> SessionRevocations: + return sessions if isinstance(sessions, CachedSessionDirectory) else NoRevocations() + @provide(scope=Scope.APP) def verify(self, users: UserDirectory, hasher: PasswordHasher) -> VerifyCredentials: return VerifyCredentials(users, hasher) @@ -100,13 +105,13 @@ class IdentityProvider(Provider): clock: Clock, ids: IdGenerator, hasher: PasswordHasher, - sessions: SessionDirectory, + revocations: SessionRevocations, settings: IdentitySettings, ) -> IdentityCommandBus: ttl = timedelta(seconds=settings.session_ttl_seconds) handlers: dict[type[Command], CommandHandler] = { RegisterUser: RegisterUserHandler(uow, clock, ids, hasher), OpenSession: OpenSessionHandler(uow, clock, ttl), - CloseSession: CloseSessionHandler(uow, sessions), + CloseSession: CloseSessionHandler(uow, revocations), } return IdentityCommandBus(handlers, uow, publisher) diff --git a/src/messenger/identity/infrastructure/postgres.py b/src/messenger/identity/infrastructure/postgres.py index ab8ddf3..eecc999 100644 --- a/src/messenger/identity/infrastructure/postgres.py +++ b/src/messenger/identity/infrastructure/postgres.py @@ -1,85 +1,126 @@ from datetime import datetime from uuid import UUID -from asyncpg import Record +from sqlalchemy import ( + CheckConstraint, + DateTime, + ForeignKey, + Index, + LargeBinary, + UniqueConstraint, + delete, + literal, + select, +) +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Mapped, mapped_column from messenger.identity.domain.errors import UsernameAlreadyTaken from messenger.identity.domain.models import Session, User from messenger.identity.domain.values import DisplayName, Username -from messenger.shared.infrastructure.postgres import Connection, Database, PostgresUnitOfWork - -INSERT_USER = """ -INSERT INTO users (id, username, display_name, password_hash, created_at) -VALUES ($1, $2, $3, $4, $5) -ON CONFLICT (username) DO NOTHING -RETURNING true -""" -USER_BY_ID = """ -SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1 -""" -USER_BY_USERNAME = """ -SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1 -""" -INSERT_SESSION = """ -WITH expired AS (DELETE FROM sessions WHERE user_id = $2 AND expires_at <= $3) -INSERT INTO sessions (token_hash, user_id, created_at, expires_at) VALUES ($1, $2, $3, $4) -""" -DELETE_SESSION = "DELETE FROM sessions WHERE token_hash = $1" -ACTIVE_SESSION = """ -SELECT token_hash, user_id, created_at, expires_at -FROM sessions -WHERE token_hash = $1 AND expires_at > $2 -""" +from messenger.shared.infrastructure.postgres import ( + Base, + Database, + DatabaseSession, + PostgresUnitOfWork, +) -def user_of(row: Record | None) -> User | None: +class UserRow(Base): + __tablename__ = "users" + __table_args__ = ( + UniqueConstraint("username", name="users_username_key"), + CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="users_username_format"), + CheckConstraint( + "char_length(display_name) BETWEEN 1 AND 100", name="users_display_name_check" + ), + CheckConstraint("password_hash LIKE '$argon2id$%'", name="users_password_hash_check"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + username: Mapped[str] + display_name: Mapped[str] + password_hash: Mapped[str] + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class SessionRow(Base): + __tablename__ = "sessions" + __table_args__ = ( + CheckConstraint("octet_length(token_hash) = 32", name="sessions_token_hash_check"), + CheckConstraint("expires_at > created_at", name="sessions_check"), + Index("sessions_user_idx", "user_id"), + ) + + token_hash: Mapped[bytes] = mapped_column(LargeBinary, primary_key=True) + user_id: Mapped[UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +def user_of(row: UserRow | None) -> User | None: if row is None: return None return User( - id=row["id"], - username=Username(row["username"]), - display_name=DisplayName(row["display_name"]), - created_at=row["created_at"], - password_hash=row["password_hash"], + id=row.id, + username=Username(row.username), + display_name=DisplayName(row.display_name), + created_at=row.created_at, + password_hash=row.password_hash, ) class PostgresUserRepository: - __slots__ = ("_connection",) + __slots__ = ("_session",) - def __init__(self, connection: Connection) -> None: - self._connection = connection + def __init__(self, session: DatabaseSession) -> None: + self._session = session async def add(self, user: User) -> None: - inserted = await self._connection.fetchval( - INSERT_USER, - user.id, - user.username.value, - user.display_name.value, - user.password_hash, - user.created_at, + statement = ( + insert(UserRow) + .values( + id=user.id, + username=user.username.value, + display_name=user.display_name.value, + password_hash=user.password_hash, + created_at=user.created_at, + ) + .on_conflict_do_nothing(index_elements=["username"]) + .returning(literal(True)) ) + inserted = (await self._session.execute(statement)).scalar_one_or_none() if inserted is None: raise UsernameAlreadyTaken("username уже занят") class PostgresSessionRepository: - __slots__ = ("_connection",) + __slots__ = ("_session",) - def __init__(self, connection: Connection) -> None: - self._connection = connection + def __init__(self, session: DatabaseSession) -> None: + self._session = session async def add(self, session: Session) -> None: - await self._connection.execute( - INSERT_SESSION, - session.token_hash, - session.user_id, - session.created_at, - session.expires_at, + expired = ( + delete(SessionRow) + .where( + SessionRow.user_id == session.user_id, SessionRow.expires_at <= session.created_at + ) + .cte("expired") + ) + await self._session.execute( + insert(SessionRow) + .values( + token_hash=session.token_hash, + user_id=session.user_id, + created_at=session.created_at, + expires_at=session.expires_at, + ) + .add_cte(expired) ) async def remove(self, token_hash: bytes) -> None: - await self._connection.execute(DELETE_SESSION, token_hash) + await self._session.execute(delete(SessionRow).where(SessionRow.token_hash == token_hash)) class PostgresIdentityUnitOfWork(PostgresUnitOfWork): @@ -87,11 +128,11 @@ class PostgresIdentityUnitOfWork(PostgresUnitOfWork): @property def users(self) -> PostgresUserRepository: - return PostgresUserRepository(self.connection) + return PostgresUserRepository(self.session) @property def sessions(self) -> PostgresSessionRepository: - return PostgresSessionRepository(self.connection) + return PostgresSessionRepository(self.session) class PostgresUserDirectory: @@ -101,12 +142,18 @@ class PostgresUserDirectory: self._database = database async def find(self, user_id: UUID) -> User | None: - async with self._database.connection() as connection: - return user_of(await connection.fetchrow(USER_BY_ID, user_id)) + async with self._database.session() as session: + row = ( + await session.execute(select(UserRow).where(UserRow.id == user_id)) + ).scalar_one_or_none() + return user_of(row) async def find_by_username(self, username: str) -> User | None: - async with self._database.connection() as connection: - return user_of(await connection.fetchrow(USER_BY_USERNAME, username)) + async with self._database.session() as session: + row = ( + await session.execute(select(UserRow).where(UserRow.username == username)) + ).scalar_one_or_none() + return user_of(row) class PostgresSessionDirectory: @@ -116,9 +163,20 @@ class PostgresSessionDirectory: self._database = database async def find(self, token_hash: bytes, now: datetime) -> Session | None: - async with self._database.connection() as connection: - row = await connection.fetchrow(ACTIVE_SESSION, token_hash, now) - return None if row is None else Session(**dict(row)) - - async def invalidate(self, session: Session) -> None: - return None + async with self._database.session() as session: + row = ( + await session.execute( + select(SessionRow).where( + SessionRow.token_hash == token_hash, + SessionRow.expires_at > now, + ) + ) + ).scalar_one_or_none() + if row is None: + return None + return Session( + token_hash=row.token_hash, + user_id=row.user_id, + created_at=row.created_at, + expires_at=row.expires_at, + ) diff --git a/src/messenger/identity/infrastructure/valkey.py b/src/messenger/identity/infrastructure/valkey.py index c3d8020..4a596d6 100644 --- a/src/messenger/identity/infrastructure/valkey.py +++ b/src/messenger/identity/infrastructure/valkey.py @@ -37,6 +37,14 @@ def decode(token_hash: bytes, value: str) -> Session: ) +class NoRevocations: + async def revoke(self, session: Session) -> None: + return None + + async def restore(self, session: Session) -> None: + return None + + class CachedSessionDirectory: __slots__ = ("_client", "_cooldown", "_limit", "_logger", "_origin", "_resume_at") @@ -70,7 +78,7 @@ class CachedSessionDirectory: await self._quietly(self._client.set(key, encode(session), pxat=until, nx=True)) return session - async def invalidate(self, session: Session) -> None: + async def revoke(self, session: Session) -> None: if not self._closed(): raise Unavailable("кэш сессий недоступен") try: @@ -81,6 +89,9 @@ class CachedSessionDirectory: self._trip(error) raise Unavailable("кэш сессий недоступен") from error + async def restore(self, session: Session) -> None: + await self._quietly(self._client.delete(key_of(session.token_hash))) + async def _read(self, key: str) -> str | None: if not self._closed(): return None diff --git a/src/messenger/main.py b/src/messenger/main.py index dc365ea..21866ec 100644 --- a/src/messenger/main.py +++ b/src/messenger/main.py @@ -42,7 +42,7 @@ from messenger.shared.entrypoints.telemetry import instrument_http from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger from messenger.shared.infrastructure.telemetry import start_telemetry -TRANSPORT_FIELDS = frozenset({"dropped_total", "queue", "instance"}) +TRANSPORT_FIELDS = frozenset({"dropped_total"}) LOG_FIELDS = ( PLATFORM_FIELDS | TRANSPORT_FIELDS | IDENTITY_FIELDS | MESSAGING_FIELDS | ATTACHMENT_FIELDS @@ -153,6 +153,7 @@ def create_app( guard=guard, instance=platform.instance_id, limits=((UPLOAD_PATH, attachments.max_upload_bytes),), + min_rate=transport.min_transfer_rate_bytes, ) errors.install(app, logger, ERROR_CODES, challenge_for, platform.instance_id) app.include_router(health.router) diff --git a/src/messenger/messaging/application/handlers.py b/src/messenger/messaging/application/handlers.py index 948fea8..e4e52c1 100644 --- a/src/messenger/messaging/application/handlers.py +++ b/src/messenger/messaging/application/handlers.py @@ -1,9 +1,14 @@ from datetime import timedelta from messenger.messaging.application.commands import CreateChat, PostMessage -from messenger.messaging.application.ports import AttachmentStates, MessagingUnitOfWork -from messenger.messaging.domain.errors import AttachmentUnavailable, ChatNotFound -from messenger.messaging.domain.models import Chat, Message, Receipt +from messenger.messaging.application.ports import ChatDirectory, MessageLog, MessagingUnitOfWork +from messenger.messaging.domain.errors import ( + ChatNotFound, + InvalidAttachmentList, + InvalidMessageText, +) +from messenger.messaging.domain.models import Chat, Message, attachment_list +from messenger.messaging.domain.values import MessageText from messenger.shared.application.ports import Clock, IdGenerator @@ -30,44 +35,41 @@ class CreateChatHandler: class PostMessageHandler: - __slots__ = ("_attachments", "_clock", "_ids", "_ttl", "_uow") + __slots__ = ("_chats", "_clock", "_ids", "_log", "_ttl", "_uow") def __init__( self, uow: MessagingUnitOfWork, - attachments: AttachmentStates, + log: MessageLog, + chats: ChatDirectory, clock: Clock, ids: IdGenerator, + *, ttl: timedelta, ) -> None: self._uow = uow - self._attachments = attachments + self._log = log + self._chats = chats self._clock = clock self._ids = ids self._ttl = ttl async def __call__(self, command: PostMessage) -> Message: - async with self._uow.transaction(): - chat = await self._uow.chats.get(command.chat_id) - if chat is None: - raise ChatNotFound("чат не найден") - now = self._clock.now() - message = chat.post( - message_id=self._ids.new_id(), - sender_id=command.sender_id, - text=command.text, - now=now, - attachment_ids=command.attachment_ids, - ) - known = await self._uow.chats.receipt( - chat.id, command.sender_id, command.key, now - self._ttl - ) - if known is not None: - return known.replay(command.fingerprint) - if message.attachment_ids and await self._attachments.unusable( - chat.id, message.attachment_ids - ): - raise AttachmentUnavailable("вложение не обработано и не может быть отправлено") - await self._uow.chats.append(Receipt(command.key, command.fingerprint, message)) - await self._uow.commit() + try: + text = MessageText(command.text) + attachments = attachment_list(text, command.attachment_ids) + except InvalidMessageText, InvalidAttachmentList: + if await self._chats.chat_for_member(command.chat_id, command.sender_id) is None: + raise ChatNotFound("чат не найден") from None + raise + now = self._clock.now() + draft = Message( + self._ids.new_id(), command.chat_id, command.sender_id, text, now, attachments + ) + receipt = await self._log.post(draft, command.key, command.fingerprint, now - self._ttl) + if receipt is None: + raise ChatNotFound("чат не найден") + message = receipt.replay(command.fingerprint) + if message.id == draft.id: + self._uow.record(message.posted()) return message diff --git a/src/messenger/messaging/application/ports.py b/src/messenger/messaging/application/ports.py index bf7b6a8..3651e0c 100644 --- a/src/messenger/messaging/application/ports.py +++ b/src/messenger/messaging/application/ports.py @@ -19,14 +19,13 @@ class CursorCodec(Protocol): class ChatRepository(Protocol): async def add(self, chat: Chat) -> None: ... - async def get(self, chat_id: UUID) -> Chat | None: ... - async def receipt( - self, chat_id: UUID, sender_id: UUID, key: str, since: datetime +@runtime_checkable +class MessageLog(Protocol): + async def post( + self, message: Message, key: str, fingerprint: bytes, since: datetime ) -> Receipt | None: ... - async def append(self, receipt: Receipt) -> None: ... - @runtime_checkable class MessagingUnitOfWork(UnitOfWork, Protocol): diff --git a/src/messenger/messaging/application/queries.py b/src/messenger/messaging/application/queries.py index 8e16e94..5fbb20a 100644 --- a/src/messenger/messaging/application/queries.py +++ b/src/messenger/messaging/application/queries.py @@ -43,7 +43,7 @@ class ListChats: async def __call__(self, user_id: UUID, *, limit: int, cursor: str | None) -> Page[Chat]: scope = chats_scope(user_id) - after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None + after: HistoryKey | None = None if cursor is None else self._cursors.decode(scope, cursor) items, has_more = await self._chats.chats_of(user_id, after=after, limit=limit) next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None return Page(items=items, next_cursor=next_cursor) @@ -67,7 +67,7 @@ class ReadHistory: if await self._chats.chat_for_member(chat_id, user_id) is None: raise ChatNotFound("чат не найден") scope = messages_scope(chat_id) - after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None + after: HistoryKey | None = None if cursor is None else self._cursors.decode(scope, cursor) items, has_more = await self._chats.history(chat_id, after=after, limit=limit) next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None return Page(items=items, next_cursor=next_cursor) diff --git a/src/messenger/messaging/domain/models.py b/src/messenger/messaging/domain/models.py index f98b843..1221b0f 100644 --- a/src/messenger/messaging/domain/models.py +++ b/src/messenger/messaging/domain/models.py @@ -1,10 +1,9 @@ from collections.abc import Sequence from dataclasses import dataclass -from datetime import datetime, timedelta +from datetime import datetime from uuid import UUID from messenger.messaging.domain.errors import ( - ChatNotFound, IdempotencyConflict, InvalidAttachmentList, InvalidChatData, @@ -20,8 +19,6 @@ from messenger.messaging.domain.values import ( ) from messenger.shared.domain.events import DomainEvent -TICK = timedelta(microseconds=1) - @dataclass(frozen=True, slots=True) class Message: @@ -36,6 +33,15 @@ class Message: def key(self) -> HistoryKey: return (self.created_at, self.id.int) + def posted(self) -> MessagePosted: + return MessagePosted( + message_id=self.id, + chat_id=self.chat_id, + sender_id=self.sender_id, + text_length=len(self.text.value), + created_at=self.created_at, + ) + def attachment_list(text: MessageText, attachment_ids: Sequence[UUID]) -> tuple[UUID, ...]: if len(attachment_ids) > MESSAGE_ATTACHMENTS_MAX: @@ -62,15 +68,7 @@ class Receipt: class Chat: - __slots__ = ( - "_events", - "_members", - "created_at", - "id", - "last_message_at", - "member_ids", - "title", - ) + __slots__ = ("_events", "created_at", "id", "member_ids", "title") def __init__( self, @@ -79,14 +77,11 @@ class Chat: title: ChatTitle, member_ids: Sequence[UUID], created_at: datetime, - last_message_at: datetime | None = None, ) -> None: self.id = chat_id self.title = title self.member_ids: tuple[UUID, ...] = tuple(member_ids) self.created_at = created_at - self.last_message_at = last_message_at - self._members: frozenset[UUID] = frozenset(member_ids) self._events: list[DomainEvent] = [] @classmethod @@ -131,47 +126,6 @@ class Chat: def record(self, event: DomainEvent) -> None: self._events.append(event) - def is_member(self, user_id: UUID) -> bool: - return user_id in self._members - - def require_member(self, user_id: UUID) -> None: - if not self.is_member(user_id): - raise ChatNotFound("чат не найден") - - def post( - self, - *, - message_id: UUID, - sender_id: UUID, - text: str, - now: datetime, - attachment_ids: Sequence[UUID] = (), - ) -> Message: - self.require_member(sender_id) - body = MessageText(text) - attachments = attachment_list(body, attachment_ids) - last = self.last_message_at - created_at = now if last is None or now > last else last + TICK - message = Message( - id=message_id, - chat_id=self.id, - sender_id=sender_id, - text=body, - created_at=created_at, - attachment_ids=attachments, - ) - self.last_message_at = created_at - self.record( - MessagePosted( - message_id=message.id, - chat_id=self.id, - sender_id=sender_id, - text_length=len(message.text.value), - created_at=created_at, - ) - ) - return message - def collect_events(self) -> list[DomainEvent]: events, self._events = self._events, [] return events diff --git a/src/messenger/messaging/entrypoints/ioc.py b/src/messenger/messaging/entrypoints/ioc.py index a2e21dd..8732dd7 100644 --- a/src/messenger/messaging/entrypoints/ioc.py +++ b/src/messenger/messaging/entrypoints/ioc.py @@ -9,6 +9,7 @@ from messenger.messaging.application.ports import ( AttachmentStates, ChatDirectory, CursorCodec, + MessageLog, MessagingUnitOfWork, ) from messenger.messaging.application.queries import GetChat, ListChats, ReadHistory @@ -16,6 +17,7 @@ from messenger.messaging.config import MessagingSettings from messenger.messaging.infrastructure.cursor import SignedCursorCodec from messenger.messaging.infrastructure.postgres import ( PostgresChatDirectory, + PostgresMessageLog, PostgresMessagingUnitOfWork, ) from messenger.shared.application.commands import Command @@ -37,6 +39,10 @@ class MessagingProvider(Provider): def directory(self, database: Database) -> ChatDirectory: return PostgresChatDirectory(database) + @provide(scope=Scope.APP) + def log(self, database: Database, attachments: AttachmentStates) -> MessageLog: + return PostgresMessageLog(database, attachments) + @provide(scope=Scope.APP) def cursors(self, settings: MessagingSettings) -> CursorCodec: return SignedCursorCodec(settings.cursor_secret.get_secret_value().encode()) @@ -61,7 +67,8 @@ class MessagingProvider(Provider): def command_bus( self, uow: MessagingUnitOfWork, - attachments: AttachmentStates, + log: MessageLog, + directory: ChatDirectory, publisher: EventPublisher, clock: Clock, ids: IdGenerator, @@ -70,6 +77,6 @@ class MessagingProvider(Provider): ttl = timedelta(seconds=settings.idempotency_ttl_seconds) handlers: dict[type[Command], CommandHandler] = { CreateChat: CreateChatHandler(uow, clock, ids), - PostMessage: PostMessageHandler(uow, attachments, clock, ids, ttl), + PostMessage: PostMessageHandler(uow, log, directory, clock, ids, ttl=ttl), } return MessagingCommandBus(handlers, uow, publisher) diff --git a/src/messenger/messaging/entrypoints/schemas.py b/src/messenger/messaging/entrypoints/schemas.py index 6b7f1a3..ffe319f 100644 --- a/src/messenger/messaging/entrypoints/schemas.py +++ b/src/messenger/messaging/entrypoints/schemas.py @@ -37,7 +37,7 @@ class CreateMessageRequest(BaseModel): class PageQuery(BaseModel): limit: int = Field(default=50, ge=1, le=100) - cursor: str | None = None + cursor: str | None = Field(default=None, min_length=1) class ChatOut(BaseModel): diff --git a/src/messenger/messaging/infrastructure/postgres.py b/src/messenger/messaging/infrastructure/postgres.py index 3c8b94b..a547a68 100644 --- a/src/messenger/messaging/infrastructure/postgres.py +++ b/src/messenger/messaging/infrastructure/postgres.py @@ -1,122 +1,266 @@ from collections.abc import Sequence -from datetime import datetime +from dataclasses import replace +from datetime import datetime, timedelta +from typing import cast from uuid import UUID -from asyncpg import ForeignKeyViolationError, Record +from sqlalchemy import ( + CheckConstraint, + DateTime, + ForeignKey, + ForeignKeyConstraint, + Index, + LargeBinary, + RowMapping, + Select, + SmallInteger, + Table, + UniqueConstraint, + bindparam, + column, + func, + insert, + lambda_stmt, + literal, + select, + true, + tuple_, + update, +) +from sqlalchemy.dialects.postgresql import ARRAY +from sqlalchemy.dialects.postgresql import UUID as PGUUID +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.sql.elements import Label -from messenger.messaging.domain.errors import InvalidChatData, UnknownAttachment +from messenger.messaging.application.ports import AttachmentStates +from messenger.messaging.domain.errors import ( + AttachmentUnavailable, + InvalidChatData, + UnknownAttachment, +) from messenger.messaging.domain.keys import HistoryKey from messenger.messaging.domain.models import Chat, Message, Receipt from messenger.messaging.domain.values import ChatTitle, MessageText +from messenger.shared.application.errors import Unavailable from messenger.shared.infrastructure.postgres import ( - Connection, + Base, Database, + DatabaseSession, EventSource, PostgresUnitOfWork, ) -INSERT_CHAT = "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)" -INSERT_MEMBERS = """ -INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) -SELECT $1, member.user_id, member.position, $3 -FROM unnest($2::uuid[]) WITH ORDINALITY AS member (user_id, position) -""" -LOCK_CHAT = """ -SELECT c.id, c.title, c.created_at, c.last_message_at, - ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids -FROM chats c -WHERE c.id = $1 -FOR UPDATE -""" -ATTACHMENT_FKEY = "message_attachments_attachment_fkey" -RECEIPT = """ -SELECT k.fingerprint, m.id, m.chat_id, m.sender_id, m.text, m.created_at, - ARRAY(SELECT a.attachment_id FROM message_attachments a - WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids -FROM idempotency_keys k JOIN messages m ON m.id = k.message_id -WHERE k.chat_id = $1 AND k.user_id = $2 AND k.key = $3 AND k.created_at > $4 -""" -APPEND_MESSAGE = """ -WITH message AS ( - INSERT INTO messages (id, chat_id, sender_id, text, created_at) - VALUES ($1, $2, $3, $4, $5) - RETURNING id, chat_id, sender_id, created_at -), links AS ( - INSERT INTO message_attachments (message_id, position, chat_id, attachment_id) - SELECT message.id, link.position - 1, message.chat_id, link.attachment_id - FROM message, unnest($8::uuid[]) WITH ORDINALITY AS link (attachment_id, position) -), receipt AS ( - INSERT INTO idempotency_keys (chat_id, user_id, key, fingerprint, message_id, created_at) - SELECT chat_id, sender_id, $6, $7, id, created_at FROM message - ON CONFLICT (chat_id, user_id, key) DO UPDATE - SET fingerprint = EXCLUDED.fingerprint, - message_id = EXCLUDED.message_id, - created_at = EXCLUDED.created_at -) -UPDATE chats SET last_message_at = message.created_at -FROM message -WHERE chats.id = message.chat_id -""" -CHAT_OF_MEMBER = """ -SELECT c.id, c.title, c.created_at, - ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids -FROM chat_members m JOIN chats c ON c.id = m.chat_id -WHERE m.chat_id = $1 AND m.user_id = $2 -""" -CHATS_FIRST = """ -SELECT c.id, c.title, c.created_at, - ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids -FROM chat_members m JOIN chats c ON c.id = m.chat_id -WHERE m.user_id = $1 -ORDER BY m.chat_created_at, m.chat_id -LIMIT $2 -""" -CHATS_AFTER = """ -SELECT c.id, c.title, c.created_at, - ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids -FROM chat_members m JOIN chats c ON c.id = m.chat_id -WHERE m.user_id = $1 AND (m.chat_created_at, m.chat_id) > ($2, $3) -ORDER BY m.chat_created_at, m.chat_id -LIMIT $4 -""" -HISTORY_FIRST = """ -SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, - ARRAY(SELECT a.attachment_id FROM message_attachments a - WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids -FROM messages m -WHERE m.chat_id = $1 -ORDER BY m.created_at, m.id -LIMIT $2 -""" -HISTORY_AFTER = """ -SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, - ARRAY(SELECT a.attachment_id FROM message_attachments a - WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids -FROM messages m -WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3) -ORDER BY m.created_at, m.id -LIMIT $4 -""" +FOREIGN_KEY_VIOLATED = "23503" -def chat_of(row: Record) -> Chat: +class ChatRow(Base): + __tablename__ = "chats" + __table_args__ = ( + CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="chats_title_check"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + title: Mapped[str] + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + last_message_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + +class ChatMemberRow(Base): + __tablename__ = "chat_members" + __table_args__ = ( + CheckConstraint("position BETWEEN 1 AND 100", name="chat_members_position_check"), + Index("chat_members_user_idx", "user_id", "chat_created_at", "chat_id"), + ) + + chat_id: Mapped[UUID] = mapped_column( + ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True + ) + user_id: Mapped[UUID] = mapped_column(ForeignKey("users.id"), primary_key=True) + position: Mapped[int] = mapped_column(SmallInteger) + chat_created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class MessageRow(Base): + __tablename__ = "messages" + __table_args__ = ( + CheckConstraint("char_length(text) <= 4000", name="messages_text_check"), + ForeignKeyConstraint( + ["chat_id", "sender_id"], ["chat_members.chat_id", "chat_members.user_id"] + ), + Index("messages_history_idx", "chat_id", "created_at", "id"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + chat_id: Mapped[UUID] + sender_id: Mapped[UUID] + text: Mapped[str] + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class IdempotencyKeyRow(Base): + __tablename__ = "idempotency_keys" + __table_args__ = ( + CheckConstraint("key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="idempotency_keys_key_check"), + CheckConstraint( + "octet_length(fingerprint) = 32", name="idempotency_keys_fingerprint_check" + ), + ForeignKeyConstraint( + ["chat_id", "user_id"], ["chat_members.chat_id", "chat_members.user_id"] + ), + ) + + chat_id: Mapped[UUID] = mapped_column(primary_key=True) + user_id: Mapped[UUID] = mapped_column(primary_key=True) + key: Mapped[str] = mapped_column(primary_key=True) + fingerprint: Mapped[bytes] = mapped_column(LargeBinary) + message_id: Mapped[UUID] = mapped_column(ForeignKey("messages.id", ondelete="CASCADE")) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class MessageAttachmentRow(Base): + __tablename__ = "message_attachments" + __table_args__ = ( + CheckConstraint("position BETWEEN 0 AND 9", name="message_attachments_position_check"), + UniqueConstraint("message_id", "attachment_id"), + ForeignKeyConstraint( + ["attachment_id", "chat_id"], + ["attachments.id", "attachments.chat_id"], + name="message_attachments_attachment_fkey", + ondelete="RESTRICT", + ), + Index("message_attachments_attachment_idx", "attachment_id"), + ) + + message_id: Mapped[UUID] = mapped_column( + ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True + ) + position: Mapped[int] = mapped_column(SmallInteger, primary_key=True) + chat_id: Mapped[UUID] + attachment_id: Mapped[UUID] + + +def post_message() -> Select: + chats, members = cast(Table, ChatRow.__table__), cast(Table, ChatMemberRow.__table__) + keys, messages = cast(Table, IdempotencyKeyRow.__table__), cast(Table, MessageRow.__table__) + sender = bindparam("sender_id", type_=PGUUID) + chat_id = bindparam("chat_id", type_=PGUUID) + message_id = bindparam("message_id", type_=PGUUID) + chat = ( + select( + chats.c.id, + func.greatest( + bindparam("stamp", type_=DateTime(timezone=True)), + chats.c.last_message_at + timedelta(microseconds=1), + ).label("stamp"), + ) + .join(members, (members.c.chat_id == chats.c.id) & (members.c.user_id == sender)) + .where(chats.c.id == chat_id) + .with_for_update(of=chats) + .cte("chat") + ) + receipt_insert = pg_insert(keys).from_select( + ["chat_id", "user_id", "key", "fingerprint", "message_id", "created_at"], + select( + chat.c.id, + sender, + bindparam("key"), + bindparam("fingerprint", type_=LargeBinary), + message_id, + chat.c.stamp, + ), + ) + receipt = ( + receipt_insert.on_conflict_do_update( + index_elements=[keys.c.chat_id, keys.c.user_id, keys.c.key], + set_={ + name: receipt_insert.excluded[name] + for name in ("fingerprint", "message_id", "created_at") + }, + where=keys.c.created_at <= bindparam("since", type_=DateTime(timezone=True)), + ) + .returning(keys.c.created_at) + .cte("receipt") + ) + message = ( + insert(messages) + .from_select( + ["id", "chat_id", "sender_id", "text", "created_at"], + select(message_id, chat_id, sender, bindparam("text"), receipt.c.created_at), + ) + .returning(messages.c.chat_id, messages.c.created_at) + .cte("message") + ) + attachments = ( + func.unnest(bindparam("attachment_ids", type_=ARRAY(PGUUID))) + .table_valued(column("attachment_id", PGUUID), with_ordinality="position") + .render_derived(name="attachment") + ) + links = ( + insert(cast(Table, MessageAttachmentRow.__table__)) + .from_select( + ["message_id", "position", "chat_id", "attachment_id"], + select( + message_id, attachments.c.position - 1, chat_id, attachments.c.attachment_id + ).select_from(message.join(attachments, true())), + ) + .cte("links") + ) + touched = ( + update(chats) + .where(chats.c.id == message.c.chat_id) + .values(last_message_at=message.c.created_at) + .cte("touched") + ) + return select( + select(chat.c.id).exists().label("member"), + select(message.c.created_at).scalar_subquery().label("created_at"), + ).add_cte(links, touched) + + +POST_MESSAGE = lambda_stmt(post_message, enable_tracking=False) + + +def member_ids() -> Label[Sequence[UUID]]: + return func.array( + select(ChatMemberRow.user_id) + .where(ChatMemberRow.chat_id == ChatRow.id) + .order_by(ChatMemberRow.position) + .correlate(ChatRow) + .scalar_subquery(), + type_=ARRAY(PGUUID(as_uuid=True)), + ).label("member_ids") + + +def attachment_ids() -> Label[Sequence[UUID]]: + return func.array( + select(MessageAttachmentRow.attachment_id) + .where(MessageAttachmentRow.message_id == MessageRow.id) + .order_by(MessageAttachmentRow.position) + .correlate(MessageRow) + .scalar_subquery(), + type_=ARRAY(PGUUID(as_uuid=True)), + ).label("attachment_ids") + + +def chat_of(row: RowMapping) -> Chat: return Chat( chat_id=row["id"], title=ChatTitle(row["title"]), member_ids=row["member_ids"], created_at=row["created_at"], - last_message_at=row.get("last_message_at"), ) -def message_of(row: Record) -> Message: +def message_of(row: RowMapping) -> Message: return Message( id=row["id"], chat_id=row["chat_id"], sender_id=row["sender_id"], text=MessageText(row["text"]), - created_at=row["created_at"], attachment_ids=tuple(row["attachment_ids"]), + created_at=row["created_at"], ) @@ -125,54 +269,110 @@ def page[ItemT](rows: Sequence[ItemT], limit: int) -> tuple[list[ItemT], bool]: class PostgresChatRepository: - __slots__ = ("_connection", "_seen") + __slots__ = ("_seen", "_session") - def __init__(self, connection: Connection, seen: list[EventSource]) -> None: - self._connection = connection + def __init__(self, session: DatabaseSession, seen: list[EventSource]) -> None: + self._session = session self._seen = seen async def add(self, chat: Chat) -> None: - await self._connection.execute(INSERT_CHAT, chat.id, chat.title.value, chat.created_at) + await self._session.execute( + insert(ChatRow).values(id=chat.id, title=chat.title.value, created_at=chat.created_at) + ) try: - await self._connection.execute( - INSERT_MEMBERS, chat.id, list(chat.member_ids), chat.created_at + await self._session.execute( + insert(ChatMemberRow), + [ + { + "chat_id": chat.id, + "user_id": member_id, + "position": position, + "chat_created_at": chat.created_at, + } + for position, member_id in enumerate(chat.member_ids, 1) + ], ) - except ForeignKeyViolationError as error: + except IntegrityError as error: + if getattr(error.orig, "sqlstate", None) != FOREIGN_KEY_VIOLATED: + raise raise InvalidChatData("поле member_ids содержит неизвестного пользователя") from error self._seen.append(chat) - async def get(self, chat_id: UUID) -> Chat | None: - row = await self._connection.fetchrow(LOCK_CHAT, chat_id) - if row is None: - return None - chat = chat_of(row) - self._seen.append(chat) - return chat - async def receipt( - self, chat_id: UUID, sender_id: UUID, key: str, since: datetime +class PostgresMessageLog: + __slots__ = ("_attachments", "_database") + + def __init__(self, database: Database, attachments: AttachmentStates) -> None: + self._database = database + self._attachments = attachments + + async def post( + self, message: Message, key: str, fingerprint: bytes, since: datetime ) -> Receipt | None: - row = await self._connection.fetchrow(RECEIPT, chat_id, sender_id, key, since) - return None if row is None else Receipt(key, row["fingerprint"], message_of(row)) - - async def append(self, receipt: Receipt) -> None: - message = receipt.message - try: - await self._connection.execute( - APPEND_MESSAGE, - message.id, - message.chat_id, - message.sender_id, - message.text.value, - message.created_at, - receipt.key, - receipt.fingerprint, - list(message.attachment_ids), + chat_id, sender_id = message.chat_id, message.sender_id + async with self._database.session() as session: + try: + row = ( + ( + await session.execute( + POST_MESSAGE, + { + "message_id": message.id, + "chat_id": chat_id, + "sender_id": sender_id, + "text": message.text.value, + "stamp": message.created_at, + "key": key, + "fingerprint": fingerprint, + "since": since, + "attachment_ids": list(message.attachment_ids), + }, + ) + ) + .mappings() + .first() + ) + except IntegrityError as error: + cause = getattr(error.orig, "__cause__", None) + if getattr(cause, "constraint_name", None) != "message_attachments_attachment_fkey": + raise + raise UnknownAttachment("вложение не найдено в этом чате") from error + if row is None or not row["member"]: + return None + if row["created_at"] is not None: + if message.attachment_ids and await self._attachments.unusable( + chat_id, message.attachment_ids + ): + raise AttachmentUnavailable("вложение не обработано и не может быть отправлено") + await session.commit() + return Receipt(key, fingerprint, replace(message, created_at=row["created_at"])) + known = ( + ( + await session.execute( + select( + IdempotencyKeyRow.fingerprint, + MessageRow.id, + MessageRow.chat_id, + MessageRow.sender_id, + MessageRow.text, + MessageRow.created_at, + attachment_ids(), + ) + .join(MessageRow, MessageRow.id == IdempotencyKeyRow.message_id) + .where( + IdempotencyKeyRow.chat_id == chat_id, + IdempotencyKeyRow.user_id == sender_id, + IdempotencyKeyRow.key == key, + IdempotencyKeyRow.created_at > since, + ) + ) + ) + .mappings() + .first() ) - except ForeignKeyViolationError as error: - if getattr(error, "constraint_name", None) != ATTACHMENT_FKEY: - raise - raise UnknownAttachment("вложение не найдено в этом чате") from error + if known is None: + raise Unavailable("ключ идемпотентности занят незавершённой записью") + return Receipt(key, known["fingerprint"], message_of(known)) class PostgresMessagingUnitOfWork(PostgresUnitOfWork): @@ -180,7 +380,7 @@ class PostgresMessagingUnitOfWork(PostgresUnitOfWork): @property def chats(self) -> PostgresChatRepository: - return PostgresChatRepository(self.connection, self.seen) + return PostgresChatRepository(self.session, self.seen) class PostgresChatDirectory: @@ -190,30 +390,62 @@ class PostgresChatDirectory: self._database = database async def chat_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None: - async with self._database.connection() as connection: - row = await connection.fetchrow(CHAT_OF_MEMBER, chat_id, user_id) + async with self._database.session() as session: + row = ( + ( + await session.execute( + select(ChatRow.id, ChatRow.title, ChatRow.created_at, member_ids()) + .select_from(ChatMemberRow) + .join(ChatRow, ChatRow.id == ChatMemberRow.chat_id) + .where(ChatMemberRow.chat_id == chat_id, ChatMemberRow.user_id == user_id) + ) + ) + .mappings() + .first() + ) return None if row is None else chat_of(row) async def chats_of( self, user_id: UUID, *, after: HistoryKey | None, limit: int ) -> tuple[list[Chat], bool]: - async with self._database.connection() as connection: - if after is None: - rows = await connection.fetch(CHATS_FIRST, user_id, limit + 1) - else: - rows = await connection.fetch( - CHATS_AFTER, user_id, after[0], UUID(int=after[1]), limit + 1 - ) + statement = ( + select(ChatRow.id, ChatRow.title, ChatRow.created_at, member_ids()) + .select_from(ChatMemberRow) + .join(ChatRow, ChatRow.id == ChatMemberRow.chat_id) + .where(ChatMemberRow.user_id == user_id) + .order_by(ChatMemberRow.chat_created_at, ChatMemberRow.chat_id) + .limit(limit + 1) + ) + if after is not None: + statement = statement.where( + tuple_(ChatMemberRow.chat_created_at, ChatMemberRow.chat_id) + > tuple_(literal(after[0]), literal(UUID(int=after[1]))) + ) + async with self._database.session() as session: + rows = (await session.execute(statement)).mappings().all() return page([chat_of(row) for row in rows], limit) async def history( self, chat_id: UUID, *, after: HistoryKey | None, limit: int ) -> tuple[list[Message], bool]: - async with self._database.connection() as connection: - if after is None: - rows = await connection.fetch(HISTORY_FIRST, chat_id, limit + 1) - else: - rows = await connection.fetch( - HISTORY_AFTER, chat_id, after[0], UUID(int=after[1]), limit + 1 - ) + statement = ( + select( + MessageRow.id, + MessageRow.chat_id, + MessageRow.sender_id, + MessageRow.text, + MessageRow.created_at, + attachment_ids(), + ) + .where(MessageRow.chat_id == chat_id) + .order_by(MessageRow.created_at, MessageRow.id) + .limit(limit + 1) + ) + if after is not None: + statement = statement.where( + tuple_(MessageRow.created_at, MessageRow.id) + > tuple_(literal(after[0]), literal(UUID(int=after[1]))) + ) + async with self._database.session() as session: + rows = (await session.execute(statement)).mappings().all() return page([message_of(row) for row in rows], limit) diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py deleted file mode 100644 index 27037f6..0000000 --- a/src/messenger/migrate.py +++ /dev/null @@ -1,70 +0,0 @@ -import asyncio -import hashlib -from importlib import resources - -import asyncpg - -from messenger.shared.config import PlatformSettings -from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger - -LOCK_KEY = 20_260_002 -JOURNAL = """ -CREATE TABLE IF NOT EXISTS schema_migrations ( - version text PRIMARY KEY, - checksum text NOT NULL, - applied_at timestamptz NOT NULL DEFAULT now() -) -""" - - -class MigrationChangedError(Exception): - pass - - -def migrations() -> list[tuple[str, str]]: - folder = resources.files("messenger") / "migrations" - found = (item for item in folder.iterdir() if item.name.endswith(".sql")) - return sorted((item.name.removesuffix(".sql"), item.read_text()) for item in found) - - -async def migrate(dsn: str) -> list[str]: - connection = await asyncpg.connect(dsn, timeout=10) - try: - await connection.execute("SELECT pg_advisory_lock($1)", LOCK_KEY) - await connection.execute(JOURNAL) - applied = dict(await connection.fetch("SELECT version, checksum FROM schema_migrations")) - done: list[str] = [] - for version, sql in migrations(): - checksum = hashlib.sha256(sql.encode()).hexdigest() - if version in applied: - if applied[version] != checksum: - raise MigrationChangedError(version) - continue - async with connection.transaction(): - await connection.execute(sql) - await connection.execute( - "INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", - version, - checksum, - ) - done.append(version) - return done - finally: - await connection.close() - - -def main() -> None: - settings = PlatformSettings() - logger = StructlogLogger(level=settings.log_level, allowed_fields=PLATFORM_FIELDS | {"version"}) - try: - done = asyncio.run(migrate(settings.database_url.get_secret_value())) - except MigrationChangedError as error: - logger.error("migration.changed", version=str(error)) - raise SystemExit(1) from None - for version in done: - logger.info("migration.applied", version=version) - logger.info("migration.completed", reason="up to date" if not done else "applied") - - -if __name__ == "__main__": - main() diff --git a/src/messenger/shared/application/messagebus.py b/src/messenger/shared/application/messagebus.py index 856824b..bf658d2 100644 --- a/src/messenger/shared/application/messagebus.py +++ b/src/messenger/shared/application/messagebus.py @@ -38,30 +38,23 @@ class CommandBus: class EventBus: - __slots__ = ("_deferred", "_dropped", "_logger", "_projections", "_queue", "_worker") + __slots__ = ("_deferred", "_dropped", "_logger", "_queue", "_worker") def __init__(self, logger: Logger, *, max_queue: int = 10_000) -> None: self._logger = logger self._queue: asyncio.Queue[tuple[DomainEvent, contextvars.Context]] = asyncio.Queue( maxsize=max_queue ) - self._projections: dict[type[DomainEvent], list[Any]] = {} self._deferred: dict[type[DomainEvent], list[Any]] = {} self._worker: asyncio.Task[None] | None = None self._dropped = 0 - def project[EventT: DomainEvent]( - self, event_type: type[EventT], handler: EventHandler[EventT] - ) -> None: - self._projections.setdefault(event_type, []).append(handler) - def subscribe[EventT: DomainEvent]( self, event_type: type[EventT], handler: EventHandler[EventT] ) -> None: self._deferred.setdefault(event_type, []).append(handler) async def emit(self, event: DomainEvent) -> bool: - await self._run(self._projections.get(type(event), []), event, "projection") if not self._deferred.get(type(event)): return True try: @@ -100,11 +93,11 @@ class EventBus: event, context = await self._queue.get() handlers = self._deferred.get(type(event), []) try: - await asyncio.create_task(self._run(handlers, event, "subscriber"), context=context) + await asyncio.create_task(self._run(handlers, event), context=context) finally: self._queue.task_done() - async def _run(self, handlers: Sequence[Any], event: DomainEvent, kind: str) -> None: + async def _run(self, handlers: Sequence[Any], event: DomainEvent) -> None: for handler in handlers: try: await handler(event) @@ -112,7 +105,6 @@ class EventBus: self._logger.error( "event.handler_failed", event_name=event.name, - reason=kind, error=type(error).__name__, ) diff --git a/src/messenger/shared/application/ports.py b/src/messenger/shared/application/ports.py index 1658686..53a4df3 100644 --- a/src/messenger/shared/application/ports.py +++ b/src/messenger/shared/application/ports.py @@ -26,6 +26,11 @@ class Logger(Protocol): def error(self, event: str, /, **fields: Any) -> None: ... +@runtime_checkable +class Readiness(Protocol): + async def check(self) -> None: ... + + @runtime_checkable class UnitOfWork(Protocol): def transaction(self) -> AbstractAsyncContextManager[None]: ... diff --git a/src/messenger/shared/entrypoints/actor.py b/src/messenger/shared/entrypoints/actor.py index 0a81d11..b60a4a3 100644 --- a/src/messenger/shared/entrypoints/actor.py +++ b/src/messenger/shared/entrypoints/actor.py @@ -9,7 +9,7 @@ from messenger.shared.entrypoints.errors import Unauthenticated async def current_actor(request: Request) -> UUID: actor = getattr(request.state, "actor", None) if not isinstance(actor, UUID): - raise Unauthenticated("заголовок идентификации обязателен и должен быть действительным") + raise Unauthenticated("сессия не подтверждена") return actor diff --git a/src/messenger/shared/entrypoints/config.py b/src/messenger/shared/entrypoints/config.py index c3e7766..05f3cb4 100644 --- a/src/messenger/shared/entrypoints/config.py +++ b/src/messenger/shared/entrypoints/config.py @@ -7,4 +7,4 @@ class TransportSettings(Settings): docs: bool = False max_body_bytes: int = Field(default=1_048_576, ge=4_096) request_timeout_seconds: float = Field(default=8.0, gt=0, le=300) - shutdown_timeout_seconds: int = Field(default=5, ge=1, le=10) + min_transfer_rate_bytes: int = Field(default=131_072, ge=1_024) diff --git a/src/messenger/shared/entrypoints/errors.py b/src/messenger/shared/entrypoints/errors.py index e32c6e7..5efde31 100644 --- a/src/messenger/shared/entrypoints/errors.py +++ b/src/messenger/shared/entrypoints/errors.py @@ -46,7 +46,7 @@ class Unauthenticated(Exception): def classify(error: DomainError, codes: ErrorCodes) -> tuple[str, int]: for kind in type(error).__mro__: - known = codes.get(kind) # type: ignore[arg-type] + known = codes.get(kind) if known is not None: return known return "internal_error", status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/src/messenger/shared/entrypoints/health.py b/src/messenger/shared/entrypoints/health.py index b2da72c..2986ccc 100644 --- a/src/messenger/shared/entrypoints/health.py +++ b/src/messenger/shared/entrypoints/health.py @@ -1,14 +1,10 @@ -from threading import Event - from dishka.integrations.fastapi import FromDishka, inject from fastapi import APIRouter -from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Readiness from messenger.shared.entrypoints.schemas import HealthOut -from messenger.shared.infrastructure.postgres import Database router = APIRouter(tags=["health"]) -DRAINING = Event() @router.get("/health/live") @@ -18,8 +14,6 @@ async def live() -> HealthOut: @router.get("/health/ready") @inject -async def ready(database: FromDishka[Database]) -> HealthOut: - if DRAINING.is_set(): - raise Unavailable("Реплика выходит из ротации") - await database.ping() +async def ready(readiness: FromDishka[Readiness]) -> HealthOut: + await readiness.check() return HealthOut() diff --git a/src/messenger/shared/entrypoints/ioc.py b/src/messenger/shared/entrypoints/ioc.py index 2c99ab1..9a86fe1 100644 --- a/src/messenger/shared/entrypoints/ioc.py +++ b/src/messenger/shared/entrypoints/ioc.py @@ -3,11 +3,12 @@ from collections.abc import AsyncIterator from dishka import Provider, Scope, provide from messenger.shared.application.messagebus import EventBus, InMemoryEventPublisher -from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger +from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger, Readiness from messenger.shared.config import PlatformSettings from messenger.shared.infrastructure.clock import MonotonicUtcClock from messenger.shared.infrastructure.ids import Uuid7Generator from messenger.shared.infrastructure.postgres import Database +from messenger.shared.infrastructure.readiness import DatabaseReadiness class PlatformProvider(Provider): @@ -45,6 +46,10 @@ class PlatformProvider(Provider): yield database await database.close() + @provide(scope=Scope.APP) + def readiness(self, database: Database) -> Readiness: + return DatabaseReadiness(database) + @provide(scope=Scope.APP) def event_bus(self, logger: Logger, settings: PlatformSettings) -> EventBus: return EventBus(logger, max_queue=settings.event_queue_max) diff --git a/src/messenger/shared/entrypoints/pipeline.py b/src/messenger/shared/entrypoints/pipeline.py index 513d812..628cfe1 100644 --- a/src/messenger/shared/entrypoints/pipeline.py +++ b/src/messenger/shared/entrypoints/pipeline.py @@ -5,6 +5,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass from uuid import uuid4 +from starlette.exceptions import HTTPException from starlette.types import ASGIApp, Message, Receive, Scope, Send from messenger.shared.application.errors import Unavailable @@ -58,8 +59,44 @@ async def send_error( await send({"type": "http.response.body", "body": body}) +class Pace: + __slots__ = ("_deadline", "_moved", "_opened", "_rate", "_timeout") + + def __init__(self, deadline: asyncio.Timeout, timeout: float, rate: float) -> None: + self._deadline = deadline + self._timeout = timeout + self._rate = rate + self._opened = asyncio.get_running_loop().time() + self._moved = 0 + + def around(self, receive: Receive, send: Send) -> tuple[Receive, Send]: + async def paced_receive() -> Message: + return self._count(await receive()) + + async def paced_send(message: Message) -> None: + await send(self._count(message)) + + return paced_receive, paced_send + + def _count(self, message: Message) -> Message: + if size := len(message.get("body", b"")): + self._moved += size + moved = min(asyncio.get_running_loop().time(), self._opened + self._moved / self._rate) + self._deadline.reschedule(moved + self._timeout) + return message + + class RequestPipeline: - __slots__ = ("_app", "_guard", "_instance", "_limits", "_logger", "_max_bytes", "_timeout") + __slots__ = ( + "_app", + "_guard", + "_instance", + "_limits", + "_logger", + "_max_bytes", + "_min_rate", + "_timeout", + ) def __init__( self, @@ -70,12 +107,14 @@ class RequestPipeline: guard: Guard, instance: str, limits: BodyLimits = (), + min_rate: float = 131_072, ) -> None: self._app = app self._logger = logger self._max_bytes = max_bytes self._limits = limits self._timeout = timeout + self._min_rate = min_rate self._guard = guard self._instance = instance.encode() @@ -87,9 +126,7 @@ class RequestPipeline: incoming = header(scope, HEADER_BYTES) request_id = incoming[:MAX_REQUEST_ID] if incoming else uuid4().hex tag_request(request_id) - state = scope.setdefault("state", {}) - state["request_id"] = request_id - state["actor"] = None + scope.setdefault("state", {}).update(request_id=request_id, actor=None) started = time.perf_counter() status_holder = [500] answered = [False] @@ -108,18 +145,23 @@ class RequestPipeline: message = {**message, "headers": headers} await send(message) - declared = header(scope, b"content-length") - if declared.isdigit() and int(declared) > self._limit(scope["path"]): - await send_error(wrapped_send, 413, "payload_too_large", request_id) - self._log(scope, status_holder[0], request_id, started) - return + limit = self._limit(scope["path"]) + received = [0] + + async def limited_receive() -> Message: + message = await receive() + received[0] += len(message.get("body", b"")) + if received[0] > limit: + raise HTTPException(413) + return message rejection: Rejection | None = None try: - async with asyncio.timeout(self._timeout): - rejection = await self._guard(scope) + async with asyncio.timeout(self._timeout) as deadline: + rejection = await self._guard(scope) or self._oversized(scope, limit) if rejection is None: - await self._app(scope, receive, wrapped_send) + pace = Pace(deadline, self._timeout, self._min_rate) + await self._app(scope, *pace.around(limited_receive, wrapped_send)) except Unavailable as error: self._logger.warning( "dependency.unavailable", @@ -128,18 +170,21 @@ class RequestPipeline: request_id=request_id, ) rejection = Rejection(503, "unavailable") - except TimeoutError: + except (TimeoutError, asyncio.CancelledError) as error: + cancelled = isinstance(error, asyncio.CancelledError) self._logger.warning( - "request.timeout", + "request.cancelled" if cancelled else "request.timeout", method=scope["method"], route=self._route(scope), request_id=request_id, - limit=self._timeout, + limit=None if cancelled else self._timeout, duration_ms=round((time.perf_counter() - started) * 1000, 3), ) if not answered[0]: await send_error(wrapped_send, 503, "unavailable", request_id) self._log(scope, status_holder[0], request_id, started) + if cancelled: + raise return except Exception: self._logger.error( @@ -162,6 +207,12 @@ class RequestPipeline: return limit return self._max_bytes + def _oversized(self, scope: Scope, limit: int) -> Rejection | None: + declared = header(scope, b"content-length") + if declared.isdigit() and int(declared) > limit: + return Rejection(413, "payload_too_large") + return None + def _route(self, scope: Scope) -> str: route = scope.get("route") return getattr(route, "path", scope["path"]) diff --git a/src/messenger/shared/infrastructure/logging.py b/src/messenger/shared/infrastructure/logging.py index 89a8ed1..1334913 100644 --- a/src/messenger/shared/infrastructure/logging.py +++ b/src/messenger/shared/infrastructure/logging.py @@ -1,6 +1,6 @@ import logging import sys -from typing import Any +from typing import Any, TextIO import orjson import structlog @@ -62,7 +62,11 @@ def add_trace_context(_logger: WrappedLogger, _method: str, event_dict: EventDic return event_dict -def configure(level: str = "INFO", allowed_fields: frozenset[str] = PLATFORM_FIELDS) -> None: +def configure( + level: str = "INFO", + allowed_fields: frozenset[str] = PLATFORM_FIELDS, + stream: TextIO | None = None, +) -> None: shared = [ structlog.contextvars.merge_contextvars, structlog.processors.add_log_level, @@ -75,7 +79,7 @@ def configure(level: str = "INFO", allowed_fields: frozenset[str] = PLATFORM_FIE wrapper_class=structlog.stdlib.BoundLogger, cache_logger_on_first_use=False, ) - handler = logging.StreamHandler(sys.stdout) + handler = logging.StreamHandler(stream or sys.stdout) handler.setFormatter( structlog.stdlib.ProcessorFormatter( foreign_pre_chain=shared, @@ -111,8 +115,9 @@ class StructlogLogger: name: str = "messenger", level: str = "INFO", allowed_fields: frozenset[str] = PLATFORM_FIELDS, + stream: TextIO | None = None, ) -> None: - configure(level, allowed_fields) + configure(level, allowed_fields, stream) self._logger = structlog.get_logger(name) def info(self, event: str, /, **fields: Any) -> None: diff --git a/src/messenger/shared/infrastructure/postgres.py b/src/messenger/shared/infrastructure/postgres.py index ba28cd6..ccbcfd4 100644 --- a/src/messenger/shared/infrastructure/postgres.py +++ b/src/messenger/shared/infrastructure/postgres.py @@ -1,32 +1,45 @@ +import asyncio from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from typing import Any, Protocol +from typing import ClassVar, Protocol -import asyncpg +from sqlalchemy import Text, make_url, select +from sqlalchemy.exc import DBAPIError, IntegrityError, InterfaceError, OperationalError +from sqlalchemy.exc import TimeoutError as PoolTimeout +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.orm import DeclarativeBase from messenger.shared.application.errors import Unavailable from messenger.shared.domain.events import DomainEvent -FAILURES = ( - OSError, - asyncpg.InterfaceError, - asyncpg.PostgresConnectionError, - asyncpg.exceptions.OperatorInterventionError, - asyncpg.exceptions.InsufficientResourcesError, - asyncpg.exceptions.LockNotAvailableError, -) +FAILURES = (OSError, InterfaceError, OperationalError, PoolTimeout, TimeoutError) PING_TIMEOUT_SECONDS = 0.4 -type Connection = asyncpg.Connection +type DatabaseSession = AsyncSession class EventSource(Protocol): def collect_events(self) -> list[DomainEvent]: ... +class Base(DeclarativeBase): + type_annotation_map: ClassVar = {str: Text} + + +def unavailable(error: BaseException) -> bool: + if isinstance(error, IntegrityError): + return False + if isinstance(error, FAILURES): + return True + if isinstance(error, DBAPIError) and error.connection_invalidated: + return True + state = getattr(error.orig if isinstance(error, DBAPIError) else error, "sqlstate", None) + return isinstance(state, str) and (state.startswith(("08", "53", "57")) or state == "55P03") + + class Database: - __slots__ = ("_acquire_timeout", "_pool") + __slots__ = ("_engine", "_sessions") def __init__( self, @@ -37,63 +50,71 @@ class Database: acquire_timeout: float, command_timeout: float, ) -> None: - self._acquire_timeout = acquire_timeout - self._pool = asyncpg.create_pool( - dsn, - min_size=0, - max_size=max_size, - timeout=connect_timeout, - command_timeout=command_timeout, - max_inactive_connection_lifetime=60, - server_settings={ - "statement_timeout": str(int(command_timeout * 1000)), - "lock_timeout": str(int(command_timeout * 1000)), - "idle_in_transaction_session_timeout": "10000", + self._engine = create_async_engine( + make_url(dsn).set(drivername="postgresql+asyncpg"), + pool_size=max_size, + max_overflow=0, + pool_timeout=acquire_timeout, + pool_recycle=60, + pool_pre_ping=True, + hide_parameters=True, + connect_args={ + "timeout": connect_timeout, + "command_timeout": command_timeout, + "server_settings": { + "statement_timeout": str(int(command_timeout * 1000)), + "lock_timeout": str(int(command_timeout * 1000)), + "idle_in_transaction_session_timeout": "10000", + }, }, ) + self._sessions = async_sessionmaker(self._engine, expire_on_commit=False) async def open(self) -> None: - await self._pool + pass async def close(self) -> None: - await self._pool.close() + await self._engine.dispose() @asynccontextmanager - async def connection(self, wait: float | None = None) -> AsyncIterator[Connection]: + async def session(self) -> AsyncIterator[DatabaseSession]: try: - async with self._pool.acquire(timeout=wait or self._acquire_timeout) as connection: - yield connection - except FAILURES as error: + async with self._sessions() as session: + yield session + except Exception as error: + if not unavailable(error): + raise raise Unavailable("PostgreSQL недоступен") from error async def ping(self) -> None: - async with self.connection(PING_TIMEOUT_SECONDS) as connection: - await connection.fetchval("SELECT 1", timeout=PING_TIMEOUT_SECONDS) + try: + async with asyncio.timeout(PING_TIMEOUT_SECONDS): + async with self.session() as session: + await session.execute(select(1)) + except TimeoutError as error: + raise Unavailable("PostgreSQL недоступен") from error class PostgresUnitOfWork: - __slots__ = ("_committed", "_connection", "_database", "_events", "_transaction", "seen") + __slots__ = ("_committed", "_database", "_events", "_session", "seen") def __init__(self, database: Database) -> None: self._database = database - self._connection: Connection | None = None - self._transaction: Any = None + self._session: DatabaseSession | None = None self._committed = False self._events: list[DomainEvent] = [] self.seen: list[EventSource] = [] @property - def connection(self) -> Connection: - if self._connection is None: + def session(self) -> DatabaseSession: + if self._session is None: raise RuntimeError("репозиторий используется вне транзакции") - return self._connection + return self._session @asynccontextmanager async def transaction(self) -> AsyncIterator[None]: - async with self._database.connection() as connection: - self._connection = connection - self._transaction = connection.transaction() - await self._transaction.start() + async with self._database.session() as session: + self._session = session self._committed = False mark = len(self._events) try: @@ -101,12 +122,11 @@ class PostgresUnitOfWork: finally: if not self._committed: del self._events[mark:] - await self._transaction.rollback() self.seen.clear() - self._connection = None + self._session = None async def commit(self) -> None: - await self._transaction.commit() + await self.session.commit() self._committed = True for source in self.seen: self._events.extend(source.collect_events()) diff --git a/src/messenger/shared/infrastructure/readiness.py b/src/messenger/shared/infrastructure/readiness.py new file mode 100644 index 0000000..4a2a02f --- /dev/null +++ b/src/messenger/shared/infrastructure/readiness.py @@ -0,0 +1,19 @@ +from threading import Event + +from messenger.shared.application.errors import Unavailable +from messenger.shared.infrastructure.postgres import Database + +DRAINING = Event() + + +class DatabaseReadiness: + __slots__ = ("_database", "_draining") + + def __init__(self, database: Database, draining: Event = DRAINING) -> None: + self._database = database + self._draining = draining + + async def check(self) -> None: + if self._draining.is_set(): + raise Unavailable("Реплика выходит из ротации") + await self._database.ping() diff --git a/student-tests/adapters/test_rabbitmq.py b/student-tests/adapters/test_rabbitmq.py index c112e29..9c9d9a4 100644 --- a/student-tests/adapters/test_rabbitmq.py +++ b/student-tests/adapters/test_rabbitmq.py @@ -17,7 +17,7 @@ from dishka import Provider, Scope, make_async_container, provide from dishka_faststream import setup_dishka from faststream.rabbit import Channel, RabbitBroker -from conftest import ROOT, STARTUP_SECONDS, container, image, published +from conftest import ROOT, STARTUP_SECONDS, container, free_port, image, published from fakes import NullLogger from messenger.attachments.application.bus import AttachmentsCommandBus from messenger.attachments.application.commands import ProcessJob @@ -92,7 +92,7 @@ def rabbitmq() -> Iterator[dict[str, Any]]: "100:101", *volumes, "-p", - "127.0.0.1::5672", + f"127.0.0.1:{free_port()}:5672", image(RABBITMQ_IMAGE), ) try: diff --git a/student-tests/adapters/test_s3_storage.py b/student-tests/adapters/test_s3_storage.py index 8722a26..112801d 100644 --- a/student-tests/adapters/test_s3_storage.py +++ b/student-tests/adapters/test_s3_storage.py @@ -9,7 +9,7 @@ from uuid import uuid4 import httpx import pytest -from conftest import STARTUP_SECONDS, container, image, published +from conftest import STARTUP_SECONDS, container, free_port, image, published from messenger.attachments.application.errors import ObjectMissingError from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client from messenger.shared.application.errors import Unavailable @@ -23,7 +23,7 @@ BUCKET = "messenger-attachments" def rustfs() -> Iterator[str]: name = f"mtusi-lab6-test-rustfs-{uuid4().hex[:8]}" keys = ["-e", f"RUSTFS_ACCESS_KEY={ROOT_KEY}", "-e", f"RUSTFS_SECRET_KEY={ROOT_SECRET}"] - container(name, *keys, "-p", "127.0.0.1::9000", image(RUSTFS_IMAGE)) + container(name, *keys, "-p", f"127.0.0.1:{free_port()}:9000", image(RUSTFS_IMAGE)) try: endpoint = f"http://127.0.0.1:{published(name, 9000)}" deadline = time.monotonic() + STARTUP_SECONDS @@ -71,6 +71,27 @@ async def test_round_trip_of_the_largest_object(rustfs: str) -> None: await objects.delete(key) +async def test_listing_is_paged_in_byte_order_and_stat_reads_size(rustfs: str) -> None: + prefix = f"reconcile-{uuid4().hex}/" + keys = [f"{prefix}{name}" for name in ("b", "a", "Z", "a-1", "é")] + async with storage(rustfs) as objects: + for size, key in enumerate(keys, start=1): + await objects.put(key, b"x" * size, "image/png") + pages = [page async for page in objects.listing(prefix, 2)] + assert [len(page) for page in pages] == [2, 2, 1] + listed = [item.key for page in pages for item in page] + assert listed == sorted(keys, key=str.encode) + sizes = {item.key: item.size for page in pages for item in page} + assert sizes == {key: size for size, key in enumerate(keys, start=1)} + found = await objects.stat(keys[0]) + assert found is not None + assert found.size == 1 + assert await objects.stat(f"{prefix}absent") is None + for key in keys: + await objects.delete(key) + assert not [item async for page in objects.listing(prefix, 2) for item in page] + + async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None: started = time.monotonic() async with storage_without_bucket("http://127.0.0.1:1") as objects: @@ -78,7 +99,11 @@ async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None: await objects.put("attachments/x", b"x", "image/png") with pytest.raises(Unavailable): await objects.open("attachments/x") - assert time.monotonic() - started < 8 + with pytest.raises(Unavailable): + await objects.stat("attachments/x") + with pytest.raises(Unavailable): + await anext(aiter(objects.listing("attachments/", 10))) + assert time.monotonic() - started < 12 @asynccontextmanager diff --git a/student-tests/api/test_access.py b/student-tests/api/test_access.py index 8a62362..339bc00 100644 --- a/student-tests/api/test_access.py +++ b/student-tests/api/test_access.py @@ -1,8 +1,11 @@ -from uuid import uuid4 +import time +from uuid import UUID, uuid4 from httpx import AsyncClient +from sqlalchemy import select -from conftest import actor, keyed, make_chat, register +from conftest import actor, keyed, make_chat, register, sql_connection +from messenger.messaging.infrastructure.postgres import ChatRow async def test_outsider_gets_404_on_chat(client: AsyncClient) -> None: @@ -82,3 +85,36 @@ async def test_403_is_never_used(client: AsyncClient) -> None: ] for path in paths: assert (await client.get(path, headers=actor(eve))).status_code != 403 + + +async def test_request_shape_is_checked_before_access(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + for chat_id in (chat["id"], uuid4()): + path = f"/api/v1/chats/{chat_id}/messages" + unkeyed = await client.post(path, json={"text": "чужое"}, headers=actor(eve)) + extra = await client.post(path, json={"text": "x", "a": 1}, headers=keyed(eve)) + assert unkeyed.status_code == extra.status_code == 400 + assert unkeyed.json()["error"]["code"] == extra.json()["error"]["code"] == "invalid_request" + blank = await client.post( + f"/api/v1/chats/{chat['id']}/messages", json={"text": " "}, headers=keyed(eve) + ) + assert blank.status_code == 404 + + +async def test_outsider_does_not_wait_for_members_lock(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + async with sql_connection() as holder: + await holder.execute( + select(ChatRow.id).where(ChatRow.id == UUID(chat["id"])).with_for_update() + ) + started = time.perf_counter() + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", json={"text": "чужое"}, headers=keyed(eve) + ) + elapsed = time.perf_counter() - started + assert response.status_code == 404 + assert elapsed < 0.5 diff --git a/student-tests/api/test_attachments.py b/student-tests/api/test_attachments.py index a4d3339..9ac3170 100644 --- a/student-tests/api/test_attachments.py +++ b/student-tests/api/test_attachments.py @@ -9,7 +9,7 @@ import pytest from httpx import AsyncClient, Response from PIL import Image -from conftest import actor, make_chat, register, resolve, scalar +from conftest import actor, make_chat, register, resolve, running_app, scalar from fakes import MemoryObjectStorage from messenger.attachments.application.ports import ObjectStorage from pictures import GIF, JPEG, SAMPLE, WEBP, png @@ -140,6 +140,18 @@ async def test_access_is_checked_before_content(client: AsyncClient) -> None: assert anonymous.headers["www-authenticate"] == 'Bearer realm="messenger"' +async def test_token_is_checked_before_the_declared_size() -> None: + async with running_app(max_upload_bytes=4096) as http: + alice, _, _, chat = await trio(http) + oversized = png(16, 16, size=8192) + anonymous = await http.post( + f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", content=oversized, headers=PNG + ) + assert anonymous.status_code == 401 + assert anonymous.headers["www-authenticate"] == 'Bearer realm="messenger"' + assert (await upload(http, alice, chat, oversized)).status_code == 413 + + async def test_exact_limit_is_accepted(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) response = await upload(client, alice, chat, png(16, 16, size=LIMIT)) diff --git a/student-tests/api/test_bounded_contexts.py b/student-tests/api/test_bounded_contexts.py index b2836e6..9893511 100644 --- a/student-tests/api/test_bounded_contexts.py +++ b/student-tests/api/test_bounded_contexts.py @@ -33,20 +33,6 @@ def test_identity_does_not_know_about_messaging() -> None: assert not {k: v for k, v in offenders.items() if v} -def test_messaging_touches_identity_only_through_events() -> None: - allowed = f"{IDENTITY}.domain.events" - offenders: dict[str, list[str]] = {} - for path in modules_under("messaging"): - borrowed = [ - module - for module in imports_of(path) - if module.startswith(IDENTITY) and module != allowed - ] - if borrowed: - offenders[str(path.relative_to(SOURCE))] = borrowed - assert not offenders - - def test_attachments_and_other_zones_do_not_import_each_other() -> None: offenders: dict[str, list[str]] = {} for zone, foreign in ( @@ -126,6 +112,7 @@ def test_shared_core_holds_only_platform() -> None: "logging.py", "postgres.py", "telemetry.py", + "readiness.py", } @@ -269,6 +256,16 @@ def sql_tables(zone: str, tables: tuple[str, ...]) -> set[str]: for node in ast.walk(tree): if isinstance(node, ast.Constant) and isinstance(node.value, str): words.update(re.findall(r"\b(?:FROM|JOIN|INTO|UPDATE)\s+(\w+)", node.value)) + if ( + isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "__tablename__" + for target in node.targets + ) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ): + words.add(node.value.value) return words & set(tables) @@ -289,3 +286,11 @@ def test_messaging_does_not_import_identity_at_all() -> None: for path in modules_under("messaging") } assert not {key: value for key, value in borrowed.items() if value} + + +def test_sqlalchemy_stays_in_infrastructure_and_asyncpg_is_only_a_driver() -> None: + for path in SOURCE.rglob("*.py"): + modules = {module.split(".")[0] for module in imports_of(path)} + assert "asyncpg" not in modules + if "infrastructure" not in path.parts: + assert "sqlalchemy" not in modules diff --git a/student-tests/api/test_cache.py b/student-tests/api/test_cache.py index 63f845d..01bf599 100644 --- a/student-tests/api/test_cache.py +++ b/student-tests/api/test_cache.py @@ -2,13 +2,18 @@ import asyncio import time from datetime import datetime from hashlib import sha256 +from uuid import UUID +import pytest from redis.asyncio import Redis +from sqlalchemy import select -from conftest import actor, bearer, register, resolve, running_app, scalar +from conftest import actor, bearer, keyed, make_chat, register, resolve, running_app, scalar from messenger.identity.application.ports import SessionDirectory from messenger.identity.domain.models import Session +from messenger.identity.infrastructure.postgres import PostgresSessionRepository, SessionRow from messenger.identity.infrastructure.valkey import CachedSessionDirectory +from messenger.shared.application.errors import Unavailable BEARER = 'Bearer realm="messenger"' @@ -31,8 +36,9 @@ async def test_second_check_is_served_from_cache(valkey: dict[str, str]) -> None assert (await cache.info("stats"))["keyspace_hits"] == hits + 1 key = key_of(token_of(alice)) stored = str(await cache.get(key)) - sql = "SELECT expires_at FROM sessions WHERE user_id = $1::uuid" - expires_at: datetime = await scalar(sql, alice["id"]) + expires_at: datetime = await scalar( + select(SessionRow.expires_at).where(SessionRow.user_id == UUID(alice["id"])) + ) assert alice["id"] in stored and token_of(alice) not in stored assert await cache.pexpiretime(key) <= int(expires_at.timestamp() * 1000) await cache.aclose() @@ -53,6 +59,29 @@ async def test_logout_writes_tombstone_before_database(valkey: dict[str, str]) - await cache.aclose() +async def test_logout_rejected_by_database_changes_nothing( + valkey: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + async def broken(self: PostgresSessionRepository, token_hash: bytes) -> None: + raise Unavailable("PostgreSQL недоступен") + + cache = Redis.from_url(valkey["url"]) + logout = "/api/v1/auth/sessions/current" + async with running_app(cache_url=valkey["url"]) as http: + alice = await register(http, "alice") + assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200 + with monkeypatch.context() as patch: + patch.setattr(PostgresSessionRepository, "remove", broken) + failed = await http.delete(logout, headers=actor(alice)) + assert failed.status_code == 503 + assert failed.json()["error"]["code"] == "unavailable" + assert await cache.get(key_of(token_of(alice))) is None + assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200 + assert (await http.delete(logout, headers=actor(alice))).status_code == 204 + assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 401 + await cache.aclose() + + async def test_fill_racing_with_logout_does_not_revive_token(valkey: dict[str, str]) -> None: async with running_app(cache_url=valkey["url"]) as http: alice = await register(http, "alice") @@ -69,9 +98,6 @@ async def test_fill_racing_with_logout_does_not_revive_token(valkey: dict[str, s await released.wait() return stale[-1] - async def invalidate(self, session: Session) -> None: - await origin.invalidate(session) - directory._origin = SlowOrigin() reader = asyncio.create_task(http.get("/api/v1/users/me", headers=actor(alice))) await entered.wait() @@ -106,6 +132,10 @@ async def test_unreachable_cache_is_bypassed_without_anonymous_access() -> None: stranger = await http.get("/api/v1/users/me", headers=bearer("x" * 43)) elapsed = time.perf_counter() - started ready = await http.get("/health/ready") + chat = await make_chat(http, alice) + posted = await http.post( + f"/api/v1/chats/{chat['id']}/messages", json={"text": "без кэша"}, headers=keyed(alice) + ) logout = await http.delete("/api/v1/auth/sessions/current", headers=actor(alice)) still = await http.get("/api/v1/users/me", headers=actor(alice)) assert me.json() == alice @@ -113,6 +143,7 @@ async def test_unreachable_cache_is_bypassed_without_anonymous_access() -> None: assert stranger.headers["WWW-Authenticate"] == BEARER assert elapsed < 1 assert ready.status_code == 200 + assert posted.status_code == 201 assert logout.status_code == 503 assert logout.json()["error"]["code"] == "unavailable" assert still.status_code == 200 diff --git a/student-tests/api/test_concurrency.py b/student-tests/api/test_concurrency.py index aeeeb4e..756bf6b 100644 --- a/student-tests/api/test_concurrency.py +++ b/student-tests/api/test_concurrency.py @@ -1,10 +1,13 @@ import asyncio +from datetime import UTC, datetime, timedelta from urllib.parse import quote from uuid import uuid4 +import pytest from httpx import AsyncClient from conftest import PASSWORD, actor, keyed, make_chat, register +from messenger.shared.infrastructure.clock import MonotonicUtcClock CLIENTS = 10 PER_CLIENT = 10 @@ -60,6 +63,22 @@ async def test_hundred_messages_from_ten_clients(client: AsyncClient) -> None: assert len(set(stamps)) == TOTAL, "created_at внутри чата обязан строго возрастать" +async def test_history_follows_the_clock_but_never_moves_back( + monkeypatch: pytest.MonkeyPatch, client: AsyncClient +) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + base = datetime.now(UTC) + timedelta(minutes=10) + stamps = [] + for moment in (base, base - timedelta(minutes=5), base + timedelta(seconds=5)): + monkeypatch.setattr(MonotonicUtcClock, "now", lambda _self, moment=moment: moment) + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", json={"text": "часы"}, headers=keyed(alice) + ) + stamps.append(datetime.fromisoformat(response.json()["created_at"])) + assert stamps == [base, base + timedelta(microseconds=1), base + timedelta(seconds=5)] + + async def test_parallel_registration_keeps_username_unique(client: AsyncClient) -> None: body = {"username": f"race_{uuid4().hex[:12]}", "display_name": "Гонка", "password": PASSWORD} responses = await asyncio.gather(*(client.post("/api/v1/users", json=body) for _ in range(20))) diff --git a/student-tests/api/test_consistency.py b/student-tests/api/test_consistency.py index 5a616a2..bbf6988 100644 --- a/student-tests/api/test_consistency.py +++ b/student-tests/api/test_consistency.py @@ -1,10 +1,13 @@ import asyncio from urllib.parse import quote -from uuid import uuid4 +from uuid import UUID, uuid4 from httpx import AsyncClient +from sqlalchemy import func, select from conftest import PASSWORD, actor, keyed, make_chat, new_user, register, running_app, scalar +from messenger.identity.infrastructure.postgres import UserRow +from messenger.messaging.infrastructure.postgres import ChatMemberRow, ChatRow REGISTRATIONS = 25 @@ -65,8 +68,8 @@ async def test_rejected_registration_leaves_no_trace() -> None: "/api/v1/users", json={"username": "AB", "display_name": "", "password": PASSWORD} ) assert invalid.status_code == 400 - sql = "SELECT count(*) FROM users WHERE username = $1" - assert await scalar(sql, alice["username"]) == 1 + sql = select(func.count()).select_from(UserRow).where(UserRow.username == alice["username"]) + assert await scalar(sql) == 1 async def test_failed_chat_creation_rolls_back_both_tables() -> None: @@ -83,9 +86,15 @@ async def test_failed_chat_creation_rolls_back_both_tables() -> None: assert rejected.json()["error"]["code"] == "invalid_request" after = (await client.get("/api/v1/chats?limit=100", headers=actor(alice))).json() assert after["items"] == before["items"] - assert await scalar("SELECT count(*) FROM chats WHERE title = $1", title) == 0 - members = "SELECT count(*) FROM chat_members WHERE user_id = $1::uuid" - assert await scalar(members, alice["id"]) == 0 + assert ( + await scalar(select(func.count()).select_from(ChatRow).where(ChatRow.title == title)) == 0 + ) + members = ( + select(func.count()) + .select_from(ChatMemberRow) + .where(ChatMemberRow.user_id == UUID(alice["id"])) + ) + assert await scalar(members) == 0 async def test_rejected_message_does_not_enter_history() -> None: diff --git a/student-tests/api/test_drain.py b/student-tests/api/test_drain.py index 299addf..dd0af03 100644 --- a/student-tests/api/test_drain.py +++ b/student-tests/api/test_drain.py @@ -5,14 +5,15 @@ import socket import subprocess import sys import time +from pathlib import Path import httpx -from conftest import DATABASE, new_user -from messenger.shared.entrypoints.health import DRAINING +from conftest import DATABASE, free_port, new_user +from messenger.shared.infrastructure.readiness import DRAINING -DRAIN = 1.5 -SHUTDOWN = 4 +DRAIN = 1.0 +SHUTDOWN = 3 GUNICORN = ( "-m", "gunicorn", @@ -36,19 +37,24 @@ async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient assert (await client.get("/health/ready")).status_code == 200 -def free_port() -> int: - with socket.socket() as probe: - probe.bind(("127.0.0.1", 0)) - return probe.getsockname()[1] - - def status(port: int, path: str) -> int: try: - return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code + return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1, trust_env=False).status_code except httpx.TransportError: return 0 +def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]: + data = b"" + while chunk := connection.recv(4096): + data += chunk + connection.close() + head, _, body = data.partition(b"\r\n\r\n") + status, *lines = head.decode("latin-1").split("\r\n") + headers = dict(line.lower().split(": ", 1) for line in lines) + return status, headers, json.loads(body) if body else {} + + def begin_registration(port: int) -> tuple[socket.socket, bytes]: body = json.dumps(new_user("drain")).encode() head = ( @@ -60,7 +66,7 @@ def begin_registration(port: int) -> tuple[socket.socket, bytes]: return connection, body[8:] -def test_sigterm_leaves_rotation_then_closes_and_exits() -> None: +def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests(tmp_path: Path) -> None: port = free_port() settings = { "HOST": "127.0.0.1", @@ -74,23 +80,27 @@ def test_sigterm_leaves_rotation_then_closes_and_exits() -> None: "DRAIN_SECONDS": str(DRAIN), "SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN), } + log_path = tmp_path / "gunicorn.log" + log = log_path.open("w") server = subprocess.Popen( [sys.executable, *GUNICORN], env={**os.environ, **settings}, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stdout=log, + stderr=log, ) try: deadline = time.monotonic() + 30 while status(port, "/health/ready") != 200: - assert time.monotonic() < deadline, "gunicorn не стал готов" + assert server.poll() is None and time.monotonic() < deadline, log_path.read_text() time.sleep(0.2) pending, rest = begin_registration(port) + stuck, _ = begin_registration(port) sent = time.monotonic() server.send_signal(signal.SIGTERM) time.sleep(0.3) during = status(port, "/health/ready"), status(port, "/health/live") while status(port, "/health/live") == 200: + assert time.monotonic() - sent < SHUTDOWN + 2, log_path.read_text() time.sleep(0.05) closed = time.monotonic() - sent pending.sendall(rest) @@ -98,13 +108,23 @@ def test_sigterm_leaves_rotation_then_closes_and_exits() -> None: pending.close() code = server.wait(timeout=SHUTDOWN + 2) exited = time.monotonic() - sent + cancelled, headers, error = answer_of(stuck) finally: if server.poll() is None: server.kill() + server.wait(timeout=5) + log.close() print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с") print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}") + print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}") + print(f" content-type {headers.get('content-type')}, тело {error}") assert during == (503, 200) assert answer == "HTTP/1.1 201 Created" + assert cancelled == "HTTP/1.1 503 Service Unavailable" + assert headers["content-type"] == "application/json" + assert headers["x-instance-id"] + assert error["error"]["code"] == "unavailable" + assert error["error"]["request_id"] == headers["x-request-id"] assert DRAIN - 0.2 <= closed < SHUTDOWN assert exited < SHUTDOWN assert code == 0 diff --git a/student-tests/api/test_events.py b/student-tests/api/test_events.py index a697556..3652bce 100644 --- a/student-tests/api/test_events.py +++ b/student-tests/api/test_events.py @@ -77,18 +77,6 @@ async def test_queue_overflow_drops_events_without_raising() -> None: assert bus.pending <= 16 -async def test_projection_runs_before_publish_returns() -> None: - bus = EventBus(NullLogger()) - seen: list[str] = [] - - async def projection(_event: MessagePosted) -> None: - seen.append("projected") - - bus.project(MessagePosted, projection) - await InMemoryEventPublisher(bus).publish([posted()]) - assert seen == ["projected"] - - async def test_publication_does_not_delay_response() -> None: app = create_app(**split_settings(log_level="CRITICAL")) async with ( diff --git a/student-tests/api/test_idempotency.py b/student-tests/api/test_idempotency.py index 02f94a9..8ea53d9 100644 --- a/student-tests/api/test_idempotency.py +++ b/student-tests/api/test_idempotency.py @@ -4,8 +4,10 @@ from uuid import uuid4 import pytest from httpx import AsyncClient, Response +from sqlalchemy import func, select from conftest import actor, keyed, make_chat, register, running_app, scalar +from messenger.messaging.infrastructure.postgres import IdempotencyKeyRow INVALID_KEYS = ["", "k" * 129, "два слова", "a b", "a/b", "ключ"] @@ -93,12 +95,22 @@ async def test_key_is_scoped_by_user_and_chat(client: AsyncClient) -> None: bob = await register(client, "bob") chat = await make_chat(client, alice, bob) other = await make_chat(client, alice) - ids = { - (await send(client, chat, alice, "алиса", "shared-1")).json()["id"], - (await send(client, chat, bob, "боб", "shared-1")).json()["id"], - (await send(client, other, alice, "другой чат", "shared-1")).json()["id"], - } + cases = [(chat, alice, "алиса"), (chat, bob, "боб"), (other, alice, "другой чат")] + messages = [] + for target, user, text in cases: + response = await send(client, target, user, text, "shared-1") + assert response.status_code == 201 + message = response.json() + assert (message["chat_id"], message["sender_id"], message["text"]) == ( + target["id"], + user["id"], + text, + ) + messages.append(message) + ids = {message["id"] for message in messages} assert len(ids) == 3 + for (target, user, text), message in zip(cases, messages, strict=True): + assert (await send(client, target, user, text, "shared-1")).json() == message async def test_replay_checks_session_and_access_again(client: AsyncClient) -> None: @@ -142,4 +154,9 @@ async def test_concurrent_retries_through_two_replicas_store_one_message() -> No } assert all(response.json() == responses[0].json() for response in responses) assert [item["text"] for item in await history(second, chat, alice)] == [text] - assert await scalar("SELECT count(*) FROM idempotency_keys WHERE key = $1", key) == 1 + assert ( + await scalar( + select(func.count()).select_from(IdempotencyKeyRow).where(IdempotencyKeyRow.key == key) + ) + == 1 + ) diff --git a/student-tests/api/test_job_returning.py b/student-tests/api/test_job_returning.py new file mode 100644 index 0000000..f1f28e3 --- /dev/null +++ b/student-tests/api/test_job_returning.py @@ -0,0 +1,62 @@ +import asyncio +import json +import os +import sys + +from httpx import AsyncClient +from test_pipeline import accepted, job_of + +from conftest import DATABASE + +PROBE = """ +import asyncio, os +from datetime import timedelta +from uuid import uuid4 +from messenger.worker import create_worker +from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.shared.infrastructure.postgres import Database + +async def main(): + db = Database(os.environ['TEST_DATABASE_URL'], max_size=2, connect_timeout=2, + acquire_timeout=2, command_timeout=3) + job = JobMessage.model_validate_json(os.environ['TEST_JOB']).job() + async def claim(): + uow = PostgresAttachmentsUnitOfWork(db) + async with uow.transaction(): + claimed = await uow.jobs.claim(job, uuid4(), timedelta(seconds=5)) + assert claimed is not None + assert claimed.lease.job.attempt == 1 + assert claimed.lease.job.created_at == job.created_at + try: + await asyncio.gather(claim(), claim()) + finally: + await db.close() +asyncio.run(main()) +""" + + +async def test_cold_worker_claim_returning_is_stable(client: AsyncClient) -> None: + _, _, ident = await accepted(client) + job = await job_of(ident) + for seed in range(24): + child = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + PROBE, + env={ + **os.environ, + "PYTHONHASHSEED": str(seed), + "TEST_DATABASE_URL": DATABASE["url"], + "TEST_JOB": json.dumps(job.payload()), + }, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + _, errors = await asyncio.wait_for(child.communicate(), 20) + finally: + if child.returncode is None: + child.kill() + await child.wait() + assert child.returncode == 0, f"seed={seed}: {errors.decode()}" diff --git a/student-tests/api/test_migrations.py b/student-tests/api/test_migrations.py index 50af1a5..6172f0d 100644 --- a/student-tests/api/test_migrations.py +++ b/student-tests/api/test_migrations.py @@ -1,240 +1,542 @@ +import asyncio +import hashlib +import os +import shutil +import sys +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from pathlib import Path from uuid import uuid4 -import asyncpg import pytest +from alembic import command +from alembic.autogenerate import compare_metadata +from alembic.migration import MigrationContext +from alembic.script import ScriptDirectory +from alembic.util import CommandError +from sqlalchemy import MetaData, Table, case, func, insert, inspect, literal, make_url, select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.pool import NullPool -from conftest import DATABASE, scalar -from messenger.migrate import MigrationChangedError, migrate, migrations +from conftest import DATABASE, ROOT, migration_config, sql_connection, upgrade +from messenger.attachments.infrastructure.postgres import ( + AttachmentRow, + JobEventRow, + JobRow, + OutboxRow, + VariantRow, +) +from messenger.identity.infrastructure.postgres import SessionRow, UserRow +from messenger.messaging.infrastructure.postgres import ( + ChatMemberRow, + ChatRow, + IdempotencyKeyRow, + MessageAttachmentRow, + MessageRow, +) +from messenger.shared.infrastructure.postgres import Base -COUNTS = """ -SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM sessions), - (SELECT count(*) FROM chats), (SELECT count(*) FROM chat_members), - (SELECT count(*) FROM messages), (SELECT count(*) FROM idempotency_keys), - (SELECT count(*) FROM attachments), (SELECT count(*) FROM message_attachments), - (SELECT count(*) FROM attachment_variants), (SELECT count(*) FROM jobs), - (SELECT count(*) FROM outbox), (SELECT count(*) FROM job_events) -""" -PIPELINE_COLUMNS = { - "attachments": { - "id", - "status", - "published_version", - "error_code", - "staging_key", - "original_key", - }, - "attachment_variants": { - "attachment_id", - "pipeline_version", - "name", - "sha256", - "size_bytes", - "object_key", - }, - "jobs": {"job_id", "attachment_id", "pipeline_version", "status", "attempt"}, - "outbox": {"job_id", "payload", "published_at"}, - "job_events": {"job_id", "attachment_id", "kind", "attempt"}, -} +VERSIONS = [ + "0001_init", + "0002_idempotency_keys", + "0003_attachments", + "0004_jobs", + "0005_failed_registry", + "0006_attachment_quarantine", +] +TABLES = [ + UserRow, + SessionRow, + ChatRow, + ChatMemberRow, + MessageRow, + IdempotencyKeyRow, + AttachmentRow, + JobRow, + OutboxRow, + JobEventRow, + VariantRow, + MessageAttachmentRow, +] -async def counts() -> list[int]: - connection = await asyncpg.connect(DATABASE["url"]) +@pytest.fixture +async def empty_database(database: str) -> AsyncIterator[str]: + name = f"migration_test_{uuid4().hex}" + url = make_url(database).set(drivername="postgresql+asyncpg") + engine = create_async_engine(url, isolation_level="AUTOCOMMIT", poolclass=NullPool) try: - return list(await connection.fetchrow(COUNTS)) + async with engine.connect() as connection: + await connection.exec_driver_sql(f"CREATE DATABASE {name}") + yield url.set(database=name).render_as_string(hide_password=False) finally: - await connection.close() + async with engine.connect() as connection: + await connection.exec_driver_sql(f"DROP DATABASE {name} WITH (FORCE)") + await engine.dispose() + + +async def legacy(dsn: str, versions: int = 6) -> None: + async with sql_connection(dsn) as connection: + await connection.execute( + text( + "CREATE TABLE schema_migrations (version text PRIMARY KEY, checksum text NOT NULL, " + "applied_at timestamptz NOT NULL DEFAULT now())" + ) + ) + journal = await connection.run_sync( + lambda sync: Table("schema_migrations", MetaData(), autoload_with=sync) + ) + for version in VERSIONS[:versions]: + sql = (ROOT / "student-tests" / "fixtures" / "legacy" / f"{version}.sql").read_text() + for statement in sql.split(";"): + if statement.strip(): + await connection.execute(text(statement)) + await connection.execute( + insert(journal).values( + version=version, checksum=hashlib.sha256(sql.encode()).hexdigest() + ) + ) + + +async def seed(dsn: str, versions: int = 6) -> None: + user, chat, message = uuid4(), uuid4(), uuid4() + stamp = datetime.now(UTC) + rows = [ + ( + UserRow, + { + "id": user, + "username": "legacy_user", + "display_name": "Старые данные", + "password_hash": "$argon2id$legacy", + "created_at": stamp, + }, + ), + ( + SessionRow, + { + "token_hash": b"s" * 32, + "user_id": user, + "created_at": stamp, + "expires_at": stamp + timedelta(days=1), + }, + ), + ( + ChatRow, + { + "id": chat, + "title": "Существующий чат", + "created_at": stamp, + "last_message_at": stamp, + }, + ), + ( + ChatMemberRow, + {"chat_id": chat, "user_id": user, "position": 1, "chat_created_at": stamp}, + ), + ( + MessageRow, + { + "id": message, + "chat_id": chat, + "sender_id": user, + "text": "Сохранить", + "created_at": stamp, + }, + ), + ( + IdempotencyKeyRow, + { + "chat_id": chat, + "user_id": user, + "key": "legacy-key", + "fingerprint": b"f" * 32, + "message_id": message, + "created_at": stamp, + }, + ), + ] + async with sql_connection(dsn) as connection: + for model, row in rows[: 5 if versions == 1 else 6]: + await connection.execute(insert(model).values(**row)) + if versions >= 3: + attachment = uuid4() + row = { + "id": attachment, + "chat_id": chat, + "owner_id": user, + "filename": "legacy.png", + "content_type": "image/png", + "size_bytes": 17, + "sha256": "a" * 64, + "width": 1, + "height": 1, + "object_key": f"attachments/{attachment}", + "status": "ready", + "created_at": stamp, + "updated_at": stamp, + } + if versions >= 4: + row.update(original_key=row["object_key"], published_version=1) + if versions >= 6: + row.update(quarantine_reason="missing", quarantined_at=stamp) + table = await connection.run_sync( + lambda sync: Table("attachments", MetaData(), autoload_with=sync) + ) + await connection.execute(insert(table).values(**row)) + await connection.execute( + insert(MessageAttachmentRow).values( + message_id=message, position=0, chat_id=chat, attachment_id=attachment + ) + ) + if versions >= 4: + job = uuid4() + await connection.execute( + insert(JobRow).values( + job_id=job, + attachment_id=attachment, + pipeline_version=1, + status="processing", + attempt=2, + claims=3, + lease_token=uuid4(), + lease_until=stamp + timedelta(minutes=1), + created_at=stamp, + updated_at=stamp, + ) + ) + await connection.execute( + insert(OutboxRow).values( + job_id=job, + payload={"job_id": str(job)}, + available_at=stamp, + ) + ) + await connection.execute( + insert(JobEventRow).values( + job_id=job, + attachment_id=attachment, + kind="claimed", + attempt=2, + ) + ) + await connection.execute( + insert(VariantRow).values( + attachment_id=attachment, + pipeline_version=1, + name="thumbnail", + object_key=f"variants/{attachment}/1/thumbnail.jpg", + content_type="image/jpeg", + size_bytes=17, + width=1, + height=1, + sha256="a" * 64, + created_at=stamp, + ) + ) + + +async def contents(dsn: str) -> dict: + async with sql_connection(dsn) as connection: + tables = await connection.run_sync(lambda sync: inspect(sync).get_table_names()) + result = {} + for model in TABLES: + if model.__tablename__ not in tables: + continue + table = await connection.run_sync( + lambda sync, name=model.__tablename__: Table(name, MetaData(), autoload_with=sync) + ) + missing = {} + if model is AttachmentRow and "published_version" not in table.c: + missing = { + "original_key": case((table.c.status == "ready", table.c.object_key)), + "staging_key": case((table.c.status == "pending", table.c.object_key)), + "published_version": case((table.c.status == "ready", 1), else_=0), + } + columns = [ + table.c[col.name] + if col.name in table.c + else missing.get(col.name, literal(None)).label(col.name) + for col in model.__table__.columns + ] + result[model.__tablename__] = ( + await connection.execute(select(*columns).order_by(*table.primary_key)) + ).all() + return result + + +async def revision(dsn: str) -> str | None: + async with sql_connection(dsn) as connection: + return await connection.run_sync( + lambda sync: MigrationContext.configure(sync).get_current_revision() + ) async def test_migrations_are_ordered_and_versioned() -> None: - versions = [version for version, _ in migrations()] - assert versions == sorted(versions) - assert versions[:5] == [ - "0001_init", - "0002_idempotency_keys", - "0003_attachments", - "0004_jobs", - "0005_failed_registry", - ] - plan = await scalar("SELECT indexdef FROM pg_indexes WHERE indexname = 'jobs_failed_idx'") - assert "WHERE (status = 'failed'::text)" in plan - applied = await scalar("SELECT array_agg(version ORDER BY version) FROM schema_migrations") - assert applied == versions + scripts = ScriptDirectory.from_config(migration_config(DATABASE["url"])) + assert [script.revision for script in reversed(list(scripts.walk_revisions()))] == VERSIONS + assert await revision(DATABASE["url"]) == VERSIONS[-1] -async def test_rerun_changes_nothing() -> None: - before = await counts() - assert await migrate(DATABASE["url"]) == [] - assert await counts() == before +async def test_fresh_schema_matches_models(empty_database: str) -> None: + await upgrade(empty_database) + async with sql_connection(empty_database) as connection: + assert ( + await connection.run_sync( + lambda sync: compare_metadata(MigrationContext.configure(sync), Base.metadata) + ) + == [] + ) + await asyncio.to_thread(command.check, migration_config(empty_database)) -async def test_changed_migration_is_refused() -> None: - connection = await asyncpg.connect(DATABASE["url"]) +async def test_rerun_changes_nothing(empty_database: str) -> None: + await upgrade(empty_database) + await seed(empty_database) + before = await contents(empty_database) + await upgrade(empty_database) + assert await contents(empty_database) == before + assert await revision(empty_database) == VERSIONS[-1] + + +@pytest.mark.parametrize("versions", [1, 2, 3, 4, 5, 6]) +async def test_legacy_upgrade_preserves_every_row(empty_database: str, versions: int) -> None: + await legacy(empty_database, versions) + await seed(empty_database, versions) + before = await contents(empty_database) + async with sql_connection(empty_database) as connection: + journal = ( + await connection.execute(text("SELECT * FROM schema_migrations ORDER BY version")) + ).all() + await upgrade(empty_database) + after = await contents(empty_database) + assert {name: after[name] for name in before} == before + assert await revision(empty_database) == VERSIONS[-1] + async with sql_connection(empty_database) as connection: + assert ( + await connection.execute(text("SELECT * FROM schema_migrations ORDER BY version")) + ).all() == journal + assert ( + await connection.scalar( + text("SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'alembic_verify_%'") + ) + == 0 + ) + await upgrade(empty_database) + assert await contents(empty_database) == after + + +@pytest.mark.parametrize( + "damage", + [ + "DROP INDEX messages_history_idx", + "ALTER TABLE messages DROP CONSTRAINT messages_chat_id_sender_id_fkey", + "ALTER TABLE users DROP CONSTRAINT users_username_format", + "ALTER TABLE users ALTER COLUMN display_name TYPE varchar", + 'ALTER TABLE users ALTER COLUMN username TYPE text COLLATE "C"', + "ALTER TABLE sessions SET UNLOGGED", + "ALTER TABLE sessions ALTER COLUMN expires_at DROP NOT NULL", + "UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0001_init'", + "DELETE FROM schema_migrations WHERE version = '0001_init'", + "UPDATE schema_migrations SET version = 'unknown' WHERE version = '0002_idempotency_keys'", + "DROP TABLE schema_migrations", + "ALTER TABLE attachments DROP CONSTRAINT attachments_owner_fkey", + "ALTER TABLE attachments DROP CONSTRAINT attachments_quarantine_check", + "ALTER TABLE message_attachments DROP CONSTRAINT message_attachments_attachment_fkey", + 'ALTER TABLE attachments ALTER COLUMN object_key TYPE text COLLATE "default"', + "DROP INDEX attachments_pending_idx", + "UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0003_attachments'", + ], +) +async def test_legacy_drift_is_refused_without_stamping(empty_database: str, damage: str) -> None: + await legacy(empty_database) + await seed(empty_database) + before = await contents(empty_database) + async with sql_connection(empty_database) as connection: + await connection.execute(text(damage)) + with pytest.raises(RuntimeError, match=r"[Ll]egacy|[Ee]xisting|Unknown"): + await upgrade(empty_database) + assert await revision(empty_database) is None + assert await contents(empty_database) == before + + +async def test_unknown_revision_is_refused(empty_database: str) -> None: + await upgrade(empty_database) + await seed(empty_database) + before = await contents(empty_database) + async with sql_connection(empty_database) as connection: + await connection.execute(text("UPDATE alembic_version SET version_num = 'unknown'")) + with pytest.raises(CommandError, match="unknown"): + await upgrade(empty_database) + assert await contents(empty_database) == before + assert await revision(empty_database) == "unknown" + + +@pytest.mark.parametrize("existing", [False, True]) +async def test_simultaneous_migration_processes_serialize( + empty_database: str, existing: bool +) -> None: + if existing: + await legacy(empty_database) + await seed(empty_database) + before = await contents(empty_database) + processes = [] try: - original = await connection.fetchval( - "SELECT checksum FROM schema_migrations WHERE version = '0001_init'" - ) - await connection.execute( - "UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0001_init'" - ) - with pytest.raises(MigrationChangedError): - await migrate(DATABASE["url"]) - await connection.execute( - "UPDATE schema_migrations SET checksum = $1 WHERE version = '0001_init'", original - ) + for _ in range(2): + processes.append( + await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "alembic", + "upgrade", + "head", + cwd=ROOT, + env={**os.environ, "DATABASE_URL": empty_database}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + ) + async with asyncio.timeout(30): + output = await asyncio.gather(*(process.communicate() for process in processes)) + assert [process.returncode for process in processes] == [0, 0], output finally: - await connection.close() + for process in processes: + if process.returncode is None: + process.kill() + await process.wait() + after = await contents(empty_database) + assert {name: after[name] for name in before} == before + assert await revision(empty_database) == VERSIONS[-1] async def test_sender_must_be_member_even_bypassing_api() -> None: - connection = await asyncpg.connect(DATABASE["url"]) - try: - with pytest.raises(asyncpg.ForeignKeyViolationError): + with pytest.raises(IntegrityError) as failure: + async with sql_connection() as connection: await connection.execute( - "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " - "VALUES (gen_random_uuid(), gen_random_uuid(), gen_random_uuid(), 'x', now())" + insert(MessageRow).values( + id=uuid4(), chat_id=uuid4(), sender_id=uuid4(), text="x", created_at=func.now() + ) ) - finally: - await connection.close() + assert getattr(failure.value.orig, "sqlstate", None) == "23503" async def test_key_needs_a_member_and_a_message() -> None: - connection = await asyncpg.connect(DATABASE["url"]) - try: - with pytest.raises(asyncpg.ForeignKeyViolationError): + with pytest.raises(IntegrityError) as failure: + async with sql_connection() as connection: await connection.execute( - "INSERT INTO idempotency_keys " - "(chat_id, user_id, key, fingerprint, message_id, created_at) " - "VALUES (gen_random_uuid(), gen_random_uuid(), 'k-1', " - "sha256('x'), gen_random_uuid(), now())" + insert(IdempotencyKeyRow).values( + chat_id=uuid4(), + user_id=uuid4(), + key="k-1", + fingerprint=b"x" * 32, + message_id=uuid4(), + created_at=func.now(), + ) ) - finally: - await connection.close() + assert getattr(failure.value.orig, "sqlstate", None) == "23503" + + +async def test_failed_revision_rolls_back_ddl_and_version( + empty_database: str, tmp_path: Path +) -> None: + await upgrade(empty_database) + await seed(empty_database) + before = await contents(empty_database) + scripts = tmp_path / "alembic" + shutil.copytree(ROOT / "alembic", scripts, ignore=shutil.ignore_patterns("__pycache__")) + (scripts / "versions" / "0003_failure.py").write_text( + "from alembic import op\n" + "import sqlalchemy as sa\n" + "revision = '0003_failure'\n" + "down_revision = '0006_attachment_quarantine'\n" + "def upgrade():\n" + " op.create_table('must_rollback', sa.Column('id', sa.Integer, primary_key=True))\n" + " raise RuntimeError('migration failed')\n" + ) + config = migration_config(empty_database) + config.set_main_option("script_location", str(scripts)) + with pytest.raises(RuntimeError, match="migration failed"): + await asyncio.to_thread(command.upgrade, config, "head") + assert await revision(empty_database) == VERSIONS[-1] + assert await contents(empty_database) == before + async with sql_connection(empty_database) as connection: + assert not await connection.run_sync(lambda sync: inspect(sync).has_table("must_rollback")) async def test_attachment_owner_must_be_member() -> None: - connection = await asyncpg.connect(DATABASE["url"]) - try: - with pytest.raises(asyncpg.ForeignKeyViolationError, match="attachments_owner_fkey"): + with pytest.raises(IntegrityError, match="attachments_owner_fkey"): + async with sql_connection() as connection: await connection.execute( - "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, " - "size_bytes, sha256, width, height, object_key, status, created_at, updated_at) " - "VALUES (gen_random_uuid(), gen_random_uuid(), gen_random_uuid(), 'a.png', " - "'image/png', 1, repeat('0', 64), 1, 1, 'attachments/x', 'pending', now(), now())" + insert(AttachmentRow).values( + id=uuid4(), + chat_id=uuid4(), + owner_id=uuid4(), + filename="a.png", + content_type="image/png", + size_bytes=1, + sha256="0" * 64, + width=1, + height=1, + object_key="attachments/x", + status="pending", + created_at=func.now(), + updated_at=func.now(), + ) ) - finally: - await connection.close() -async def test_message_links_only_attachments_of_its_chat() -> None: - user, chat_a, chat_b, attachment, message = (uuid4() for _ in range(5)) - seed = [ - ( - "INSERT INTO users (id, username, display_name, password_hash, created_at) " - "VALUES ($1, $2, 'A', '$argon2id$x', now())", - user, - f"u_{user.hex[:12]}", - ), - ("INSERT INTO chats (id, title, created_at) VALUES ($1, 'A', now())", chat_a), - ("INSERT INTO chats (id, title, created_at) VALUES ($1, 'B', now())", chat_b), - ( - "INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) " - "VALUES ($1, $3, 1, now()), ($2, $3, 1, now())", - chat_a, - chat_b, - user, - ), - ( - "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, " - "sha256, width, height, object_key, status, created_at, updated_at) VALUES ($1, $2, " - "$3, 'a.png', 'image/png', 1, repeat('0', 64), 1, 1, $4, 'ready', now(), now())", - attachment, - chat_a, - user, - f"attachments/{attachment}", - ), - ( - "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " - "VALUES ($1, $2, $3, '', now())", - message, - chat_b, - user, - ), - ] - connection = await asyncpg.connect(DATABASE["url"]) - transaction = connection.transaction() - await transaction.start() - try: - for sql, *arguments in seed: - await connection.execute(sql, *arguments) - assert await connection.fetchval("SELECT text FROM messages WHERE id = $1", message) == "" - with pytest.raises( - asyncpg.ForeignKeyViolationError, match="message_attachments_attachment_fkey" - ): +async def test_message_links_only_attachments_of_its_chat(empty_database: str) -> None: + await upgrade(empty_database) + await seed(empty_database) + with pytest.raises(IntegrityError, match="message_attachments_attachment_fkey"): + async with sql_connection(empty_database) as connection: + attachment = (await connection.execute(select(AttachmentRow.id))).scalar_one() + user = (await connection.execute(select(UserRow.id))).scalar_one() + chat, message = uuid4(), uuid4() await connection.execute( - "INSERT INTO message_attachments (message_id, position, chat_id, attachment_id) " - "VALUES ($1, 0, $2, $3)", - message, - chat_b, - attachment, + insert(ChatRow).values(id=chat, title="Other", created_at=func.now()) + ) + await connection.execute( + insert(ChatMemberRow).values( + chat_id=chat, user_id=user, position=1, chat_created_at=func.now() + ) + ) + await connection.execute( + insert(MessageRow).values( + id=message, chat_id=chat, sender_id=user, text="", created_at=func.now() + ) + ) + assert ( + await connection.scalar(select(MessageRow.text).where(MessageRow.id == message)) + == "" + ) + await connection.execute( + insert(MessageAttachmentRow).values( + message_id=message, + position=0, + chat_id=chat, + attachment_id=attachment, + ) ) - finally: - await transaction.rollback() - await connection.close() -async def test_pipeline_tables_have_the_columns_the_suite_reads() -> None: - connection = await asyncpg.connect(DATABASE["url"]) - try: - rows = await connection.fetch( - "SELECT table_name, column_name FROM information_schema.columns " - "WHERE table_schema = current_schema()" - ) - finally: - await connection.close() - found: dict[str, set[str]] = {} - for row in rows: - found.setdefault(row["table_name"], set()).add(row["column_name"]) - assert all(columns <= found.get(table, set()) for table, columns in PIPELINE_COLUMNS.items()) +async def test_pipeline_tables_have_the_columns_the_suite_reads(empty_database: str) -> None: + await upgrade(empty_database) + async with sql_connection(empty_database) as connection: + for model in (JobRow, OutboxRow, VariantRow, JobEventRow): + columns = await connection.run_sync( + lambda sync, name=model.__tablename__: inspect(sync).get_columns(name) + ) + assert {col["name"] for col in columns} == set(model.__table__.columns.keys()) -async def test_one_variant_row_per_effect_key() -> None: - user, chat, attachment = uuid4(), uuid4(), uuid4() - variant = ( - "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, " - "content_type, size_bytes, width, height, sha256, created_at) " - "VALUES ($1, 1, 'thumbnail', $2, 'image/jpeg', 1, 1, 1, repeat('a', 64), now())" - ) - connection = await asyncpg.connect(DATABASE["url"]) - transaction = connection.transaction() - await transaction.start() - try: - await connection.execute( - "INSERT INTO users (id, username, display_name, password_hash, created_at) " - "VALUES ($1, $2, 'A', '$argon2id$x', now())", - user, - f"u_{user.hex[:12]}", - ) - await connection.execute( - "INSERT INTO chats (id, title, created_at) VALUES ($1, 'A', now())", chat - ) - await connection.execute( - "INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) " - "VALUES ($1, $2, 1, now())", - chat, - user, - ) - await connection.execute( - "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, " - "sha256, width, height, staging_key, status, created_at, updated_at) VALUES ($1, $2, " - "$3, 'a.png', 'image/png', 1, repeat('0', 64), 1, 1, $4, 'queued', now(), now())", - attachment, - chat, - user, - f"staging/{attachment}", - ) - await connection.execute(variant, attachment, f"variants/{attachment}/1/a") - with pytest.raises(asyncpg.UniqueViolationError, match="attachment_variants_pkey"): - await connection.execute(variant, attachment, f"variants/{attachment}/1/b") - finally: - await transaction.rollback() - await connection.close() +async def test_one_variant_row_per_effect_key(empty_database: str) -> None: + await upgrade(empty_database) + await seed(empty_database) + async with sql_connection(empty_database) as connection: + row = (await connection.execute(select(VariantRow.__table__))).mappings().one() + duplicate = dict(row, object_key="variants/duplicate") + with pytest.raises(IntegrityError, match="attachment_variants_pkey"): + async with sql_connection(empty_database) as connection: + await connection.execute(insert(VariantRow).values(**duplicate)) diff --git a/student-tests/api/test_operability.py b/student-tests/api/test_operability.py index 91adaeb..a6fcffb 100644 --- a/student-tests/api/test_operability.py +++ b/student-tests/api/test_operability.py @@ -1,10 +1,20 @@ import json import time +from collections.abc import AsyncIterator from uuid import uuid4 from httpx import ASGITransport, AsyncClient -from conftest import PASSWORD, actor, keyed, make_chat, register, running_app, split_settings +from conftest import ( + PASSWORD, + actor, + keyed, + make_chat, + new_user, + register, + running_app, + split_settings, +) from messenger.main import create_app MARKER_TEXT = "маркер-который-не-должен-попасть-в-логи-9f3a" @@ -100,6 +110,23 @@ async def test_body_limit_returns_413() -> None: assert response.json()["error"]["code"] == "payload_too_large" +async def test_chunked_body_is_limited_too() -> None: + async def chunks(body: bytes) -> AsyncIterator[bytes]: + for start in range(0, len(body), 1000): + yield body[start : start + 1000] + + user = json.dumps(new_user("chunked")).encode() + async with running_app(max_body_bytes=4096) as http: + large = await http.post("/api/v1/users", content=chunks(b"x" * 5000)) + small = await http.post( + "/api/v1/users", content=chunks(user), headers={"Content-Type": "application/json"} + ) + assert "content-length" not in large.request.headers + assert large.status_code == 413 + assert large.json()["error"]["code"] == "payload_too_large" + assert small.status_code == 201 + + async def test_ready_turns_unavailable_without_database() -> None: async with running_app(database_url="postgresql://u:p@127.0.0.1:1/none") as http: started = time.perf_counter() diff --git a/student-tests/api/test_pagination.py b/student-tests/api/test_pagination.py index 980aa55..fa6a1e1 100644 --- a/student-tests/api/test_pagination.py +++ b/student-tests/api/test_pagination.py @@ -102,14 +102,15 @@ async def test_cursor_from_another_chat_is_rejected(client: AsyncClient) -> None assert response.json()["error"]["code"] == "invalid_request" -@pytest.mark.parametrize("cursor", ["garbage", "1.2.3.4", "%20"]) +@pytest.mark.parametrize("cursor", ["garbage", "1.2.3.4", "%20", ""]) async def test_broken_cursor_is_rejected(client: AsyncClient, cursor: str) -> None: alice = await register(client, "alice") chat = await make_chat(client, alice) - response = await client.get( - f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", headers=actor(alice) - ) - assert response.status_code == 400 + await fill(client, chat, alice, 1) + for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"): + response = await client.get(f"{path}?limit=1&cursor={cursor}", headers=actor(alice)) + assert response.status_code == 400, f"{path}?cursor={cursor} → {response.status_code}" + assert response.json()["error"]["code"] == "invalid_request" @pytest.mark.parametrize("limit", ["0", "-1", "101", "abc", "1.5", ""]) diff --git a/student-tests/api/test_pipeline.py b/student-tests/api/test_pipeline.py index 811abcf..7395c34 100644 --- a/student-tests/api/test_pipeline.py +++ b/student-tests/api/test_pipeline.py @@ -4,12 +4,11 @@ import re from typing import Any from uuid import uuid4 -import asyncpg import pytest from httpx import AsyncClient from test_attachments import storage, trio, upload -from conftest import DATABASE, actor +from conftest import actor, sql_connection from messenger.attachments.application.pipeline import TRANSIENT_ERROR from messenger.attachments.application.ports import Rendition from messenger.attachments.domain.errors import ImageTooDemanding @@ -21,11 +20,9 @@ from pipeline import pipeline_of async def rows(sql: str, *args: Any) -> list[dict[str, Any]]: - connection = await asyncpg.connect(DATABASE["url"]) - try: - return [dict(row) for row in await connection.fetch(sql, *args)] - finally: - await connection.close() + async with sql_connection() as connection: + result = await connection.exec_driver_sql(sql, args) + return [dict(row) for row in result.mappings()] if result.returns_rows else [] async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]: diff --git a/student-tests/api/test_pool.py b/student-tests/api/test_pool.py index e27586f..094ea34 100644 --- a/student-tests/api/test_pool.py +++ b/student-tests/api/test_pool.py @@ -1,6 +1,38 @@ import time +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from uuid import UUID, uuid4 -from conftest import actor, register, resolve, running_app +import pytest +from sqlalchemy import func, select +from sqlalchemy.exc import DBAPIError, IntegrityError + +from conftest import ( + actor, + keyed, + make_chat, + new_user, + register, + resolve, + running_app, + sql_connection, +) +from messenger.identity.domain.models import Session +from messenger.identity.infrastructure.postgres import ( + PostgresIdentityUnitOfWork, + PostgresSessionDirectory, + UserRow, +) +from messenger.messaging.application.ports import AttachmentStates +from messenger.messaging.domain.models import Message +from messenger.messaging.domain.values import MessageText +from messenger.messaging.infrastructure.postgres import ( + ChatRow, + IdempotencyKeyRow, + PostgresMessageLog, +) +from messenger.shared.application.errors import Unavailable +from messenger.shared.domain.events import DomainEvent from messenger.shared.infrastructure.postgres import Database @@ -8,7 +40,8 @@ async def test_exhausted_pool_answers_503_in_bounded_time() -> None: async with running_app(db_pool_max_size=1, db_acquire_timeout_seconds=0.3) as http: alice = await register(http, "alice") database = await resolve(http, Database) - async with database.connection(): + async with database.session() as session: + await session.execute(select(1)) started = time.perf_counter() starved = await http.get("/api/v1/users/me", headers=actor(alice)) elapsed = time.perf_counter() - started @@ -22,6 +55,178 @@ async def test_exhausted_pool_answers_503_in_bounded_time() -> None: async def test_statement_timeout_is_set_on_connections() -> None: async with running_app(db_command_timeout_seconds=1.5) as http: database = await resolve(http, Database) - async with database.connection() as connection: - assert await connection.fetchval("SHOW statement_timeout") == "1500ms" - assert await connection.fetchval("SHOW lock_timeout") == "1500ms" + async with database.session() as session: + statement_timeout = await session.scalar( + select(func.current_setting("statement_timeout")) + ) + lock_timeout = await session.scalar(select(func.current_setting("lock_timeout"))) + idle_timeout = await session.scalar( + select(func.current_setting("idle_in_transaction_session_timeout")) + ) + assert statement_timeout == "1500ms" + assert lock_timeout == "1500ms" + assert idle_timeout == "10s" + + +async def test_idle_session_does_not_acquire_connection() -> None: + async with running_app(db_pool_max_size=1) as http: + database = await resolve(http, Database) + async with database.session(), database.session() as active: + assert await active.scalar(select(1)) == 1 + await database.ping() + + +async def test_closed_idle_connection_is_replaced_before_registration() -> None: + async with running_app(db_pool_max_size=1) as http: + database = await resolve(http, Database) + async with database.session() as session: + pid = await session.scalar(select(func.pg_backend_pid())) + async with sql_connection() as connection: + assert await connection.scalar(select(func.pg_terminate_backend(pid, 1000))) + body = new_user("reconnect") + response = await http.post("/api/v1/users", json=body) + assert response.status_code == 201 + assert response.headers["X-Instance-Id"] and response.headers["X-Request-Id"] + async with database.session() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(UserRow) + .where(UserRow.username == body["username"]) + ) + == 1 + ) + + +async def test_readiness_timeout_releases_waiter() -> None: + async with running_app(db_pool_max_size=1, db_acquire_timeout_seconds=2) as http: + database = await resolve(http, Database) + async with database.session() as session: + await session.execute(select(1)) + started = time.perf_counter() + response = await http.get("/health/ready") + elapsed = time.perf_counter() - started + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert 0.3 < elapsed < 1 + assert (await http.get("/health/ready")).status_code == 200 + + +async def test_server_statement_timeout_returns_connection() -> None: + async with running_app(db_pool_max_size=1, db_command_timeout_seconds=1.5) as http: + database = await resolve(http, Database) + with pytest.raises(Unavailable) as failure: + async with database.session() as session: + await session.execute(select(func.set_config("statement_timeout", "100ms", True))) + await session.execute(select(func.pg_sleep(2))) + assert isinstance(failure.value.__cause__, DBAPIError) + assert getattr(failure.value.__cause__.orig, "sqlstate", None) == "57014" + await database.ping() + + +@pytest.mark.parametrize("commit", [False, True]) +async def test_unit_of_work_persists_sessions_and_events_only_on_commit(commit: bool) -> None: + async with running_app(db_pool_max_size=1) as http: + alice = await register(http, "alice") + database = await resolve(http, Database) + uow = PostgresIdentityUnitOfWork(database) + now = datetime.now(UTC) + _, session = Session.issue(user_id=UUID(alice["id"]), issued_at=now, ttl=timedelta(hours=1)) + event = DomainEvent() + async with uow.transaction(): + await uow.sessions.add(session) + uow.record(event) + if commit: + await uow.commit() + stored = await PostgresSessionDirectory(database).find(session.token_hash, now) + assert stored == (session if commit else None) + assert uow.collect_new_events() == ([event] if commit else []) + assert uow.collect_new_events() == [] + with pytest.raises(RuntimeError, match="вне транзакции"): + _ = uow.session + + +async def test_integrity_failure_rolls_back_and_keeps_pool_usable() -> None: + async with running_app(db_pool_max_size=1) as http: + database = await resolve(http, Database) + uow = PostgresIdentityUnitOfWork(database) + _, session = Session.issue( + user_id=uuid4(), issued_at=datetime.now(UTC), ttl=timedelta(hours=1) + ) + with pytest.raises(IntegrityError) as failure: + async with uow.transaction(): + uow.record(DomainEvent()) + await uow.sessions.add(session) + await uow.commit() + assert getattr(failure.value.orig, "sqlstate", None) == "23503" + assert uow.collect_new_events() == [] + assert uow.seen == [] + await database.ping() + + +@pytest.mark.parametrize( + ("command_timeout", "request_timeout"), [(0.15, 5), (1.5, 0.3)], ids=["database", "request"] +) +async def test_blocked_message_times_out_without_taking_key( + command_timeout: float, request_timeout: float +) -> None: + async with running_app( + db_command_timeout_seconds=command_timeout, request_timeout_seconds=request_timeout + ) as http: + alice = await register(http, "alice") + chat = await make_chat(http, alice) + database = await resolve(http, Database) + headers = keyed(alice) + path = f"/api/v1/chats/{chat['id']}/messages" + async with database.session() as session: + await session.execute( + select(ChatRow.id).where(ChatRow.id == UUID(chat["id"])).with_for_update() + ) + started = time.perf_counter() + blocked = await http.post(path, json={"text": "повтор"}, headers=headers) + assert time.perf_counter() - started < 2 + assert blocked.status_code == 503 + assert blocked.json()["error"]["code"] == "unavailable" + recovered = await http.post(path, json={"text": "повтор"}, headers=headers) + replay = await http.post(path, json={"text": "повтор"}, headers=headers) + history = await http.get(path, headers=actor(alice)) + assert recovered.status_code == replay.status_code == 201 + assert replay.json() == recovered.json() + assert history.json()["items"] == [recovered.json()] + + +async def test_message_integrity_failure_rolls_back_receipt_and_timestamp() -> None: + async with running_app() as http: + alice = await register(http, "alice") + chat = await make_chat(http, alice) + database = await resolve(http, Database) + log = PostgresMessageLog(database, await resolve(http, AttachmentStates)) + now = datetime.now(UTC) + since = now - timedelta(days=1) + message = Message(uuid4(), UUID(chat["id"]), UUID(alice["id"]), MessageText("текст"), now) + fingerprint = b"x" * 32 + first = await log.post(message, "first", fingerprint, since) + assert first is not None + with pytest.raises(IntegrityError): + await log.post(message, "retry", fingerprint, since) + async with database.session() as session: + assert ( + await session.scalar( + select(ChatRow.last_message_at).where(ChatRow.id == message.chat_id) + ) + == first.message.created_at + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(IdempotencyKeyRow) + .where( + IdempotencyKeyRow.chat_id == message.chat_id, + IdempotencyKeyRow.key == "retry", + ) + ) + == 0 + ) + retry = await log.post(replace(message, id=uuid4()), "retry", fingerprint, since) + assert retry is not None + assert retry.message.created_at > first.message.created_at diff --git a/student-tests/api/test_reconciler.py b/student-tests/api/test_reconciler.py new file mode 100644 index 0000000..e961a22 --- /dev/null +++ b/student-tests/api/test_reconciler.py @@ -0,0 +1,432 @@ +import asyncio +import hashlib +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any +from uuid import UUID, uuid4 + +import pytest +from httpx import AsyncClient +from sqlalchemy import make_url +from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.pool import NullPool +from test_attachments import storage, trio, upload + +from conftest import actor, keyed, running_app, upgrade +from fakes import MemoryObjectStorage, NullLogger, RecordingPublisher +from messenger.attachments.application.commands import ReconcileStorage +from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler +from messenger.attachments.domain.events import AttachmentDiscarded +from messenger.attachments.entrypoints import reconcile as entrypoint +from messenger.attachments.entrypoints.reconcile import LOCK_KEY +from messenger.attachments.infrastructure.postgres import ( + PostgresAttachmentsUnitOfWork, + PostgresSingleRun, +) +from messenger.shared.application.errors import Unavailable +from messenger.shared.infrastructure.clock import MonotonicUtcClock +from messenger.shared.infrastructure.postgres import Database +from pictures import SAMPLE +from pipeline import settle + +GRACE = timedelta(seconds=60) +OLD = timedelta(hours=2) +AGED_PENDING = """UPDATE attachments SET status = 'pending', staging_key = object_key, +original_key = NULL, updated_at = created_at, +created_at = created_at - interval '2 hours' WHERE id = $1""" +LATE_ROW = """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, width, height, object_key, staging_key, status, created_at, updated_at) +SELECT gen_random_uuid(), chat_id, owner_id, 'late.png', content_type, size_bytes, sha256, + width, height, $2, $2, 'pending', now(), now() +FROM attachments WHERE id = $1""" + + +@pytest.fixture +def dsn(database: str) -> str: + name = f"reconcile_{uuid4().hex[:12]}" + + async def create() -> None: + engine = create_async_engine( + make_url(database).set(drivername="postgresql+asyncpg"), + isolation_level="AUTOCOMMIT", + poolclass=NullPool, + ) + try: + async with engine.connect() as connection: + await connection.exec_driver_sql(f"CREATE DATABASE {name}") + finally: + await engine.dispose() + await upgrade(target) + + target = f"{database.rsplit('/', 1)[0]}/{name}" + asyncio.run(create()) + return target + + +@dataclass(slots=True) +class Stand: + http: AsyncClient + store: MemoryObjectStorage + database: Database + users: tuple[dict[str, Any], ...] + published: RecordingPublisher + + async def run(self, *, apply: bool = False, deep: bool = False) -> Reconciliation: + reconciler = StorageReconciler( + lambda: PostgresAttachmentsUnitOfWork(self.database), + self.store, + PostgresSingleRun(self.database, LOCK_KEY), + MonotonicUtcClock(), + logger=NullLogger(), + publisher=self.published, + ) + command = ReconcileStorage( + apply=apply, grace=GRACE, skew=timedelta(0), deep=deep, page_size=2 + ) + return await reconciler(command) + + async def sql(self, query: str, *args: Any) -> Any: + async with self.database.session() as session: + connection = await session.connection() + result = await connection.exec_driver_sql(query, args) + rows = result.mappings().all() if result.returns_rows else [] + await session.commit() + return rows + + async def upload(self) -> dict[str, Any]: + alice, _, _, chat = self.users + response = await upload(self.http, alice, chat) + assert response.status_code == 202, response.text + attachment = response.json() + legacy_key = key(attachment) + staging_key = f"staging/{attachment['id']}" + self.store.objects[legacy_key] = self.store.objects.pop(staging_key) + self.store.modified[legacy_key] = self.store.modified.pop(staging_key) + await self.sql( + "DELETE FROM outbox WHERE job_id IN (SELECT job_id FROM jobs WHERE attachment_id = $1)", + attachment["id"], + ) + await self.sql("DELETE FROM jobs WHERE attachment_id = $1", attachment["id"]) + await self.sql( + "UPDATE attachments SET status = 'ready', object_key = $2, original_key = $2, " + "staging_key = NULL, published_version = 1 WHERE id = $1", + attachment["id"], + legacy_key, + ) + return attachment + + def age(self, key: str) -> None: + self.store.modified[key] = datetime.now(UTC) - OLD + + def state(self) -> dict[str, bytes]: + return {key: data for key, (data, _) in self.store.objects.items()} + + +@asynccontextmanager +async def stand(dsn: str) -> AsyncIterator[Stand]: + async with running_app(database_url=dsn) as http: + database = Database( + dsn, max_size=12, connect_timeout=2, acquire_timeout=2, command_timeout=5 + ) + await database.open() + try: + yield Stand(http, await storage(http), database, await trio(http), RecordingPublisher()) + finally: + await database.close() + + +def key(attachment: dict[str, Any]) -> str: + return f"attachments/{attachment['id']}" + + +async def damage(s: Stand) -> dict[str, Any]: + alice, _, _, chat = s.users + healthy, missing, sized, flipped, stale, fresh = [await s.upload() for _ in range(6)] + posted = await s.http.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "", "attachment_ids": [missing["id"]]}, + headers=keyed(alice), + ) + assert posted.status_code == 201, posted.text + del s.store.objects[key(missing)] + s.store.objects[key(sized)] = (SAMPLE[:100], "image/png") + s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png") + await s.sql(AGED_PENDING, stale["id"]) + await s.sql( + "UPDATE attachments SET status = 'pending', staging_key = object_key, " + "original_key = NULL WHERE id = $1", + fresh["id"], + ) + orphan, newborn = f"attachments/{uuid4()}", f"attachments/{uuid4()}" + s.store.objects[orphan] = s.store.objects[newborn] = (SAMPLE, "image/png") + s.store.modified[newborn] = datetime.now(UTC) + s.age(orphan) + return { + "healthy": healthy, + "missing": missing, + "sized": sized, + "flipped": flipped, + "stale": stale, + "fresh": fresh, + "orphan": orphan, + "newborn": newborn, + "message": posted.json(), + } + + +EVERY_CLASS = { + "orphan_stale": 1, + "orphan_fresh": 1, + "missing": 1, + "size_mismatch": 1, + "sha_mismatch": 1, + "pending_stale": 1, + "pending_fresh": 1, +} + + +async def test_dry_run_reports_every_class_and_changes_nothing(dsn: str) -> None: + async with stand(dsn) as s: + await damage(s) + objects = s.state() + rows = await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id") + shallow = await s.run() + deep = await s.run(deep=True) + assert s.state() == objects + assert ( + await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id") == rows + ) + assert dict(deep.found) == EVERY_CLASS + assert dict(shallow.found) == {k: v for k, v in EVERY_CLASS.items() if k != "sha_mismatch"} + assert not deep.fixed + assert not shallow.fixed + assert shallow.requests == {"s3_list": 4, "sql": 3} + assert deep.requests["s3_get"] == 2 + + +async def test_apply_fixes_what_it_found_and_repeats_as_noop(dsn: str) -> None: + async with stand(dsn) as s: + damaged = await damage(s) + alice, bob, _, chat = s.users + first = await s.run(apply=True, deep=True) + second = await s.run(apply=True, deep=True) + reasons = { + str(row["id"]): row["quarantine_reason"] + for row in await s.sql("SELECT id, quarantine_reason FROM attachments") + } + history = await s.http.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(bob)) + meta = await s.http.get( + f"/api/v1/attachments/{damaged['flipped']['id']}", headers=actor(bob) + ) + broken = await s.http.get( + f"/api/v1/attachments/{damaged['flipped']['id']}/content", headers=actor(bob) + ) + intact = await s.http.get( + f"/api/v1/attachments/{damaged['healthy']['id']}/content", headers=actor(alice) + ) + objects = s.state() + assert {name: first.fixed[name] for name in EVERY_CLASS} == { + **EVERY_CLASS, + "orphan_fresh": 0, + "pending_fresh": 0, + } + assert not second.fixed + assert dict(second.found) == {"orphan_fresh": 1, "pending_fresh": 1, "quarantined": 3} + assert reasons[damaged["missing"]["id"]] == "missing" + assert reasons[damaged["sized"]["id"]] == "size_mismatch" + assert reasons[damaged["flipped"]["id"]] == "sha_mismatch" + assert reasons[damaged["healthy"]["id"]] is None + assert [ + (event.attachment_id, event.reason) + for event in s.published.events + if isinstance(event, AttachmentDiscarded) + ] == [(UUID(damaged["stale"]["id"]), "reconcile")] + assert reasons[damaged["fresh"]["id"]] is None + assert damaged["stale"]["id"] not in reasons + assert key(damaged["stale"]) not in objects + assert damaged["orphan"] not in objects + assert damaged["newborn"] in objects + assert key(damaged["flipped"]) in objects + assert history.json()["items"][0]["attachment_ids"] == [damaged["missing"]["id"]] + assert meta.status_code == 200 + assert meta.json()["sha256"] == damaged["flipped"]["sha256"] + assert broken.status_code == 404 + assert broken.json()["error"]["code"] == "not_found" + assert intact.status_code == 200 + assert intact.content == SAMPLE + + +async def test_restored_object_leaves_quarantine(dsn: str) -> None: + async with stand(dsn) as s: + _, bob, _, _ = s.users + lost, flipped = await s.upload(), await s.upload() + del s.store.objects[key(lost)] + s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png") + isolated = await s.run(apply=True, deep=True) + s.store.objects[key(lost)] = s.store.objects[key(flipped)] = (SAMPLE, "image/png") + s.store.modified[key(lost)] = s.store.modified[key(flipped)] + shallow = await s.run(apply=True) + again = await s.run(apply=True) + content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob)) + assert isolated.fixed["missing"] == isolated.fixed["sha_mismatch"] == 1 + assert shallow.fixed["healed"] == 2 + assert shallow.requests["s3_get"] == 2 + assert not again.fixed + assert again.requests["s3_get"] == 0 + assert hashlib.sha256(content.content).hexdigest() == lost["sha256"] + + +async def test_other_bytes_of_the_same_length_keep_the_quarantine(dsn: str) -> None: + async with stand(dsn) as s: + _, bob, _, _ = s.users + lost = await s.upload() + del s.store.objects[key(lost)] + isolated = await s.run(apply=True) + s.store.objects[key(lost)] = (bytes(len(SAMPLE)), "image/png") + s.age(key(lost)) + shallow = await s.run(apply=True) + content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob)) + reason = await s.sql("SELECT quarantine_reason FROM attachments WHERE id = $1", lost["id"]) + assert isolated.fixed["missing"] == 1 + assert not shallow.fixed + assert shallow.found["quarantined"] == 1 + assert shallow.requests["s3_get"] == 1 + assert content.status_code == 404 + assert reason[0]["quarantine_reason"] == "missing" + + +async def test_pages_of_rows_are_not_followed_by_an_empty_one(dsn: str) -> None: + async with stand(dsn) as s: + empty = await s.run() + for _ in range(4): + await s.upload() + full = await s.run() + assert (empty.rows, empty.requests["sql"]) == (0, 1) + assert (full.rows, full.requests["sql"]) == (4, 2) + + +def test_any_pass_failure_ends_as_reconcile_failed( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def broken(*_args: Any) -> Reconciliation: + cancelled = TimeoutError("statement timeout") + raise ExceptionGroup("pass", [cancelled]) + + monkeypatch.setattr(entrypoint, "reconcile", broken) + for name, value in { + "DATABASE_URL": "postgresql://unused@127.0.0.1:1/unused", + "S3_ACCESS_KEY_ID": "reconciler", + "S3_SECRET_ACCESS_KEY": "reconciler-secret", + }.items(): + monkeypatch.setenv(name, value) + with pytest.raises(SystemExit) as stopped: + entrypoint.main(["--apply"]) + captured = capsys.readouterr() + assert stopped.value.code == 1 + assert captured.out == "" + assert "reconcile.failed" in captured.err + assert "TimeoutError" in captured.err + assert "Traceback" not in captured.err + + +async def test_row_written_after_listing_keeps_the_object(dsn: str) -> None: + async with stand(dsn) as s: + first, last = await s.upload(), await s.upload() + fresh, late = f"{key(first)}-a", f"{key(first)}-b" + for name in (fresh, late): + s.store.objects[name] = (SAMPLE, "image/png") + s.store.modified[name] = datetime.now(UTC) + s.age(late) + + async def row_appears() -> None: + s.store.after_page = None + await s.sql(LATE_ROW, first["id"], late) + + s.store.after_page = row_appears + report = await s.run(apply=True) + objects = s.state() + assert key(first) < fresh < late < key(last) + assert report.found["orphan_stale"] == 1 + assert report.rechecked["orphan_stale"] == 1 + assert not report.fixed + assert late in objects + + +async def test_second_pass_is_skipped_while_one_runs(dsn: str) -> None: + async with stand(dsn) as s: + orphan = f"attachments/{uuid4()}" + s.store.objects[orphan] = (SAMPLE, "image/png") + s.age(orphan) + async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held: + busy = await s.run(apply=True) + free = await s.run(apply=True) + assert held + assert busy.skipped + assert not busy.found + assert not free.skipped + assert free.fixed["orphan_stale"] == 1 + + +async def test_storage_failure_mid_pass_ends_it_as_unavailable(dsn: str) -> None: + async with stand(dsn) as s: + await damage(s) + + async def broken() -> None: + raise Unavailable("S3 недоступно") from TimeoutError() + + s.store.after_page = broken + with pytest.raises(Unavailable) as failed: + await s.run(apply=True, deep=True) + assert isinstance(failed.value.__cause__, TimeoutError) + + +async def test_cancelled_pass_releases_its_autocommit_lock(dsn: str) -> None: + async with stand(dsn) as s: + entered = asyncio.Event() + + async def hold() -> None: + async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held: + assert held + entered.set() + await asyncio.Event().wait() + + task = asyncio.create_task(hold()) + try: + await asyncio.wait_for(entered.wait(), 2) + states = await s.sql( + "SELECT a.state FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid " + "WHERE l.locktype = 'advisory' AND l.objid = $1", + LOCK_KEY, + ) + assert [row["state"] for row in states] == ["idle"] + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + locks = await s.sql( + "SELECT count(*) AS total FROM pg_locks WHERE locktype = 'advisory' AND objid = $1", + LOCK_KEY, + ) + assert locks[0]["total"] == 0 + + +async def test_legacy_reconciler_does_not_touch_pipeline_objects(dsn: str) -> None: + async with stand(dsn) as s: + alice, _, _, chat = s.users + response = await upload(s.http, alice, chat) + assert response.status_code == 202 + await settle(s.http) + before = s.state() + assert {key.split("/", 1)[0] for key in before} == {"staging", "originals", "variants"} + for object_key in before: + s.age(object_key) + report = await s.run(apply=True, deep=True) + assert report.rows == report.objects == 0 + assert not report.fixed and not report.errors + assert s.state() == before + content = await s.http.get( + f"/api/v1/attachments/{response.json()['id']}/content", headers=actor(alice) + ) + assert content.status_code == 200 and content.content == SAMPLE diff --git a/student-tests/api/test_sessions.py b/student-tests/api/test_sessions.py index 6397ad1..8b9334d 100644 --- a/student-tests/api/test_sessions.py +++ b/student-tests/api/test_sessions.py @@ -1,8 +1,11 @@ import asyncio from datetime import datetime, timedelta +from hashlib import sha256 +from uuid import UUID import pytest from httpx import AsyncClient +from sqlalchemy import func, select from conftest import ( PASSWORD, @@ -16,6 +19,7 @@ from conftest import ( running_app, scalar, ) +from messenger.identity.infrastructure.postgres import SessionRow, UserRow BASIC = 'Basic realm="messenger", charset="UTF-8"' BEARER = 'Bearer realm="messenger"' @@ -52,7 +56,7 @@ async def test_registration_response_hides_password(client: AsyncClient) -> None async def test_password_is_stored_as_argon2id(client: AsyncClient) -> None: alice = await register(client, "alice") - stored = await scalar("SELECT password_hash FROM users WHERE id = $1::uuid", alice["id"]) + stored = await scalar(select(UserRow.password_hash).where(UserRow.id == UUID(alice["id"]))) assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") assert PASSWORD not in stored @@ -60,10 +64,9 @@ async def test_password_is_stored_as_argon2id(client: AsyncClient) -> None: async def test_session_stores_only_token_digest(client: AsyncClient) -> None: alice = await register(client, "alice") token = actor(alice)["Authorization"].split()[1] - sql = "SELECT count(*) FROM sessions WHERE token_hash = sha256(convert_to($1, 'UTF8'))" - assert await scalar(sql, token) == 1 - raw = "SELECT count(*) FROM sessions WHERE encode(token_hash, 'escape') = $1" - assert await scalar(raw, token) == 0 + rows = select(func.count()).select_from(SessionRow) + assert await scalar(rows.where(SessionRow.token_hash == sha256(token.encode()).digest())) == 1 + assert await scalar(rows.where(SessionRow.token_hash == token.encode())) == 0 async def test_wrong_password_and_unknown_user_look_the_same(client: AsyncClient) -> None: @@ -154,8 +157,8 @@ async def test_expired_sessions_are_purged_on_next_login() -> None: await login(client, body["username"]) await asyncio.sleep(1.2) await login(client, body["username"]) - sql = "SELECT count(*) FROM sessions WHERE user_id = $1::uuid" - assert await scalar(sql, user["id"]) == 1 + sql = select(func.count()).select_from(SessionRow).where(SessionRow.user_id == UUID(user["id"])) + assert await scalar(sql) == 1 async def test_chat_routes_require_bearer(client: AsyncClient) -> None: diff --git a/student-tests/api/test_timeout.py b/student-tests/api/test_timeout.py index 9118d66..c38e288 100644 --- a/student-tests/api/test_timeout.py +++ b/student-tests/api/test_timeout.py @@ -4,10 +4,14 @@ from contextlib import asynccontextmanager from httpx import AsyncClient -from conftest import keyed, make_chat, register, resolve, running_app +from conftest import actor, keyed, make_chat, register, resolve, running_app, scalar +from fakes import MemoryObjectStorage +from messenger.attachments.application.ports import ObjectStorage from messenger.messaging.application import handlers from messenger.messaging.domain.events import MessagePosted from messenger.shared.application.messagebus import EventBus +from pictures import SAMPLE, png +from pipeline import settle MARKER = "текст-который-не-должен-утечь-при-таймауте" @@ -95,3 +99,50 @@ async def test_healthcheck_stays_fast(client: AsyncClient) -> None: started = loop.time() assert (await client.get("/health/live")).status_code == 200 assert loop.time() - started < 1.0 + + +async def trickled(data: bytes, size: int, delay: float) -> AsyncIterator[bytes]: + for start in range(0, len(data), size): + await asyncio.sleep(delay) + yield data[start : start + size] + + +async def upload(client: AsyncClient, user: dict, chat: dict, body: AsyncIterator[bytes]): + return await client.post( + f"/api/v1/chats/{chat['id']}/attachments?filename=slow.png", + content=body, + headers={**actor(user), "Content-Type": "image/png"}, + ) + + +async def test_steady_slow_transfer_outlives_the_request_timeout() -> None: + picture = png(16, 16, size=1 << 20) + async with running_app(request_timeout_seconds=0.4, min_transfer_rate_bytes=16_384) as client: + alice, chat = await prepared(client) + loop = asyncio.get_running_loop() + started = loop.time() + uploaded = await upload(client, alice, chat, trickled(picture, 65_536, 0.1)) + assert uploaded.status_code == 202, uploaded.text + assert loop.time() - started > 1.0 + await settle(client) + store = await resolve(client, ObjectStorage) + assert isinstance(store, MemoryObjectStorage) + store.delay = 0.1 + started = loop.time() + content = await client.get( + f"/api/v1/attachments/{uploaded.json()['id']}/content", headers=actor(alice) + ) + assert (content.status_code, content.content) == (200, picture) + assert loop.time() - started > 1.0 + + +async def test_body_slower_than_the_minimal_rate_is_cut() -> None: + async with running_app(request_timeout_seconds=0.3, min_transfer_rate_bytes=1 << 20) as client: + alice, chat = await prepared(client) + before = await scalar("SELECT count(*) FROM attachments") + loop = asyncio.get_running_loop() + started = loop.time() + response = await upload(client, alice, chat, trickled(SAMPLE, 4_096, 0.1)) + assert (response.status_code, response.json()["error"]["code"]) == (503, "unavailable") + assert loop.time() - started < 1.0 + assert await scalar("SELECT count(*) FROM attachments") == before diff --git a/student-tests/api/test_upload_consistency.py b/student-tests/api/test_upload_consistency.py index b65bb08..7cd73ee 100644 --- a/student-tests/api/test_upload_consistency.py +++ b/student-tests/api/test_upload_consistency.py @@ -3,12 +3,11 @@ from collections.abc import AsyncIterator from typing import Any from uuid import uuid4 -import asyncpg from httpx import AsyncClient from starlette.types import Message from test_attachments import storage, trio, upload -from conftest import APPS, DATABASE, actor, keyed, resolve, running_app, scalar +from conftest import APPS, actor, keyed, resolve, running_app, scalar, sql_connection from messenger.attachments.domain.events import AttachmentDiscarded from messenger.attachments.entrypoints.sweeper import UploadSweeper from messenger.shared.application.messagebus import EventBus @@ -19,27 +18,27 @@ AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE async def execute(sql: str, *args: Any) -> None: - connection = await asyncpg.connect(DATABASE["url"]) - try: - await connection.execute(sql, *args) - finally: - await connection.close() + async with sql_connection() as connection: + await connection.exec_driver_sql(sql, args) def sweeper(http: AsyncClient) -> UploadSweeper: return APPS[id(http)].state.sweeper -async def discarded(http: AsyncClient) -> list[AttachmentDiscarded]: - seen: list[AttachmentDiscarded] = [] - - async def keep(event: AttachmentDiscarded) -> None: - seen.append(event) - - (await resolve(http, EventBus)).project(AttachmentDiscarded, keep) +async def discarded(http: AsyncClient) -> asyncio.Queue[AttachmentDiscarded]: + seen: asyncio.Queue[AttachmentDiscarded] = asyncio.Queue() + (await resolve(http, EventBus)).subscribe(AttachmentDiscarded, seen.put) return seen +async def delivered(seen: asyncio.Queue[AttachmentDiscarded], count: int) -> list[str]: + async with asyncio.timeout(2): + events = [await seen.get() for _ in range(count)] + assert seen.empty() + return sorted(f"{event.reason} {event.attachment_id}" for event in events) + + async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str: store = await storage(http) store.failing = True @@ -67,15 +66,18 @@ async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> N trigger = f"lab4_fail_{uuid4().hex[:8]}" await execute( f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ - BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$; - CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW + BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$""" + ) + await execute( + f"""CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW WHEN (NEW.filename = '{name}' AND NEW.status = 'queued') EXECUTE FUNCTION {trigger}()""" ) before, seen = set(store.objects), await discarded(client) try: response = await upload(client, alice, chat, filename=name) finally: - await execute(f"DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}()") + await execute(f"DROP TRIGGER {trigger} ON attachments") + await execute(f"DROP FUNCTION {trigger}()") assert response.status_code == 503 assert response.json()["error"]["code"] == "unavailable" assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 @@ -88,7 +90,7 @@ async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> N == 0 ) assert set(store.objects) == before - assert [event.reason for event in seen] == ["compensation"] + assert [event.split()[0] for event in await delivered(seen, 1)] == ["compensation"] async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None: @@ -152,8 +154,7 @@ async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None: seen = await discarded(client) first, second = await asyncio.gather(sweeper(client).sweep(), sweeper(client).sweep()) assert first + second == 5 - assert sorted(str(event.attachment_id) for event in seen) == sorted(pending) - assert {event.reason for event in seen} == {"abandoned"} + assert await delivered(seen, 5) == sorted(f"abandoned {item}" for item in pending) assert await sweeper(client).sweep() == 0 objects = (await storage(client)).objects assert not any(f"staging/{attachment_id}" in objects for attachment_id in pending) diff --git a/student-tests/api/test_versioned_pipeline.py b/student-tests/api/test_versioned_pipeline.py new file mode 100644 index 0000000..ceb5852 --- /dev/null +++ b/student-tests/api/test_versioned_pipeline.py @@ -0,0 +1,200 @@ +import asyncio +import json +from uuid import UUID, uuid4 + +import pytest +from httpx import AsyncClient +from test_attachments import storage +from test_pipeline import accepted, job_of, rows + +from conftest import DATABASE, actor +from messenger.admin import serve +from messenger.attachments.application.commands import ReprocessAttachment +from messenger.attachments.application.pipeline import ( + AFTER_CLAIM, + AFTER_S3_WRITE, + ReprocessAttachmentHandler, +) +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.errors import ImageTooDemanding +from messenger.attachments.domain.jobs import Outcome, ReprocessResult +from messenger.shared.infrastructure.clock import MonotonicUtcClock +from messenger.shared.infrastructure.ids import Uuid7Generator +from pipeline import InlinePipeline, InlineRenderer, pipeline_of + + +async def reprocess(pipeline: InlinePipeline, ident: str, version: int = 2) -> ReprocessResult: + handler = ReprocessAttachmentHandler(pipeline.uow(), Uuid7Generator(), MonotonicUtcClock()) + return await handler(ReprocessAttachment(UUID(ident), version, "test")) + + +async def attachment(ident: str) -> dict: + (row,) = await rows( + "SELECT status, published_version, error_code FROM attachments WHERE id = $1", ident + ) + return row + + +async def versions(pipeline: InlinePipeline, ident: str) -> dict: + return { + job.pipeline_version: job + for job in await pipeline.publish() + if str(job.attachment_id) == ident + } + + +async def test_concurrent_reprocess_creates_one_job_and_outbox(client: AsyncClient) -> None: + _, _, ident = await accepted(client) + pipeline = await pipeline_of(client) + results = await asyncio.gather(*(reprocess(pipeline, ident) for _ in range(6))) + assert results.count(ReprocessResult.QUEUED) == 1 + assert results.count(ReprocessResult.NOOP) == 5 + jobs = await rows( + "SELECT pipeline_version, count(*) FROM jobs WHERE attachment_id = $1 GROUP BY 1", ident + ) + assert {r["pipeline_version"]: r["count"] for r in jobs} == {1: 1, 2: 1} + assert len(await versions(pipeline, ident)) == 2 + + +@pytest.mark.parametrize("order", [(1, 2), (2, 1)]) +async def test_version_order_never_reverts_published_bytes( + client: AsyncClient, order: tuple[int, int] +) -> None: + _, bob, ident = await accepted(client) + pipeline = await pipeline_of(client) + assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED + jobs = await versions(pipeline, ident) + for version in order: + outcome = await pipeline.process(jobs[version]) + assert outcome is (Outcome.SUPERSEDED if order == (2, 1) and version == 1 else Outcome.DONE) + assert (await attachment(ident))["status"] == "ready" + await pipeline.clean() + assert (await attachment(ident))["published_version"] == 2 + public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() + stored = await rows( + "SELECT pipeline_version, name, sha256 FROM attachment_variants WHERE attachment_id = $1", + ident, + ) + current = {r["name"]: r["sha256"] for r in stored if r["pipeline_version"] == 2} + older = {r["name"]: r["sha256"] for r in stored if r["pipeline_version"] == 1} + assert current != older + assert {r["name"]: r["sha256"] for r in public["variants"]} == current + assert await pipeline.process(jobs[1]) is Outcome.DUPLICATE + assert await reprocess(pipeline, ident) is ReprocessResult.NOOP + assert (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() == public + + +class PausedEffect: + def __init__(self, point: str) -> None: + self.point = point + self.reached = asyncio.Event() + self.resume = asyncio.Event() + + async def reach(self, point: str) -> None: + if point == self.point: + self.reached.set() + await self.resume.wait() + + +@pytest.mark.parametrize("point", [AFTER_CLAIM, AFTER_S3_WRITE]) +async def test_old_holder_finishes_after_new_publication(client: AsyncClient, point: str) -> None: + _, bob, ident = await accepted(client) + old, new = await pipeline_of(client), await pipeline_of(client) + old.faults = pause = PausedEffect(point) + job = await job_of(ident) + running = asyncio.create_task(old.process(job)) + try: + await asyncio.wait_for(pause.reached.wait(), 5) + assert await reprocess(new, ident) is ReprocessResult.QUEUED + jobs = await versions(new, ident) + assert await new.process(jobs[2]) is Outcome.DONE + public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() + assert await new.clean() == 0 + pause.resume.set() + assert await asyncio.wait_for(running, 5) is Outcome.SUPERSEDED + assert await new.clean() == 1 + assert (await attachment(ident))["published_version"] == 2 + assert ( + await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob)) + ).json() == public + finally: + pause.resume.set() + await running + + +class RejectingRenderer: + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + raise ImageTooDemanding("test limit") + + +async def test_failed_old_version_keeps_other_version_active(client: AsyncClient) -> None: + _, _, ident = await accepted(client) + pipeline = await pipeline_of(client) + assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED + jobs = await versions(pipeline, ident) + pipeline.renderer = RejectingRenderer() + assert await pipeline.process(jobs[1]) is Outcome.FAILED + assert (await attachment(ident))["status"] in ("queued", "processing") + pipeline.renderer = InlineRenderer() + assert await pipeline.process(jobs[2]) is Outcome.DONE + assert (await attachment(ident))["published_version"] == 2 + + +async def test_concurrent_failures_leave_no_stuck_attachment(client: AsyncClient) -> None: + _, _, ident = await accepted(client) + pipeline = await pipeline_of(client) + assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED + jobs = await versions(pipeline, ident) + pipeline.renderer = RejectingRenderer() + assert await asyncio.gather(*(pipeline.process(job) for job in jobs.values())) == [ + Outcome.FAILED, + Outcome.FAILED, + ] + assert (await attachment(ident))["status"] == "failed" + assert await reprocess(pipeline, ident) is ReprocessResult.CONFLICT + assert await pipeline.publish() == [] + + +async def test_failed_new_version_keeps_ready_original(client: AsyncClient) -> None: + _, bob, ident = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.drain() + public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() + assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED + assert (await attachment(ident))["status"] == "ready" + jobs = await versions(pipeline, ident) + pipeline.renderer = RejectingRenderer() + assert await pipeline.process(jobs[2]) is Outcome.FAILED + assert (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() == public + + +async def test_reprocess_rejects_unknown_id_and_version(client: AsyncClient) -> None: + pipeline = await pipeline_of(client) + assert await reprocess(pipeline, str(uuid4())) is ReprocessResult.NOT_FOUND + with pytest.raises(ValueError, match="версия"): + await reprocess(pipeline, str(uuid4()), 3) + + +async def test_operator_cli_schedules_version_once( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + _, _, ident = await accepted(client) + monkeypatch.setenv("DATABASE_URL", DATABASE["url"]) + monkeypatch.setenv("LOG_LEVEL", "WARNING") + args = ["reprocess", "--attachment-id", ident, "--pipeline-version", "2"] + for expected in ("queued", "noop"): + assert await serve(args) == 0 + report = json.loads(capsys.readouterr().out.splitlines()[-1]) + assert report["result"] == expected + assert report["pipeline_version"] == 2 + + +async def test_ready_reprocess_survives_cleanup_before_metadata_commit(client: AsyncClient) -> None: + _, _, ident = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.drain() + await (await storage(client)).delete(f"staging/{ident}") + assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED + jobs = await versions(pipeline, ident) + assert await pipeline.process(jobs[2]) is Outcome.DONE + assert (await attachment(ident))["published_version"] == 2 diff --git a/student-tests/conftest.py b/student-tests/conftest.py index 8cb762f..251703b 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -1,5 +1,6 @@ import asyncio import re +import socket import subprocess import time from base64 import b64encode @@ -9,19 +10,23 @@ from pathlib import Path from typing import Any from uuid import uuid4 -import asyncpg import pytest +from alembic import command +from alembic.config import Config from fastapi import FastAPI from httpx import ASGITransport, AsyncClient +from sqlalchemy import make_url +from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine +from sqlalchemy.pool import NullPool from fakes import MemoryStorageProvider from messenger.attachments.config import AttachmentsSettings from messenger.identity.config import IdentitySettings from messenger.main import create_app from messenger.messaging.config import MessagingSettings -from messenger.migrate import migrate from messenger.shared.config import PlatformSettings from messenger.shared.entrypoints.config import TransportSettings +from messenger.shared.infrastructure.postgres import unavailable ROOT = Path(__file__).resolve().parents[1] POSTGRES_IMAGE = re.compile(r"image: (postgres:\S+)") @@ -38,18 +43,53 @@ def docker(*args: str) -> str: return done.stdout.strip() +def migration_config(dsn: str) -> Config: + config = Config(str(ROOT / "alembic.ini")) + config.attributes["database_url"] = dsn + return config + + +async def upgrade(dsn: str, revision: str = "head") -> None: + await asyncio.to_thread(command.upgrade, migration_config(dsn), revision) + + async def migrated(dsn: str) -> None: deadline = time.monotonic() + STARTUP_SECONDS while True: try: - await migrate(dsn) + await upgrade(dsn) return - except OSError, asyncpg.PostgresError, asyncpg.InterfaceError: - if time.monotonic() > deadline: + except Exception as error: + if not unavailable(error) or time.monotonic() > deadline: raise await asyncio.sleep(0.5) +@asynccontextmanager +async def sql_connection(dsn: str | None = None) -> AsyncIterator[AsyncConnection]: + engine = create_async_engine( + make_url(dsn or DATABASE["url"]).set(drivername="postgresql+asyncpg"), + poolclass=NullPool, + hide_parameters=True, + ) + try: + async with engine.begin() as connection: + yield connection + finally: + await engine.dispose() + + +def free_port() -> int: + for port in range(26000, 27000): + with socket.socket() as probe: + try: + probe.bind(("127.0.0.1", port)) + return port + except OSError: + continue + raise RuntimeError("No free test port in 26000–26999") + + @pytest.fixture(autouse=True) def database_for_api(request: pytest.FixtureRequest) -> None: if "api" in request.node.path.parts: @@ -78,7 +118,7 @@ def answering(name: str, *probe: str) -> bool: @pytest.fixture(scope="session") def valkey() -> Iterator[dict[str, str]]: name = f"mtusi-lab6-test-valkey-{uuid4().hex[:8]}" - container(name, "-p", "127.0.0.1::6379", image(VALKEY_IMAGE)) + container(name, "-p", f"127.0.0.1:{free_port()}:6379", image(VALKEY_IMAGE)) try: deadline = time.monotonic() + STARTUP_SECONDS while not answering(name, "valkey-cli", "ping"): @@ -86,7 +126,7 @@ def valkey() -> Iterator[dict[str, str]]: time.sleep(0.2) yield {"name": name, "url": f"redis://127.0.0.1:{published(name, 6379)}/0"} finally: - subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + subprocess.run(["docker", "rm", "-fv", name], capture_output=True, check=False) @pytest.fixture(scope="session") @@ -94,14 +134,16 @@ def database() -> Iterator[str]: name = f"mtusi-lab6-test-{uuid4().hex[:8]}" tuning = ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] environment = ["-e", "POSTGRES_PASSWORD=test", "-e", "POSTGRES_DB=messenger"] - container(name, *environment, "-p", "127.0.0.1::5432", image(POSTGRES_IMAGE), *tuning) + container( + name, *environment, "-p", f"127.0.0.1:{free_port()}:5432", image(POSTGRES_IMAGE), *tuning + ) try: dsn = f"postgresql://postgres:test@127.0.0.1:{published(name, 5432)}/messenger" asyncio.run(migrated(dsn)) DATABASE["url"] = dsn yield dsn finally: - subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + subprocess.run(["docker", "rm", "-fv", name], capture_output=True, check=False) def split_settings(**overrides: Any) -> dict[str, Any]: @@ -152,12 +194,12 @@ async def client() -> AsyncIterator[AsyncClient]: yield http -async def scalar(sql: str, *args: Any) -> Any: - connection = await asyncpg.connect(DATABASE["url"]) - try: - return await connection.fetchval(sql, *args) - finally: - await connection.close() +async def scalar(statement: Any, *args: Any) -> Any: + async with sql_connection() as connection: + if isinstance(statement, str): + result = await connection.exec_driver_sql(statement, args) + return result.scalar() if result.returns_rows else None + return await connection.scalar(statement) def basic(username: str, password: str) -> dict[str, str]: diff --git a/student-tests/fakes.py b/student-tests/fakes.py index 2b0e23f..deb86af 100644 --- a/student-tests/fakes.py +++ b/student-tests/fakes.py @@ -1,11 +1,12 @@ -from collections.abc import AsyncIterator, Awaitable, Callable -from datetime import datetime, timedelta +import asyncio +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from datetime import UTC, datetime, timedelta from typing import Any from dishka import Provider, Scope, provide from messenger.attachments.application.errors import ObjectMissingError -from messenger.attachments.application.ports import ObjectContent, ObjectStorage +from messenger.attachments.application.ports import ObjectContent, ObjectStorage, StoredObject from messenger.shared.application.errors import Unavailable @@ -30,33 +31,53 @@ class FrozenClock: return self._current -async def pieces(data: bytes, size: int = 65_536) -> AsyncIterator[bytes]: +async def pieces(data: bytes, size: int = 65_536, delay: float = 0.0) -> AsyncIterator[bytes]: for start in range(0, len(data), size): + await asyncio.sleep(delay) yield data[start : start + size] class MemoryObjectStorage: - __slots__ = ("after_put", "failing", "objects") + __slots__ = ("after_page", "after_put", "delay", "failing", "modified", "objects") def __init__(self) -> None: self.objects: dict[str, tuple[bytes, str]] = {} + self.modified: dict[str, datetime] = {} self.failing = False + self.delay = 0.0 self.after_put: Callable[[str], Awaitable[None]] | None = None + self.after_page: Callable[[], Awaitable[None]] | None = None async def put(self, key: str, data: bytes, content_type: str) -> None: if self.failing: raise Unavailable("S3 недоступно") self.objects[key] = (bytes(data), content_type) + self.modified[key] = datetime.now(UTC) if self.after_put is not None: await self.after_put(key) + async def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]: + if self.failing: + raise Unavailable("S3 недоступно") + keys = sorted(key for key in self.objects if key.startswith(prefix)) + for start in range(0, len(keys), page_size): + yield [self._stored(key) for key in keys[start : start + page_size]] + if self.after_page is not None: + await self.after_page() + + async def stat(self, key: str) -> StoredObject | None: + return self._stored(key) if key in self.objects else None + + def _stored(self, key: str) -> StoredObject: + return StoredObject(key, len(self.objects[key][0]), self.modified[key]) + async def open(self, key: str) -> ObjectContent: if self.failing: raise Unavailable("S3 недоступно") if key not in self.objects: raise ObjectMissingError(key) data = self.objects[key][0] - return ObjectContent(size=len(data), chunks=pieces(data)) + return ObjectContent(size=len(data), chunks=pieces(data, delay=self.delay)) async def read(self, key: str) -> bytes: content = await self.open(key) @@ -73,9 +94,20 @@ class MemoryObjectStorage: if self.failing: raise Unavailable("S3 недоступно") self.objects.pop(key, None) + self.modified.pop(key, None) class MemoryStorageProvider(Provider): @provide(scope=Scope.APP) def storage(self) -> ObjectStorage: return MemoryObjectStorage() + + +class RecordingPublisher: + __slots__ = ("events",) + + def __init__(self) -> None: + self.events: list[Any] = [] + + async def publish(self, events: Sequence[Any]) -> None: + self.events.extend(events) diff --git a/src/messenger/migrations/0001_init.sql b/student-tests/fixtures/legacy/0001_init.sql similarity index 100% rename from src/messenger/migrations/0001_init.sql rename to student-tests/fixtures/legacy/0001_init.sql diff --git a/src/messenger/migrations/0002_idempotency_keys.sql b/student-tests/fixtures/legacy/0002_idempotency_keys.sql similarity index 100% rename from src/messenger/migrations/0002_idempotency_keys.sql rename to student-tests/fixtures/legacy/0002_idempotency_keys.sql diff --git a/src/messenger/migrations/0003_attachments.sql b/student-tests/fixtures/legacy/0003_attachments.sql similarity index 100% rename from src/messenger/migrations/0003_attachments.sql rename to student-tests/fixtures/legacy/0003_attachments.sql diff --git a/src/messenger/migrations/0004_jobs.sql b/student-tests/fixtures/legacy/0004_jobs.sql similarity index 100% rename from src/messenger/migrations/0004_jobs.sql rename to student-tests/fixtures/legacy/0004_jobs.sql diff --git a/src/messenger/migrations/0005_failed_registry.sql b/student-tests/fixtures/legacy/0005_failed_registry.sql similarity index 100% rename from src/messenger/migrations/0005_failed_registry.sql rename to student-tests/fixtures/legacy/0005_failed_registry.sql diff --git a/student-tests/fixtures/legacy/0006_attachment_quarantine.sql b/student-tests/fixtures/legacy/0006_attachment_quarantine.sql new file mode 100644 index 0000000..7108f5f --- /dev/null +++ b/student-tests/fixtures/legacy/0006_attachment_quarantine.sql @@ -0,0 +1,6 @@ +ALTER TABLE attachments + ADD COLUMN quarantine_reason text + CHECK (quarantine_reason IN ('missing', 'size_mismatch', 'sha_mismatch')), + ADD COLUMN quarantined_at timestamptz, + ADD CONSTRAINT attachments_quarantine_check + CHECK ((quarantine_reason IS NULL) = (quarantined_at IS NULL)); diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py index bd91fde..09d8689 100644 --- a/student-tests/pipeline.py +++ b/student-tests/pipeline.py @@ -56,7 +56,7 @@ class InlineRenderer: __slots__ = () async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: - return render_variants(data, LIMITS) + return render_variants(data, LIMITS, pipeline_version) class CapturingPublisher: diff --git a/student-tests/stack/test_attachment_faults.py b/student-tests/stack/test_attachment_faults.py index be9af4b..d32c798 100644 --- a/student-tests/stack/test_attachment_faults.py +++ b/student-tests/stack/test_attachment_faults.py @@ -137,14 +137,23 @@ def replica_at(address: str) -> str: def raw( - path: str, headers: dict[str, str], body: bytes, *, hang_up: bool + path: str, + headers: dict[str, str], + body: bytes, + *, + hang_up: bool, + piece: int = 0, + delay: float = 0.0, ) -> tuple[int, dict[str, Any], float]: parts = urlsplit(BASE_URL) head = f"POST {path} HTTP/1.1\r\nHost: {parts.netloc}\r\nConnection: close\r\n" head += "".join(f"{name}: {value}\r\n" for name, value in headers.items()) + "\r\n" started = time.perf_counter() with socket.create_connection((parts.hostname, parts.port or 80), timeout=30) as sock: - sock.sendall(head.encode("latin-1") + body) + sock.sendall(head.encode("latin-1")) + for start in range(0, len(body), piece or max(len(body), 1)): + sock.sendall(body[start : start + (piece or len(body))]) + time.sleep(delay) if hang_up: return 0, {}, 0.0 response = HTTPResponse(sock) @@ -152,6 +161,56 @@ def raw( return response.status, json.loads(response.read()), time.perf_counter() - started +READER = """ +import hashlib, http.client, sys, time +path, token, request_id, piece, delay = sys.argv[1:6] +connection = http.client.HTTPConnection("lb", 8080, timeout=30) +headers = {"Authorization": "Bearer " + token, "X-Request-Id": request_id} +connection.request("GET", path, headers=headers) +response, digest, started = connection.getresponse(), hashlib.sha256(), time.monotonic() +while chunk := response.read(int(piece)): + digest.update(chunk) + time.sleep(float(delay)) +print(response.status, digest.hexdigest(), round(time.monotonic() - started, 2)) +""" + + +def slow_read(path: str, token: str, request_id: str, piece: int, delay: float) -> list[str]: + template = "{{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}" + networks = subprocess.run( + ["docker", "inspect", "-f", template, compose("ps", "-q", "lb")], + check=True, + capture_output=True, + text=True, + ).stdout.split() + image = subprocess.run( + ["docker", "inspect", "-f", "{{.Config.Image}}", compose("ps", "-q", "api").split()[0]], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + command = [ + "docker", + "run", + "--rm", + "--network", + next(name for name in networks if name.endswith("_edge")), + "--sysctl", + "net.ipv4.tcp_rmem=4096 16384 65536", + "--entrypoint", + "python", + image, + "-c", + READER, + path, + token, + request_id, + str(piece), + str(delay), + ] + return subprocess.run(command, check=True, capture_output=True, text=True).stdout.split() + + @pytest.fixture async def http() -> AsyncIterator[AsyncClient]: async with AsyncClient(base_url=BASE_URL, timeout=30, trust_env=False) as client: @@ -470,3 +529,49 @@ async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, fault assert elapsed <= REQUEST_TIMEOUT + 2 assert seen(name) is None assert (await upload(http, s["alice"], s["chats"][0])).status_code == 202 + + +async def test_token_is_checked_before_the_declared_size(http: AsyncClient) -> None: + s = await scene(http) + headers = {"Content-Type": "image/png", "Content-Length": str(LIMIT + 1)} + path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=a.png" + status, body, _ = await asyncio.to_thread(raw, path, headers, SAMPLE[:1024], hang_up=False) + print(f"без токена, Content-Length {LIMIT + 1}: {status} {body['error']['code']}") + assert (status, body["error"]["code"]) == (401, "unauthorized") + + +async def test_slow_but_steady_transfers_outlive_the_request_timeout(http: AsyncClient) -> None: + since = moment() + s = await scene(http) + picture = png(16, 16, size=3 << 20) + sent, read = f"slow-upload-{uuid4().hex}", f"slow-download-{uuid4().hex}" + headers = { + **bearer(s["alice"]), + "Content-Type": "image/png", + "Content-Length": str(len(picture)), + "X-Request-Id": sent, + } + path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=slow.png" + status, body, up = await asyncio.to_thread( + raw, path, headers, picture, hang_up=False, piece=64 << 10, delay=0.25 + ) + assert status == 202, body + assert body["sha256"] == sha256(picture).hexdigest() + await waited(lambda: status_of(body["id"]) == "ready", 60) + content = f"/api/v1/attachments/{body['id']}/content" + code, digest, down = await asyncio.to_thread(slow_read, content, s["bob"], read, 64 << 10, 0.33) + assert (code, digest) == ("200", body["sha256"]) + + def handled() -> dict[str, tuple[int, float]]: + return { + e["request_id"]: (e["status"], e["duration_ms"] / 1000) + for e in logs(since) + if e.get("event") == "request.handled" and e.get("request_id") in (sent, read) + } + + await waited(lambda: len(handled()) == 2, 15) + replica = handled() + print(f"upload 3 MiB ≈ 256 KiB/s: клиент {up:.1f} с, реплика {replica[sent]}") + print(f"скачивание 3 MiB ≈ 192 KiB/s из сети edge: клиент {down} с, реплика {replica[read]}") + assert replica[sent][0] == 202 and replica[sent][1] > REQUEST_TIMEOUT + assert replica[read][0] == 200 and replica[read][1] > REQUEST_TIMEOUT diff --git a/student-tests/stack/test_faults.py b/student-tests/stack/test_faults.py index 6e78de4..5a67c49 100644 --- a/student-tests/stack/test_faults.py +++ b/student-tests/stack/test_faults.py @@ -1,4 +1,5 @@ import asyncio +import time from collections.abc import AsyncIterator from uuid import uuid4 @@ -18,6 +19,7 @@ from stand import ( new_user, psql, report, + restart_stack, started_at, timed, valkey, @@ -55,7 +57,8 @@ async def test_chat_creation_rolls_back_on_unknown_member(http: AsyncClient) -> json={"title": title, "member_ids": [alice["id"], str(uuid4())]}, headers=bearer(token), ) - print(f"чат с несуществующим участником: {response.status_code}") + code = response.json()["error"]["code"] + print(f"чат с несуществующим участником: {response.status_code} {code}") assert response.status_code == 400 assert response.json()["error"]["code"] == "invalid_request" assert psql("SELECT count(*) FROM chats WHERE title = :'title'", title=title) == "0" @@ -208,6 +211,13 @@ async def test_postgres_stopped_gives_bounded_503_and_recovers(http: AsyncClient me = await http.get("/api/v1/users/me", headers=bearer(token)) print(f"postgres запущен: готовность через {recovered:.1f} с, прежний токен {me.status_code}") assert me.json() == user + instances = set() + for _ in range(40): + response = await http.post("/api/v1/users", json=new_user("recovered")) + assert response.status_code == 201 + instances.add(response.headers["X-Instance-Id"]) + print(f"postgres запущен: 40 регистраций 201 через {len(instances)} реплики без рестарта API") + assert len(instances) >= 2 assert started_at("api") == api_started @@ -227,3 +237,43 @@ async def test_api_restart_keeps_session_without_extending_it(http: AsyncClient) print(f"expires_at при входе {session['expires_at']}, в БД после рестарта {stored}") assert me.status_code == 200 assert stored.rstrip("Z").rstrip("0") == session["expires_at"].rstrip("Z").rstrip("0") + + +async def test_full_restart_keeps_session_until_logout(http: AsyncClient) -> None: + user, token = await enroll(http, "fullrestart") + restart_stack() + recovered = await wait_ready(http) + me = await http.get("/api/v1/users/me", headers=bearer(token)) + logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token)) + after = await http.get("/api/v1/users/me", headers=bearer(token)) + print( + f"restart.sh: готовность через {recovered:.1f} с, прежний токен {me.status_code}, " + f"выход {logout.status_code}, затем {after.status_code}" + ) + assert me.json() == user + assert logout.status_code == 204 + assert after.status_code == 401 + + +async def test_session_expires_after_restart_at_original_time(http: AsyncClient) -> None: + user, token = await enroll(http, "shortrestart") + expires_at = float( + psql( + "UPDATE sessions s SET expires_at = now() + interval '20 seconds' FROM users u " + "WHERE u.id = s.user_id AND u.username = :'name' " + "RETURNING extract(epoch FROM s.expires_at)", + name=user["username"], + ).splitlines()[0] + ) + compose("restart", "api") + recovered = await wait_ready(http) + fresh = await http.get("/api/v1/users/me", headers=bearer(token)) + await asyncio.sleep(max(0.0, expires_at - time.time()) + 0.5) + stale = await http.get("/api/v1/users/me", headers=bearer(token)) + print( + f"срок сессии сокращён до 20 с, рестарт api: готовность через {recovered:.1f} с, " + f"до срока {fresh.status_code}, после срока {stale.status_code}" + ) + assert fresh.status_code == 200 + assert stale.status_code == 401 + assert stale.headers["WWW-Authenticate"] == BEARER diff --git a/student-tests/stand.py b/student-tests/stand.py index 7264b1d..a194f5f 100644 --- a/student-tests/stand.py +++ b/student-tests/stand.py @@ -32,6 +32,10 @@ def compose(*args: str, stdin: str | None = None) -> str: return done.stdout.strip() +def restart_stack() -> None: + subprocess.run(["sh", "scripts/restart.sh"], cwd=ROOT, check=True, capture_output=True) + + def psql(sql: str, **variables: str) -> str: assigned = [item for name, value in variables.items() for item in ("-v", f"{name}={value}")] command = ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tA"] diff --git a/student-tests/tools/seed_storage.py b/student-tests/tools/seed_storage.py new file mode 100644 index 0000000..9deb592 --- /dev/null +++ b/student-tests/tools/seed_storage.py @@ -0,0 +1,139 @@ +import argparse +import asyncio +import hashlib +import json +import time +from datetime import UTC, datetime, timedelta +from io import BytesIO +from uuid import uuid4 + +from PIL import Image +from sqlalchemy import func, insert, make_url, select +from sqlalchemy.ext.asyncio import create_async_engine + +from messenger.attachments.config import AttachmentsSettings +from messenger.attachments.infrastructure.postgres import AttachmentRow +from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client +from messenger.identity.infrastructure.postgres import UserRow +from messenger.messaging.infrastructure.postgres import ChatMemberRow +from messenger.shared.config import PlatformSettings + +MODELS = (UserRow, ChatMemberRow, AttachmentRow) +COLUMNS = ( + "id", + "chat_id", + "owner_id", + "filename", + "content_type", + "size_bytes", + "sha256", + "width", + "height", + "object_key", + "status", + "created_at", + "updated_at", +) + + +def picture() -> bytes: + buffer = BytesIO() + Image.effect_noise((8, 8), 64).convert("RGB").save(buffer, "PNG") + return buffer.getvalue() + + +async def seed(count: int, concurrency: int) -> dict[str, object]: + platform, settings = PlatformSettings(), AttachmentsSettings() + body = picture() + digest = hashlib.sha256(body).hexdigest() + engine = create_async_engine( + make_url(platform.database_url.get_secret_value()).set(drivername="postgresql+asyncpg"), + hide_parameters=True, + ) + try: + async with engine.connect() as connection: + member = ( + ( + await connection.execute( + select(ChatMemberRow.chat_id, ChatMemberRow.user_id).limit(1) + ) + ) + .mappings() + .first() + ) + if member is None: + raise SystemExit("нет ни одного чата: сначала make test") + ids = [uuid4() for _ in range(count)] + pending = iter(ids) + completed = 0 + started = time.monotonic() + async with s3_client( + endpoint=settings.s3_endpoint, + region=settings.s3_region, + access_key_id=settings.s3_access_key_id.get_secret_value(), + secret_access_key=settings.s3_secret_access_key.get_secret_value(), + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_read_timeout_seconds, + max_attempts=3, + ) as client: + storage = S3ObjectStorage(client, settings.s3_bucket) + + async def worker() -> None: + nonlocal completed + for attachment_id in pending: + await storage.put(f"attachments/{attachment_id}", body, "image/png") + completed += 1 + if completed % 10000 == 0 or completed == count: + print(json.dumps({"progress": completed, "target": count}), flush=True) + + await asyncio.gather(*(worker() for _ in range(concurrency))) + stored = time.monotonic() + stamp = datetime.now(UTC) - timedelta(hours=2) + records = [ + ( + attachment_id, + member["chat_id"], + member["user_id"], + f"seed-{number}.png", + "image/png", + len(body), + digest, + 8, + 8, + f"attachments/{attachment_id}", + "ready", + stamp, + stamp, + ) + for number, attachment_id in enumerate(ids) + ] + async with engine.begin() as connection: + for start in range(0, count, 1000): + await connection.execute( + insert(AttachmentRow), + [dict(zip(COLUMNS, row, strict=True)) for row in records[start : start + 1000]], + ) + total = await connection.scalar(select(func.count()).select_from(AttachmentRow)) + finally: + await engine.dispose() + return { + "added": count, + "rows_total": total, + "object_bytes": len(body), + "seconds": { + "s3_put": round(stored - started, 3), + "pg_insert": round(time.monotonic() - stored, 3), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Наполнить bucket и attachments N вложениями") + parser.add_argument("--count", type=int, required=True) + parser.add_argument("--concurrency", type=int, default=32) + args = parser.parse_args() + print(json.dumps(asyncio.run(seed(args.count, args.concurrency)))) + + +if __name__ == "__main__": + main() diff --git a/student-tests/unit/test_domain.py b/student-tests/unit/test_domain.py index 9546277..eaf5fd8 100644 --- a/student-tests/unit/test_domain.py +++ b/student-tests/unit/test_domain.py @@ -9,13 +9,12 @@ from messenger.identity.domain.errors import InvalidUserData from messenger.identity.domain.models import Session, User, token_digest from messenger.identity.domain.values import DisplayName, Password, Username from messenger.messaging.domain.errors import ( - ChatNotFound, IdempotencyConflict, InvalidAttachmentList, InvalidChatData, InvalidMessageText, ) -from messenger.messaging.domain.models import Chat, Receipt +from messenger.messaging.domain.models import Chat, Message, Receipt, attachment_list from messenger.messaging.domain.values import MessageText from messenger.shared.infrastructure.clock import MonotonicUtcClock @@ -90,17 +89,6 @@ def test_message_text_rejects_whitespace() -> None: assert MessageText("").value == "" -def make_chat(*members: object) -> Chat: - ids = [uuid4() for _ in members] if members else [uuid4()] - return Chat.create( - chat_id=uuid4(), - title="Чат", - member_ids=ids, - creator_id=ids[0], - created_at=MOMENT, - ) - - def test_chat_requires_creator_in_members() -> None: members = [uuid4(), uuid4()] with pytest.raises(InvalidChatData): @@ -137,87 +125,38 @@ def test_chat_rejects_more_than_hundred_members() -> None: ) -def test_outsider_cannot_post() -> None: - chat = make_chat(1) - with pytest.raises(ChatNotFound): - chat.post(message_id=uuid4(), sender_id=uuid4(), text="привет", now=MOMENT) - - -def test_post_never_moves_history_back() -> None: - chat = make_chat(1) - sender = chat.member_ids[0] - first = chat.post( - message_id=uuid4(), sender_id=sender, text="первое", now=MOMENT + timedelta(seconds=1) - ) - second = chat.post(message_id=uuid4(), sender_id=sender, text="второе", now=MOMENT) - assert second.created_at == first.created_at + timedelta(microseconds=1) - assert chat.last_message_at == second.created_at - - -def test_post_keeps_clock_when_it_is_ahead() -> None: - chat = make_chat(1) - sender = chat.member_ids[0] - first = chat.post(message_id=uuid4(), sender_id=sender, text="первое", now=MOMENT) - later = MOMENT + timedelta(seconds=5) - second = chat.post(message_id=uuid4(), sender_id=sender, text="второе", now=later) - assert first.created_at == MOMENT - assert second.created_at == later - - -def test_post_validates_membership_before_text() -> None: - chat = make_chat(1) - with pytest.raises(ChatNotFound): - chat.post(message_id=uuid4(), sender_id=uuid4(), text="", now=MOMENT) - - def test_empty_text_needs_an_attachment() -> None: - chat = make_chat(1) - sender = chat.member_ids[0] - with pytest.raises(InvalidMessageText): - chat.post(message_id=uuid4(), sender_id=sender, text="", now=MOMENT) ids = (uuid4(), uuid4()) - message = chat.post( - message_id=uuid4(), sender_id=sender, text="", now=MOMENT, attachment_ids=ids - ) - assert message.attachment_ids == ids with pytest.raises(InvalidMessageText): - chat.post(message_id=uuid4(), sender_id=sender, text=" ", now=MOMENT, attachment_ids=ids) + attachment_list(MessageText(""), ()) + assert attachment_list(MessageText(""), ids) == ids @pytest.mark.parametrize("ids", [[uuid4() for _ in range(11)], [UUID_TWICE, UUID_TWICE]]) def test_attachment_list_is_bounded_and_unique(ids: list[UUID]) -> None: - chat = make_chat(1) with pytest.raises(InvalidAttachmentList): - chat.post( - message_id=uuid4(), - sender_id=chat.member_ids[0], - text="x", - now=MOMENT, - attachment_ids=ids, - ) - - -def test_membership_is_checked_before_attachments() -> None: - chat = make_chat(1) - with pytest.raises(ChatNotFound): - chat.post( - message_id=uuid4(), - sender_id=uuid4(), - text="", - now=MOMENT, - attachment_ids=[uuid4() for _ in range(11)], - ) + attachment_list(MessageText("x"), ids) def test_receipt_replays_only_the_same_request() -> None: - chat = make_chat(1) - message = chat.post(message_id=uuid4(), sender_id=chat.member_ids[0], text="раз", now=MOMENT) + message = Message(uuid4(), uuid4(), uuid4(), MessageText("раз"), MOMENT) receipt = Receipt("key-1", b"a" * 32, message) assert receipt.replay(b"a" * 32) is message with pytest.raises(IdempotencyConflict): receipt.replay(b"b" * 32) +def test_posted_message_describes_itself() -> None: + message = Message(uuid4(), uuid4(), uuid4(), MessageText("пять"), MOMENT) + event = message.posted() + assert (event.message_id, event.chat_id, event.sender_id) == ( + message.id, + message.chat_id, + message.sender_id, + ) + assert (event.text_length, event.created_at, event.name) == (4, MOMENT, "message.posted") + + def test_user_registration_emits_event() -> None: user, event = User.register( user_id=uuid4(), diff --git a/student-tests/unit/test_probe.py b/student-tests/unit/test_probe.py new file mode 100644 index 0000000..c0764d6 --- /dev/null +++ b/student-tests/unit/test_probe.py @@ -0,0 +1,83 @@ +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + +SPEC = importlib.util.spec_from_file_location( + "grade4_probe", Path(__file__).resolve().parents[2] / "scripts/probe.py" +) +assert SPEC is not None and SPEC.loader is not None +probe = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(probe) +ARGS = SimpleNamespace( + label="old", command=[], expect="replace", max_failed_ratio=0.01, max_recovery=10 +) + + +def result_of(probes: list[tuple]) -> dict: + result = probe.summarize( + ARGS, probes, [(99, 2), (101, 2)], {"start": 100, "end": 101, "exit": 0, "load_end": 120}, 2 + ) + result["k6"] = {"exit": 0, "failed_ratio": 0, "requests": 100, "dropped_iterations": 0} + result["history"] = {"exact_match": True, "unique": True} + return result + + +@pytest.mark.parametrize( + ("attempts", "recovery", "passed"), + [ + ([(100, 503, 20, "old")], None, False), + ([(100, 503, 20, "old"), (109, 201, 2000, "new")], 11, False), + ([(100, 503, 20, "old"), (101, 201, 500, "new")], 1.5, True), + ([(101, 201, 500, "new")], 0, True), + ([], 0, False), + ], +) +def test_recovery_requires_completed_success(attempts: list, recovery: float, passed: bool) -> None: + result = result_of([(99, 201, 1, "old"), *attempts]) + assert result["probe"]["recovery_s"] == recovery + assert probe.verdict(ARGS, result)["write_recovery_ok"] is passed + + +@pytest.mark.parametrize( + ("field", "value", "check"), + [("requests", 0, "load_delivered"), ("dropped_iterations", 1, "load_delivered")], +) +def test_incomplete_load_fails(field: str, value: int, check: str) -> None: + result = result_of([(99, 201, 1, "old"), (102, 201, 1, "new")]) + result["k6"][field] = value + assert not probe.verdict(ARGS, result)[check] + + +def test_missing_lb_sample_fails() -> None: + result = result_of([(99, 201, 1, "old"), (102, 201, 1, "new")]) + result["lb_samples_missing"] = 1 + assert not probe.verdict(ARGS, result)["lb_observed"] + + +@pytest.mark.parametrize("corruption", ["none", "missing", "duplicate", "changed", "http", "cycle"]) +def test_full_history_matches_confirmed_bodies( + monkeypatch: pytest.MonkeyPatch, corruption: str +) -> None: + messages = [{"id": str(n), "text": "original"} for n in range(101)] + stored = list(reversed(messages)) + if corruption == "missing": + stored.pop() + if corruption == "duplicate": + stored.append(stored[0]) + if corruption == "changed": + stored[0] = dict(stored[0], text="changed") + pages = iter( + [ + (200, "", {"items": stored[:100], "next_cursor": "next"}), + ( + 503 if corruption == "http" else 200, + "", + {"items": stored[100:], "next_cursor": "next" if corruption == "cycle" else None}, + ), + ] + ) + monkeypatch.setattr(probe, "call", lambda *_a, **_kw: next(pages)) + result = probe.history_check({"auth": {}}, "chat", messages) + assert (result["exact_match"] and result["unique"]) is (corruption == "none") diff --git a/student-tests/unit/test_reconcile_measurements.py b/student-tests/unit/test_reconcile_measurements.py new file mode 100644 index 0000000..4625782 --- /dev/null +++ b/student-tests/unit/test_reconcile_measurements.py @@ -0,0 +1,69 @@ +import importlib.util +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + + +@pytest.fixture +def track(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + script = Path(__file__).resolve().parents[2] / "scripts/reconcile_track.py" + spec = importlib.util.spec_from_file_location("reconcile_measurements", script) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("disturbed", "quiet", "accepted", "discarded"), + [ + ([False], True, True, 0), + ([True, False], True, True, 1), + ([True, True], True, False, 2), + ([True], False, False, 1), + ], +) +def test_only_clean_measurements_are_accepted( + track: ModuleType, + monkeypatch: pytest.MonkeyPatch, + *, + disturbed: list[bool], + quiet: bool, + accepted: bool, + discarded: int, +) -> None: + reports = [{"cost": {"cpu_seconds": 1}, "attempt": i} for i in range(len(disturbed))] + monkeypatch.setattr(track, "sample", lambda: None) + monkeypatch.setattr(track, "reconcile", Mock(side_effect=reports)) + monkeypatch.setattr(track, "screen", Mock(side_effect=[{"disturbed": d} for d in disturbed])) + monkeypatch.setattr(track, "wait_quiet", lambda *_: quiet) + chosen, rejected = track.measure([], {}, SimpleNamespace(retries=1), set()) + assert (chosen is not None) == accepted + assert len(rejected) == discarded + assert all(r["screen"]["disturbed"] for r in rejected) + if chosen is not None: + assert not chosen["screen"]["disturbed"] + assert chosen not in rejected + + +def test_existing_series_is_preserved_before_docker_access( + track: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("RECONCILE_EVIDENCE_DIR", str(tmp_path)) + folder = tmp_path / "cost" + folder.mkdir() + raw = folder / "n1000-shallow-1.json" + raw.write_text("previous evidence") + query = Mock(side_effect=AssertionError("must not access the stand")) + monkeypatch.setattr(track, "psql", query) + with pytest.raises(FileExistsError): + track.cost(SimpleNamespace(sizes=[1000])) + assert raw.read_text() == "previous evidence" + query.assert_not_called() + + +def test_quiet_deadline_is_not_success(track: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(track.time, "monotonic", lambda: 10) + assert not track.wait_quiet(SimpleNamespace(quiet_timeout=0), set()) diff --git a/student-tests/unit/test_render_pool.py b/student-tests/unit/test_render_pool.py index a22ec6d..ead86c1 100644 --- a/student-tests/unit/test_render_pool.py +++ b/student-tests/unit/test_render_pool.py @@ -20,7 +20,7 @@ LIMITS = RenderLimits(max_bytes=10_485_760, max_pixels=40_000_000) HOG_BYTES = 512 << 20 -def misbehaving(data: bytes, limits: RenderLimits) -> list[Rendition]: +def misbehaving(data: bytes, limits: RenderLimits, pipeline_version: int) -> list[Rendition]: if data == b"slow": time.sleep(30) if data == b"crash": @@ -29,7 +29,7 @@ def misbehaving(data: bytes, limits: RenderLimits) -> list[Rendition]: raise MemoryError if data == b"hog": bytearray(HOG_BYTES) - return render_variants(data, limits) + return render_variants(data, limits, pipeline_version) @pytest.fixture @@ -89,3 +89,11 @@ async def test_address_space_limit_applies_to_the_render_process( ) -> None: with pytest.raises(ImageTooDemanding): await renderer.render(b"hog", 1) + + +async def test_process_uses_the_requested_recipe(renderer: ProcessVariantRenderer) -> None: + first = await renderer.render(SAMPLE, 1) + second = await renderer.render(SAMPLE, 2) + assert second == render_variants(SAMPLE, LIMITS, 2) + assert first != second + assert await renderer.render(SAMPLE, 2) == second diff --git a/student-tests/unit/test_scaling.py b/student-tests/unit/test_scaling.py new file mode 100644 index 0000000..a862032 --- /dev/null +++ b/student-tests/unit/test_scaling.py @@ -0,0 +1,80 @@ +import importlib.util +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + + +@pytest.fixture +def scaling(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + scripts = Path(__file__).resolve().parents[2] / "scripts" + monkeypatch.syspath_prepend(str(scripts)) + spec = importlib.util.spec_from_file_location("scaling", scripts / "scaling.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_retry_preserves_previous_raw_data(scaling: ModuleType, tmp_path: Path) -> None: + base = {"variant": "base", "config": "1x1"} + first = scaling.attempt_folder(tmp_path, base, 100, 1) + (first / "k6.txt").write_text("failed attempt") + second = scaling.attempt_folder(tmp_path, base, 100, 1) + assert first != second + assert (first / "k6.txt").read_text() == "failed attempt" + + +@pytest.mark.parametrize("recover", [False, True]) +def test_disturbed_runs_never_enter_summary( + scaling: ModuleType, monkeypatch: pytest.MonkeyPatch, recover: bool +) -> None: + args = SimpleNamespace( + base_env={}, out="unused", retries=1, pause=0, rates="100", rejected=Mock() + ) + rejected = {"raw_dir": "attempt1", "disturbed": True, "saturated": False} + accepted = {"raw_dir": "attempt2", "disturbed": False, "saturated": False} + monkeypatch.setattr(scaling, "layout", lambda *_: ({}, {})) + monkeypatch.setattr(scaling, "apply", lambda *_: {}) + monkeypatch.setattr(scaling, "deployment", lambda *_: []) + monkeypatch.setattr(scaling, "wait_quiet", lambda *_: None) + monkeypatch.setattr( + scaling, "step", Mock(side_effect=[rejected, accepted if recover else rejected]) + ) + output = Mock() + if recover: + scaling.block(args, output, "1x1", "base", [1]) + output.writerows.assert_called_once_with([accepted]) + args.rejected.writerow.assert_called_once_with(rejected) + else: + with pytest.raises(SystemExit, match="exhausted retries"): + scaling.block(args, output, "1x1", "base", [1]) + output.writerows.assert_not_called() + assert args.rejected.writerow.call_count == 2 + + +@pytest.mark.parametrize( + ("fixed", "expected"), + [ + (True, [("4x1", [2]), ("8x0.5", [2]), ("4x1", [3]), ("8x0.5", [3])]), + (False, [("4x1", [2, 3]), ("8x0.5", [2, 3])]), + ], +) +def test_series_order( + scaling: ModuleType, + monkeypatch: pytest.MonkeyPatch, + fixed: bool, + expected: list[tuple[str, list[int]]], +) -> None: + args = SimpleNamespace( + first_repeat=2, + repeats=2, + configs="4x1,8x0.5", + variants=["pinned"], + rates="3040,4560" if fixed else None, + ) + block = Mock() + monkeypatch.setattr(scaling, "block", block) + scaling.series(args, Mock()) + assert [(call.args[2], list(call.args[4])) for call in block.call_args_list] == expected diff --git a/student-tests/unit/test_timeouts.py b/student-tests/unit/test_timeouts.py index 4050372..3e6e917 100644 --- a/student-tests/unit/test_timeouts.py +++ b/student-tests/unit/test_timeouts.py @@ -46,15 +46,16 @@ def test_timeouts_shrink_inwards_and_shutdown_fits_the_stop_budget() -> None: database = api["DB_ACQUIRE_TIMEOUT_SECONDS"] + api["DB_COMMAND_TIMEOUT_SECONDS"] drain, shutdown = api["DRAIN_SECONDS"], api["SHUTDOWN_TIMEOUT_SECONDS"] print( - f"обнаружение {detection} с (зависание {hung} с), окно {drain} с, остановка {shutdown} с," - f" хук {hook} с, stop_grace_period {grace} с; LB server {lb['timeout server']} с >" + f"обнаружение {detection} с (зависание {hung} с), окно {drain} с," + f" дообработка {shutdown - drain - 1} с, остановка {shutdown} с, хук {hook} с," + f" stop_grace_period {grace} с; LB server {lb['timeout server']} с >" f" запрос {api['REQUEST_TIMEOUT_SECONDS']} с > БД {database} с;" f" readiness {2 * PING_TIMEOUT_SECONDS} с < check {lb['timeout check']} с" ) assert lb["timeout server"] > api["REQUEST_TIMEOUT_SECONDS"] > database assert lb["timeout check"] > 2 * PING_TIMEOUT_SECONDS assert drain > detection - assert shutdown - drain >= database + assert shutdown - drain - 1 >= database assert shutdown < hook < grace < 10 assert lb["hard-stop-after"] < seconds(stand["lb"]["stop_grace_period"]) assert (detection, hung) == (1.5, 3.5) diff --git a/tools/plot_track1.py b/tools/plot_track1.py new file mode 100644 index 0000000..9c24ec3 --- /dev/null +++ b/tools/plot_track1.py @@ -0,0 +1,414 @@ +import argparse +import collections +import csv +import statistics +import subprocess +import time +from pathlib import Path + +import matplotlib.pyplot as plt + +ORDER = ("1x1", "2x1", "4x1", "8x1", "2x2", "1x4") +PALETTE = ("#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948") +MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*") +USL_POINTS = 3 +MIN_REPEATS = 3 +COMPONENTS = (("api", "API"), ("postgres", "PostgreSQL"), ("lb", "HAProxy"), ("k6", "k6")) +INK, MUTED, GRID, SURFACE = "#0b0b0b", "#52514e", "#e4e3df", "#fcfcfb" +STAMP = ( + f"date: {time.strftime('%FT%TZ', time.gmtime())} commit: " + + subprocess.check_output(["/usr/bin/env", "git", "rev-parse", "HEAD"], text=True).strip() +) +SUMMARY = [ + "series", + "variant", + "config", + "rates", + "capacity_rate", + "capacity_label", + "capacity_ok_rps", + "capacity_p95_ms", + "capacity_p99_ms", + "peak_rate", + "peak_ok_rps", + "peak_ok_min", + "peak_ok_max", + "peak_p95_ms", + "peak_failed_ratio", + "peak_dropped", + "cpu_api", + "cpu_api_limit", + "cpu_postgres", + "cpu_lb", + "cpu_k6", + "pg_active_avg", + "pg_lock_avg", + "pg_idle_tx_avg", + "pg_idle_avg", + "pool_total", + "pg_backends_max", + "lb_qcur_max", +] + +plt.rcParams.update( + { + "figure.facecolor": SURFACE, + "axes.facecolor": SURFACE, + "axes.edgecolor": MUTED, + "axes.labelcolor": MUTED, + "axes.titlecolor": INK, + "axes.titleweight": "bold", + "axes.grid": True, + "grid.color": GRID, + "grid.linewidth": 0.8, + "axes.spines.top": False, + "axes.spines.right": False, + "xtick.color": MUTED, + "ytick.color": MUTED, + "legend.frameon": False, + "lines.linewidth": 2, + "lines.markersize": 7, + "font.size": 10, + } +) + + +VARIANTS = { + "before": {"color": MUTED, "marker": "o"}, + "after": {"color": PALETTE[1], "marker": "s"}, +} + + +def style(name): + index = ORDER.index(name) if name in ORDER else len(ORDER) + return {"color": PALETTE[index], "marker": MARKERS[index]} + + +def load(paths): + groups = collections.defaultdict(lambda: collections.defaultdict(list)) + for path in paths: + with Path(path).open() as handle: + for row in csv.DictReader(line for line in handle if not line.startswith("#")): + if row["disturbed"] == "True": + raise ValueError(f"disturbed run in accepted sample: {path}") + key = (Path(path).parent.name, row["variant"], row["config"]) + groups[key][int(row["rate"])].append(row) + for key, rates in groups.items(): + for rate, rows in rates.items(): + if len(rows) < MIN_REPEATS or len({row["repeat"] for row in rows}) != len(rows): + raise ValueError(f"need at least three distinct repeats: {key}, {rate}") + return groups + + +def med(rows, field): + values = [float(r[field]) for r in rows] + middle = statistics.median(values) + return middle, middle - min(values), max(values) - middle + + +def label(key): + return key[2] if key[1] == "base" else f"{key[2]} {key[1]}" + + +def summarize(key, rates): + series, variant, config = key + sat = { + rate: 2 * sum(r["saturated"] == "True" for r in rows) > len(rows) + for rate, rows in rates.items() + } + capacity = max((rate for rate, bad in sat.items() if not bad), default=None) + peak = max(rates, key=lambda rate: med(rates[rate], "ok_rps")[0]) + top, first = rates[peak], rates[peak][0] + replicas, workers, pool = (int(first[f]) for f in ("replicas", "workers", "pool")) + row = { + "series": series, + "variant": variant, + "config": config, + "rates": " ".join(str(r) for r in sorted(rates)), + "peak_rate": peak, + "peak_ok_rps": med(top, "ok_rps")[0], + "peak_ok_min": min(float(r["ok_rps"]) for r in top), + "peak_ok_max": max(float(r["ok_rps"]) for r in top), + "peak_p95_ms": med(top, "p95_ms")[0], + "peak_failed_ratio": med(top, "failed_ratio")[0], + "peak_dropped": med(top, "dropped_iterations")[0], + "cpu_api_limit": round(replicas * float(first["cpus"]) * 100), + "pool_total": replicas * workers * pool, + } + for field in [ + "cpu_api", + "cpu_postgres", + "cpu_lb", + "cpu_k6", + "pg_active_avg", + "pg_lock_avg", + "pg_idle_tx_avg", + "pg_idle_avg", + "pg_backends_max", + "lb_qcur_max", + ]: + row[field] = round(med(top, field)[0], 2) + if capacity is not None: + for field, name in (("ok_rps", "ok_rps"), ("p95_ms", "p95_ms"), ("p99_ms", "p99_ms")): + row[f"capacity_{name}"] = med(rates[capacity], field)[0] + row["capacity_rate"] = capacity + row["capacity_label"] = ( + f"≥{capacity}" if capacity in (min(rates), max(rates)) else str(capacity) + ) + return row + + +def direct_label(axis, x, y, text): + axis.annotate( + text, (x, y), xytext=(6, 0), textcoords="offset points", va="center", color=INK, fontsize=9 + ) + + +def save(figure, path): + figure.tight_layout() + figure.savefig(path, dpi=150, metadata={"Description": STAMP}) + plt.close(figure) + + +def lines(axis, keys, groups, field, categorical=False): + for index, key in enumerate(keys): + rates = sorted(groups[key]) + values = [med(groups[key][rate], field) for rate in rates] + look = VARIANTS.get(key[1]) or style(key[2]) + if categorical: + look = {"color": PALETTE[index], "marker": MARKERS[index]} + axis.errorbar( + [index] if categorical else rates, + [v[0] for v in values], + yerr=[[v[1] for v in values], [v[2] for v in values]], + capsize=3, + label=label(key), + markeredgecolor=SURFACE, + markeredgewidth=1.5, + **look, + ) + if categorical: + axis.set_xticks(range(len(keys)), [label(k) for k in keys], rotation=25, ha="right") + + +def throughput(series, keys, groups, out): + figure, axes = plt.subplots(1, 3, figsize=(17, 4.8)) + top = max(rate for key in keys for rate in groups[key]) + categorical = len({rate for key in keys for rate in groups[key]}) == 1 + if not categorical: + axes[0].plot( + [0, top], [0, top], color=MUTED, linestyle="--", linewidth=1, label="RPS = нагрузка" + ) + lines(axes[0], keys, groups, "ok_rps", categorical) + axes[0].set_title("Успешные RPS") + for axis, field, title in ((axes[1], "p95_ms", "p95, мс"), (axes[2], "p99_ms", "p99, мс")): + lines(axis, keys, groups, field, categorical) + if not categorical: + axis.set_yscale("log") + axis.set_title(title) + for axis in axes: + axis.set_xlabel( + f"Вариант при {top} запросах/с" if categorical else "Предложенная нагрузка, запросов/с" + ) + if not categorical: + axis.legend(loc="best", fontsize=9) + figure.suptitle(f"{series}: медиана повторов, усы — min–max", color=INK) + save(figure, out / f"{series}-throughput.png") + + +def cpu(series, keys, groups, out): + figure, axes = plt.subplots(1, len(keys), figsize=(5 * len(keys), 4.2), squeeze=False) + for axis, key in zip(axes[0], keys, strict=True): + rates = sorted(groups[key]) + first = groups[key][rates[0]][0] + limit = int(first["replicas"]) * float(first["cpus"]) * 100 + for index, (field, name) in enumerate(COMPONENTS): + values = [med(groups[key][rate], f"cpu_{field}") for rate in rates] + axis.errorbar( + rates, + [v[0] for v in values], + yerr=[[v[1] for v in values], [v[2] for v in values]], + capsize=3, + color=PALETTE[index], + marker=MARKERS[index], + label=name, + ) + axis.axhline(limit, color=PALETTE[0], linestyle=":", linewidth=1.2, label="лимит API") + axis.axhline(200, color=PALETTE[1], linestyle=":", linewidth=1.2, label="лимит PostgreSQL") + axis.set_title(f"CPU, % ядра — {label(key)}") + axis.set_xlabel("Предложенная нагрузка, запросов/с") + axis.legend(fontsize=8, loc="best") + figure.suptitle("Медиана CPU по повторам, усы — min–max", color=INK) + save(figure, out / f"{series}-cpu.png") + + +def usl(points): + base = points[0][1] / points[0][0] + pairs = [(n - 1, n * (n - 1), n * base / x - 1) for n, x in points] + a, b, c = (sum(p[i] * p[j] for p in pairs) for i, j in ((0, 0), (0, 1), (1, 1))) + d, e = (sum(p[i] * p[2] for p in pairs) for i in (0, 1)) + det = a * c - b * b + sigma, kappa = (d * c - b * e) / det, (a * e - b * d) / det + peak = ((1 - sigma) / kappa) ** 0.5 if kappa > 0 else None + return base, sigma, kappa, peak + + +def scaling(rows, out): + by = {(r["series"], r["config"]): r for r in rows if r["variant"] == "base"} + growth = [ + by.get(("series-b", c)) or by.get(("series-a", c)) for c in ("1x1", "2x1", "4x1", "8x1") + ] + growth = [r for r in growth if r] + if not growth: + return + figure, axis = plt.subplots(figsize=(7.5, 4.8)) + cpus = [r["cpu_api_limit"] / 100 for r in growth] + unit = growth[0]["peak_ok_rps"] / cpus[0] + axis.plot(cpus, [unit * c for c in cpus], color=MUTED, linestyle="--", linewidth=1) + axis.annotate("линейный рост", (cpus[-1], unit * cpus[-1]), color=MUTED, fontsize=9) + fit = None + if len(growth) >= USL_POINTS: + fit = usl([(c, r["peak_ok_rps"]) for c, r in zip(cpus, growth, strict=True)]) + base, sigma, kappa, _ = fit + grid = [1 + i * (cpus[-1] - 1) / 60 for i in range(61)] + model = [base * n / (1 + sigma * (n - 1) + kappa * n * (n - 1)) for n in grid] + axis.plot( + grid, model, color=PALETTE[6], linewidth=1.5, label=f"USL: σ={sigma:.3f}, κ={kappa:.4f}" + ) + axis.errorbar( + cpus, + [r["peak_ok_rps"] for r in growth], + yerr=[ + [r["peak_ok_rps"] - r["peak_ok_min"] for r in growth], + [r["peak_ok_max"] - r["peak_ok_rps"] for r in growth], + ], + color=PALETTE[0], + marker="o", + capsize=3, + label="серия B: N × 1 CPU", + ) + for r, c in zip(growth, cpus, strict=True): + direct_label(axis, c, r["peak_ok_rps"], r["config"]) + for config in ("2x2", "1x4"): + r = by.get(("series-a", config)) + if r: + look = style(config) + axis.errorbar( + 4, + r["peak_ok_rps"], + yerr=[[r["peak_ok_rps"] - r["peak_ok_min"]], [r["peak_ok_max"] - r["peak_ok_rps"]]], + capsize=3, + linestyle="none", + label=f"серия A: {config}", + **look, + ) + after = [r for r in rows if r["variant"] == "after"] + for r in after: + width = r["cpu_api_limit"] / 100 + axis.errorbar( + width, + r["peak_ok_rps"], + yerr=[[r["peak_ok_rps"] - r["peak_ok_min"]], [r["peak_ok_max"] - r["peak_ok_rps"]]], + capsize=3, + linestyle="none", + label=f"{r['config']} после улучшения", + **VARIANTS["after"], + ) + axis.set_xlabel("Суммарный CPU реплик API") + axis.set_ylabel("Пик успешных RPS") + axis.set_title("Пропускная способность от ресурсов API") + axis.set_xticks(cpus) + axis.legend(loc="best", fontsize=9) + save(figure, out / "scaling.png") + return fit + + +def utilisation(rows, out): + picked = [r for r in rows if r["variant"] == "base" and r["series"] != "compare"] + seen, ordered = set(), [] + for config in ORDER: + for r in picked: + if r["config"] == config and config not in seen: + seen.add(config) + ordered.append(r) + if not ordered: + return + metrics = ( + ("CPU API, % лимита", lambda r: 100 * r["cpu_api"] / r["cpu_api_limit"]), + ("CPU PostgreSQL, % лимита", lambda r: r["cpu_postgres"] / 2), + ("CPU HAProxy, % лимита", lambda r: r["cpu_lb"]), + ("ждут блокировку, % пула", lambda r: 100 * r["pg_lock_avg"] / r["pool_total"]), + ) + figure, axis = plt.subplots(figsize=(10, 4.8)) + width = 0.8 / len(metrics) + for index, (name, value) in enumerate(metrics): + xs = [i + (index - 1.5) * width for i in range(len(ordered))] + axis.bar( + xs, + [value(r) for r in ordered], + width=width - 0.02, + color=PALETTE[index], + label=name, + edgecolor=SURFACE, + ) + axis.set_xticks(range(len(ordered)), [r["config"] for r in ordered]) + axis.axhline(100, color=MUTED, linestyle=":", linewidth=1) + axis.set_ylabel("%") + axis.set_ylim(0, 115) + axis.set_title("Загрузка звеньев в точке пика успешных RPS", pad=34) + axis.legend(fontsize=9, ncols=4, loc="lower left", bbox_to_anchor=(0, 1.0)) + save(figure, out / "utilisation.png") + + +def write(rows, out, fit): + with (out / "capacity.csv").open("w", newline="") as handle: + handle.write(f"# {STAMP}\n") + writer = csv.DictWriter(handle, fieldnames=SUMMARY, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + head = ( + "| серия | вариант | конфигурация | потолок по SLO, rps | p95 там, мс | пик успешных RPS " + "[min–max] | при нагрузке | CPU API / лимит | CPU PG | CPU LB | ждут блокировку / пул |" + ) + lines_ = [head, "|" + " --- |" * 11] + for r in rows: + lines_.append( + f"| {r['series']} | {r['variant']} | {r['config']} | {r.get('capacity_label', '—')} | " + f"{r.get('capacity_p95_ms', '—')} | {r['peak_ok_rps']:.0f} " + f"[{r['peak_ok_min']:.0f}–{r['peak_ok_max']:.0f}] | {r['peak_rate']} | " + f"{r['cpu_api']:.0f} / {r['cpu_api_limit']} | {r['cpu_postgres']:.0f} | " + f"{r['cpu_lb']:.0f} | {r['pg_lock_avg']:.1f} / {r['pool_total']} |" + ) + if fit: + _, sigma, kappa, peak = fit + best = f"{peak:.1f}" if peak else "—" + lines_.append( + f"\nОписательная USL-аппроксимация серии B: σ = {sigma:.4f}, κ = {kappa:.5f}, " + f"модельный N* = {best} CPU. Это не измеренный оптимум и не доказательство причины." + ) + (out / "capacity.md").write_text(STAMP + "\n\n" + "\n".join(lines_) + "\n") + print("\n".join(lines_)) + + +def main(): + parser = argparse.ArgumentParser(description="Графики и сводка трека 1 из summary.csv") + parser.add_argument("summaries", nargs="+") + parser.add_argument("--out", required=True) + args = parser.parse_args() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + groups = load(args.summaries) + rank = {name: index for index, name in enumerate(ORDER)} + keys = sorted(groups, key=lambda k: (k[0], k[1] != "before", rank.get(k[2], 99))) + rows = [summarize(key, groups[key]) for key in keys] + for series in dict.fromkeys(k[0] for k in keys): + chosen = [k for k in keys if k[0] == series] + throughput(series, chosen, groups, out) + cpu(series, chosen, groups, out) + fit = scaling(rows, out) + utilisation(rows, out) + write(rows, out, fit) + + +if __name__ == "__main__": + main() diff --git a/tools/summarize_reconciler.py b/tools/summarize_reconciler.py new file mode 100644 index 0000000..a61c630 --- /dev/null +++ b/tools/summarize_reconciler.py @@ -0,0 +1,111 @@ +import argparse +import json +import math +from pathlib import Path +from statistics import median + +SIZES = (1000, 10000, 30000, 100000) +REPEATS = 3 +LINEAR_BOUNDS = (7, 13) +RSS_BOUND = 1.1 +DEEP_ORDER = 10 + + +def main() -> None: + parser = argparse.ArgumentParser(description="Validate and compare complete reconciler series") + parser.add_argument("current", type=Path) + parser.add_argument("baseline", type=Path) + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args() + current = json.loads((args.current / "summary.json").read_text()) + baseline = json.loads(args.baseline.read_text()) + rows = {(r["objects"], r["mode"]): r for r in current["results"]} + old = {(r["objects"], r["mode"]): r for r in baseline["results"]} + expected = {(n, mode) for n in SIZES for mode in ("shallow", "deep")} | {(10000, "page100")} + if set(rows) != expected or any(r["runs"] != REPEATS for r in rows.values()): + parser.error("expected all nine points with three repeats") + comparisons = [] + for (n, mode), row in sorted(rows.items()): + runs = [] + for repeat in range(1, REPEATS + 1): + raw = json.loads((args.current / f"n{n}-{mode}-{repeat}.json").read_text()) + runs.append(raw) + pages = math.ceil(n / (100 if mode == "page100" else 1000)) + requests = { + "s3_list": pages, + "sql": pages, + "s3_head": 0, + "s3_delete": 0, + "s3_get": n if mode == "deep" else 0, + } + if ( + raw["cost"]["requests"] != requests + or raw["screen"]["disturbed"] + or raw["scanned"] != {"objects": n, "rows": n} + or raw["errors"] + or raw["exit_code"] + or raw["skipped"] + or any(raw["found"].values()) + or raw["commit"] != current["commit"] + ): + parser.error(f"invalid accepted measurement: {n}/{mode}/{repeat}") + times = row["seconds_all"] + if ( + times != [r["cost"]["seconds"] for r in runs] + or row["seconds_median"] != median(times) + or row["max_rss_mib_median"] != median(r["cost"]["max_rss_mib"] for r in runs) + ): + parser.error(f"summary differs from raw measurements: {n}/{mode}") + previous = old[(n, mode)] + comparisons.append( + { + **row, + "seconds_range": [min(times), max(times)], + "spread_percent": round((max(times) - min(times)) / row["seconds_median"] * 100, 2), + "historical_seconds": previous["seconds_median"], + "historical_rss_mib": previous["max_rss_mib_median"], + "time_ratio_to_historical": row["seconds_median"] / previous["seconds_median"], + "rss_ratio_to_historical": row["max_rss_mib_median"] + / previous["max_rss_mib_median"], + } + ) + ratio = rows[(100000, "shallow")]["seconds_median"] / rows[(10000, "shallow")]["seconds_median"] + rss = ( + rows[(100000, "shallow")]["max_rss_mib_median"] + / rows[(1000, "shallow")]["max_rss_mib_median"] + ) + deep = { + str(n): rows[(n, "deep")]["seconds_median"] / rows[(n, "shallow")]["seconds_median"] + for n in SIZES + } + result = { + "current_commit": current["commit"], + "baseline_commit": baseline["commit"], + "accepted": len(rows) * REPEATS, + "rejected_files": len(list((args.current / "disturbed").glob("*.json"))), + "requests_and_consistency_verified_for_every_raw_run": True, + "h2": { + "t100k_over_t10k": ratio, + "within_7_to_13": LINEAR_BOUNDS[0] <= ratio <= LINEAR_BOUNDS[1], + "rss100k_over_rss1k": rss, + "rss_growth_below_10_percent": rss < RSS_BOUND, + "deep_over_shallow": deep, + "deep_at_least_tenfold": all(v >= DEEP_ORDER for v in deep.values()), + }, + "comparison": comparisons, + "limitation": ( + "Historical comparison is not an isolated causal A/B experiment; " + "warm seeded data in Docker Desktop." + ), + } + with args.out.open("x") as output: + output.write(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print( + json.dumps( + {k: v for k, v in result.items() if k != "comparison"}, ensure_ascii=False, indent=2 + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tools/summarize_upload_memory.py b/tools/summarize_upload_memory.py new file mode 100644 index 0000000..e494c6f --- /dev/null +++ b/tools/summarize_upload_memory.py @@ -0,0 +1,50 @@ +import argparse +import json +import statistics +from collections import defaultdict +from pathlib import Path + +REPEATS = 3 + + +def spread(values: list[float]) -> dict[str, float]: + return {"median": statistics.median(values), "min": min(values), "max": max(values)} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Summarize three independent upload memory runs") + parser.add_argument("root", type=Path) + args = parser.parse_args() + groups = defaultdict(list) + for path in sorted(args.root.glob("run*/slots*.json")): + run = json.loads(path.read_text()) + key = (run["upload_concurrency"], round(run["body_bytes"] / 1048576), run["concurrency"]) + groups[key].append((path, run)) + expected = {(s, m, c) for s in (3, 64) for m in (1, 10) for c in (10, 20, 40)} + if set(groups) != expected or any(len(runs) != REPEATS for runs in groups.values()): + parser.error("expected 12 points with three repeats each") + summary = [] + for (slots, size, concurrency), rows in sorted(groups.items()): + runs = [r for _, r in rows] + summary.append( + { + "slots": slots, + "mib": size, + "concurrency": concurrency, + "files": [str(p.relative_to(args.root)) for p, _ in rows], + "commits": sorted({r["commit"] for r in runs}), + "status_codes": [r["status_codes"] for r in runs], + "api_peak_mib": spread( + [max(m["cgroup_memory_peak_mib"] for m in r["memory"].values()) for r in runs] + ), + "lb_peak_mib": spread([r["lb"]["cgroup_memory_peak_mib"] for r in runs]), + "p95_ms": spread([r["latency_ms"]["p95"] for r in runs]), + "api_oom": sum(m["oom_kill"] for r in runs for m in r["memory"].values()), + "lb_oom": sum(r["lb"]["oom_kill"] for r in runs), + } + ) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index 7ffd41b..51b2ea1 100644 --- a/uv.lock +++ b/uv.lock @@ -135,6 +135,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "alembic" +version = "1.16.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/ca/4dc52902cf3491892d464f5265a81e9dff094692c8a049a3ed6a05fe7ee8/alembic-1.16.5.tar.gz", hash = "sha256:a88bb7f6e513bd4301ecf4c7f2206fe93f9913f9b48dac3b78babde2d6fe765e", size = 1969868, upload-time = "2025-08-27T18:02:05.668Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/4a/4c61d4c84cfd9befb6fa08a702535b27b21fff08c946bc2f6139decbf7f7/alembic-1.16.5-py3-none-any.whl", hash = "sha256:e845dfe090c5ffa7b92593ae6687c5cb1a101e91fa53868497dbd79847f9dbe3", size = 247355, upload-time = "2025-08-27T18:02:07.37Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.5" @@ -523,6 +537,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/6d/78dc67a9e779e594b49bf62604034d3a6aa428a55afc729cafda4b7333e1/googleapis_common_protos-1.76.0-py3-none-any.whl", hash = "sha256:59e13344a9f3651aa201a03e2bee815a3bf43a4baf1f9ff8a9b62a840ce25fac", size = 307707, upload-time = "2026-10-22T23:43:10.445Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/6e/0091f175ccd02b02bc8811bbcbcc6ac2e980be116e3b2f7a736ca322bf84/greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575", size = 207653, upload-time = "2026-09-14T15:42:51.806Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/c0/d254544ae2b8bdd311aef000fafc02828c2771b17d994b3075620ea7cc6e/greenlet-3.5.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cddea1b8339451c2fb3388e138347b6126744f33b611bdb55b7357361cfef46", size = 295221, upload-time = "2026-09-14T14:25:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/18/18/eb54be16b9cc3971e09ca5b73334e1b8c804a4630d9addaaf218a4fe300f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c59acfa8eb73a1e0d484392dc002bdf001fd4ce73394e0132df3d1ab6093d7cb", size = 660992, upload-time = "2026-09-14T15:12:04.876Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b4/e193efe65671dcf294bc51fcc59efb52d154adf8612c4ea016da0d2c486c/greenlet-3.5.6-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3b4a01c6da07ef9f80d4fe8933b994bc99747bcea3eab0330a9c34d3c12655b", size = 673428, upload-time = "2026-09-14T15:20:45.756Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/631bb45fafde1dca782152377c0676d182ec924820064047f533a3627b28/greenlet-3.5.6-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd0b83bed3405b586a3133629f1d1a5bc7bfd64822a3b7ab342bdc68e6dbc61b", size = 677688, upload-time = "2026-09-14T15:25:09.279Z" }, + { url = "https://files.pythonhosted.org/packages/45/ac/28fa7a9e50f2859466214c4ac584d776db52c1604ad4dd158960a5af2a1f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a09d59bef1db94f384b5bcc2d523694d338f3df6b757aeeaf7baca5d0c0be88", size = 670773, upload-time = "2026-09-14T14:36:02.577Z" }, + { url = "https://files.pythonhosted.org/packages/40/30/2b0a73e68e1e18e30b601d0d183cfdfc2beca4de5a6843c630f0fc9fb90c/greenlet-3.5.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:fdacf26402389bdd89857ad3c045a26fe8f3314f9a8b28226f82f88463a65b77", size = 480475, upload-time = "2026-09-14T15:28:40.741Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cd/fb7d6cdd86ff3427c1494854f0e35437eba05142be91f530f6da75e09e19/greenlet-3.5.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8b7c73d1cef3d9ae963e9ff03f6222df43efbb9054ffd2f1969c935b7fc84c02", size = 1631900, upload-time = "2026-09-14T15:10:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/143bdbb20a516628cb15074ae52ed17d850b450292609c7a6fccac6dbece/greenlet-3.5.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8b27df301f56e3b3d2298095c8f7d6b68f2521f6b1693e901fa039bdbae34424", size = 1693740, upload-time = "2026-09-14T14:35:52.959Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/019642432e6ae283301df1361227d47610709d2dc69a38f95edef266d713/greenlet-3.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:f8f0bd690e1a41294ac87905e8121c81a3761ec2583c768f13467428606c8c7a", size = 327473, upload-time = "2026-09-14T14:28:12.948Z" }, + { url = "https://files.pythonhosted.org/packages/e9/7f/8aafc7bf70c948786dba7221d0dc0838e5329bebc6d434ef2208b4f0e760/greenlet-3.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:8cda13494d86a4f12429641117cb6ac4bbbc9c30a33f711f7d3a2e5fbe4b0b7e", size = 311095, upload-time = "2026-09-14T14:28:00.7Z" }, + { url = "https://files.pythonhosted.org/packages/14/7e/7a205688a5b3074933b18a906608d46d106e9a79d776bdab5a4abf4b4feb/greenlet-3.5.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:97c5a53e8c1754df58e73f047a99e287d4da1bdfe64b0072fb25c87000897951", size = 305352, upload-time = "2026-09-14T14:21:31.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/cb/9c4a57a9d9dd0256e20b8f7f4f06554c2c92badebf0ab73ce344321b78b9/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fea4427d1ffdb3b523d7daa6712038428a4c16c450b9777bdd1221cfee0eab49", size = 672671, upload-time = "2026-09-14T15:12:06.347Z" }, + { url = "https://files.pythonhosted.org/packages/97/52/c6729681ebbd298f4decd28746815acc8a0b0a0fde21d2df33776fd4d042/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73a29b5ba642e35433166a03a3e02935e7238c4b3467fbd77523b99edea23e5b", size = 679489, upload-time = "2026-09-14T15:20:47.291Z" }, + { url = "https://files.pythonhosted.org/packages/71/76/3c11c21e0716b1f1dc7c1a4b3d690abb1d3b448c69a9d32049fecb64010a/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:61a61b4a95a4f97922c3a6f5606d3e360851584bd47e500a5161373c53810e3d", size = 681303, upload-time = "2026-09-14T15:25:11.088Z" }, + { url = "https://files.pythonhosted.org/packages/58/c5/2b6c721ba8b8963da42d5a0f57f25b8aaeb1fe9bdd156875e57f3be648a2/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:460e70b033aba8ed47e2ac9b5d0d2157b05a34fbfa30a241400aef4118902cdc", size = 676608, upload-time = "2026-09-14T14:36:03.959Z" }, + { url = "https://files.pythonhosted.org/packages/3f/26/3ae402202452cd5941bbbd483e5a74297e2397e7aa3182c2a5e3ab7d5666/greenlet-3.5.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:fe3170a69fe039b18ad18171e66faa9a75f6fe9d78f968fd9b54e09fbd714d81", size = 510112, upload-time = "2026-09-14T15:28:42.112Z" }, + { url = "https://files.pythonhosted.org/packages/b2/04/0d018e0d05bcdde19a0fcb907834155f1fc853a9bedd3f3f5e6acadcae19/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ca80a49b53ed1d22f7282da7255f7bb2fd1935fd0f623d8613fda38745f18961", size = 1641479, upload-time = "2026-09-14T15:10:11.216Z" }, + { url = "https://files.pythonhosted.org/packages/59/bb/f02ef9073919158f6403fe3701d4ed4403d646720e7201dfc6e9d264bac3/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:916f92f2a8db10508f739d0b5e00b83defe5d1115a997c54532a6d7cf8c95404", size = 1698758, upload-time = "2026-09-14T14:35:54.336Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/1f48fe647473a2dcccfd1839b2ff2c78eb57009be776b4da071e901c9bff/greenlet-3.5.6-cp314-cp314t-win_amd64.whl", hash = "sha256:886bcf1870af74c32bc310fd00a6b803445e17e51b7d5a107c7b35c0f362cc16", size = 331574, upload-time = "2026-09-14T14:27:18.451Z" }, +] + [[package]] name = "gunicorn" version = "26.2.0" @@ -618,12 +659,136 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-path" +version = "0.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pathable" }, + { name = "pyyaml" }, + { name = "referencing" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/45/41ebc679c2a4fced6a722f624c18d658dee42612b83ea24c1caf7c0eb3a8/jsonschema_path-0.3.4.tar.gz", hash = "sha256:8365356039f16cc65fddffafda5f58766e34bebab7d6d105616ab52bc4297001", size = 11159, upload-time = "2025-01-24T14:33:16.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/58/3485da8cb93d2f393bce453adeef16896751f14ba3e2024bc21dc9597646/jsonschema_path-0.3.4-py3-none-any.whl", hash = "sha256:f502191fdc2b22050f9a81c9237be9d27145b9001c55842bece5e94e382e52f8", size = 14810, upload-time = "2025-01-24T14:33:14.652Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "lazy-object-proxy" +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/8e/f26b09778fcb26261fdeb4cc2b011975292316529bff9b541a5a02acb976/lazy_object_proxy-1.13.0.tar.gz", hash = "sha256:ce34046cb021d69c72d05ddb83821ecfd4d2d0a5d43a93c5cd01c9e34779cf14", size = 45649, upload-time = "2026-10-17T16:38:01.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/0b/410ea9a56ad59421d8969535099560f0bad7162d483869e1c07ff5880f35/lazy_object_proxy-1.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:62612150e2b67d1cb26c2201d020773e9bab9c596d783d55ada42885a695f749", size = 26856, upload-time = "2026-10-17T16:37:48.864Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/79ab37585905cf6841134806e73ace576d61a87c4eee333be4622161b23f/lazy_object_proxy-1.13.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ad7630266b5775463f2a1806165c1d9dff07780cf1bba3c53f311e1d026e8921", size = 70985, upload-time = "2026-10-20T01:05:49.274Z" }, + { url = "https://files.pythonhosted.org/packages/18/2e/66bbb45f20aab9e81139e2bf9fea1a29850b5af769cbf7f181221ce062dc/lazy_object_proxy-1.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73f05b09e37b0eb0ca7d6e80e2b0689c49bedac8c5d69667c1ebb9f02a4ed5c7", size = 70984, upload-time = "2026-10-20T01:05:50.548Z" }, + { url = "https://files.pythonhosted.org/packages/74/e0/63dc39750c7b266e9e25022962eba32ba80e4f17fdcd674fc4ad4143fd90/lazy_object_proxy-1.13.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:91a725d3c64dfe13d365cf78c469c2a8f95884859379aeca89cb8d2ed291649a", size = 65689, upload-time = "2026-10-20T01:05:51.765Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6c/b94a8779eec1331477557ca3ea4742860d59dc9bcd4dcf21bcef3cd8c4f2/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e28a75d33378141927c08f3eb291cc98d0a22776670ae330f8347f898cd3b175", size = 69492, upload-time = "2026-10-20T01:05:53.116Z" }, + { url = "https://files.pythonhosted.org/packages/60/99/6c3ff4a32f15e807ffb868c47de136ebd4be31e7320af42b699dc43a7495/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a785fe44b61fcbffc0e264fe6b735a909e0b5ef922f0e976e281bbe8477fb47a", size = 65019, upload-time = "2026-10-20T01:05:54.462Z" }, + { url = "https://files.pythonhosted.org/packages/c4/1a/447ff40629a11f879f194e839d59e5a026dd26f4d0eb8672de858644be85/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e321425f76451885c93083ecec0850b64de0f83bb94c113e930e8a0a630e45ab", size = 70334, upload-time = "2026-10-20T01:05:55.619Z" }, + { url = "https://files.pythonhosted.org/packages/70/52/2ca84fb914b17d4b02193cbf8b6ba9344b36459944a19711d3da1d14b03d/lazy_object_proxy-1.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:0e52094b7fff1f26f4e227b88be3f350817078e4bb79b4c80846fca0d15efae4", size = 26122, upload-time = "2026-10-17T16:37:53.02Z" }, + { url = "https://files.pythonhosted.org/packages/84/c7/ebbeb84639e84fbf6f07a48373f76fe048d23fbcfd4557509b48f35d284d/lazy_object_proxy-1.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:b2dd1c37d64cf9f3f4a2072971639408989b0803a3b7f05e75c1fe28d1a93c1f", size = 24746, upload-time = "2026-10-17T16:37:53.954Z" }, + { url = "https://files.pythonhosted.org/packages/38/ac/8ff35cc4d1316e056144edfb816dbff9b5db9ccd7e98c69f6b602c11bc29/lazy_object_proxy-1.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:348370676ea6d9ed08f9b74545c37e5be4cf009c45b9d50c6aff476fb7fcc7ec", size = 27524, upload-time = "2026-10-17T16:37:49.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/c5/8afe34ead41fdd48a207c4535192a4e931c56c791d5f0a57c0abd4558c1c/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:dee2fb3326cd81384d2b752c3f6888d7ed9eba2cdbf0583097b5168b8bb4a703", size = 80523, upload-time = "2026-10-20T01:05:57.129Z" }, + { url = "https://files.pythonhosted.org/packages/38/fd/0e68a385daaf141b1b1d943b132b8a9887bd53abb88886dab286cca9b62e/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b4c446cbc8e411b116c9ee9b475cb664b8bc5c3d912bcbf1bab5c785fc69a02", size = 82823, upload-time = "2026-10-20T01:05:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/91/ce/648e09f90dc59cb6007d9093c2cc17d72e2d93c06553828037126205ea7c/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1e671e0e9f2793759f382a69a3853c8742474bc011865ed3595187340ac5956d", size = 74655, upload-time = "2026-10-20T01:05:59.717Z" }, + { url = "https://files.pythonhosted.org/packages/10/4d/811442390862dbf85e00520ecb0cb58c7dbdf824dc563ad933729f9e20f3/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a0155084a5dc8ce3e1a3d6e3f2fb4d84d5fc09e66ed84dc0e5f8a2697d421ea0", size = 80848, upload-time = "2026-10-20T01:06:00.884Z" }, + { url = "https://files.pythonhosted.org/packages/24/4e/c30ea3678b50a7ed7774760fbde917106cbb860edb88cc31a17a8385b3d4/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2d07e0bde2738922a7117fccc1a82514aefbc9c07ee8ff0bb84a1c606dbb5ac8", size = 74127, upload-time = "2026-10-20T01:06:01.949Z" }, + { url = "https://files.pythonhosted.org/packages/19/d0/862f0f96b031dab7faa036679e2076bbd6a18ea06e34fb1400fca65b03f0/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da9f1b8780678af93c4ddba46826166c53a25d8d71133fc818542bf7b481e033", size = 79497, upload-time = "2026-10-20T01:06:03.122Z" }, + { url = "https://files.pythonhosted.org/packages/d5/50/20454483d8f95a11710b951518f86aaaaaae9195982fe27c12eeed130660/lazy_object_proxy-1.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f21b4711e6744d46a4672497ca2db390cbdb03241d9b1654bbf38dbbc4db2a5e", size = 26670, upload-time = "2026-10-17T16:37:50.977Z" }, + { url = "https://files.pythonhosted.org/packages/4a/df/c1ff843fd9c4978d9a50ef2bceda3897d71645d4127ec1413aa15204cb64/lazy_object_proxy-1.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:de295ae25e5c1fd712088547f251c4b2976d2f60df28057b6c416e771a99842b", size = 25297, upload-time = "2026-10-17T16:37:52.088Z" }, +] + +[[package]] +name = "mako" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/09/e07c4b5579a79f4b16f8d4f29f6c54514ac787c4ad506b8c4f28a0e6b0bf/mako-1.4.3.tar.gz", hash = "sha256:cd6537fe88d5fec315c55c2f8529bc4ce7a9a352ad7db3eeaa6a66e2dd4ec37a", size = 412799, upload-time = "2026-09-22T20:54:31.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/a0/053d6af3e8f871e0073b4a36732d9e65be77a72e5434c31b94f6af78a6bb/mako-1.4.3-py3-none-any.whl", hash = "sha256:723296007c870bfd6b3f0c3230dba7198096e5269297ebf5e4eff9e7ffa39d4f", size = 80164, upload-time = "2026-09-22T20:54:33.128Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/c3/a944f3b0df22bd129e96915b9f4e98d2eeca6516687d7618304a966c3c74/markupsafe-3.0.4-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed", size = 12787, upload-time = "2026-10-02T23:05:30.971Z" }, + { url = "https://files.pythonhosted.org/packages/d4/d6/a44863f69d88b6c7e27889108f70d47aed259edf89d5df3c5fca1eac87d6/markupsafe-3.0.4-cp314-cp314-android_24_x86_64.whl", hash = "sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59", size = 12525, upload-time = "2026-10-02T23:05:32.263Z" }, + { url = "https://files.pythonhosted.org/packages/17/8f/168ba80e532dd6a93f96f8f706f1ad41d7990b6e1aeedc1cc0d211a33497/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453", size = 11589, upload-time = "2026-10-02T23:05:33.251Z" }, + { url = "https://files.pythonhosted.org/packages/32/b3/aa2c95a574d3af39403a469b295886eb9b6d448da568cbebb5a2cbfdc2e5/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b", size = 11801, upload-time = "2026-10-02T23:05:34.315Z" }, + { url = "https://files.pythonhosted.org/packages/60/d0/34b810107d83840e768bf485de795893ebbae35b26ab061b487adfa0a692/markupsafe-3.0.4-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6", size = 11447, upload-time = "2026-10-02T23:05:35.302Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ab/2f8488f0f817a39fca068d2b17daf446bf5cdb3eae28c3720af534d873b4/markupsafe-3.0.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634", size = 11722, upload-time = "2026-10-02T23:05:36.363Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/e2d117b048d47282ade906fbfd92814cbee5647afc13fda88a3406039372/markupsafe-3.0.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f", size = 12061, upload-time = "2026-10-02T23:05:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a8/73a81135e85ba66217f5af7facb03bbb386807e1a729ab64532e4c802652/markupsafe-3.0.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9", size = 24434, upload-time = "2026-10-02T23:05:38.407Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ca/fa9216dd01efee2dfdacafe7df32b4d0170fbac694b0c258a193d6e53999/markupsafe-3.0.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f", size = 27183, upload-time = "2026-10-02T23:05:39.581Z" }, + { url = "https://files.pythonhosted.org/packages/fa/4e/a469509e538d37af51103b17b073126973f2b1cbf197ff32c7ddf025cfe5/markupsafe-3.0.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c", size = 24813, upload-time = "2026-10-02T23:05:40.671Z" }, + { url = "https://files.pythonhosted.org/packages/8f/db/d7282caf7ab03af44d5d6fdbaa019b35c7d7f1c90588b839c07cba640d6a/markupsafe-3.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300", size = 23049, upload-time = "2026-10-02T23:05:41.864Z" }, + { url = "https://files.pythonhosted.org/packages/30/f3/b6a425206e6964efda6acee544d0eb01d1501784d0b8e2dcc74986f33b17/markupsafe-3.0.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0", size = 22088, upload-time = "2026-10-02T23:05:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8a/84d3582fc1f0d5bd466cdf2eebf175e172158a6e70701aacec1de1b35430/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977", size = 23846, upload-time = "2026-10-02T23:05:44.098Z" }, + { url = "https://files.pythonhosted.org/packages/1c/65/db101cce51b7ba4864ac491a9859d297dd1adf0e55b103fee9db9c47c527/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7", size = 22674, upload-time = "2026-10-02T23:05:45.23Z" }, + { url = "https://files.pythonhosted.org/packages/e0/49/ddee9813d71db0c7a5c9d97c832125e6758a0c844777f1cf076569bb0e22/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17", size = 24104, upload-time = "2026-10-02T23:05:46.398Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0e/7d8518d726726870a2399d69fd30d0fa36c5e57a2132c336b58d7c491073/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c", size = 21637, upload-time = "2026-10-02T23:05:47.48Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b0/b505e8a361ba557dbf3b3aa7331ea39b00d2022a26e925ff8463b9714bb3/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4", size = 23081, upload-time = "2026-10-02T23:05:48.611Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ea/9cc3cea873f980c75cbdb6f4277ce30ee955de38be0b3d02f14c108e0698/markupsafe-3.0.4-cp314-cp314-win32.whl", hash = "sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c", size = 14272, upload-time = "2026-10-02T23:05:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/80/f0/5792ff768a410f93ee3f84fc19345295ffc352d2c936b424cb37e514714c/markupsafe-3.0.4-cp314-cp314-win_amd64.whl", hash = "sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe", size = 14501, upload-time = "2026-10-02T23:05:50.788Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cf/3d074a8edffcc6899355232ff2543ae8d929733239596423b7db79698bc9/markupsafe-3.0.4-cp314-cp314-win_arm64.whl", hash = "sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a", size = 14364, upload-time = "2026-10-02T23:05:51.857Z" }, + { url = "https://files.pythonhosted.org/packages/d9/31/87ce42159aae2163cf3bbbd0c44bc87780510eecab1ea3859099aed95dcb/markupsafe-3.0.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2", size = 11759, upload-time = "2026-10-02T23:05:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/5f/53/b047207eeb7752e960aca3eb1df5fb7eefa7dd4c62ac49bb156456c8a702/markupsafe-3.0.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977", size = 12096, upload-time = "2026-10-02T23:05:54.066Z" }, + { url = "https://files.pythonhosted.org/packages/ee/51/4326c88a13c7b755657d44b4bb986f8c3d9843ecba7e22d98661d87f9a57/markupsafe-3.0.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289", size = 25699, upload-time = "2026-10-02T23:05:55.15Z" }, + { url = "https://files.pythonhosted.org/packages/f2/bb/990581b7474bfcf2cf34bed6ba5ea23bd87adb9d671213d68e88620e7a6b/markupsafe-3.0.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe", size = 27804, upload-time = "2026-10-02T23:05:56.29Z" }, + { url = "https://files.pythonhosted.org/packages/6b/89/89491878c28e8291f5aa2fffe2c2d57230d10ae366d55dd810b840513d78/markupsafe-3.0.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a", size = 25912, upload-time = "2026-10-02T23:05:57.416Z" }, + { url = "https://files.pythonhosted.org/packages/30/77/680998b54efdea06fc114565cd739b6d059f826a0279219b218dfa750d29/markupsafe-3.0.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733", size = 23623, upload-time = "2026-10-02T23:05:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/ae/75/2709f5ac5de9467b40b10e2bb8f89cc63dfb74582e09aa734b1124a217de/markupsafe-3.0.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34", size = 23301, upload-time = "2026-10-02T23:05:59.94Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c8/39eadc6c5b14c9c7679bfb98f4d4c6a97863b5beb91839aca4d2d6e16e55/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978", size = 24950, upload-time = "2026-10-02T23:06:01.289Z" }, + { url = "https://files.pythonhosted.org/packages/1a/5e/01037f8a43e8ccb0bffb4fbdc5212db05bf080fdd7286cd392332d58128a/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc", size = 23330, upload-time = "2026-10-02T23:06:02.441Z" }, + { url = "https://files.pythonhosted.org/packages/d4/f4/23e83ce0596bb0cbe670502d31df8f757bbd01a392aa486fa3b40d1ed399/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc", size = 25239, upload-time = "2026-10-02T23:06:03.579Z" }, + { url = "https://files.pythonhosted.org/packages/88/5b/3708897368073cc683d524750474f41a77d2986152c380dcc55b20fdf340/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932", size = 22767, upload-time = "2026-10-02T23:06:04.699Z" }, + { url = "https://files.pythonhosted.org/packages/c6/61/ebda1307864b409e6b3115757a3d4a09cca46cfb6cc65191b5de226b424b/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6", size = 23668, upload-time = "2026-10-02T23:06:05.9Z" }, + { url = "https://files.pythonhosted.org/packages/09/15/98075cceac3b5ba0dbb8e4762a847be967d2befc349a2cf2d0ac77f62c9d/markupsafe-3.0.4-cp314-cp314t-win32.whl", hash = "sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691", size = 14301, upload-time = "2026-10-02T23:06:07.109Z" }, + { url = "https://files.pythonhosted.org/packages/0b/a3/768b560fcc4156685cb563d922b217810cfa7bc135773367f62f1f9d2078/markupsafe-3.0.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464", size = 14526, upload-time = "2026-10-02T23:06:08.276Z" }, + { url = "https://files.pythonhosted.org/packages/93/63/da554b4c97a6b0ea3229ca7fe8cbfb620be81613d517f482e85958550537/markupsafe-3.0.4-cp314-cp314t-win_arm64.whl", hash = "sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c", size = 14421, upload-time = "2026-10-02T23:06:09.402Z" }, +] + [[package]] name = "mtusi-messenger" version = "1.6.0" source = { editable = "." } dependencies = [ { name = "aiobotocore" }, + { name = "alembic" }, { name = "argon2-cffi" }, { name = "asyncpg" }, { name = "dishka" }, @@ -641,6 +806,7 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "redis" }, + { name = "sqlalchemy", extra = ["asyncio"] }, { name = "structlog" }, { name = "uvicorn", extra = ["standard"] }, { name = "uvicorn-worker" }, @@ -649,6 +815,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "httpx" }, + { name = "openapi-spec-validator" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "ruff" }, @@ -658,6 +825,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "aiobotocore", specifier = ">=3.10.0" }, + { name = "alembic", specifier = "==1.16.5" }, { name = "argon2-cffi", specifier = ">=25.1.0" }, { name = "asyncpg", specifier = ">=0.32.0" }, { name = "dishka", specifier = ">=1.7.0" }, @@ -675,6 +843,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.11.0" }, { name = "pydantic-settings", specifier = ">=2.11.0" }, { name = "redis", specifier = ">=8.1.0" }, + { name = "sqlalchemy", extras = ["asyncio"], specifier = "==2.0.45" }, { name = "structlog", specifier = ">=26.1.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.37.0" }, { name = "uvicorn-worker", specifier = ">=0.4.0" }, @@ -683,6 +852,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "httpx", specifier = ">=0.28.0" }, + { name = "openapi-spec-validator", specifier = "==0.7.2" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-asyncio", specifier = ">=1.2.0" }, { name = "ruff", specifier = ">=0.14.0" }, @@ -741,6 +911,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/96/1aaad0e5df33f1f5af21775c411349c0234a6edd581ea403bc81919a6eab/multidict-7.2.1-py3-none-any.whl", hash = "sha256:08ea94663d9e7c25d14d78222f7aee906c472d4837766bec62477c38f6e419dc", size = 28145, upload-time = "2026-10-24T20:40:59.159Z" }, ] +[[package]] +name = "openapi-schema-validator" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "jsonschema-specifications" }, + { name = "rfc3339-validator" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/f3/5507ad3325169347cd8ced61c232ff3df70e2b250c49f0fe140edb4973c6/openapi_schema_validator-0.6.3.tar.gz", hash = "sha256:f37bace4fc2a5d96692f4f8b31dc0f8d7400fd04f3a937798eaf880d425de6ee", size = 11550, upload-time = "2025-01-10T18:08:22.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/c6/ad0fba32775ae749016829dace42ed80f4407b171da41313d1a3a5f102e4/openapi_schema_validator-0.6.3-py3-none-any.whl", hash = "sha256:f3b9870f4e556b5a62a1c39da72a6b4b16f3ad9c73dc80084b1b11e74ba148a3", size = 8755, upload-time = "2025-01-10T18:08:19.758Z" }, +] + +[[package]] +name = "openapi-spec-validator" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "jsonschema-path" }, + { name = "lazy-object-proxy" }, + { name = "openapi-schema-validator" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/af/fe2d7618d6eae6fb3a82766a44ed87cd8d6d82b4564ed1c7cfb0f6378e91/openapi_spec_validator-0.7.2.tar.gz", hash = "sha256:cc029309b5c5dbc7859df0372d55e9d1ff43e96d678b9ba087f7c56fc586f734", size = 36855, upload-time = "2025-06-07T14:48:56.299Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/dd/b3fd642260cb17532f66cc1e8250f3507d1e580483e209dc1e9d13bd980d/openapi_spec_validator-0.7.2-py3-none-any.whl", hash = "sha256:4bbdc0894ec85f1d1bea1d6d9c8b2c3c8d7ccaa13577ef40da9c006c9fd0eb60", size = 39713, upload-time = "2025-06-07T14:48:54.077Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.45.1" @@ -960,6 +1159,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/14/1dfc08b743ba995a38dee0ea09beb46a05c7fe8ac53d729095905f7bf11d/pamqp-4.0.1-py3-none-any.whl", hash = "sha256:a547f45128b06e42ce8d7a739b0cfcc40f2c724770622eaaff4a3f587b1cf7d0", size = 32773, upload-time = "2026-07-06T16:37:50.623Z" }, ] +[[package]] +name = "pathable" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/67/93/8f2c2075b180c12c1e9f6a09d1a985bc2036906b13dff1d8917e395f2048/pathable-0.4.4.tar.gz", hash = "sha256:6905a3cd17804edfac7875b5f6c9142a218c7caef78693c2dbbbfbac186d88b2", size = 8124, upload-time = "2025-01-10T18:43:13.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/eb/b6260b31b1a96386c0a880edebe26f89669098acea8e0318bff6adb378fd/pathable-0.4.4-py3-none-any.whl", hash = "sha256:5ae9e94793b6ef5a4cbe0a7ce9dbbefc1eec38df253763fd0aeeacf2762dbbc2", size = 9592, upload-time = "2025-01-10T18:43:11.88Z" }, +] + [[package]] name = "pillow" version = "13.0.0" @@ -1228,6 +1436,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, ] +[[package]] +name = "referencing" +version = "0.36.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0", size = 26775, upload-time = "2025-01-25T08:48:14.241Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1243,6 +1464,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa", size = 3490, upload-time = "2021-05-12T16:37:52.536Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz", hash = "sha256:4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12", size = 63948, upload-time = "2026-10-04T16:32:36.469Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/f2/67b94be1532767803415c1c5a1fd88ea487643d74a673cec1ba140af77bb/rpds_py-2026.9.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c9d1aca01f49170fdcf5c92761b1fafe97f554b721ca4570c5949fff778f0d4b", size = 347178, upload-time = "2026-10-04T16:30:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/04/37/b751de2b59b0197a1d92a5dd491de88e8a5e928c2e6562581974f1e85263/rpds_py-2026.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3f0e9ac28fc067d4d34b88ae43c48e9489455c97fee9633d851f7eeed5a05d35", size = 341878, upload-time = "2026-10-04T16:30:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/72/e2/5873bc4643c250db9e05d48dc0c93763d4aa68bc3b81164cb1af3b45b284/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07deecbfce94c78473018bc7d10b337cc651d12df87a1eb2cb3e4024bc9c33d0", size = 373963, upload-time = "2026-10-04T16:30:04.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/03/5acf7632158247f3f6386ff0af3a1ee48167d575037e8d0920594b76d92b/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:821b2755db9194409254012f429c56643416fb96ef9be090be82ec8826b7f477", size = 377975, upload-time = "2026-10-04T16:30:05.555Z" }, + { url = "https://files.pythonhosted.org/packages/e6/00/63fda451b8bffa5808fc8bb311ee7c073b340974b09f273c7b2a145d3d62/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c91c210ae7645626c608400e3519b4a642f837cce09ca830db3beb2e9f274d4", size = 490347, upload-time = "2026-10-04T16:30:07.156Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ae/c093ffd070ba0fb02f76c565d06fecc65ad6e4afdbae78f7031076d3cdac/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54ac2158a6f96cfbabff0b2eedaf94b90c5ec7ca8317fcadc61e1c2b2e0ff6ef", size = 392962, upload-time = "2026-10-04T16:30:08.77Z" }, + { url = "https://files.pythonhosted.org/packages/22/9d/d08a1128ab199b2f0cf25bfeb0639bd05119fff4b7c47bec24ef9a8ec23f/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd", size = 371623, upload-time = "2026-10-04T16:30:10.501Z" }, + { url = "https://files.pythonhosted.org/packages/53/c7/4758ddcbb75609414bbccfcb11d612436f9b3ee821bd2f33f0f1604ee648/rpds_py-2026.9.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:8aa5dda18d39b6143eb24809d158f9252c88f402749b6f1b62a506cc7d96cc35", size = 376997, upload-time = "2026-10-04T16:30:12.124Z" }, + { url = "https://files.pythonhosted.org/packages/87/e4/947bd7f608ff60faf46dc9d389c3dffd0e3d767d78a0be19978448ef0ce7/rpds_py-2026.9.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5c90e7fa02e8f5de0d10c17595c568ada48c5302e749462c0ea1a4c362111a86", size = 400169, upload-time = "2026-10-04T16:30:13.804Z" }, + { url = "https://files.pythonhosted.org/packages/4b/35/fe93e020a0543b5670472c18d7e6af3197c08da571240ce1965c84f85c0f/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6d198bad4e49dd6732fbd636e2fc5c082f45c8cad0b4acb756b00c82c76072e", size = 551615, upload-time = "2026-10-04T16:30:15.332Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4f/5d2a0136bb03b2a56a39dc6ff92d58a6e3e53a2e17079238b86228882f16/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:96beca19ec79de272e8668585380ff9092c47077c1d7a1e098e00bbd921f4785", size = 614655, upload-time = "2026-10-04T16:30:16.92Z" }, + { url = "https://files.pythonhosted.org/packages/09/1c/3f1025aaf70d9bf7272cc41f8b64ee76b48bf01726248138430e16f23b38/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5cf77eb04f20b720be95265a3e00eb2a14814074255cc27069c551b2db53118", size = 579602, upload-time = "2026-10-04T16:30:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/53/0d/5c72e6204f76610608706da32b6b7e11ef7e10317558a7bb15a122e008dc/rpds_py-2026.9.1-cp314-cp314-win32.whl", hash = "sha256:a03d57b86d2a51d0a66c92177e2be154ad015f357791d306e714569999cdb4cc", size = 206315, upload-time = "2026-10-04T16:30:20.05Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0b/489d48abbcc7d70cf3fbf662d9d22abf1f4650761c0a9ae05260800800d3/rpds_py-2026.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:837c6b305e26fe0f75b15c92cf3b2ba29e0ae19dc40b1c557b026cb426347d0c", size = 222852, upload-time = "2026-10-04T16:30:21.604Z" }, + { url = "https://files.pythonhosted.org/packages/91/16/bbb05a7e6a10cf79ba639be7f799d770ee15f64175cc61d081b218dd402a/rpds_py-2026.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:fce4b85234a0cbad67bf8e6e1201ee815d172c9aebad75f25645bc4d834f8e31", size = 218709, upload-time = "2026-10-04T16:30:23.036Z" }, + { url = "https://files.pythonhosted.org/packages/22/ac/ac507a0a4ec478ca470440a09583db4be5259ba7670aeba0620822f1e57a/rpds_py-2026.9.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3a72c11530d71abfb66c8d7696a2f86c43e63fca8b948f1a784ac490f4ec688e", size = 349596, upload-time = "2026-10-04T16:30:24.558Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f2/817a46b658d5070f477f722c298ee9a24525b0e4017347964146ef5fdd0e/rpds_py-2026.9.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:068c37bba854ec2fe42f7365c640af11dd9895890ccbf2df5070d0c059bd7f96", size = 343114, upload-time = "2026-10-04T16:30:26.048Z" }, + { url = "https://files.pythonhosted.org/packages/6c/42/6ade976b13ac1b4cb3bf2eb603f1be2fe74df19a29988d4c2b386be59d6f/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fca4eb6df565e2a928f1c7dad92d27db8f9df0f449e76423ed5d7e713ed445", size = 374197, upload-time = "2026-10-04T16:30:27.699Z" }, + { url = "https://files.pythonhosted.org/packages/d9/70/77cdf1d3f1a07faabe936016ae623aec7981f73108a8fe7a203ed2e21998/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c933c6678c6f116ff8af47a4c6db0868b8ace74af0343016c0ef00f00272ea69", size = 378025, upload-time = "2026-10-04T16:30:29.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/18a44a3beaa9b7931acb04af7bd9539836477c630a794452d4826d6185d4/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:028ad274ea951dac64491b5d1e65712a4aeabfdbdb9fccf797b57bd899b0c495", size = 491969, upload-time = "2026-10-04T16:30:30.995Z" }, + { url = "https://files.pythonhosted.org/packages/73/27/fb39cfd6bddaf741b024f813890374578ff8ac1f1adc473659c048b03b05/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:740d0a99cf9de0b17a3943388e9294a59becf75e7c43421f387bd3c7a9901f7c", size = 393905, upload-time = "2026-10-04T16:30:32.628Z" }, + { url = "https://files.pythonhosted.org/packages/ed/71/0fa7bb77b57af0d710273964180d11b503f62b8a5c358eb2d8c3f62feff6/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0da298fb372dc192610a4b9ecbc68a0cd8b675bbbd1fc519d01b41cfd658333e", size = 374569, upload-time = "2026-10-04T16:30:34.257Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/f41cfac269af3b449513ef1bc3d7f32fde52abbbd2d01c7e76b47743acd9/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:eb61be926bb81567c1f48bdc8aa22b9855048dc2efd53871f9f7e6e9a5632346", size = 377482, upload-time = "2026-10-04T16:30:35.997Z" }, + { url = "https://files.pythonhosted.org/packages/b4/fc/312b49006e7f8f9ca5f96647577b8aa6f3df30519c46bd448f5c425af0b2/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:42e75466f83cd43f6026c81eab74246efb2bdadafb307b85700632d06c68f299", size = 400040, upload-time = "2026-10-04T16:30:37.76Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a6/18cca7a878dc7fa95165a83343fd4d7b65643fd22e54e47340a451121d5c/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:617f59cde379b4f648a09797b7f683d04b90a46344cddab85639da5aff0f5531", size = 551849, upload-time = "2026-10-04T16:30:39.443Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/1f5685e20f39604691bdc3c05aaa6b8bd2f954e9e996477adf2376768e33/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3edae8c5ddfdb6985d49ae9d150516e5076888879022f91a26c2de9276ce0bdb", size = 614842, upload-time = "2026-10-04T16:30:41.231Z" }, + { url = "https://files.pythonhosted.org/packages/c6/25/98652109fd9f7e10268dd4571aa52b81987001b806f37ef1a18260de714a/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0f045bb053c9057720d72c56dffe30dffdc05997b2897a827b9325f0ab6623fa", size = 582606, upload-time = "2026-10-04T16:30:43.345Z" }, + { url = "https://files.pythonhosted.org/packages/19/03/11ca09099bab5f53373a80a334c424ec917c13d050760a59499d2af5171e/rpds_py-2026.9.1-cp314-cp314t-win32.whl", hash = "sha256:bf35d0568abda97233239ce32896d3ad53fccc537832c104e30c94aa5fb93569", size = 203341, upload-time = "2026-10-04T16:30:44.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8a/88909e3ffd9f47f5b58211473875d8c3c09079f0058c46fb72c55a702a26/rpds_py-2026.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:1e8d4d79d828299bf44a55db22a9388ab967b49d17132c88eab0f4360b48da8e", size = 222843, upload-time = "2026-10-04T16:30:46.486Z" }, +] + [[package]] name = "ruff" version = "0.16.9" @@ -1277,6 +1547,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "sqlalchemy" +version = "2.0.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/f9/5e4491e5ccf42f5d9cfc663741d261b3e6e1683ae7812114e7636409fcc6/sqlalchemy-2.0.45.tar.gz", hash = "sha256:1632a4bda8d2d25703fdad6363058d882541bdaaee0e5e3ddfa0cd3229efce88", size = 9869912, upload-time = "2025-12-09T21:05:16.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/64/4e1913772646b060b025d3fc52ce91a58967fe58957df32b455de5a12b4f/sqlalchemy-2.0.45-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f46ec744e7f51275582e6a24326e10c49fbdd3fc99103e01376841213028774", size = 3272404, upload-time = "2025-12-09T22:11:09.662Z" }, + { url = "https://files.pythonhosted.org/packages/b3/27/caf606ee924282fe4747ee4fd454b335a72a6e018f97eab5ff7f28199e16/sqlalchemy-2.0.45-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:883c600c345123c033c2f6caca18def08f1f7f4c3ebeb591a63b6fceffc95cce", size = 3277057, upload-time = "2025-12-09T22:13:56.213Z" }, + { url = "https://files.pythonhosted.org/packages/85/d0/3d64218c9724e91f3d1574d12eb7ff8f19f937643815d8daf792046d88ab/sqlalchemy-2.0.45-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2c0b74aa79e2deade948fe8593654c8ef4228c44ba862bb7c9585c8e0db90f33", size = 3222279, upload-time = "2025-12-09T22:11:11.1Z" }, + { url = "https://files.pythonhosted.org/packages/24/10/dd7688a81c5bc7690c2a3764d55a238c524cd1a5a19487928844cb247695/sqlalchemy-2.0.45-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a420169cef179d4c9064365f42d779f1e5895ad26ca0c8b4c0233920973db74", size = 3244508, upload-time = "2025-12-09T22:13:57.932Z" }, + { url = "https://files.pythonhosted.org/packages/aa/41/db75756ca49f777e029968d9c9fee338c7907c563267740c6d310a8e3f60/sqlalchemy-2.0.45-cp314-cp314-win32.whl", hash = "sha256:e50dcb81a5dfe4b7b4a4aa8f338116d127cb209559124f3694c70d6cd072b68f", size = 2113204, upload-time = "2025-12-09T21:39:38.365Z" }, + { url = "https://files.pythonhosted.org/packages/89/a2/0e1590e9adb292b1d576dbcf67ff7df8cf55e56e78d2c927686d01080f4b/sqlalchemy-2.0.45-cp314-cp314-win_amd64.whl", hash = "sha256:4748601c8ea959e37e03d13dcda4a44837afcd1b21338e637f7c935b8da06177", size = 2138785, upload-time = "2025-12-09T21:39:39.503Z" }, + { url = "https://files.pythonhosted.org/packages/42/39/f05f0ed54d451156bbed0e23eb0516bcad7cbb9f18b3bf219c786371b3f0/sqlalchemy-2.0.45-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd337d3526ec5298f67d6a30bbbe4ed7e5e68862f0bf6dd21d289f8d37b7d60b", size = 3522029, upload-time = "2025-12-09T22:13:32.09Z" }, + { url = "https://files.pythonhosted.org/packages/54/0f/d15398b98b65c2bce288d5ee3f7d0a81f77ab89d9456994d5c7cc8b2a9db/sqlalchemy-2.0.45-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9a62b446b7d86a3909abbcd1cd3cc550a832f99c2bc37c5b22e1925438b9367b", size = 3475142, upload-time = "2025-12-09T22:13:33.739Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e1/3ccb13c643399d22289c6a9786c1a91e3dcbb68bce4beb44926ac2c557bf/sqlalchemy-2.0.45-py3-none-any.whl", hash = "sha256:5225a288e4c8cc2308dbdd874edad6e7d0fd38eac1e9e5f23503425c8eee20d0", size = 1936672, upload-time = "2025-12-09T21:54:52.608Z" }, +] + +[package.optional-dependencies] +asyncio = [ + { name = "greenlet" }, +] + [[package]] name = "starlette" version = "1.7.0"