forked from mtusi2026/mtusi2026lab2
Forward the lab 4 foundation and migrate the lab 5 pipeline to SQLAlchemy
This commit is contained in:
@@ -42,3 +42,5 @@ compose*.yaml
|
||||
lab.json
|
||||
requirements-dev.txt
|
||||
.claude
|
||||
|
||||
.verification/
|
||||
|
||||
+16
-3
@@ -27,8 +27,8 @@ STAGING_RETENTION_SECONDS=86400
|
||||
# LB queue per replica, drain window before closing the listener on SIGTERM, PostgreSQL
|
||||
# limits, load generator limits (profile bench), cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone:
|
||||
# only the load balancer is published.
|
||||
LB_HTTP_PORT=8080
|
||||
LB_STATS_PORT=8404
|
||||
LB_HTTP_PORT=18085
|
||||
LB_STATS_PORT=25085
|
||||
API_CPUS=1
|
||||
API_MEMORY=512m
|
||||
WEB_CONCURRENCY=2
|
||||
@@ -58,7 +58,7 @@ MALLOC_MMAP_THRESHOLD_=131072
|
||||
# Solution-specific for lab5: RabbitMQ management UI on 127.0.0.1, backoff base of a transient
|
||||
# retry (doubles per attempt up to 60 s), relay poll and sweep intervals, worker limits and the
|
||||
# PostgreSQL pool of relay and worker. FAULT_POINT is set only by infra/compose.faults.yaml.
|
||||
BROKER_MANAGEMENT_PORT=15672
|
||||
BROKER_MANAGEMENT_PORT=25086
|
||||
JOB_BACKOFF_BASE_SECONDS=2
|
||||
OUTBOX_POLL_SECONDS=1
|
||||
SWEEP_INTERVAL_SECONDS=5
|
||||
@@ -71,3 +71,16 @@ PIPELINE_DB_POOL_MAX_SIZE=4
|
||||
RENDER_TIMEOUT_SECONDS=30
|
||||
RENDER_MEMORY_MB=448
|
||||
RENDER_TASKS_PER_CHILD=100
|
||||
# Minimal client transfer rate, bytes/s: every body byte moved in either direction extends the
|
||||
# request deadline by 1/rate, so slow but steady uploads and downloads outlive REQUEST_TIMEOUT_SECONDS.
|
||||
MIN_TRANSFER_RATE_BYTES=131072
|
||||
# Solution-specific for lab4 track 2: limits of the one-off reconciler (profile reconcile), its S3
|
||||
# read timeout for listing pages, optional CPU pinning of the measured services (empty = no pinning)
|
||||
# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for the 100k point).
|
||||
RECONCILE_CPUS=1
|
||||
RECONCILE_MEMORY=256m
|
||||
RECONCILE_S3_READ_TIMEOUT_SECONDS=10
|
||||
RECONCILE_CPUSET=
|
||||
S3_CPUSET=
|
||||
PG_CPUSET=
|
||||
S3_CPUS=1
|
||||
|
||||
@@ -59,6 +59,9 @@ jobs:
|
||||
- name: Несколько реплик на одном токене и гонка одного ключа
|
||||
run: make lb-check
|
||||
|
||||
- name: Повтор неподтверждённой записи после потерянного ответа и убитой реплики
|
||||
run: make retry-check
|
||||
|
||||
- name: Остановка реплики под запросами, затем возврат
|
||||
run: |
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
@@ -84,7 +87,17 @@ jobs:
|
||||
|
||||
- name: Процессы работают не от root
|
||||
run: |
|
||||
docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}'
|
||||
for id in $(docker compose ps -q); do
|
||||
docker top "$id" -eo pid,uid,comm | awk 'NR>1 && $2==0 {print; bad=1} END {exit bad}' || exit 1
|
||||
done
|
||||
|
||||
- name: Одноразовые сервисы не от root
|
||||
run: |
|
||||
for service in migrate s3-init; do
|
||||
user=$(docker inspect -f '{{.Config.User}}' "$(docker compose ps -aq "$service")")
|
||||
echo "$service: $user"
|
||||
case "${user%%:*}" in ""|0|root) exit 1 ;; esac
|
||||
done
|
||||
|
||||
- name: Корректная остановка relay и worker по SIGTERM
|
||||
run: |
|
||||
|
||||
@@ -38,3 +38,5 @@ coverage/
|
||||
|
||||
# Локальные настройки агента
|
||||
.claude/settings.local.json
|
||||
|
||||
.verification/
|
||||
|
||||
+4
-1
@@ -27,7 +27,7 @@ ENV PYTHONUNBUFFERED=1 \
|
||||
WEB_CONCURRENCY=2 \
|
||||
REQUEST_TIMEOUT_SECONDS=8 \
|
||||
DRAIN_SECONDS=2 \
|
||||
SHUTDOWN_TIMEOUT_SECONDS=5
|
||||
SHUTDOWN_TIMEOUT_SECONDS=6
|
||||
|
||||
RUN groupadd --system --gid 10001 messenger \
|
||||
&& useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
|
||||
@@ -36,6 +36,9 @@ COPY --from=builder --chown=root:root /opt/venv /opt/venv
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --chown=root:root alembic.ini ./
|
||||
COPY --chown=root:root alembic ./alembic
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
BASE_URL ?= http://$(shell docker compose port lb 8080 2>/dev/null || echo localhost:8080)
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
@@ -8,13 +8,14 @@ VENV ?= .venv
|
||||
UV ?= uv
|
||||
COMPOSE ?= docker compose
|
||||
RATE ?= 100
|
||||
API_IMAGE ?= mtusi-messenger:lab5
|
||||
FROM_REV ?=
|
||||
SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||
scripts/upload_memory.py scripts/make_images.py scripts/pipeline_suite.py \
|
||||
scripts/pipeline_evidence.py
|
||||
SHIFT ?= 0
|
||||
SCRIPTS_PY = scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
|
||||
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
SCENARIO ?= failover-upload
|
||||
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
|
||||
PYTEST_ARGS ?=
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover \
|
||||
install lint format typecheck student-test \
|
||||
@@ -23,7 +24,11 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
|
||||
limits detection failover-each rolling-update lab4-level4 \
|
||||
scenario s3-check attachments-evidence upload-faults upload-memory \
|
||||
images queues topology worker-stop worker-start broker-restart suite-preflight level3 \
|
||||
pipeline-evidence level4 replay failed dlq
|
||||
pipeline-evidence level4 replay failed dlq \
|
||||
faults-evidence retry-check reconcile reconcile-demo reconcile-cost load-evidence \
|
||||
clean-clone-evidence overload-leftovers
|
||||
|
||||
.NOTPARALLEL:
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -36,10 +41,23 @@ help:
|
||||
@echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/"
|
||||
@echo "lab5: queues, topology, worker-stop/worker-start, broker-restart, images, suite-preflight, level3, level4"
|
||||
@echo "lab5 оператор: failed (реестр), replay ID=<attachment_id>, dlq (просмотр DLQ без извлечения)"
|
||||
@echo "retry-check: потерянный ответ на живой и на убитой реплике, повтор тем же ключом"
|
||||
@echo "lab4-level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps"
|
||||
@echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence"
|
||||
@echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда"
|
||||
@echo "upload-memory ARGS='--out evidence/upload-memory/runN': пик памяти реплик при 10/20/40 загрузках 1 и 10 MiB"
|
||||
@echo "load-evidence: tests/load.py ×3 клиентом-контейнером в сети edge стенда → evidence/load.txt, evidence/load/"
|
||||
@echo "overload-leftovers: после upload-memory — намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt"
|
||||
@echo "clean-clone-evidence: блок «Запуск из чистого clone» в свежем клоне рядом → evidence/clean-clone.txt"
|
||||
@echo "reconcile ARGS='--apply --deep': сверка PostgreSQL ↔ S3 (без ARGS — dry-run); трек 2:"
|
||||
@echo "reconcile-demo: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/"
|
||||
@echo "reconcile-cost: стоимость обхода 1k/10k/30k/100k объектов × 3 прогона на чистом стенде → evidence/reconcile/cost/"
|
||||
@echo "BASE_URL=$(BASE_URL) (порт lb из docker compose); failover-evidence SHIFT=1 — сдвиг round robin на вторую реплику"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
|
||||
validate: PYTHON := $(if $(wildcard $(VENV)/bin/python),$(VENV)/bin/python,$(PYTHON))
|
||||
validate:
|
||||
$(PYTHON) scripts/validate_openapi.py
|
||||
|
||||
@@ -63,17 +81,17 @@ install:
|
||||
lint:
|
||||
$(VENV)/bin/ruff check src student-tests $(SCRIPTS_PY)
|
||||
$(VENV)/bin/ruff format --check src student-tests $(SCRIPTS_PY)
|
||||
$(VENV)/bin/ty check src student-tests
|
||||
$(VENV)/bin/ty check src student-tests alembic
|
||||
|
||||
format:
|
||||
$(VENV)/bin/ruff check --fix src student-tests $(SCRIPTS_PY)
|
||||
$(VENV)/bin/ruff format src student-tests $(SCRIPTS_PY)
|
||||
|
||||
typecheck:
|
||||
$(VENV)/bin/ty check src student-tests
|
||||
$(VENV)/bin/ty check src student-tests alembic
|
||||
|
||||
student-test:
|
||||
$(VENV)/bin/pytest
|
||||
$(VENV)/bin/pytest $(PYTEST_ARGS)
|
||||
|
||||
secrets:
|
||||
sh scripts/secrets.sh
|
||||
@@ -82,13 +100,14 @@ secrets:
|
||||
up: secrets
|
||||
$(COMPOSE) build
|
||||
$(COMPOSE) up -d --wait
|
||||
scripts/lb-wait.sh
|
||||
|
||||
down:
|
||||
$(COMPOSE) down
|
||||
$(COMPOSE) --profile bench --profile tools --profile reconcile down
|
||||
|
||||
reset-data:
|
||||
@echo "Удаляются volumes с учебными данными стенда"
|
||||
$(COMPOSE) down -v --remove-orphans
|
||||
$(COMPOSE) --profile bench --profile tools --profile reconcile down -v --remove-orphans
|
||||
|
||||
migrate:
|
||||
$(COMPOSE) run --rm migrate
|
||||
@@ -105,19 +124,23 @@ restore:
|
||||
lb-check:
|
||||
$(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json
|
||||
|
||||
retry-check:
|
||||
$(PYTHON) scripts/probe.py retry --out evidence/retry.json
|
||||
|
||||
limits:
|
||||
$(PYTHON) scripts/limits.py --out evidence/limits.txt
|
||||
|
||||
detection:
|
||||
sh scripts/detection.sh evidence/detection
|
||||
rm -rf evidence/detection
|
||||
for run in 1 2 3; do sh scripts/detection.sh evidence/detection/run$$run || exit 1; done
|
||||
|
||||
failover-each:
|
||||
RATE=$(RATE) sh scripts/failover-each.sh evidence/failover-each
|
||||
|
||||
rolling-update:
|
||||
ifneq ($(FROM_REV),)
|
||||
git archive "$(FROM_REV)" | docker build -q -t "$(API_IMAGE)" -
|
||||
$(COMPOSE) up -d --no-deps --force-recreate --wait api
|
||||
git archive "$(FROM_REV)" | docker build -q -t "mtusi-messenger:$(FROM_REV)" -
|
||||
API_IMAGE="mtusi-messenger:$(FROM_REV)" $(COMPOSE) up -d --no-deps --force-recreate --wait api
|
||||
scripts/lb-wait.sh
|
||||
endif
|
||||
$(COMPOSE) build -q api
|
||||
@@ -144,10 +167,28 @@ upload-faults:
|
||||
status=$$?; cat evidence/upload-faults.txt; exit $$status
|
||||
|
||||
upload-memory:
|
||||
$(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)"
|
||||
$(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)" $(ARGS)
|
||||
|
||||
load-evidence:
|
||||
sh scripts/load-evidence.sh
|
||||
|
||||
overload-leftovers:
|
||||
sh scripts/overload-leftovers.sh
|
||||
|
||||
clean-clone-evidence:
|
||||
sh scripts/clean-clone.sh
|
||||
|
||||
reconcile:
|
||||
$(COMPOSE) run --rm --no-deps reconciler $(ARGS)
|
||||
|
||||
reconcile-demo:
|
||||
$(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)" $(ARGS)
|
||||
|
||||
reconcile-cost:
|
||||
$(PYTHON) scripts/reconcile_track.py cost $(ARGS)
|
||||
|
||||
failover-evidence:
|
||||
sh scripts/failover-evidence.sh "$(BASE_URL)"
|
||||
sh scripts/failover-evidence.sh "$(BASE_URL)" "$(SHIFT)"
|
||||
|
||||
stack-evidence:
|
||||
sh scripts/evidence-stack.sh
|
||||
@@ -166,6 +207,12 @@ cache-evidence:
|
||||
test-faults:
|
||||
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack
|
||||
|
||||
faults-evidence:
|
||||
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; echo "\$$ make test-faults"; \
|
||||
$(MAKE) --no-print-directory test-faults 2>&1; } | sed 's#$(CURDIR)#<repo>#g' > evidence/faults.txt
|
||||
cat evidence/faults.txt
|
||||
grep -qE '^=+ [0-9]+ passed in' evidence/faults.txt
|
||||
|
||||
seed:
|
||||
$(PSQL) -f - < scripts/sql/seed-history.sql
|
||||
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
[alembic]
|
||||
script_location = %(here)s/alembic
|
||||
prepend_sys_path = %(here)s/src %(here)s/alembic
|
||||
path_separator = space
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
import asyncio
|
||||
|
||||
from alembic import context
|
||||
from alembic.script import ScriptDirectory
|
||||
from legacy import adopt_legacy
|
||||
from sqlalchemy import Connection, func, make_url, pool, select
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from messenger.attachments.infrastructure.postgres import (
|
||||
AttachmentRow,
|
||||
JobEventRow,
|
||||
JobRow,
|
||||
OutboxRow,
|
||||
VariantRow,
|
||||
)
|
||||
from messenger.identity.infrastructure.postgres import SessionRow, UserRow
|
||||
from messenger.messaging.infrastructure.postgres import (
|
||||
ChatMemberRow,
|
||||
ChatRow,
|
||||
IdempotencyKeyRow,
|
||||
MessageAttachmentRow,
|
||||
MessageRow,
|
||||
)
|
||||
from messenger.shared.config import PlatformSettings
|
||||
|
||||
LOCK_KEY = 20_260_002
|
||||
MODELS = (
|
||||
UserRow,
|
||||
SessionRow,
|
||||
ChatRow,
|
||||
ChatMemberRow,
|
||||
MessageRow,
|
||||
IdempotencyKeyRow,
|
||||
AttachmentRow,
|
||||
JobRow,
|
||||
OutboxRow,
|
||||
JobEventRow,
|
||||
VariantRow,
|
||||
MessageAttachmentRow,
|
||||
)
|
||||
target_metadata = UserRow.metadata
|
||||
config = context.config
|
||||
|
||||
|
||||
def include_object(_obj, name, _type, _reflected, _compare_to):
|
||||
return name != "schema_migrations"
|
||||
|
||||
|
||||
def run_migrations(connection: Connection) -> None:
|
||||
with connection.engine.connect() as lock:
|
||||
lock.execute(select(func.pg_advisory_lock(LOCK_KEY)))
|
||||
lock.commit()
|
||||
try:
|
||||
with connection.begin():
|
||||
adopt_legacy(connection, ScriptDirectory.from_config(config))
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
transaction_per_migration=True,
|
||||
compare_type=True,
|
||||
include_object=include_object,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
finally:
|
||||
lock.execute(select(func.pg_advisory_unlock(LOCK_KEY)))
|
||||
lock.commit()
|
||||
|
||||
|
||||
async def run_async_migrations() -> None:
|
||||
dsn = config.attributes.get("database_url")
|
||||
if dsn is None:
|
||||
dsn = PlatformSettings().database_url.get_secret_value()
|
||||
engine = create_async_engine(
|
||||
make_url(dsn).set(drivername="postgresql+asyncpg"),
|
||||
poolclass=pool.NullPool,
|
||||
hide_parameters=True,
|
||||
connect_args={
|
||||
"timeout": 10,
|
||||
"server_settings": {"lock_timeout": "10000", "statement_timeout": "60000"},
|
||||
},
|
||||
)
|
||||
try:
|
||||
async with engine.connect() as connection:
|
||||
await connection.run_sync(run_migrations)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
context.configure(
|
||||
dialect_name="postgresql",
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
transaction_per_migration=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
elif config.attributes.get("connection") is not None:
|
||||
run_migrations(config.attributes["connection"])
|
||||
else:
|
||||
asyncio.run(run_async_migrations())
|
||||
@@ -0,0 +1,94 @@
|
||||
from uuid import uuid4
|
||||
|
||||
from alembic.migration import MigrationContext
|
||||
from alembic.operations import Operations
|
||||
from alembic.script import ScriptDirectory
|
||||
from sqlalchemy import Connection, MetaData, Table, func, inspect, select, text
|
||||
from sqlalchemy.schema import CreateSchema
|
||||
|
||||
LEGACY = {
|
||||
"0001_init": "9a13618f078ae9056431391e2e5197a31b981ae1fff047a9b3421ac750d3fd0d",
|
||||
"0002_idempotency_keys": "d9159a35553249a1e51af5f2e4e7c39b014e97593c49748e13f3799839063dd0",
|
||||
"0003_attachments": "c4417f560538a083efba1c67f660147dc94532fcbb2fa91f3f3e8258a0f01f2e",
|
||||
"0004_jobs": "c3887fa7a886ae8153a216aa43fd4c3c6190a2074e8ce20c3e77f29bfb01550e",
|
||||
"0005_failed_registry": "02b80061a08a6929feb092148c453ad970e3d001901c651b5a73b90b1e40f145",
|
||||
"0006_attachment_quarantine": (
|
||||
"541f5399d87fac3584c4102b4e0bf92c23cd372cc20a7d56dad03e771773202c"
|
||||
),
|
||||
}
|
||||
SNAPSHOT = text("""
|
||||
WITH relations AS (
|
||||
SELECT c.* FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = :schema AND c.relkind IN ('r', 'p')
|
||||
AND c.relname NOT IN ('schema_migrations', 'alembic_version')
|
||||
)
|
||||
SELECT 'table' AS kind, c.relname AS name,
|
||||
concat_ws('|', c.relkind, c.relpersistence,
|
||||
c.relrowsecurity, c.relforcerowsecurity) AS definition
|
||||
FROM relations c
|
||||
UNION ALL
|
||||
SELECT 'column', c.relname || '.' || a.attname,
|
||||
concat_ws('|', a.attnum, format_type(a.atttypid, a.atttypmod), a.attnotnull,
|
||||
a.attidentity, a.attgenerated, a.attcollation::regcollation,
|
||||
pg_get_expr(d.adbin, d.adrelid))
|
||||
FROM relations c JOIN pg_attribute a ON a.attrelid = c.oid
|
||||
LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum
|
||||
WHERE a.attnum > 0 AND NOT a.attisdropped
|
||||
UNION ALL
|
||||
SELECT 'constraint', c.relname || '.' || k.conname,
|
||||
concat_ws('|', pg_get_constraintdef(k.oid), k.convalidated)
|
||||
FROM relations c JOIN pg_constraint k ON k.conrelid = c.oid
|
||||
UNION ALL
|
||||
SELECT 'index', c.relname || '.' || x.relname,
|
||||
concat_ws('|', pg_get_indexdef(i.indexrelid), i.indisvalid, i.indisready)
|
||||
FROM relations c JOIN pg_index i ON i.indrelid = c.oid
|
||||
JOIN pg_class x ON x.oid = i.indexrelid
|
||||
UNION ALL
|
||||
SELECT 'trigger', c.relname || '.' || t.tgname,
|
||||
concat_ws('|', pg_get_triggerdef(t.oid), t.tgenabled)
|
||||
FROM relations c JOIN pg_trigger t ON t.tgrelid = c.oid WHERE NOT t.tgisinternal
|
||||
ORDER BY kind, name
|
||||
""")
|
||||
|
||||
|
||||
def snapshot(connection: Connection, schema: str) -> list[tuple[str, str, str]]:
|
||||
return [
|
||||
(kind, name, definition.replace(f"{schema}.", ""))
|
||||
for kind, name, definition in connection.execute(SNAPSHOT, {"schema": schema})
|
||||
]
|
||||
|
||||
|
||||
def adopt_legacy(connection: Connection, scripts: ScriptDirectory) -> None:
|
||||
migration = MigrationContext.configure(connection)
|
||||
if migration.get_current_heads():
|
||||
return
|
||||
tables = set(inspect(connection).get_table_names(schema="public"))
|
||||
if tables <= {"alembic_version"}:
|
||||
return
|
||||
if "schema_migrations" not in tables:
|
||||
raise RuntimeError("Existing schema has no legacy migration journal")
|
||||
journal = Table("schema_migrations", MetaData(), schema="public", autoload_with=connection)
|
||||
applied = dict(connection.execute(select(journal.c.version, journal.c.checksum)).tuples().all())
|
||||
expected = dict(list(LEGACY.items())[: len(applied)])
|
||||
if applied != expected:
|
||||
raise RuntimeError("Unknown or changed legacy migrations")
|
||||
if not applied:
|
||||
if tables - {"schema_migrations", "alembic_version"}:
|
||||
raise RuntimeError("Existing schema has an empty migration journal")
|
||||
return
|
||||
revision = list(expected)[-1]
|
||||
actual = snapshot(connection, "public")
|
||||
reference = f"alembic_verify_{uuid4().hex}"
|
||||
savepoint = connection.begin_nested()
|
||||
try:
|
||||
connection.execute(CreateSchema(reference))
|
||||
connection.execute(select(func.set_config("search_path", reference, True)))
|
||||
with Operations.context(MigrationContext.configure(connection)):
|
||||
for script in reversed(list(scripts.walk_revisions(base="base", head=revision))):
|
||||
script.module.upgrade()
|
||||
wanted = snapshot(connection, reference)
|
||||
finally:
|
||||
savepoint.rollback()
|
||||
if actual != wanted:
|
||||
raise RuntimeError("Legacy schema differs from the recorded migrations")
|
||||
migration.stamp(scripts, revision)
|
||||
@@ -0,0 +1,16 @@
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,87 @@
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0001_init"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"users",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("username", sa.Text(), nullable=False),
|
||||
sa.Column("display_name", sa.Text(), nullable=False),
|
||||
sa.Column("password_hash", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint("password_hash LIKE '$argon2id$%'", name="users_password_hash_check"),
|
||||
sa.CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="users_username_format"),
|
||||
sa.CheckConstraint(
|
||||
"char_length(display_name) BETWEEN 1 AND 100", name="users_display_name_check"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("username", name="users_username_key"),
|
||||
)
|
||||
op.create_table(
|
||||
"sessions",
|
||||
sa.Column("token_hash", sa.LargeBinary(), nullable=False),
|
||||
sa.Column("user_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint("expires_at > created_at", name="sessions_check"),
|
||||
sa.CheckConstraint("octet_length(token_hash) = 32", name="sessions_token_hash_check"),
|
||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("token_hash"),
|
||||
)
|
||||
op.create_index("sessions_user_idx", "sessions", ["user_id"], unique=False)
|
||||
op.create_table(
|
||||
"chats",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("title", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("last_message_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="chats_title_check"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"chat_members",
|
||||
sa.Column("chat_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("user_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("position", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("chat_created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint("position BETWEEN 1 AND 100", name="chat_members_position_check"),
|
||||
sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["user_id"],
|
||||
["users.id"],
|
||||
),
|
||||
sa.PrimaryKeyConstraint("chat_id", "user_id"),
|
||||
)
|
||||
op.create_index(
|
||||
"chat_members_user_idx",
|
||||
"chat_members",
|
||||
["user_id", "chat_created_at", "chat_id"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_table(
|
||||
"messages",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("chat_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("sender_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("text", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint("char_length(text) BETWEEN 1 AND 4000", name="messages_text_check"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chat_id", "sender_id"],
|
||||
["chat_members.chat_id", "chat_members.user_id"],
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index(
|
||||
"messages_history_idx", "messages", ["chat_id", "created_at", "id"], unique=False
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise RuntimeError("Downgrade would delete application data")
|
||||
@@ -0,0 +1,33 @@
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0002_idempotency_keys"
|
||||
down_revision = "0001_init"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"idempotency_keys",
|
||||
sa.Column("chat_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("user_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("key", sa.Text(), nullable=False),
|
||||
sa.Column("fingerprint", sa.LargeBinary(), nullable=False),
|
||||
sa.Column("message_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint("key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="idempotency_keys_key_check"),
|
||||
sa.CheckConstraint(
|
||||
"octet_length(fingerprint) = 32", name="idempotency_keys_fingerprint_check"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chat_id", "user_id"],
|
||||
["chat_members.chat_id", "chat_members.user_id"],
|
||||
),
|
||||
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("chat_id", "user_id", "key"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise RuntimeError("Downgrade would delete application data")
|
||||
@@ -0,0 +1,77 @@
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0003_attachments"
|
||||
down_revision = "0002_idempotency_keys"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_constraint("messages_text_check", "messages", type_="check")
|
||||
op.create_check_constraint("messages_text_check", "messages", "char_length(text) <= 4000")
|
||||
op.create_table(
|
||||
"attachments",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("chat_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("owner_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("filename", sa.Text(), nullable=False),
|
||||
sa.Column("content_type", sa.Text(), nullable=False),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("sha256", sa.Text(), nullable=False),
|
||||
sa.Column("width", sa.Integer(), nullable=False),
|
||||
sa.Column("height", sa.Integer(), nullable=False),
|
||||
sa.Column("object_key", sa.Text(collation="C"), nullable=False),
|
||||
sa.Column("status", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("object_key", name="attachments_object_key_key"),
|
||||
sa.UniqueConstraint("id", "chat_id", name="attachments_chat_key"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chat_id", "owner_id"],
|
||||
["chat_members.chat_id", "chat_members.user_id"],
|
||||
name="attachments_owner_fkey",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_check"
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_check"
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_check"
|
||||
),
|
||||
sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachments_sha256_check"),
|
||||
sa.CheckConstraint("width > 0", name="attachments_width_check"),
|
||||
sa.CheckConstraint("height > 0", name="attachments_height_check"),
|
||||
sa.CheckConstraint("status IN ('pending', 'ready')", name="attachments_status_check"),
|
||||
)
|
||||
op.create_index(
|
||||
"attachments_pending_idx",
|
||||
"attachments",
|
||||
["created_at"],
|
||||
postgresql_where=sa.text("status = 'pending'"),
|
||||
)
|
||||
op.create_table(
|
||||
"message_attachments",
|
||||
sa.Column("message_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("position", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("chat_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("attachment_id", sa.Uuid(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("message_id", "position"),
|
||||
sa.UniqueConstraint("message_id", "attachment_id"),
|
||||
sa.CheckConstraint("position BETWEEN 0 AND 9", name="message_attachments_position_check"),
|
||||
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["attachment_id", "chat_id"],
|
||||
["attachments.id", "attachments.chat_id"],
|
||||
name="message_attachments_attachment_fkey",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
)
|
||||
op.create_index("message_attachments_attachment_idx", "message_attachments", ["attachment_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise RuntimeError("Downgrade would delete application data")
|
||||
@@ -0,0 +1,146 @@
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision = "0004_jobs"
|
||||
down_revision = "0003_attachments"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_constraint("attachments_status_check", "attachments", type_="check")
|
||||
op.create_check_constraint(
|
||||
"attachments_status_check",
|
||||
"attachments",
|
||||
"status IN ('pending', 'queued', 'processing', 'ready', 'failed')",
|
||||
)
|
||||
for name in ("staging_key", "original_key"):
|
||||
op.add_column("attachments", sa.Column(name, sa.Text(collation="C"), nullable=True))
|
||||
op.create_unique_constraint(f"attachments_{name}_key", "attachments", [name])
|
||||
op.add_column(
|
||||
"attachments",
|
||||
sa.Column("published_version", sa.Integer(), nullable=False, server_default="0"),
|
||||
)
|
||||
op.create_check_constraint(
|
||||
"attachments_published_version_check", "attachments", "published_version >= 0"
|
||||
)
|
||||
op.add_column("attachments", sa.Column("error_code", sa.Text(), nullable=True))
|
||||
op.create_check_constraint(
|
||||
"attachments_error_code_check",
|
||||
"attachments",
|
||||
"error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')",
|
||||
)
|
||||
op.alter_column("attachments", "object_key", nullable=True)
|
||||
attachments = sa.table(
|
||||
"attachments",
|
||||
sa.column("status"),
|
||||
sa.column("object_key"),
|
||||
sa.column("original_key"),
|
||||
sa.column("staging_key"),
|
||||
sa.column("published_version"),
|
||||
)
|
||||
op.execute(
|
||||
attachments.update()
|
||||
.where(attachments.c.status == "ready")
|
||||
.values(original_key=attachments.c.object_key, published_version=1)
|
||||
)
|
||||
op.execute(
|
||||
attachments.update()
|
||||
.where(attachments.c.status == "pending")
|
||||
.values(staging_key=attachments.c.object_key)
|
||||
)
|
||||
op.create_index(
|
||||
"attachments_staged_idx",
|
||||
"attachments",
|
||||
["updated_at"],
|
||||
postgresql_where=sa.column("staging_key").is_not(None),
|
||||
)
|
||||
op.create_table(
|
||||
"attachment_variants",
|
||||
sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False),
|
||||
sa.Column("pipeline_version", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.Text(), nullable=False),
|
||||
sa.Column("object_key", sa.Text(collation="C"), nullable=False),
|
||||
sa.Column("content_type", sa.Text(), nullable=False),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("width", sa.Integer(), nullable=False),
|
||||
sa.Column("height", sa.Integer(), nullable=False),
|
||||
sa.Column("sha256", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("attachment_id", "pipeline_version", "name"),
|
||||
sa.UniqueConstraint("object_key", name="attachment_variants_object_key_key"),
|
||||
sa.CheckConstraint(
|
||||
"pipeline_version >= 1", name="attachment_variants_pipeline_version_check"
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"name IN ('thumbnail', 'crop', 'optimized')", name="attachment_variants_name_check"
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"content_type IN ('image/jpeg', 'image/png', 'image/webp')",
|
||||
name="attachment_variants_content_type_check",
|
||||
),
|
||||
sa.CheckConstraint("size_bytes >= 1", name="attachment_variants_size_bytes_check"),
|
||||
sa.CheckConstraint("width >= 1", name="attachment_variants_width_check"),
|
||||
sa.CheckConstraint("height >= 1", name="attachment_variants_height_check"),
|
||||
sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachment_variants_sha256_check"),
|
||||
)
|
||||
op.create_table(
|
||||
"jobs",
|
||||
sa.Column("job_id", sa.Uuid(), primary_key=True),
|
||||
sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False),
|
||||
sa.Column("pipeline_version", sa.Integer(), nullable=False),
|
||||
sa.Column("status", sa.Text(), nullable=False),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False),
|
||||
sa.Column("claims", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("lease_token", sa.Uuid(), nullable=True),
|
||||
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("attachment_id", "pipeline_version", name="jobs_effect_key"),
|
||||
sa.CheckConstraint("pipeline_version >= 1", name="jobs_pipeline_version_check"),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('queued', 'processing', 'done', 'failed', 'superseded')",
|
||||
name="jobs_status_check",
|
||||
),
|
||||
sa.CheckConstraint("attempt >= 1", name="jobs_attempt_check"),
|
||||
sa.CheckConstraint("claims >= 0", name="jobs_claims_check"),
|
||||
)
|
||||
op.create_index(
|
||||
"jobs_lease_idx",
|
||||
"jobs",
|
||||
["lease_until"],
|
||||
postgresql_where=sa.column("status") == "processing",
|
||||
)
|
||||
op.create_table(
|
||||
"outbox",
|
||||
sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True),
|
||||
sa.Column("job_id", sa.Uuid(), sa.ForeignKey("jobs.job_id"), nullable=False),
|
||||
sa.Column("payload", postgresql.JSONB(), nullable=False),
|
||||
sa.Column("available_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("published_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"outbox_pending_idx",
|
||||
"outbox",
|
||||
["available_at", "id"],
|
||||
postgresql_where=sa.column("published_at").is_(None),
|
||||
)
|
||||
op.create_index("outbox_job_idx", "outbox", ["job_id"])
|
||||
op.create_table(
|
||||
"job_events",
|
||||
sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True),
|
||||
sa.Column("job_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("attachment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("kind", sa.Text(), nullable=False),
|
||||
sa.Column("attempt", sa.Integer(), nullable=True),
|
||||
sa.Column("detail", postgresql.JSONB(), nullable=False, server_default="{}"),
|
||||
sa.Column("at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("job_events_attachment_idx", "job_events", ["attachment_id", "id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise RuntimeError("Downgrade would delete pipeline data")
|
||||
@@ -0,0 +1,20 @@
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0005_failed_registry"
|
||||
down_revision = "0004_jobs"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_index(
|
||||
"jobs_failed_idx",
|
||||
"jobs",
|
||||
[sa.column("updated_at").desc()],
|
||||
postgresql_where=sa.column("status") == "failed",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("jobs_failed_idx", table_name="jobs")
|
||||
@@ -0,0 +1,28 @@
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0006_attachment_quarantine"
|
||||
down_revision = "0005_failed_registry"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("attachments", sa.Column("quarantine_reason", sa.Text(), nullable=True))
|
||||
op.create_check_constraint(
|
||||
"attachments_quarantine_reason_check",
|
||||
"attachments",
|
||||
"quarantine_reason IN ('missing', 'size_mismatch', 'sha_mismatch')",
|
||||
)
|
||||
op.add_column(
|
||||
"attachments", sa.Column("quarantined_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
op.create_check_constraint(
|
||||
"attachments_quarantine_check",
|
||||
"attachments",
|
||||
"(quarantine_reason IS NULL) = (quarantined_at IS NULL)",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise RuntimeError("Downgrade would delete quarantine data")
|
||||
+58
-12
@@ -1,5 +1,5 @@
|
||||
x-app: &app
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:${COMPOSE_PROJECT_NAME}}
|
||||
networks: [backend]
|
||||
|
||||
x-s3-env: &s3-env
|
||||
@@ -19,7 +19,7 @@ x-s3-env: &s3-env
|
||||
UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100}
|
||||
|
||||
x-pipeline: &pipeline
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:${COMPOSE_PROJECT_NAME}}
|
||||
pull_policy: never
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
@@ -64,10 +64,17 @@ x-s3-client: &s3-client
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
RC_CONFIG_DIR: /tmp/rc
|
||||
user: "65534:65534"
|
||||
volumes: ["./infra/s3:/s3:ro"]
|
||||
secrets:
|
||||
[s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key,
|
||||
s3_worker_access_key_id, s3_worker_secret_access_key]
|
||||
- s3_root_user
|
||||
- s3_root_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- s3_reconciler_access_key_id
|
||||
- s3_reconciler_secret_access_key
|
||||
- s3_worker_access_key_id
|
||||
- s3_worker_secret_access_key
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
restart: "no"
|
||||
@@ -83,15 +90,16 @@ services:
|
||||
API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64}
|
||||
volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
|
||||
ports:
|
||||
- "127.0.0.1:${LB_HTTP_PORT:-8080}:8080"
|
||||
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
||||
- "127.0.0.1:${LB_HTTP_PORT:-18085}:8080"
|
||||
- "127.0.0.1:${LB_STATS_PORT:-25085}:8404"
|
||||
networks: [edge, backend]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
start_period: 60s
|
||||
start_interval: 1s
|
||||
sysctls:
|
||||
net.ipv4.ip_local_port_range: "10240 65535"
|
||||
net.ipv4.tcp_tw_reuse: "1"
|
||||
@@ -117,10 +125,11 @@ services:
|
||||
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
|
||||
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
|
||||
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
|
||||
MIN_TRANSFER_RATE_BYTES: ${MIN_TRANSFER_RATE_BYTES:-131072}
|
||||
DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1}
|
||||
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-6}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
MALLOC_MMAP_THRESHOLD_: ${MALLOC_MMAP_THRESHOLD_:-131072}
|
||||
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
|
||||
@@ -130,7 +139,7 @@ services:
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
start_interval: 1s
|
||||
start_interval: 200ms
|
||||
stop_grace_period: 8s
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
@@ -153,7 +162,8 @@ services:
|
||||
migrate:
|
||||
<<: *app
|
||||
pull_policy: never
|
||||
command: ["python", "-m", "messenger.migrate"]
|
||||
init: true
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
@@ -192,12 +202,14 @@ services:
|
||||
secrets: [postgres_password]
|
||||
volumes: [pgdata:/var/lib/postgresql/data]
|
||||
shm_size: 256m
|
||||
cpuset: ${PG_CPUSET:-}
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
start_interval: 200ms
|
||||
stop_grace_period: 10s
|
||||
restart: unless-stopped
|
||||
networks: [backend]
|
||||
@@ -245,6 +257,7 @@ services:
|
||||
RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn}
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3data:/data]
|
||||
cpuset: ${S3_CPUSET:-}
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:size=64m"]
|
||||
cap_drop: [ALL]
|
||||
@@ -260,7 +273,36 @@ services:
|
||||
networks: [backend]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"}
|
||||
limits: {cpus: "${S3_CPUS:-1}", memory: "${S3_MEMORY:-512m}"}
|
||||
|
||||
reconciler:
|
||||
<<: *app
|
||||
pull_policy: never
|
||||
profiles: [reconcile]
|
||||
init: true
|
||||
entrypoint: ["python", "-m", "messenger.attachments.entrypoints.reconcile"]
|
||||
command: ["--dry-run"]
|
||||
environment:
|
||||
<<: *s3-env
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
|
||||
S3_READ_TIMEOUT_SECONDS: ${RECONCILE_S3_READ_TIMEOUT_SECONDS:-10}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key]
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
read_only: true
|
||||
cpuset: ${RECONCILE_CPUSET:-}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${RECONCILE_CPUS:-1}", memory: "${RECONCILE_MEMORY:-256m}"}
|
||||
|
||||
s3-init:
|
||||
<<: *s3-client
|
||||
@@ -284,7 +326,7 @@ services:
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
cap_drop: [ALL]
|
||||
ports:
|
||||
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672"
|
||||
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-25086}:15672"
|
||||
networks: [backend, edge]
|
||||
volumes:
|
||||
- broker-data:/var/lib/rabbitmq
|
||||
@@ -387,6 +429,10 @@ secrets:
|
||||
file: ./secrets/broker_url_api
|
||||
broker_url_worker:
|
||||
file: ./secrets/broker_url_worker
|
||||
s3_reconciler_access_key_id:
|
||||
file: ./secrets/s3_reconciler_access_key_id
|
||||
s3_reconciler_secret_access_key:
|
||||
file: ./secrets/s3_reconciler_secret_access_key
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
||||
@@ -4,20 +4,8 @@
|
||||
{
|
||||
"Sid": "MessengerApiObjects",
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts"
|
||||
],
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
|
||||
},
|
||||
{
|
||||
"Sid": "MessengerApiBucket",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+2
-1
@@ -4,7 +4,8 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
|
||||
endpoint="${S3_ENDPOINT:-http://s3:9000}"
|
||||
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
|
||||
RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}"
|
||||
export RC_HOST_root RC_HOST_api
|
||||
RC_HOST_reconciler="${endpoint%%://*}://$(secret s3_reconciler_access_key_id):$(secret s3_reconciler_secret_access_key)@${endpoint#*://}"
|
||||
export RC_HOST_root RC_HOST_api RC_HOST_reconciler
|
||||
rc alias set anon "$endpoint" --anonymous >/dev/null
|
||||
case "${1:-}" in
|
||||
sh|wget) exec "$@" ;;
|
||||
|
||||
+12
-13
@@ -4,23 +4,22 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
|
||||
bucket="${S3_BUCKET:-messenger-attachments}"
|
||||
endpoint="${S3_ENDPOINT:-http://s3:9000}"
|
||||
policy="${S3_API_POLICY:-messenger-api}"
|
||||
reconciler="${S3_RECONCILER_POLICY:-messenger-reconciler}"
|
||||
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
|
||||
export RC_HOST_root
|
||||
api_key=$(secret s3_access_key_id)
|
||||
grant() {
|
||||
sed "s/messenger-attachments/$bucket/g" "/s3/$2" > "/tmp/$2"
|
||||
rc admin policy create root "$1" "/tmp/$2" >/dev/null
|
||||
rc admin user add root "$(secret "$3")" "$(secret "$4")" >/dev/null
|
||||
rc admin policy attach root "$1" --user "$(secret "$3")" >/dev/null
|
||||
rc admin user info root "$(secret "$3")" --json | grep -q "\"$1\""
|
||||
}
|
||||
rc ready root --timeout 10 >/dev/null
|
||||
rc bucket create --ignore-existing "root/$bucket" >/dev/null
|
||||
rc bucket anonymous set private "root/$bucket" >/dev/null
|
||||
sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json
|
||||
rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null
|
||||
rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null
|
||||
rc admin policy attach root "$policy" --user "$api_key" >/dev/null
|
||||
worker_key=$(secret s3_worker_access_key_id)
|
||||
sed "s/messenger-attachments/$bucket/g" /s3/worker-policy.json > /tmp/worker-policy.json
|
||||
rc admin policy create root messenger-worker /tmp/worker-policy.json >/dev/null
|
||||
rc admin user add root "$worker_key" "$(secret s3_worker_secret_access_key)" >/dev/null
|
||||
rc admin policy attach root messenger-worker --user "$worker_key" >/dev/null
|
||||
grant "$policy" api-policy.json s3_access_key_id s3_secret_access_key
|
||||
grant "$reconciler" reconciler-policy.json s3_reconciler_access_key_id s3_reconciler_secret_access_key
|
||||
grant messenger-worker worker-policy.json s3_worker_access_key_id s3_worker_secret_access_key
|
||||
rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null
|
||||
rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"'
|
||||
rc admin user info root "$api_key" --json | grep -q "\"$policy\""
|
||||
rc admin user info root "$worker_key" --json | grep -q '"messenger-worker"'
|
||||
echo "s3-init: bucket=$bucket anonymous=private policy=$policy worker=messenger-worker lifecycle=abort-incomplete-uploads"
|
||||
echo "s3-init: bucket=$bucket anonymous=private policies=$policy,$reconciler lifecycle=abort-incomplete-uploads"
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "MessengerReconcilerObjects",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
|
||||
},
|
||||
{
|
||||
"Sid": "MessengerReconcilerListing",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -20,7 +20,7 @@
|
||||
- Несовместимость тестов ready/queued → адаптировать собственные тесты к текущему API, публичные не менять.
|
||||
- Регрессия транзакций → проверить rollback, одновременные claim, fencing, retry и outbox существующими тестами.
|
||||
- Дрейф legacy-схемы → отказ adoption вместо угадывания версии, тесты populated migration и повторного запуска.
|
||||
- Сверка ограничена legacy-префиксом; pipeline владеет staging/originals/variants. Это ограничение передачи основы фиксируется для отдельного отчёта.
|
||||
- Сверка ограничена legacy-префиксом; pipeline владеет staging/originals/variants. Это ограничение передачи основы фиксируется для отдельного отчёта. Унаследованный `reconcile_track.py demo` также рассчитан на синхронный upload ЛР4 (`201`) и legacy-ключи; он не подтверждает критерии ЛР5 и в этой фазе не запускается.
|
||||
|
||||
## Migration Plan
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"severity": "minor",
|
||||
"file": "scripts/reconcile_track.py:396",
|
||||
"problem": "Унаследованный демонстратор сверки ЛР4 ожидает upload 201 и legacy-ключи attachments/. Для API ЛР5 (202 queued) не адаптирован; не использовать как доказательство ЛР5. Сама сверка legacy-объектов и её изоляция от pipeline проверены."
|
||||
}
|
||||
]
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
### New Capabilities
|
||||
|
||||
Нет.
|
||||
- `lab4-storage-reconciler`: унаследованная ручная сверка объектов прежнего префикса `attachments/`.
|
||||
|
||||
### Modified Capabilities
|
||||
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@
|
||||
- **WHEN** БД содержит проверенный SQL-журнал ЛР5, вложения, задания, outbox и варианты
|
||||
- **THEN** переход сохраняет строки, ограничения, состояния и ссылки на объекты
|
||||
|
||||
#### Scenario: Изменённая миграция или схема
|
||||
#### Scenario: Изменённая миграция
|
||||
|
||||
- **WHEN** контрольная сумма SQL-журнала или фактическая схема не совпадает с ожидаемой
|
||||
- **THEN** принятие истории завершается ошибкой без изменения данных и отметки версии
|
||||
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
## Purpose
|
||||
|
||||
Сохраняет совместимость перенесённой из ЛР4 сверки PostgreSQL и S3 с ранее записанными объектами и ограничивает её область, чтобы не вмешиваться в асинхронную обработку ЛР5. Основание — сохранение предыдущего решения при переносе; не является заявкой на грейд ЛР5.
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Ручная сверка legacy-объектов
|
||||
|
||||
Одноразовый reconciler SHALL по умолчанию работать в dry-run над прежним префиксом `attachments/`, выдавая JSON со счётчиками расхождений. Apply SHALL сохранять свежие объекты и повторно проверять ссылку в БД перед удалением старого orphan. Одновременный второй проход SHALL завершаться с признаком skipped.
|
||||
|
||||
#### Scenario: Dry-run без изменения данных
|
||||
|
||||
- **WHEN** оператор запускает reconciler без apply для legacy-объектов
|
||||
- **THEN** он возвращает JSON и не меняет строки или байты
|
||||
|
||||
#### Scenario: Повторная проверка защищает объект
|
||||
|
||||
- **WHEN** после листинга старого orphan появляется строка с его ключом
|
||||
- **THEN** apply сохраняет объект
|
||||
|
||||
### Requirement: Карантин сохраняет данные
|
||||
|
||||
При подтверждённом расхождении готового legacy-вложения apply SHALL поместить метаданные в карантин без удаления строки и объекта. GET `/api/v1/attachments/{id}/content` SHALL возвращать 404 с ошибкой not_found для карантина, метаданные остаются доступны участнику. Снятие карантина SHALL требовать совпадения SHA-256 даже без deep.
|
||||
|
||||
#### Scenario: Возвращены другие байты той же длины
|
||||
|
||||
- **WHEN** объект в карантине заменён байтами прежнего размера, но другого SHA-256
|
||||
- **THEN** карантин остаётся и скачивание не разрешается
|
||||
|
||||
### Requirement: Разделение со staging и pipeline
|
||||
|
||||
Перенесённая сверка SHALL NOT обходить или удалять объекты из `staging/`, `originals/` и `variants/`. Этими префиксами управляет pipeline ЛР5 и его существующая уборка staging.
|
||||
|
||||
#### Scenario: Готовое вложение ЛР5 вне legacy-префикса
|
||||
|
||||
- **WHEN** у нового вложения оригинал и варианты опубликованы pipeline
|
||||
- **THEN** legacy-сверка не считает их orphan и не изменяет их
|
||||
@@ -1,16 +1,16 @@
|
||||
## 1. Перенос основы обязательного минимума
|
||||
|
||||
- [ ] 1.1 Перенести платформу и identity из immutable snapshot; сохранить границы зон.
|
||||
- [ ] 1.2 Перенести messaging и собственные проверки идемпотентности; сохранить порт проверки failed-вложений.
|
||||
- [ ] 1.3 Перенести attachments и перевести jobs/outbox/variants на SQLAlchemy, сохранив fencing и транзакции.
|
||||
- [x] 1.1 Перенести платформу и identity из immutable snapshot; сохранить границы зон.
|
||||
- [x] 1.2 Перенести messaging и собственные проверки идемпотентности; сохранить порт проверки failed-вложений.
|
||||
- [x] 1.3 Перенести attachments и перевести jobs/outbox/variants на SQLAlchemy, сохранив fencing и транзакции.
|
||||
|
||||
## 2. Миграции и стенд
|
||||
|
||||
- [ ] 2.1 Перенести Alembic, добавить историю ЛР5 и проверяемое принятие legacy-схемы с данными.
|
||||
- [ ] 2.2 Перенести зависимости, Docker/Compose, собственные скрипты и CI с сохранением worker/relay.
|
||||
- [ ] 2.3 Адаптировать собственные тесты и проверить make check/lint/student-test.
|
||||
- [x] 2.1 Перенести Alembic, добавить историю ЛР5 и проверяемое принятие legacy-схемы с данными.
|
||||
- [x] 2.2 Перенести зависимости, Docker/Compose, собственные скрипты и CI с сохранением worker/relay.
|
||||
- [x] 2.3 Адаптировать собственные тесты и проверить make check/lint/student-test.
|
||||
|
||||
## 3. Завершение
|
||||
|
||||
- [ ] 3.1 Проверить smoke через LB 18085 на своём стенде; остановить стенд без удаления volumes.
|
||||
- [x] 3.1 Проверить smoke через LB 18085 на своём стенде; остановить стенд без удаления volumes.
|
||||
- [ ] 3.2 Зафиксировать перенос кода, синхронизировать спецификации и архивировать change.
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# Проверка переноса основы
|
||||
|
||||
Источник: immutable snapshot `d7478726a85791c21e065d58f610735a9e687433`, diff от `d2744bc`. Исходный HEAD ЛР5: `64d5c67`. Эта фаза не реализует и не подтверждает грейды 3/4/5.
|
||||
|
||||
| Команда | Фактический результат |
|
||||
| --- | --- |
|
||||
| `make lint` | ruff check/format, ty: PASS |
|
||||
| `make check` | PASS: структура skeleton, ссылки и синтаксис Python |
|
||||
| `make validate` | PASS: OpenAPI 3.1 и JSON Schema |
|
||||
| `NO_PROXY=localhost,127.0.0.1,::1 make student-test PYTEST_ARGS='--tb=short'` | 494 passed, 1 skipped, 19 deselected; 141.49 s |
|
||||
| `NO_PROXY=localhost,127.0.0.1,::1 .venv/bin/pytest student-tests/api/test_reconciler.py -k pipeline --tb=short` | 1 passed, 10 deselected; 2.49 s |
|
||||
| `COMPOSE_PROJECT_NAME=mtusi2026lab5 LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 NO_PROXY=localhost,127.0.0.1,::1 make up` | Собран образ, API ×2, worker, relay и хранилища healthy; migrate и s3-init exit 0 |
|
||||
| `NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab5 LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 make test BASE_URL=http://127.0.0.1:18085` | PASS: lab5 public API scenario |
|
||||
| `COMPOSE_PROJECT_NAME=mtusi2026lab5 LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 make down` | Контейнеры и сети своего стенда остановлены/удалены; постоянные volumes сохранены |
|
||||
| `git diff --exit-code 64d5c67 -- contracts tests README.md COURSE.md REPORT.md evidence AGENTS.md` | exit 0, защищённые файлы неизменны |
|
||||
|
||||
Миграционные тесты включают populated legacy-истории 1–6, сохранность jobs/outbox/job_events/variants, дрейф схемы и контрольных сумм, два одновременных migrate, rollback DDL и совпадение ORM metadata с БД. Тесты pipeline проверяют аренды, fencing, дубли, retry, replay и окна отказа. На существующем volume стенда получена версия `0006_attachment_quarantine`; сохранены ready/failed-вложения и done/failed-задания.
|
||||
|
||||
Пропущен только Linux-specific RLIMIT_AS на macOS. 19 stack-тестов не входят в штатный student-test; в этой фазе из них не заявляется результат. Перед успешным полным прогоном исправлены несовместимые с ЛР5 вспомогательные тесты: DML без возвращаемых строк и ожидание 201 вместо 202. Первый RustFS-прогон без NO_PROXY не прошёл readiness; с исключением localhost из прокси полный прогон прошёл.
|
||||
|
||||
Ограничение для отдельного REPORT: findings.json, severity minor — старый демонстратор сверки рассчитан на ЛР4. Legacy-сверка намеренно ограничена attachments/; staging/originals/variants обслуживает pipeline ЛР5. REPORT и evidence этой фазой не менялись. Публикации, Telegram, нагрузочных измерений и фоновых агентов не было.
|
||||
@@ -40,7 +40,7 @@ context: |
|
||||
Python 3.14 (аннотации вычисляются лениво по PEP 649, `TYPE_CHECKING` и
|
||||
`from __future__ import annotations` не используются), FastAPI, gunicorn с несколькими
|
||||
`UvicornWorker` (пакет `uvicorn-worker`), dishka (DI, интеграции FastAPI и FastStream),
|
||||
pydantic v2 + pydantic-settings, structlog, asyncpg (параметризованный SQL без ORM),
|
||||
pydantic v2 + pydantic-settings, structlog, SQLAlchemy 2 async (драйвер asyncpg), Alembic,
|
||||
redis-py (`redis.asyncio`) для Valkey, argon2-cffi, aiobotocore (S3 path-style, SigV4,
|
||||
таймауты, checksum `when_required`), Pillow (проверка в API, варианты в worker), FastStream
|
||||
0.7 с `faststream[rabbit]` поверх aio-pika (consumer worker с `AckPolicy.MANUAL` и prefetch,
|
||||
@@ -68,7 +68,7 @@ context: |
|
||||
main.py сборка HTTP-приложения: собирает вклады зон, связывает зоны портами
|
||||
relay.py сборка процесса relay: outbox → брокер, возврат просроченных аренд
|
||||
worker.py сборка процесса worker: FastStream-подписчик очереди заданий
|
||||
migrate.py одноразовая точка входа: применяет SQL-миграции под advisory lock
|
||||
alembic/ одноразовые миграции под advisory lock с проверкой legacy-схемы
|
||||
identity/ пользователи, пароли и сессии
|
||||
messaging/ чаты, история и идемпотентная отправка, ссылки сообщения на вложения
|
||||
attachments/ вложения: приём и staging, задания и outbox, обработка и варианты,
|
||||
@@ -159,7 +159,7 @@ operations:
|
||||
- "Логирование — только через порт `Logger`; второй логгер в приложении не заводится. Object key, ключи S3 и содержимое файлов в журнал не пишутся."
|
||||
- "Код приложения — в `src/`, конфигурация LB, нагрузки и S3 — в `infra/`, собственные тесты — в `student-tests/`, обезличенные доказательства — в `evidence/`."
|
||||
- "Перед коммитом прогоняй `make lint`, `make student-test` и `make check`."
|
||||
- "SQL только параметризованный (`$1`, `$2`); значения из запроса никогда не склеиваются в текст запроса."
|
||||
- "SQL через SQLAlchemy ORM/Core с параметрами; миграции Alembic; значения запроса не склеиваются в SQL."
|
||||
- "Тело загрузки читается потоком со счётчиком до лимита; изображение не декодируется целиком до проверки размера по заголовку; тяжёлая проверка идёт в потоке под ограничением параллелизма."
|
||||
- "API не обрабатывает изображения синхронно и не подключается к брокеру: задание уходит только через outbox; `202` возвращает снимок записанного объекта."
|
||||
- "Worker и relay не объявляют очереди и exchange: топология и права — в `infra/rabbitmq/`; `ack` — строго после `COMMIT`, сообщение брокера — ровно поля `jobs.schema.json`."
|
||||
|
||||
+10
-3
@@ -13,6 +13,8 @@ dependencies = [
|
||||
"gunicorn>=26.2.0",
|
||||
"orjson>=3.12.0",
|
||||
"asyncpg>=0.32.0",
|
||||
"sqlalchemy[asyncio]==2.0.45",
|
||||
"alembic==1.16.5",
|
||||
"argon2-cffi>=25.1.0",
|
||||
"uvicorn-worker>=0.4.0",
|
||||
"redis>=8.1.0",
|
||||
@@ -29,6 +31,7 @@ dev = [
|
||||
"httpx>=0.28.0",
|
||||
"ruff>=0.14.0",
|
||||
"ty>=0.0.1a21",
|
||||
"openapi-spec-validator==0.7.2",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
@@ -82,6 +85,7 @@ ignore = [
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "ARG002", "T20", "TID251"]
|
||||
"tools/*.py" = ["T20"]
|
||||
"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"]
|
||||
"scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"]
|
||||
"scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"]
|
||||
@@ -91,10 +95,10 @@ ignore = [
|
||||
"src/messenger/*/entrypoints/**" = ["TID251", "N818", "PLR0917", "S105"]
|
||||
"src/messenger/shared/config.py" = ["ARG002", "ARG003"]
|
||||
"src/messenger/main.py" = ["TID251"]
|
||||
"src/messenger/migrate.py" = ["TID251"]
|
||||
"src/messenger/relay.py" = ["TID251"]
|
||||
"src/messenger/worker.py" = ["TID251"]
|
||||
"src/messenger/admin.py" = ["TID251"]
|
||||
"alembic/**" = ["TID251"]
|
||||
"src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"]
|
||||
|
||||
[tool.ruff.lint.flake8-tidy-imports.banned-api]
|
||||
@@ -107,7 +111,10 @@ ignore = [
|
||||
known-first-party = ["messenger"]
|
||||
|
||||
[tool.ty.src]
|
||||
include = ["src", "student-tests"]
|
||||
include = ["src", "student-tests", "alembic"]
|
||||
|
||||
[tool.ty.environment]
|
||||
extra-paths = ["alembic"]
|
||||
|
||||
[tool.ty.rules]
|
||||
unresolved-import = "error"
|
||||
@@ -118,7 +125,7 @@ possibly-missing-attribute = "warn"
|
||||
testpaths = ["student-tests"]
|
||||
asyncio_mode = "auto"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
addopts = "-q --strict-markers -m 'not stack'"
|
||||
addopts = "--strict-markers -m 'not stack'"
|
||||
markers = [
|
||||
"stack: black-box проверки против поднятого compose-стенда (make test-faults)",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SETTINGS = (
|
||||
"COMPOSE_PROJECT_NAME",
|
||||
"COMPOSE_FILE",
|
||||
"LB_HTTP_PORT",
|
||||
"LB_STATS_PORT",
|
||||
"BASE_URL",
|
||||
"EVIDENCE_DIR",
|
||||
"RECONCILE_EVIDENCE_DIR",
|
||||
"NO_PROXY",
|
||||
"S3_CPUS",
|
||||
"S3_MEMORY",
|
||||
"UPLOAD_GRACE_SECONDS",
|
||||
"API_IMAGE",
|
||||
"API_REPLICAS",
|
||||
"WEB_CONCURRENCY",
|
||||
"MALLOC_MMAP_THRESHOLD_",
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Capture a command without overwriting evidence")
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("command", nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
command = args.command[1:] if args.command[:1] == ["--"] else args.command
|
||||
if not command:
|
||||
parser.error("command is required")
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
|
||||
started = time.monotonic()
|
||||
with args.output.open("x") as output:
|
||||
|
||||
def write(line):
|
||||
line = line.replace(str(ROOT), "<repo>")
|
||||
output.write(line)
|
||||
output.flush()
|
||||
sys.stdout.write(line)
|
||||
sys.stdout.flush()
|
||||
|
||||
write(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit: {commit}\n")
|
||||
write(
|
||||
"environment: "
|
||||
+ json.dumps({k: os.environ[k] for k in SETTINGS if k in os.environ})
|
||||
+ "\n"
|
||||
)
|
||||
write("$ " + shlex.join(command) + "\n")
|
||||
with subprocess.Popen(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
errors="replace",
|
||||
) as process:
|
||||
for line in process.stdout:
|
||||
write(line)
|
||||
code = process.wait()
|
||||
write(f"exit_code: {code} elapsed_seconds: {time.monotonic() - started:.3f}\n")
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
repo=$PWD
|
||||
project="hlpipe-lab5-clean-$(date -u +%Y%m%d%H%M%S)"
|
||||
dir="$repo/.verification/$project"
|
||||
out="$repo/${EVIDENCE_DIR:-evidence}/clean-clone.txt"
|
||||
sha=$(git rev-parse HEAD)
|
||||
[ ! -e "$out" ] || { echo "Choose a new EVIDENCE_DIR" >&2; exit 2; }
|
||||
mkdir -p "$(dirname "$dir")" "$(dirname "$out")"
|
||||
exec 3>&1
|
||||
exec > "$out" 2>&1
|
||||
cleanup() {
|
||||
status=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ -d "$dir/.git" ]; then
|
||||
cd "$dir"
|
||||
docker compose --profile bench --profile tools --profile reconcile down || status=1
|
||||
docker volume ls --filter "label=com.docker.compose.project=$project"
|
||||
fi
|
||||
echo "exit_code: $status; volumes retained"
|
||||
sed "s#$repo#<repo>#g" "$out" >&3
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' HUP INT TERM
|
||||
step() { echo "\$ $*"; "$@"; }
|
||||
export COMPOSE_PROJECT_NAME="$project" COMPOSE_FILE=compose.yaml
|
||||
export LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 BASE_URL=http://127.0.0.1:18085
|
||||
export NO_PROXY=localhost,127.0.0.1,::1
|
||||
unset API_IMAGE
|
||||
echo "date: $(date -u +%FT%TZ) commit: $sha project: $project"
|
||||
echo "HTTP=$LB_HTTP_PORT stats=$LB_STATS_PORT"
|
||||
step git clone --quiet --no-hardlinks "$repo" "$dir"
|
||||
cd "$dir"
|
||||
step git checkout --quiet --detach "$sha"
|
||||
step make secrets
|
||||
step docker compose --env-file .env config --quiet
|
||||
step make up
|
||||
step docker compose ps -a --format '{{.Service}} {{.State}} {{.Status}} {{.Ports}}'
|
||||
step make test
|
||||
step make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
step make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
step docker compose ps -a --format '{{.Service}} {{.Name}} {{.State}}' api
|
||||
step make restore
|
||||
step make scenario SCENARIO=failover-upload
|
||||
step make restore
|
||||
step make scenario SCENARIO=s3-outage
|
||||
step sh scripts/s3-check.sh
|
||||
step make reconcile
|
||||
step make migrate
|
||||
step make down
|
||||
step docker volume ls --filter "label=com.docker.compose.project=$project"
|
||||
@@ -8,7 +8,7 @@ replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
|
||||
detect() { python3 scripts/probe.py detect "$@"; }
|
||||
failed=0
|
||||
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
|
||||
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
|
||||
-- docker stop -t "${STOP_TIMEOUT:-7}" "$(replica)" || failed=1
|
||||
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
|
||||
-- docker kill "$(replica)" || failed=1
|
||||
hung=$(replica)
|
||||
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
out=${1:?usage: docker-stats.sh <csv>}
|
||||
vm=$(dirname "$out")/vm_stat.csv
|
||||
cd "$(dirname "$0")/.."
|
||||
project=$(docker compose config | sed -n 's/^name: //p')
|
||||
echo 'ts,name,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids' > "$out"
|
||||
echo 'ts,busy,total,cpus' > "$vm"
|
||||
while :; do
|
||||
ts=$(date +%s)
|
||||
docker compose exec -T valkey cat /proc/stat 2>/dev/null | awk -v ts="$ts" '/^cpu /{b=$2+$3+$4+$7+$8+$9; t=b+$5+$6} /^cpu[0-9]/{n++} END{if (t) print ts "," b "," t "," n}' >> "$vm" || true
|
||||
ids=$(docker ps -q --filter "label=com.docker.compose.project=$project")
|
||||
[ -z "$ids" ] || docker stats --no-stream --format "$ts,{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true
|
||||
sleep "${INTERVAL:-1}"
|
||||
done
|
||||
@@ -1,12 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; }
|
||||
value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; }
|
||||
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
|
||||
bucket=${S3_BUCKET:-messenger-attachments}
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
|
||||
echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'"
|
||||
docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}'
|
||||
for table in attachments message_attachments; do
|
||||
@@ -36,5 +38,5 @@ bucket=${S3_BUCKET:-messenger-attachments}
|
||||
docker compose config --format json | python3 -c 'import json,sys
|
||||
for name, service in sorted(json.load(sys.stdin)["services"].items()):
|
||||
print(name, sorted(s["source"] for s in service.get("secrets", [])))'
|
||||
} > evidence/storage-layout.txt
|
||||
cat evidence/storage-layout.txt
|
||||
} > "$out"/storage-layout.txt
|
||||
cat "$out"/storage-layout.txt
|
||||
|
||||
@@ -11,14 +11,25 @@ cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
|
||||
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
|
||||
cli dbsize
|
||||
cli "info stats" | grep -E '^keyspace_(hits|misses)'
|
||||
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
|
||||
hash=${key#session:}
|
||||
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
|
||||
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
|
||||
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
|
||||
echo "\$ количество отметок отзыва и закэшированных сессий"
|
||||
revoked=0; cached=0
|
||||
echo "\$ valkey-cli --scan --pattern 'session:*' и вид значения (хеш сокращён, значения не печатаются)"
|
||||
revoked=0; cached=0; key=""; stored=""
|
||||
for k in $(cli "--scan --pattern session:* --count 1000"); do
|
||||
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
|
||||
if [ "$(cli "get $k")" = "-" ]; then
|
||||
revoked=$((revoked + 1)); printf '%.16s… отметка отзыва "-"\n' "$k"; continue
|
||||
fi
|
||||
cached=$((cached + 1)); printf '%.16s… сессия\n' "$k"
|
||||
if [ -z "$key" ]; then
|
||||
stored=$(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('${k#session:}', 'hex')")
|
||||
if [ -n "$stored" ]; then key=$k; fi
|
||||
fi
|
||||
done
|
||||
echo "tombstones: $revoked, cached sessions: $cached"
|
||||
if [ "$revoked" -eq 0 ] || [ -z "$key" ]; then
|
||||
echo "нужны закэшированная сессия из БД и отметка отзыва: сначала make test" >&2
|
||||
exit 1
|
||||
fi
|
||||
expires=$(cli "pexpiretime $key")
|
||||
echo "\$ срок ключа кэша против срока сессии для $(printf '%.16s' "$key")…"
|
||||
echo "valkey pexpiretime, ms: $expires"
|
||||
echo "postgres expires_at, ms: $stored"
|
||||
if [ "$expires" -le "$stored" ]; then echo "ключ не переживает сессию: да"; else echo "ключ не переживает сессию: НЕТ"; exit 1; fi
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
base=${1:-http://$(docker compose port lb 8080)}
|
||||
stats=http://$(docker compose port lb 8404)/stats\;csv
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
|
||||
{
|
||||
header
|
||||
echo "\$ docker compose --env-file .env config --quiet && echo valid"
|
||||
@@ -25,7 +27,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
|
||||
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
|
||||
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
|
||||
} > evidence/stack.txt
|
||||
} > "$out"/stack.txt
|
||||
{
|
||||
header
|
||||
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
|
||||
@@ -38,7 +40,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
|
||||
echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)"
|
||||
echo "packages (uv.lock):"
|
||||
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
|
||||
for name in fastapi uvicorn uvicorn-worker gunicorn sqlalchemy alembic asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
|
||||
faststream aio-pika aiormq dishka-faststream; do
|
||||
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
|
||||
done
|
||||
@@ -47,5 +49,5 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
|
||||
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
|
||||
done | sort -u
|
||||
} > evidence/environment.txt
|
||||
cat evidence/stack.txt evidence/environment.txt
|
||||
} > "$out"/environment.txt
|
||||
cat "$out"/stack.txt "$out"/environment.txt
|
||||
|
||||
+16
-13
@@ -1,20 +1,22 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
|
||||
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
|
||||
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM alembic_version) versions"
|
||||
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, username = 'seed_author' AS seed_stub, count(*) AS users, min(length(password_hash)) AS min_length, max(length(password_hash)) AS max_length FROM users GROUP BY 1, 2 ORDER BY 2"
|
||||
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
|
||||
{
|
||||
header
|
||||
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do
|
||||
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments alembic_version; do
|
||||
echo "\$ psql -c '\\d+ $table'"
|
||||
psql "\\d+ $table"
|
||||
done
|
||||
echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'"
|
||||
psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations"
|
||||
} > evidence/schema.txt
|
||||
echo "\$ psql -c 'SELECT version_num FROM alembic_version'"
|
||||
psql "SELECT version_num FROM alembic_version"
|
||||
} > "$out"/schema.txt
|
||||
{
|
||||
header
|
||||
echo "\$ psql -c \"$counts\""
|
||||
@@ -23,14 +25,14 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
|
||||
docker compose run --rm migrate 2>/dev/null
|
||||
echo "\$ psql -c \"$counts\""
|
||||
psql "$counts"
|
||||
} > evidence/migrations.txt
|
||||
} > "$out"/migrations.txt
|
||||
{
|
||||
header
|
||||
echo "\$ psql -c \"$kdf\""
|
||||
psql "$kdf"
|
||||
echo "\$ psql -c \"$tokens\""
|
||||
psql "$tokens"
|
||||
} > evidence/kdf.txt
|
||||
} > "$out"/kdf.txt
|
||||
{
|
||||
header
|
||||
echo "\$ git ls-files secrets .env backups volumes"
|
||||
@@ -38,7 +40,8 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
|
||||
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
|
||||
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
|
||||
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
|
||||
echo "\$ docker volume ls --filter name=mtusi2026lab5"
|
||||
docker volume ls --filter name=mtusi2026lab5
|
||||
} > evidence/secrets.txt
|
||||
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
|
||||
project=$(docker compose config --format json | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])')
|
||||
echo "\$ docker volume ls --filter label=com.docker.compose.project=$project"
|
||||
docker volume ls --filter "label=com.docker.compose.project=$project"
|
||||
} > "$out"/secrets.txt
|
||||
echo ""$out"/{schema,migrations,kdf,secrets}.txt обновлены"
|
||||
|
||||
@@ -10,7 +10,7 @@ failed=0
|
||||
set --
|
||||
for host in $hosts; do
|
||||
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
|
||||
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
|
||||
-- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1
|
||||
scripts/restore.sh
|
||||
set -- "$@" "$out/$host/result.json"
|
||||
done
|
||||
|
||||
@@ -1,28 +1,33 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
replicas() {
|
||||
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
|
||||
}
|
||||
tmp=evidence/failover.tmp
|
||||
url=${1:-http://localhost:8080}
|
||||
shift=0
|
||||
while [ "$shift" -lt "${2:-0}" ]; do curl -fsS -o /dev/null "$url/health/live"; shift=$((shift + 1)); done
|
||||
tmp="$out"/failover.tmp
|
||||
since=$(date -u +%FT%TZ)
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
|
||||
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
|
||||
replicas
|
||||
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$url"
|
||||
echo "== after: api replicas"
|
||||
replicas
|
||||
} > "$tmp" 2>&1
|
||||
stopped=$(replicas | awk '$2 == "exited" {print $1}')
|
||||
{
|
||||
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
|
||||
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
|
||||
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|service.draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
|
||||
echo "\$ scripts/restore.sh"
|
||||
scripts/restore.sh
|
||||
echo "== after restore"
|
||||
replicas
|
||||
} >> "$tmp" 2>&1
|
||||
mv "$tmp" "evidence/failover-$stopped.txt"
|
||||
cat "evidence/failover-$stopped.txt"
|
||||
mv "$tmp" "$out/failover-$stopped.txt"
|
||||
cat "$out/failover-$stopped.txt"
|
||||
|
||||
@@ -23,6 +23,19 @@ def compose(*args, stdin=None):
|
||||
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def containers(service):
|
||||
ids = compose("ps", "-aq", service).stdout.split()
|
||||
fields = ("hostname", "status", "exit_code", "started_at", "finished_at")
|
||||
template = (
|
||||
"{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}"
|
||||
" {{.State.StartedAt}} {{.State.FinishedAt}}"
|
||||
)
|
||||
out = subprocess.run(
|
||||
["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False
|
||||
)
|
||||
return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()]
|
||||
|
||||
|
||||
def upload(c, actor, chat, expected=202):
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
||||
schema = "Attachment" if expected == 202 else None
|
||||
@@ -90,6 +103,10 @@ def failover_upload(c, args):
|
||||
recovered = time.monotonic() - started
|
||||
served = got["headers"].get("X-Instance-Id")
|
||||
check(served != target, "download served by the stopped replica")
|
||||
during = containers(args.service)
|
||||
states = {item["hostname"]: item["status"] for item in during}
|
||||
check(states.get(target) == "exited", f"target {target} is not exited: {states}")
|
||||
check(states.get(served) == "running", f"neighbour {served} is not running: {states}")
|
||||
digest = hashlib.sha256(got["bytes"]).hexdigest()
|
||||
check(
|
||||
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
|
||||
@@ -104,7 +121,9 @@ def failover_upload(c, args):
|
||||
"sha256_equal": True,
|
||||
"hook_seconds": round(hook_seconds, 3),
|
||||
"download_ok_after_seconds": round(recovered, 3),
|
||||
"replicas_at_download": during,
|
||||
"restored": args.restore,
|
||||
"replicas_after_restore": containers(args.service) if args.restore else None,
|
||||
}
|
||||
|
||||
|
||||
@@ -124,6 +143,7 @@ def s3_outage(c, args):
|
||||
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
|
||||
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
|
||||
}
|
||||
stopped = containers(args.s3_service)
|
||||
finally:
|
||||
started = time.monotonic()
|
||||
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
|
||||
@@ -131,8 +151,11 @@ def s3_outage(c, args):
|
||||
got = wait(lambda: content(c, bob, att), 30)
|
||||
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
|
||||
expected = {"upload": 503, "download": 503, "text_message": 201}
|
||||
check(all(during[k]["status"] == v for k, v in expected.items()), f"S3 outage: {during}")
|
||||
check(all(during[k]["seconds"] < 6 for k in ("upload", "download")), "S3 timeout exceeded")
|
||||
return {
|
||||
"scenario": "s3-outage",
|
||||
"s3_container_during_outage": stopped,
|
||||
"during_outage": during,
|
||||
"expected": expected,
|
||||
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
|
||||
@@ -180,6 +203,8 @@ def abort_upload(c, args):
|
||||
)
|
||||
rows = out.stdout.split()
|
||||
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
|
||||
check(out.returncode == 0, "cannot inspect aborted upload")
|
||||
check(counts.get("ready", 0) == 0, "aborted upload became ready")
|
||||
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
|
||||
return {
|
||||
"scenario": "abort-upload",
|
||||
@@ -216,10 +241,17 @@ def main():
|
||||
"s3-outage": s3_outage,
|
||||
"abort-upload": abort_upload,
|
||||
}[args.scenario]
|
||||
commit = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, check=False
|
||||
)
|
||||
stamp = {
|
||||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"commit": commit.stdout.decode().strip(),
|
||||
}
|
||||
try:
|
||||
report = {"ok": True, **run(c, args)}
|
||||
report = {"ok": True, **stamp, **run(c, args)}
|
||||
except Failure as exc:
|
||||
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
|
||||
report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + "\n")
|
||||
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
out=${1:?usage: lb-stats.sh <csv>}
|
||||
cd "$(dirname "$0")/.."
|
||||
url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv}
|
||||
curl -fsS "$url" | sed -n '1s/^# /ts,/p' > "$out"
|
||||
while :; do
|
||||
ts=$(date +%s)
|
||||
curl -fsS "$url" | grep -E '^(public|api),' | sed "s/^/$ts,/" >> "$out" || true
|
||||
sleep "${INTERVAL:-1}"
|
||||
done
|
||||
+29
-5
@@ -9,12 +9,20 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
S3_CLIENT_SECRETS = [
|
||||
"s3_access_key_id",
|
||||
"s3_reconciler_access_key_id",
|
||||
"s3_reconciler_secret_access_key",
|
||||
"s3_worker_access_key_id",
|
||||
"s3_worker_secret_access_key",
|
||||
"s3_root_password",
|
||||
"s3_root_user",
|
||||
"s3_secret_access_key",
|
||||
"s3_worker_access_key_id",
|
||||
"s3_worker_secret_access_key",
|
||||
]
|
||||
RECONCILER_SECRETS = [
|
||||
"database_url",
|
||||
"s3_reconciler_access_key_id",
|
||||
"s3_reconciler_secret_access_key",
|
||||
]
|
||||
OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli")
|
||||
POLICY = {
|
||||
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
|
||||
"api": {
|
||||
@@ -29,6 +37,11 @@ POLICY = {
|
||||
"networks": ["backend"],
|
||||
},
|
||||
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
|
||||
"reconciler": {
|
||||
"secrets": RECONCILER_SECRETS,
|
||||
"volumes": [],
|
||||
"networks": ["backend"],
|
||||
},
|
||||
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
|
||||
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
|
||||
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
|
||||
@@ -94,6 +107,9 @@ def declared(name, service, networks, problems):
|
||||
problems.append(f"{name}: publishes {ports} beyond loopback")
|
||||
if name == "api" and not service.get("read_only"):
|
||||
problems.append("api: root filesystem is writable")
|
||||
user = str(service.get("user", ""))
|
||||
if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"):
|
||||
problems.append(f"{name}: runs as root")
|
||||
internal = [n for n in nets if networks.get(n, {}).get("internal")]
|
||||
return [
|
||||
name,
|
||||
@@ -103,6 +119,7 @@ def declared(name, service, networks, problems):
|
||||
",".join(volumes) or "-",
|
||||
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
|
||||
str(service.get("read_only", False)).lower(),
|
||||
user or "image",
|
||||
]
|
||||
|
||||
|
||||
@@ -149,6 +166,8 @@ def main():
|
||||
"bench",
|
||||
"--profile",
|
||||
"tools",
|
||||
"--profile",
|
||||
"reconcile",
|
||||
"config",
|
||||
"--format",
|
||||
"json",
|
||||
@@ -162,9 +181,14 @@ def main():
|
||||
if not any(row[0] == "api" for row in containers):
|
||||
problems.append("no running api replicas to inspect")
|
||||
lines = [
|
||||
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
|
||||
"$ docker compose --profile bench --profile tools config --format json (declared)",
|
||||
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
|
||||
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} "
|
||||
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
|
||||
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
|
||||
" (declared)",
|
||||
*table(
|
||||
["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"],
|
||||
services,
|
||||
),
|
||||
"",
|
||||
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
|
||||
*table(["service", "hostname", "cpus", "memory", "published"], containers),
|
||||
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
runs="${RUNS:-3}"
|
||||
pause="${PAUSE:-15}"
|
||||
project=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$(docker compose ps -q lb)")
|
||||
network="${project}_edge"
|
||||
image=$(sed -n 's/^FROM \(python:[^ ]*\) AS runtime$/\1/p' Dockerfile)
|
||||
name="$project-load-client"
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out/load"
|
||||
[ -z "$(ls -A "$out/load")" ] || { echo "Choose an empty EVIDENCE_DIR/load" >&2; exit 2; }
|
||||
client() {
|
||||
docker run --rm --name "$name" --network "$network" --cpus 2 --memory 512m \
|
||||
--sysctl net.ipv4.tcp_tw_reuse=1 -v "$PWD/tests:/w/tests:ro" -v "$PWD/contracts:/w/contracts:ro" \
|
||||
-v "$PWD/lab.json:/w/lab.json:ro" -v "$PWD/$out/load:/out" "$image" python /w/tests/load.py "$@"
|
||||
}
|
||||
handled() {
|
||||
docker compose logs --no-log-prefix --since "$1" api | python3 -c 'import collections, json, sys
|
||||
seen = collections.Counter()
|
||||
for line in sys.stdin:
|
||||
if line.startswith("{"):
|
||||
event = json.loads(line)
|
||||
if event.get("event") == "request.handled":
|
||||
seen[(event["status"], event["method"], event["route"])] += 1
|
||||
for (status, method, route), count in sorted(seen.items()):
|
||||
print(status, method, route, count)'
|
||||
}
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) network: $network"
|
||||
echo "# make load-evidence (scripts/load-evidence.sh): закрытый контур tests/load.py, 64 потока, 8 с, все записи"
|
||||
echo "# прогона в один новый чат, $runs повтора с паузой $pause с; клиент — одноразовый контейнер в сети <проект>_edge"
|
||||
echo "# стенда (2 CPU, 512 MiB, базовый образ API по digest, tests/, contracts/ и lab.json только для чтения,"
|
||||
echo "# tcp_tw_reuse=1), адрес http://lb:8080. На 4-й секунде — docker stats; после прогона — request.handled реплик."
|
||||
echo "\$ docker ps --format '{{.Names}}' | grep -v '^$project-' # чужие контейнеры на VM во время серии"
|
||||
docker ps --format '{{.Names}}' | grep -v "^$project-" || true
|
||||
echo "\$ client() { docker run --rm --name $name --network $network --cpus 2 --memory 512m --sysctl net.ipv4.tcp_tw_reuse=1 -v \"\$PWD/tests:/w/tests:ro\" -v \"\$PWD/contracts:/w/contracts:ro\" -v \"\$PWD/lab.json:/w/lab.json:ro\" -v \"\$PWD/$out/load:/out\" $image python /w/tests/load.py \"\$@\"; }"
|
||||
run=1
|
||||
while [ "$run" -le "$runs" ]; do
|
||||
since=$(date -u +%FT%TZ)
|
||||
echo "\$ client --base-url http://lb:8080 --duration 8 --concurrency 64 --output /out/run$run.json"
|
||||
client --base-url http://lb:8080 --duration 8 --concurrency 64 --output "/out/run$run.json" > "$work/out" 2>&1 &
|
||||
pid=$!
|
||||
sleep 4
|
||||
stats=$(docker stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}}' $(docker compose ps -q) "$name" | sed "s/^$project-//" | sort)
|
||||
if wait "$pid"; then code=0; else code=$?; fi
|
||||
echo "exit $code"
|
||||
cat "$work/out"
|
||||
echo "\$ docker stats --no-stream # 4-я секунда прогона: контейнер CPU% память"
|
||||
echo "$stats"
|
||||
echo "\$ docker compose logs --since $since api | request.handled: статус метод маршрут число"
|
||||
handled "$since"
|
||||
[ "$code" -eq 0 ] || { exit "$code"; }
|
||||
[ "$run" -lt "$runs" ] && sleep "$pause"
|
||||
run=$((run + 1))
|
||||
done
|
||||
echo "\$ docker compose exec -T postgres psql -c 'SELECT count(*) FROM messages'"
|
||||
docker compose exec -T postgres psql -U messenger -d messenger -qAt -c 'SELECT count(*) FROM messages'
|
||||
} > "$out/load.txt"
|
||||
cat "$out/load.txt"
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
limit="${WAIT_SECONDS:-1500}"
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
sql() { docker compose exec -T postgres psql -U messenger -d messenger -X "$@"; }
|
||||
pending() { sql -tAc "SELECT count(*) FROM attachments WHERE status = 'pending'"; }
|
||||
statuses="SELECT status, count(*), min(created_at), max(created_at) FROM attachments GROUP BY status ORDER BY status"
|
||||
since=$(date -u +%FT%TZ)
|
||||
{
|
||||
echo "date: $since commit: $(git rev-parse HEAD)"
|
||||
echo "# make overload-leftovers (scripts/overload-leftovers.sh) после серии памяти: невидимые намерения от 503 по"
|
||||
echo "# таймауту видны сверке свежими, уборка API удаляет их после UPLOAD_GRACE_SECONDS; стенд тот же"
|
||||
echo "\$ make reconcile # dry-run"
|
||||
docker compose run --rm --no-deps -T reconciler 2>/dev/null
|
||||
echo "\$ psql -c \"$statuses\""
|
||||
sql -c "$statuses"
|
||||
echo "\$ until [ \"\$(psql -tAc \"SELECT count(*) FROM attachments WHERE status = 'pending'\")\" = 0 ]; do sleep 10; done"
|
||||
started=$(date +%s)
|
||||
until [ "$(pending)" = 0 ] || [ $(( $(date +%s) - started )) -gt "$limit" ]; do sleep 10; done
|
||||
echo "pending = $(pending) в $(date -u +%FT%TZ)"
|
||||
[ "$(pending)" = 0 ] || { echo "FAIL: pending cleanup timed out"; exit 1; }
|
||||
echo "\$ docker compose logs --since $since api | audit.attachment_discarded и attachments.swept"
|
||||
docker compose logs --no-log-prefix --since "$since" api | python3 -c 'import collections, json, sys
|
||||
reasons, swept, times = collections.Counter(), 0, []
|
||||
for line in sys.stdin:
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
event = json.loads(line)
|
||||
if event.get("event") == "audit.attachment_discarded":
|
||||
reasons[event["reason"]] += 1
|
||||
times.append(event["ts"][:19] + "Z")
|
||||
elif event.get("event") == "attachments.swept":
|
||||
swept += event["count"]
|
||||
print(dict(reasons), "swept", swept, "first", min(times, default="-"), "last", max(times, default="-"))'
|
||||
echo "\$ psql -c \"$statuses\""
|
||||
sql -c "$statuses"
|
||||
echo "\$ make reconcile # dry-run после уборки"
|
||||
docker compose run --rm --no-deps -T reconciler 2>/dev/null
|
||||
} > "$out/overload-leftovers.txt"
|
||||
cat "$out/overload-leftovers.txt"
|
||||
@@ -6,7 +6,7 @@ started() {
|
||||
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
|
||||
done | sort
|
||||
}
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
|
||||
echo "== before: service hostname StartedAt status"
|
||||
started
|
||||
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
mode=${1:?usage: pg-stats.sh activity <csv> | statements <csv> | size | reset}
|
||||
cd "$(dirname "$0")/.."
|
||||
psql() {
|
||||
docker compose exec -T postgres sh -c 'exec psql -X -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "${POSTGRES_DB:-$POSTGRES_USER}" "$@"' psql "$@"
|
||||
}
|
||||
case $mode in
|
||||
reset)
|
||||
psql -c 'set client_min_messages = warning' -c 'create extension if not exists pg_stat_statements' -c 'select pg_stat_statements_reset()' >/dev/null
|
||||
;;
|
||||
size)
|
||||
psql -At -F, -c "select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
|
||||
;;
|
||||
statements)
|
||||
psql -c "copy (select calls, round(total_exec_time::numeric, 3) as total_ms, round(mean_exec_time::numeric, 3) as mean_ms, rows, shared_blks_hit, shared_blks_read, left(regexp_replace(query, '\s+', ' ', 'g'), 200) as query from pg_stat_statements where dbid = (select oid from pg_database where datname = current_database()) and query not like '%pg_stat%' order by total_exec_time desc limit ${TOP:-20}) to stdout with csv header" > "${2:?csv}"
|
||||
;;
|
||||
activity)
|
||||
echo 'ts,state,wait_event_type,wait_event,backends,max_connections' > "${2:?csv}"
|
||||
while :; do
|
||||
ts=$(date +%s)
|
||||
psql -At -F, -c "select coalesce(state, ''), coalesce(wait_event_type, ''), coalesce(wait_event, ''), count(*), current_setting('max_connections') from pg_stat_activity where backend_type = 'client backend' and pid <> pg_backend_pid() group by 1, 2, 3" | sed "s/^/$ts,/" >> "$2" || true
|
||||
sleep "${INTERVAL:-1}"
|
||||
done
|
||||
;;
|
||||
*)
|
||||
echo "unknown mode $mode" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
+163
-7
@@ -10,12 +10,16 @@ import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
||||
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
|
||||
PSQL += ["-X", "-q", "-At"]
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -207,16 +211,112 @@ def check(args):
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def output(*command):
|
||||
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
return done.stdout + done.stderr
|
||||
|
||||
|
||||
def replicas():
|
||||
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
|
||||
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
|
||||
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
|
||||
|
||||
|
||||
def answered(container, request_id):
|
||||
for line in output("docker", "logs", container).splitlines():
|
||||
if request_id in line and "request.handled" in line:
|
||||
return json.loads(line).get("status")
|
||||
return None
|
||||
|
||||
|
||||
def lost(alice, bob, chat, kill, args):
|
||||
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
|
||||
request_id = "lost-" + key[:12]
|
||||
lock = subprocess.Popen(
|
||||
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
lock.stdin.write(
|
||||
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
|
||||
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
|
||||
)
|
||||
lock.stdin.flush()
|
||||
lock.stdout.readline()
|
||||
sent = time.time()
|
||||
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
|
||||
path = f"/api/v1/chats/{chat}/messages"
|
||||
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
|
||||
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
|
||||
waiting = set(output(*PSQL, "-c", sql).split())
|
||||
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
|
||||
if kill and holder[0]:
|
||||
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
|
||||
lock.stdin.close()
|
||||
lock.wait()
|
||||
retried = time.time()
|
||||
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
|
||||
while status != 201 and attempts < 20:
|
||||
time.sleep(0.2)
|
||||
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
|
||||
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
|
||||
copies = [m for m in page["items"] if m["text"] == text]
|
||||
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
|
||||
report = {
|
||||
"variant": "kill" if kill else "timeout",
|
||||
"lost_attempt": {
|
||||
"status": first,
|
||||
"client_timeout_s": args.timeout,
|
||||
"chat_lock_held_s": args.hold,
|
||||
"replica": holder[1],
|
||||
"replica_killed": bool(kill and holder[0]),
|
||||
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
|
||||
},
|
||||
"retry": {
|
||||
"after_s": round(retried - sent, 3),
|
||||
"status": status,
|
||||
"attempts": attempts,
|
||||
"replica": instance,
|
||||
},
|
||||
"copies_in_history": len(copies),
|
||||
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
|
||||
"written_by": None
|
||||
if stamp is None
|
||||
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
|
||||
}
|
||||
if kill and holder[0]:
|
||||
subprocess.call(
|
||||
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
||||
)
|
||||
report["pass"] = (
|
||||
first == 0
|
||||
and holder[0] is not None
|
||||
and status == 201
|
||||
and report["copies_in_history"] == 1
|
||||
and report["same_message"]
|
||||
and (kill or report["lost_attempt"]["replica_answered"] == 201)
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def retry(args):
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
chat = chat_of(alice, bob)
|
||||
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
|
||||
passed = all(v["pass"] for v in variants)
|
||||
emit({"variants": variants, "pass": passed}, args.out)
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def record(stop, alice, chat):
|
||||
probes, trace = [], []
|
||||
probes, trace, confirmed = [], [], []
|
||||
|
||||
def prober():
|
||||
key = uuid.uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.time()
|
||||
status, instance, _ = send(alice, chat, "probe " + key, key)
|
||||
status, instance, body = send(alice, chat, "probe " + key, key)
|
||||
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
|
||||
if status == 201:
|
||||
confirmed.append(body)
|
||||
key = uuid.uuid4().hex
|
||||
time.sleep(0.1)
|
||||
|
||||
@@ -228,7 +328,38 @@ def record(stop, alice, chat):
|
||||
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
return probes, trace, threads
|
||||
return probes, trace, threads, confirmed
|
||||
|
||||
|
||||
def history_check(reader, chat, confirmed):
|
||||
items, cursors = [], set()
|
||||
cursor = None
|
||||
for _ in range(len(confirmed) // 100 + 2):
|
||||
path = f"/api/v1/chats/{chat}/messages?limit=100"
|
||||
if cursor:
|
||||
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
|
||||
status, _, page = call("GET", path, headers=reader["auth"])
|
||||
if status != 200 or not isinstance(page, dict):
|
||||
break
|
||||
items.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"])
|
||||
return {
|
||||
"confirmed": len(confirmed),
|
||||
"stored": len(items),
|
||||
"exact_match": bool(confirmed) and same,
|
||||
"unique": len({m["id"] for m in items}) == len(items),
|
||||
}
|
||||
if cursor in cursors:
|
||||
break
|
||||
cursors.add(cursor)
|
||||
return {
|
||||
"confirmed": len(confirmed),
|
||||
"stored": len(items),
|
||||
"exact_match": False,
|
||||
"unique": False,
|
||||
}
|
||||
|
||||
|
||||
def write_csv(path, header, rows):
|
||||
@@ -241,6 +372,10 @@ def summarize(args, probes, trace, action, expected):
|
||||
t0, end = action["start"], action["end"]
|
||||
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
|
||||
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
|
||||
successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0]
|
||||
recovery = (
|
||||
next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0
|
||||
)
|
||||
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
|
||||
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
|
||||
outage = [t for t in down if not back or t < back[0]] or [end]
|
||||
@@ -256,13 +391,15 @@ def summarize(args, probes, trace, action, expected):
|
||||
"action_finished_before_load_end": end < action["load_end"],
|
||||
"lb_up_before": expected,
|
||||
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
|
||||
"lb_samples_missing": sum(up is None for _, up in trace),
|
||||
"detection_seconds": round(down[0] - t0, 3) if down else None,
|
||||
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
|
||||
"probe": {
|
||||
"attempts": len(probes),
|
||||
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
|
||||
"first_failure_s": round(failures[0], 3) if failures else None,
|
||||
"recovery_s": round(failures[-1], 3) if failures else 0.0,
|
||||
"first_success_s": round(successes[0][1], 3) if successes else None,
|
||||
"recovery_s": round(recovery, 3) if recovery is not None else None,
|
||||
"instances_before": dict(before),
|
||||
"instances_during_outage": dict(during),
|
||||
"instances_after": dict(after),
|
||||
@@ -275,7 +412,13 @@ def verdict(args, result):
|
||||
checks = {
|
||||
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
|
||||
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
|
||||
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
|
||||
"load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0,
|
||||
"lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0,
|
||||
"write_recovery_ok": all(
|
||||
probe[k] is not None and probe[k] <= args.max_recovery
|
||||
for k in ("first_success_s", "recovery_s")
|
||||
),
|
||||
"history_preserved": result["history"]["exact_match"] and result["history"]["unique"],
|
||||
}
|
||||
if args.expect == "dip":
|
||||
checks["detected"] = result["detection_seconds"] is not None
|
||||
@@ -296,7 +439,8 @@ def load(args):
|
||||
expected = lb_up() or 0
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
stop, action = threading.Event(), {}
|
||||
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
|
||||
chat = chat_of(alice, bob)
|
||||
probes, trace, threads, confirmed = record(stop, alice, chat)
|
||||
|
||||
def act():
|
||||
time.sleep(args.warmup + args.lead)
|
||||
@@ -327,6 +471,7 @@ def load(args):
|
||||
)
|
||||
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
|
||||
result = summarize(args, probes, trace, action, expected)
|
||||
result["history"] = history_check(bob, chat, confirmed)
|
||||
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
|
||||
result["k6"] = dict(
|
||||
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
|
||||
@@ -344,10 +489,13 @@ def detect(args):
|
||||
started = time.time()
|
||||
process = subprocess.Popen(args.command, cwd=str(ROOT))
|
||||
detected = acted = None
|
||||
missing = 0
|
||||
while time.time() - started < args.timeout and (detected is None or acted is None):
|
||||
if acted is None and process.poll() is not None:
|
||||
acted = time.time() - started
|
||||
if detected is None and (lb_up() or 0) <= until:
|
||||
up = lb_up()
|
||||
missing += up is None
|
||||
if detected is None and up is not None and up <= until:
|
||||
detected = time.time() - started
|
||||
time.sleep(0.05)
|
||||
code = process.wait()
|
||||
@@ -362,6 +510,7 @@ def detect(args):
|
||||
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
|
||||
checks = {
|
||||
"action_ok": code == 0,
|
||||
"lb_observed": before >= 2 and missing == 0,
|
||||
"detected_in_time": detected is not None and detected <= args.max + args.slack,
|
||||
"lb_answer_ok": status == args.lb_status,
|
||||
}
|
||||
@@ -374,6 +523,7 @@ def detect(args):
|
||||
"action_exit": code,
|
||||
"action_seconds": round(acted, 3),
|
||||
"lb_up_before": before,
|
||||
"lb_samples_missing": missing,
|
||||
"lb_up_counted_as_detected": until,
|
||||
"detection_seconds": None if detected is None else round(detected, 3),
|
||||
"declared_max_seconds": args.max,
|
||||
@@ -413,6 +563,10 @@ def main():
|
||||
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
||||
under.add_argument("--max-recovery", type=float, default=10.0)
|
||||
under.add_argument("command", nargs=argparse.REMAINDER)
|
||||
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
|
||||
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
|
||||
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
|
||||
again.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
||||
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
||||
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
||||
@@ -426,6 +580,8 @@ def main():
|
||||
args = parser.parse_args()
|
||||
if args.mode == "check":
|
||||
return check(args)
|
||||
if args.mode == "retry":
|
||||
return retry(args)
|
||||
args.command = [c for c in args.command if c != "--"]
|
||||
if not args.command:
|
||||
parser.error(f"{args.mode} needs an action command after --")
|
||||
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
kind="${1:?usage: reconcile-inject.sh orphan|missing|corrupt|bitflip|stale-pending|pending-object|multipart [count]}"
|
||||
count="${2:-1}"
|
||||
age="${INJECT_AGE_SECONDS:-7200}"
|
||||
prefix="${S3_PREFIX:-attachments/}"
|
||||
sql() { docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -qAt -F ' ' -U "${PGUSER:-messenger}" -d "${PGDATABASE:-messenger}" -c "$1"; }
|
||||
s3() { docker compose run --rm --no-deps -T -e ROWS="$1" s3-cli sh -c "$2"; }
|
||||
ready() { sql "SELECT id, object_key, size_bytes FROM attachments WHERE status = 'ready' ${INJECT_IDS:+AND id = ANY('{$INJECT_IDS}'::uuid[])} ORDER BY random() LIMIT $count"; }
|
||||
pending() {
|
||||
sql "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at)
|
||||
SELECT g.id, s.chat_id, s.owner_id, 'injected.png', 'image/png', 2048, repeat('0', 64), 1, 1, '$prefix' || g.id, 'pending',
|
||||
clock_timestamp() - make_interval(secs => $age), clock_timestamp() - make_interval(secs => $age)
|
||||
FROM (SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1) s,
|
||||
(SELECT gen_random_uuid() AS id FROM generate_series(1, $count)) g
|
||||
RETURNING id, object_key"
|
||||
}
|
||||
fresh() { sql "SELECT g, '$prefix' || g FROM (SELECT gen_random_uuid() AS g FROM generate_series(1, $count)) s"; }
|
||||
put_random='printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; head -c "${size:-2048}" /dev/urandom | rc pipe "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done'
|
||||
case "$kind" in
|
||||
orphan) rows=$(fresh); s3 "$rows" "$put_random" | sed "s/^/orphan /" ;;
|
||||
missing) rows=$(ready); s3 "$rows" 'printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; rc object remove --force "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' | sed "s/^/missing /" ;;
|
||||
corrupt) rows=$(ready | awk '{print $1, $2, int($3 / 2) + 1}'); s3 "$rows" "$put_random" | sed "s/^/corrupt /" ;;
|
||||
bitflip) rows=$(ready); s3 "$rows" "$put_random" | sed "s/^/bitflip /" ;;
|
||||
stale-pending) pending | sed "s/^/stale-pending /" ;;
|
||||
pending-object) rows=$(pending); s3 "$rows" "$put_random" | sed "s/^/pending-object /" ;;
|
||||
multipart)
|
||||
fresh | while read -r _ key; do
|
||||
docker compose exec -T -e KEY="$key" -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" s3 sh -c 'u="http://127.0.0.1:9000/$S3_BUCKET/$KEY"; a="$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"; s="aws:amz:${RUSTFS_REGION:-us-east-1}:s3"
|
||||
upload=$(curl -fsS --aws-sigv4 "$s" --user "$a" -X POST "$u?uploads" | sed -n "s:.*<UploadId>\(.*\)</UploadId>.*:\1:p")
|
||||
head -c 4096 /dev/urandom | curl -fsS --aws-sigv4 "$s" --user "$a" -X PUT --data-binary @- "$u?partNumber=1&uploadId=$upload" -o /dev/null' </dev/null
|
||||
echo "multipart - $key"
|
||||
done
|
||||
;;
|
||||
*) echo "unknown kind: $kind" >&2; exit 2 ;;
|
||||
esac
|
||||
Executable
+199
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SQL = (
|
||||
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256,"
|
||||
" floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint,"
|
||||
" EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id),"
|
||||
" (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL"
|
||||
" FROM attachments a ORDER BY a.object_key"
|
||||
)
|
||||
HASH = (
|
||||
'while read -r k; do printf "%s %s\\n" "$k" '
|
||||
'"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done'
|
||||
)
|
||||
CLASSES = (
|
||||
"orphan_stale",
|
||||
"orphan_fresh",
|
||||
"missing_referenced",
|
||||
"missing_unreferenced",
|
||||
"size_mismatch",
|
||||
"sha_mismatch",
|
||||
"pending_stale_with_object",
|
||||
"pending_stale_without_object",
|
||||
"pending_fresh",
|
||||
"quarantined",
|
||||
"incomplete_stale",
|
||||
"incomplete_fresh",
|
||||
"lost_ready",
|
||||
)
|
||||
TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"}
|
||||
BROKEN = (
|
||||
"missing_referenced",
|
||||
"missing_unreferenced",
|
||||
"size_mismatch",
|
||||
"sha_mismatch",
|
||||
"quarantined",
|
||||
)
|
||||
|
||||
|
||||
def timed(cmd, stdin=None):
|
||||
started = time.monotonic()
|
||||
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
||||
if out.returncode:
|
||||
raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}")
|
||||
return out.stdout, round(time.monotonic() - started, 3)
|
||||
|
||||
|
||||
def s3(*args, stdin=None):
|
||||
return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin)
|
||||
|
||||
|
||||
def age(stamp, now):
|
||||
return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp()
|
||||
|
||||
|
||||
def load(bucket, prefix, deep):
|
||||
listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}")
|
||||
uploads, uploads_seconds = s3(
|
||||
"object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}"
|
||||
)
|
||||
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
||||
psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"]
|
||||
rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL])
|
||||
objects = {}
|
||||
for item in json.loads(listing).get("items", []):
|
||||
key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"]
|
||||
objects[key] = item
|
||||
rows = {}
|
||||
for line in rows_raw.splitlines():
|
||||
rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t")
|
||||
rows[key] = {
|
||||
"id": rid,
|
||||
"status": status,
|
||||
"size": int(size),
|
||||
"sha256": sha,
|
||||
"age": int(row_age),
|
||||
"referenced": referenced == "t",
|
||||
"quarantined": quarantined == "t",
|
||||
}
|
||||
hashes, deep_seconds = {}, 0.0
|
||||
if deep:
|
||||
present = "".join(
|
||||
f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects
|
||||
)
|
||||
out, deep_seconds = s3("sh", "-c", HASH, stdin=present)
|
||||
hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
|
||||
seconds = {
|
||||
"s3_list": list_seconds,
|
||||
"s3_incomplete": uploads_seconds,
|
||||
"pg_query": pg_seconds,
|
||||
"deep_hash": deep_seconds,
|
||||
}
|
||||
return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds
|
||||
|
||||
|
||||
def row_class(key, row, objects, hashes, grace):
|
||||
if row["quarantined"]:
|
||||
return "quarantined"
|
||||
if row["status"] == "pending":
|
||||
stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object"
|
||||
return "pending_fresh" if row["age"] <= grace else stale
|
||||
if key not in objects:
|
||||
return "missing_referenced" if row["referenced"] else "missing_unreferenced"
|
||||
if objects[key]["size_bytes"] != row["size"]:
|
||||
return "size_mismatch"
|
||||
return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None
|
||||
|
||||
|
||||
def classify(objects, rows, hashes, uploads, grace):
|
||||
now = time.time()
|
||||
classes = {name: [] for name in CLASSES}
|
||||
for key, item in objects.items():
|
||||
if key not in rows:
|
||||
classes[
|
||||
"orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh"
|
||||
].append(key)
|
||||
for key, row in rows.items():
|
||||
name = row_class(key, row, objects, hashes, grace)
|
||||
if name:
|
||||
classes[name].append(key if name.startswith("orphan") else row["id"])
|
||||
for item in uploads:
|
||||
classes[
|
||||
"incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh"
|
||||
].append(item["key"])
|
||||
return classes
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(
|
||||
description="Independent PG <-> S3 invariant check for lab4 attachments"
|
||||
)
|
||||
p.add_argument(
|
||||
"--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600"))
|
||||
)
|
||||
p.add_argument(
|
||||
"--deep", action="store_true", help="stream every present object and compare sha256"
|
||||
)
|
||||
p.add_argument(
|
||||
"--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments"
|
||||
)
|
||||
p.add_argument(
|
||||
"--expect-ready",
|
||||
metavar="FILE",
|
||||
help="fail if any attachment from FILE is no longer healthy",
|
||||
)
|
||||
p.add_argument("--limit", type=int, default=20, help="max ids per class in the report")
|
||||
p.add_argument("--output", metavar="FILE")
|
||||
args = p.parse_args()
|
||||
started = time.monotonic()
|
||||
prefix = os.environ.get("S3_PREFIX", "attachments/")
|
||||
objects, rows, hashes, uploads, seconds = load(
|
||||
os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep
|
||||
)
|
||||
classes = classify(objects, rows, hashes, uploads, args.grace)
|
||||
bad = {i for c in BROKEN for i in classes[c]}
|
||||
healthy = {
|
||||
r["id"]: r["sha256"]
|
||||
for k, r in rows.items()
|
||||
if r["status"] == "ready" and k in objects and r["id"] not in bad
|
||||
}
|
||||
if args.expect_ready:
|
||||
expected = json.loads(Path(args.expect_ready).read_text())
|
||||
classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha)
|
||||
if args.save_ready:
|
||||
Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True))
|
||||
violations = [c for c in CLASSES if classes[c] and c not in TOLERATED]
|
||||
seconds["total"] = round(time.monotonic() - started, 3)
|
||||
report = {
|
||||
"checked_at": dt.datetime.now(dt.timezone.utc)
|
||||
.isoformat(timespec="seconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"grace_seconds": args.grace,
|
||||
"deep": args.deep,
|
||||
"objects": len(objects),
|
||||
"rows": len(rows),
|
||||
"ready_healthy": len(healthy),
|
||||
"counts": {c: len(v) for c, v in classes.items()},
|
||||
"samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v},
|
||||
"seconds": seconds,
|
||||
"violations": violations,
|
||||
"ok": not violations,
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + "\n")
|
||||
print(text)
|
||||
sys.exit(0 if report["ok"] else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,823 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
client = __import__("client")
|
||||
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
GRACE = 60
|
||||
FLAGS = ("--grace", str(GRACE), "--skew", "0")
|
||||
INTENT_AGE = 300
|
||||
LOCK_KEY = 20260004
|
||||
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale")
|
||||
USER_HZ = 100
|
||||
BUSY = (0, 1, 2, 5, 6, 7)
|
||||
|
||||
|
||||
def run(cmd, env=None, stdin=None):
|
||||
started = time.monotonic()
|
||||
done = subprocess.run(
|
||||
cmd,
|
||||
cwd=ROOT,
|
||||
input=stdin,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
env=dict(os.environ, **(env or {})),
|
||||
)
|
||||
out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace")
|
||||
return done.returncode, out, err, round(time.monotonic() - started, 3)
|
||||
|
||||
|
||||
def must(cmd, env=None, stdin=None):
|
||||
code, out, err, _ = run(cmd, env, stdin)
|
||||
if code:
|
||||
raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}")
|
||||
return out
|
||||
|
||||
|
||||
def compose(*args, env=None, stdin=None):
|
||||
return must(["docker", "compose", *args], env, stdin)
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-qAt",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-c",
|
||||
sql,
|
||||
).strip()
|
||||
|
||||
|
||||
def reconcile(*flags, env=None):
|
||||
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags]
|
||||
code, out, err, wall = run(cmd, env)
|
||||
if code not in (0, 1) or not out.strip():
|
||||
raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}")
|
||||
report = json.loads(out)
|
||||
audit = [json.loads(line) for line in err.splitlines() if '"audit.' in line]
|
||||
report.update(exit_code=code, wall_seconds=wall, audit=audit)
|
||||
return report
|
||||
|
||||
|
||||
def oracle(*flags):
|
||||
code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags])
|
||||
if not out.strip():
|
||||
raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}")
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def inject(kind, count, ids=(), age=None):
|
||||
env = {"INJECT_IDS": ",".join(ids)} if ids else {}
|
||||
if age is not None:
|
||||
env["INJECT_AGE_SECONDS"] = str(age)
|
||||
out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env)
|
||||
return [line.split()[1] for line in out.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def head():
|
||||
return must(["git", "rev-parse", "HEAD"]).strip()
|
||||
|
||||
|
||||
def evidence_root():
|
||||
return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
|
||||
|
||||
|
||||
def save(name, data, *, stamp=True):
|
||||
path = evidence_root() / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data}
|
||||
path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
class Checks:
|
||||
def __init__(self):
|
||||
self.items = []
|
||||
|
||||
def __call__(self, name, ok, **detail):
|
||||
self.items.append(dict(name=name, ok=bool(ok), **detail))
|
||||
sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n")
|
||||
|
||||
@property
|
||||
def ok(self):
|
||||
return all(item["ok"] for item in self.items)
|
||||
|
||||
|
||||
def chunk(tag, data):
|
||||
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
|
||||
|
||||
|
||||
def tiny_png():
|
||||
raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8))
|
||||
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(raw))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def pick(found, names):
|
||||
return {name: found.get(name, 0) for name in names}
|
||||
|
||||
|
||||
def faults(c, check, pre):
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)]
|
||||
linked = ids[:6]
|
||||
for part in (linked[:3], linked[3:]):
|
||||
c.message(alice, chat, "", attachments=part)
|
||||
targets = {
|
||||
"missing": [linked[0], linked[1], ids[6]],
|
||||
"corrupt": [linked[2], ids[7], ids[8]],
|
||||
"bitflip": [linked[3], ids[9], ids[10]],
|
||||
}
|
||||
damaged = {i for group in targets.values() for i in group}
|
||||
ready = evidence_root() / "ready-before.json"
|
||||
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
|
||||
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
|
||||
save("ready-untouched.json", untouched, stamp=False)
|
||||
started = time.monotonic()
|
||||
injected = {
|
||||
"orphan_stale": inject("orphan", 3),
|
||||
"missing": inject("missing", 3, targets["missing"]),
|
||||
"size_mismatch": inject("corrupt", 3, targets["corrupt"]),
|
||||
"sha_mismatch": inject("bitflip", 3, targets["bitflip"]),
|
||||
"pending_stale": inject("stale-pending", 3, age=INTENT_AGE)
|
||||
+ inject("pending-object", 3, age=INTENT_AGE),
|
||||
}
|
||||
time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started)))
|
||||
injected["orphan_fresh"] = inject("orphan", 2)
|
||||
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
|
||||
expected = {name: len(keys) for name, keys in injected.items()}
|
||||
expected.update(quarantined=pre["counts"]["quarantined"])
|
||||
before = oracle(
|
||||
"--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json")
|
||||
)
|
||||
dry = reconcile("--dry-run", "--deep", *FLAGS)
|
||||
save("dry-run.json", dry)
|
||||
after_dry = oracle("--grace", str(GRACE), "--deep")
|
||||
named = {k: sorted(s.rsplit("/", 1)[-1] for s in dry["samples"].get(k, [])) for k in injected}
|
||||
check(
|
||||
"dry-run находит каждый внедрённый класс, и найдено ровно внедрённое",
|
||||
pick(dry["found"], expected) == expected
|
||||
and named == {name: sorted(ids) for name, ids in injected.items()},
|
||||
expected=expected,
|
||||
found=dry["found"],
|
||||
)
|
||||
check(
|
||||
"dry-run ничего не меняет",
|
||||
not any(dry["fixed"].values())
|
||||
and after_dry["counts"] == before["counts"]
|
||||
and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]),
|
||||
objects=[before["objects"], after_dry["objects"]],
|
||||
rows=[before["rows"], after_dry["rows"]],
|
||||
)
|
||||
first = reconcile("--apply", "--deep", *FLAGS)
|
||||
save("apply-1.json", first)
|
||||
discarded = sorted(e["attachment_id"] for e in first["audit"] if e["reason"] == "reconcile")
|
||||
check(
|
||||
"apply исправляет ровно найденное dry-run, удаление намерений — в журнале аудита",
|
||||
pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE)
|
||||
and not first["errors"]
|
||||
and discarded == sorted(injected["pending_stale"]),
|
||||
dry_run=pick(dry["found"], FIXABLE),
|
||||
fixed=pick(first["fixed"], FIXABLE),
|
||||
audit_discarded=len(discarded),
|
||||
)
|
||||
second = reconcile("--apply", "--deep", *FLAGS)
|
||||
save("apply-2.json", second)
|
||||
check(
|
||||
"повторный apply — 0 действий",
|
||||
not any(second["fixed"].values()),
|
||||
fixed=second["fixed"],
|
||||
found=second["found"],
|
||||
)
|
||||
after = oracle(
|
||||
"--grace",
|
||||
str(GRACE),
|
||||
"--deep",
|
||||
"--expect-ready",
|
||||
str(evidence_root() / "ready-untouched.json"),
|
||||
"--output",
|
||||
str(evidence_root() / "check-after.json"),
|
||||
)
|
||||
tolerated = ("orphan_fresh", "pending_fresh")
|
||||
check(
|
||||
"независимая проверка после apply: нарушений нет, готовые не потеряны",
|
||||
after["ok"]
|
||||
and not after["counts"]["lost_ready"]
|
||||
and pick(after["counts"], tolerated) == pick(before["counts"], tolerated)
|
||||
and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9,
|
||||
counts=after["counts"],
|
||||
)
|
||||
marks = dict(
|
||||
line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split()
|
||||
)
|
||||
links = psql("SELECT attachment_id FROM message_attachments").split()
|
||||
kept = {i: marks.get(i) for i in sorted(damaged)}
|
||||
check(
|
||||
"строки с расхождениями и ссылки сообщений сохранены",
|
||||
all(kept.values()) and sum(i in linked for i in links) == len(linked),
|
||||
quarantine=kept,
|
||||
message_links=sum(i in linked for i in links),
|
||||
)
|
||||
meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment")
|
||||
blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob)
|
||||
intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob)
|
||||
check(
|
||||
"карантин: метаданные 200, /content 404; нетронутое скачивается",
|
||||
meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE,
|
||||
)
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
for attachment, data in ((linked[0], FIXTURE), (linked[1], os.urandom(len(FIXTURE)))):
|
||||
compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"s3-cli",
|
||||
"pipe",
|
||||
f"root/{bucket}/attachments/{attachment}",
|
||||
stdin=data,
|
||||
)
|
||||
healed = reconcile("--apply", *FLAGS)
|
||||
save("healed.json", healed)
|
||||
restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob)
|
||||
foreign = c.call("GET", f"/api/v1/attachments/{linked[1]}/content", 404, actor=bob)
|
||||
check(
|
||||
"вернули те же байты — карантин снят; другие байты той же длины — остался",
|
||||
healed["fixed"]["healed"] == 1
|
||||
and healed["samples"].get("healed") == [linked[0]]
|
||||
and linked[1] in healed["samples"].get("quarantined", [])
|
||||
and restored["bytes"] == FIXTURE
|
||||
and foreign["status"] == 404,
|
||||
fixed=healed["fixed"],
|
||||
s3_get=healed["cost"]["requests"]["s3_get"],
|
||||
)
|
||||
return {"injected": injected, "targets": targets, "linked": linked}
|
||||
|
||||
|
||||
def single(check):
|
||||
holder = subprocess.Popen(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-c",
|
||||
f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)",
|
||||
],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
time.sleep(3)
|
||||
busy = reconcile("--apply", *FLAGS)
|
||||
finally:
|
||||
holder.wait()
|
||||
save("lock-busy.json", busy)
|
||||
check(
|
||||
"второй одновременный проход пропущен",
|
||||
busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()),
|
||||
)
|
||||
|
||||
|
||||
def healthy(service, seconds=90):
|
||||
container = compose("ps", "-q", service).strip()
|
||||
probe = ["docker", "inspect", "-f", "{{.State.Health.Status}}", container]
|
||||
deadline = time.monotonic() + seconds
|
||||
while must(probe).strip() != "healthy":
|
||||
if time.monotonic() > deadline:
|
||||
raise SystemExit(f"{service} is not healthy after {seconds} s")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def interrupt(check):
|
||||
name = f"hlpipe-lab5-reconcile-sigterm-{os.getpid()}"
|
||||
compose("pause", "s3")
|
||||
try:
|
||||
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "--name", name, "reconciler"]
|
||||
runner = subprocess.Popen(
|
||||
[*cmd, "--apply", *FLAGS],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
time.sleep(4)
|
||||
state = must(["docker", "inspect", "-f", "{{.State.Status}} {{.HostConfig.Init}}", name])
|
||||
started = time.monotonic()
|
||||
must(["docker", "stop", name])
|
||||
stopped = round(time.monotonic() - started, 2)
|
||||
code = runner.wait(timeout=60)
|
||||
finally:
|
||||
compose("unpause", "s3")
|
||||
healthy("s3")
|
||||
after = reconcile("--dry-run", *FLAGS)
|
||||
status, init = state.split()
|
||||
report = {
|
||||
"s3": "paused",
|
||||
"state_before_stop": status,
|
||||
"init": init,
|
||||
"stop_seconds": stopped,
|
||||
"exit_code": code,
|
||||
"next_pass_skipped": after["skipped"],
|
||||
}
|
||||
save("sigterm.json", report)
|
||||
check(
|
||||
"SIGTERM: идущий проход останавливается за секунды, блокировка освобождена",
|
||||
status == "running" and code == 143 and stopped < 5 and not after["skipped"],
|
||||
**report,
|
||||
)
|
||||
|
||||
|
||||
def outage(check):
|
||||
before = oracle("--grace", str(GRACE))
|
||||
compose("stop", "s3")
|
||||
try:
|
||||
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", "--apply"]
|
||||
code, out, err, wall = run([*cmd, *FLAGS])
|
||||
finally:
|
||||
compose("up", "-d", "--wait", "--no-deps", "s3")
|
||||
after = oracle("--grace", str(GRACE))
|
||||
log = [line for line in err.splitlines() if "reconcile." in line]
|
||||
same = all(after[k] == before[k] for k in ("objects", "rows", "counts"))
|
||||
save("s3-down.json", {"exit_code": code, "wall_seconds": wall, "stdout": out, "log": log})
|
||||
check(
|
||||
"S3 остановлен: код 1 за ограниченное время, состояние не изменилось",
|
||||
code == 1 and wall < 30 and not out.strip() and same,
|
||||
exit_code=code,
|
||||
wall_seconds=wall,
|
||||
)
|
||||
|
||||
|
||||
def uploader(base, token, chat, *, deadline, results, lock):
|
||||
host = urlsplit(base).netloc
|
||||
conn = http.client.HTTPConnection(host, timeout=30)
|
||||
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
|
||||
sent = 0
|
||||
while time.monotonic() < deadline:
|
||||
path = f"/api/v1/chats/{chat}/attachments?filename=load.png"
|
||||
try:
|
||||
conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"})
|
||||
response = conn.getresponse()
|
||||
status, data = response.status, response.read()
|
||||
if status == 201 and sent % 10 == 0:
|
||||
body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]})
|
||||
extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}
|
||||
conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra})
|
||||
linked = conn.getresponse()
|
||||
linked.read()
|
||||
status = 201 if linked.status == 201 else -linked.status
|
||||
except (OSError, http.client.HTTPException):
|
||||
conn.close()
|
||||
conn = http.client.HTTPConnection(host, timeout=30)
|
||||
status, data = 0, b""
|
||||
with lock:
|
||||
results.append((status, json.loads(data) if status == 201 else None))
|
||||
sent += 1
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def under_load(c, check, base, seconds, threads):
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
ready = evidence_root() / "ready-load.json"
|
||||
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
|
||||
leftover = {
|
||||
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
|
||||
"pending_stale": left["pending_fresh"]
|
||||
+ left["pending_stale_with_object"]
|
||||
+ left["pending_stale_without_object"],
|
||||
}
|
||||
orphans = inject("orphan", 3)
|
||||
intents = inject("stale-pending", 3, age=INTENT_AGE)
|
||||
results, lock = [], threading.Lock()
|
||||
deadline = time.monotonic() + seconds
|
||||
workers = [
|
||||
threading.Thread(
|
||||
target=uploader,
|
||||
args=(base, alice["token"], chat["id"]),
|
||||
kwargs={"deadline": deadline, "results": results, "lock": lock},
|
||||
)
|
||||
for _ in range(threads)
|
||||
]
|
||||
for worker in workers:
|
||||
worker.start()
|
||||
runs = []
|
||||
while time.monotonic() < deadline:
|
||||
runs.append(reconcile("--apply", *FLAGS))
|
||||
for worker in workers:
|
||||
worker.join()
|
||||
runs.append(reconcile("--apply", *FLAGS))
|
||||
created = {att["id"]: att["sha256"] for status, att in results if att}
|
||||
expected = dict(json.loads(ready.read_text()), **created)
|
||||
save("ready-load.json", expected, stamp=False)
|
||||
after = oracle(
|
||||
"--grace",
|
||||
str(GRACE),
|
||||
"--deep",
|
||||
"--expect-ready",
|
||||
str(ready),
|
||||
"--output",
|
||||
str(evidence_root() / "check-under-load.json"),
|
||||
)
|
||||
statuses = {}
|
||||
for status, _ in results:
|
||||
statuses[str(status)] = statuses.get(str(status), 0) + 1
|
||||
fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE}
|
||||
rechecked = {}
|
||||
for r in runs:
|
||||
for name, count in r["rechecked"].items():
|
||||
rechecked[name] = rechecked.get(name, 0) + count
|
||||
seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")}
|
||||
summary = {
|
||||
"seconds": seconds,
|
||||
"upload_threads": threads,
|
||||
"uploads": statuses,
|
||||
"created": len(created),
|
||||
"linked_every": 10,
|
||||
"reconciler_runs": len(runs),
|
||||
"run_seconds": [r["cost"]["seconds"] for r in runs],
|
||||
"fixed_total": fixed,
|
||||
"fresh_seen_total": seen,
|
||||
"rechecked_total": rechecked,
|
||||
"errors": sum(r["errors"] for r in runs),
|
||||
"skipped": sum(1 for r in runs if r["skipped"]),
|
||||
"injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)},
|
||||
"left_from_faults": leftover,
|
||||
"check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")},
|
||||
}
|
||||
save("under-load.json", summary)
|
||||
check(
|
||||
"под нагрузкой: исправлены только внедрённые, ошибок нет",
|
||||
fixed
|
||||
== {
|
||||
**dict.fromkeys(FIXABLE, 0),
|
||||
"orphan_stale": leftover["orphan_stale"] + len(orphans),
|
||||
"pending_stale": leftover["pending_stale"] + len(intents),
|
||||
}
|
||||
and not summary["errors"]
|
||||
and not summary["skipped"],
|
||||
fixed=fixed,
|
||||
runs=len(runs),
|
||||
)
|
||||
check(
|
||||
"под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0",
|
||||
after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"},
|
||||
uploads=statuses,
|
||||
lost_ready=after["counts"]["lost_ready"],
|
||||
)
|
||||
|
||||
|
||||
def demo(args):
|
||||
evidence_root().mkdir(parents=True, exist_ok=False)
|
||||
c = client.Client(args.base_url, timeout=30)
|
||||
check = Checks()
|
||||
pre = oracle("--grace", str(GRACE))
|
||||
dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"}
|
||||
if dirty:
|
||||
raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply")
|
||||
started = time.monotonic()
|
||||
result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()}
|
||||
result["faults"] = faults(c, check, pre)
|
||||
single(check)
|
||||
interrupt(check)
|
||||
outage(check)
|
||||
under_load(c, check, args.base_url, args.load_seconds, args.load_threads)
|
||||
result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok)
|
||||
save("demo.json", result)
|
||||
print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False))
|
||||
sys.exit(0 if check.ok else 1)
|
||||
|
||||
|
||||
def pins(args):
|
||||
cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip())
|
||||
sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory}
|
||||
if args.no_pin or cpus < 8:
|
||||
return sized
|
||||
return dict(
|
||||
sized,
|
||||
S3_CPUSET=f"{cpus - 5}-{cpus - 4}",
|
||||
PG_CPUSET=f"{cpus - 3}-{cpus - 2}",
|
||||
RECONCILE_CPUSET=str(cpus - 1),
|
||||
)
|
||||
|
||||
|
||||
def cpu_set(env):
|
||||
cpus = set()
|
||||
for name in ("S3_CPUSET", "PG_CPUSET", "RECONCILE_CPUSET"):
|
||||
for part in filter(None, env.get(name, "").split(",")):
|
||||
low, _, high = part.partition("-")
|
||||
cpus.update(range(int(low), int(high or low) + 1))
|
||||
return cpus
|
||||
|
||||
|
||||
def limits():
|
||||
ids = compose("ps", "-q", "s3", "postgres").split()
|
||||
fmt = (
|
||||
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}"
|
||||
" cpuset={{.HostConfig.CpusetCpus}}"
|
||||
)
|
||||
return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines()
|
||||
|
||||
|
||||
def host(title):
|
||||
stats = must(
|
||||
[
|
||||
"docker",
|
||||
"stats",
|
||||
"--no-stream",
|
||||
"--all",
|
||||
"--format",
|
||||
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}",
|
||||
]
|
||||
)
|
||||
running = must(["docker", "ps", "--format", "{{.Names}} {{.Image}} {{.Status}}"])
|
||||
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
load = " ".join(f"{v:.2f}" for v in os.getloadavg())
|
||||
return (
|
||||
f"== {title} date: {stamp} commit: {head()} host loadavg: {load}\n"
|
||||
f"$ docker stats --no-stream --all\n{stats}\n$ docker ps\n{running}\n"
|
||||
)
|
||||
|
||||
|
||||
def usage(text):
|
||||
return int(re.search(r"^usage_usec (\d+)$", text, re.M).group(1)) / 1e6
|
||||
|
||||
|
||||
def sample():
|
||||
vm = compose(
|
||||
"exec", "-T", "postgres", "cat", "/proc/uptime", "/proc/stat", "/sys/fs/cgroup/cpu.stat"
|
||||
)
|
||||
s3 = compose("exec", "-T", "s3", "cat", "/sys/fs/cgroup/cpu.stat")
|
||||
busy = {}
|
||||
for line in vm.splitlines()[1:]:
|
||||
name, *values = line.split() or [""]
|
||||
if name.startswith("cpu") and name[3:].isdigit():
|
||||
busy[int(name[3:])] = sum(int(values[i]) for i in BUSY) / USER_HZ
|
||||
return float(vm.split()[0]), busy, usage(vm) + usage(s3)
|
||||
|
||||
|
||||
def screen(first, last, own, pinned, args, *, started=True):
|
||||
window = max(last[0] - first[0], 0.01)
|
||||
busy = {cpu: spent - first[1].get(cpu, 0.0) for cpu, spent in last[1].items()}
|
||||
ours = last[2] - first[2] + own
|
||||
other = sum(busy.values()) - ours - args.start_other * started
|
||||
on_pinned = sum(busy.get(cpu, 0.0) for cpu in pinned) - ours - args.start_pinned * started
|
||||
other, on_pinned = max(0.0, other / window), max(0.0, on_pinned / window) if pinned else 0.0
|
||||
return {
|
||||
"window_seconds": round(window, 2),
|
||||
"ours_cores": round(ours / window, 2),
|
||||
"other_cores": round(other, 2),
|
||||
"other_on_pinned_cores": round(on_pinned, 2),
|
||||
"host_loadavg_1m": round(os.getloadavg()[0], 2),
|
||||
"disturbed": other > args.max_other or on_pinned > args.max_pinned,
|
||||
}
|
||||
|
||||
|
||||
def wait_quiet(args, pinned):
|
||||
deadline, calm = time.monotonic() + args.quiet_timeout, 0
|
||||
while calm < 3 and time.monotonic() < deadline:
|
||||
first = sample()
|
||||
time.sleep(5)
|
||||
quiet = screen(first, sample(), 0.0, pinned, args, started=False)
|
||||
calm = 0 if quiet["disturbed"] else calm + 1
|
||||
sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n")
|
||||
return calm == 3
|
||||
|
||||
|
||||
def measure(flags, env, args, pinned):
|
||||
rejected = []
|
||||
for attempt in range(args.retries + 1):
|
||||
first = sample()
|
||||
report = reconcile(*flags, "--samples", "3", env=env)
|
||||
report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args)
|
||||
if not report["screen"]["disturbed"]:
|
||||
return report, rejected
|
||||
rejected.append(report)
|
||||
if attempt < args.retries and not wait_quiet(args, pinned):
|
||||
break
|
||||
return None, rejected
|
||||
|
||||
|
||||
def seed(count, env):
|
||||
if count <= 0:
|
||||
return {"added": 0}
|
||||
env = dict(env, API_CPUS="4", API_MEMORY="512m")
|
||||
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
|
||||
out = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"-v",
|
||||
tools,
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"api",
|
||||
"/tools/seed_storage.py",
|
||||
"--count",
|
||||
str(count),
|
||||
"--concurrency",
|
||||
"4",
|
||||
env=env,
|
||||
)
|
||||
records = [json.loads(line) for line in out.strip().splitlines()]
|
||||
return {**records[-1], "progress": records[:-1]}
|
||||
|
||||
|
||||
def median(runs, path):
|
||||
values = []
|
||||
for r in runs:
|
||||
value = r
|
||||
for key in path:
|
||||
value = value[key]
|
||||
values.append(value)
|
||||
return statistics.median(values)
|
||||
|
||||
|
||||
def summarize(results, rejected):
|
||||
rows = []
|
||||
for (n, mode), runs in sorted(results.items()):
|
||||
seconds = median(runs, ("cost", "seconds"))
|
||||
rows.append(
|
||||
{
|
||||
"objects": n,
|
||||
"mode": mode,
|
||||
"runs": len(runs),
|
||||
"seconds_median": seconds,
|
||||
"seconds_all": [r["cost"]["seconds"] for r in runs],
|
||||
"wall_seconds_median": median(runs, ("wall_seconds",)),
|
||||
"us_per_object": round(seconds / n * 1e6, 1),
|
||||
"cpu_seconds_median": median(runs, ("cost", "cpu_seconds")),
|
||||
"s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")),
|
||||
"sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")),
|
||||
"requests": runs[0]["cost"]["requests"],
|
||||
"max_rss_mib_median": median(runs, ("cost", "max_rss_mib")),
|
||||
"other_cores_max": max(r["screen"]["other_cores"] for r in runs),
|
||||
"other_on_pinned_cores_max": max(
|
||||
r["screen"]["other_on_pinned_cores"] for r in runs
|
||||
),
|
||||
"disturbed_kept": sum(r["screen"]["disturbed"] for r in runs),
|
||||
"rejected": rejected.get((n, mode), 0),
|
||||
"scanned": runs[0]["scanned"],
|
||||
"found_nonzero": {k: v for k, v in runs[0]["found"].items() if v},
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def table(rows):
|
||||
head = (
|
||||
"objects mode runs t_med,s wall,s us/obj cpu,s list,s sql,s lists sqls"
|
||||
" gets rss,MiB other pinned rejected"
|
||||
)
|
||||
lines = [head]
|
||||
for r in rows:
|
||||
q = r["requests"]
|
||||
lines.append(
|
||||
f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} "
|
||||
f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} "
|
||||
f"{r['cpu_seconds_median']:>5.2f} {r['s3_list_wait_median']:>6.3f} "
|
||||
f"{r['sql_wait_median']:>5.3f} {q['s3_list']:>5} {q['sql']:>4} {q['s3_get']:>5} "
|
||||
f"{r['max_rss_mib_median']:>7.1f} {r['other_cores_max']:>5.2f} "
|
||||
f"{r['other_on_pinned_cores_max']:>6.2f} {r['rejected']:>8}"
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def cost(args):
|
||||
out = evidence_root() / "cost"
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
(out / "disturbed").mkdir()
|
||||
rows = int(psql("SELECT count(*) FROM attachments"))
|
||||
if rows > args.sizes[0]:
|
||||
raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project")
|
||||
snapshot = out / "host-docker-stats.txt"
|
||||
snapshot.write_text(host("before the series"))
|
||||
env = pins(args)
|
||||
pinned = cpu_set(env)
|
||||
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env)
|
||||
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []}
|
||||
plan["screen"] = {
|
||||
"max_other_cores": args.max_other,
|
||||
"max_other_on_pinned_cores": args.max_pinned,
|
||||
"start_allowance_core_seconds": {"vm": args.start_other, "pinned": args.start_pinned},
|
||||
"retries": args.retries,
|
||||
}
|
||||
results, rejected = {}, {}
|
||||
try:
|
||||
for n in args.sizes:
|
||||
plan["seeds"].append(dict(seed(n - rows, env), target=n))
|
||||
rows = n
|
||||
time.sleep(args.settle)
|
||||
save("cost/plan.json", plan)
|
||||
if not wait_quiet(args, pinned):
|
||||
raise SystemExit("no quiet measurement window before timeout")
|
||||
modes = [("shallow", ["--dry-run"])]
|
||||
if n <= args.deep_max:
|
||||
modes.append(("deep", ["--dry-run", "--deep"]))
|
||||
if n == args.page_compare:
|
||||
modes.append(("page100", ["--dry-run", "--page-size", "100"]))
|
||||
for mode, flags in modes:
|
||||
for attempt in range(1, args.repeats + 1):
|
||||
report, dropped = measure(flags, env, args, pinned)
|
||||
name = f"n{n}-{mode}-{attempt}"
|
||||
for number, bad in enumerate(dropped, 1):
|
||||
save(f"cost/disturbed/{name}-try{number}.json", bad)
|
||||
rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped)
|
||||
if report is None:
|
||||
raise SystemExit(f"{name}: no clean run after retries; rejected preserved")
|
||||
if (
|
||||
report["exit_code"]
|
||||
or report["errors"]
|
||||
or report["skipped"]
|
||||
or report["scanned"] != {"objects": n, "rows": n}
|
||||
or any(report["found"].values())
|
||||
):
|
||||
save(f"cost/invalid/{name}.json", report)
|
||||
raise SystemExit(f"{name}: inconsistent dataset or failed pass")
|
||||
save(f"cost/{name}.json", report)
|
||||
results.setdefault((n, mode), []).append(report)
|
||||
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
|
||||
sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n")
|
||||
finally:
|
||||
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
|
||||
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
|
||||
with snapshot.open("a") as handle:
|
||||
handle.write("\n" + host("after the series"))
|
||||
summary = summarize(results, rejected)
|
||||
save("cost/summary.json", {**plan, "results": summary})
|
||||
(out / "summary.txt").write_text(
|
||||
f"date: {stamp} commit: {plan['commit']} pins: {env}\n" + table(summary)
|
||||
)
|
||||
sys.stdout.write(table(summary))
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load")
|
||||
d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
||||
d.add_argument("--load-seconds", type=int, default=75)
|
||||
d.add_argument("--load-threads", type=int, default=3)
|
||||
k = sub.add_parser("cost", help="cost of a pass for N objects, repeated")
|
||||
k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 30000, 100000])
|
||||
k.add_argument("--repeats", type=int, default=3)
|
||||
k.add_argument("--deep-max", type=int, default=100000)
|
||||
k.add_argument("--page-compare", type=int, default=10000)
|
||||
k.add_argument("--settle", type=float, default=10)
|
||||
k.add_argument("--s3-memory", default="3g")
|
||||
k.add_argument("--no-pin", action="store_true")
|
||||
k.add_argument("--max-other", type=float, default=2.0, help="foreign CPU in the VM, cores")
|
||||
k.add_argument("--max-pinned", type=float, default=0.25, help="foreign CPU on pinned cores")
|
||||
k.add_argument("--start-other", type=float, default=1.5, help="container start, core-s")
|
||||
k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores")
|
||||
k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run")
|
||||
k.add_argument("--quiet-timeout", type=float, default=900.0)
|
||||
for command in (d, k):
|
||||
command.add_argument(
|
||||
"--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
|
||||
)
|
||||
args = p.parse_args()
|
||||
os.environ["RECONCILE_EVIDENCE_DIR"] = args.out
|
||||
if args.command == "demo":
|
||||
demo(args)
|
||||
else:
|
||||
cost(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -2,7 +2,8 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
service=${API_SERVICE:-api}
|
||||
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
|
||||
show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; }
|
||||
scripts/stand-usage.sh &
|
||||
echo "$(date -u +%T) migrate once before the first replacement"
|
||||
docker compose run --rm --no-deps -T migrate 2>&1
|
||||
old=$(docker compose ps -q "$service")
|
||||
@@ -13,8 +14,9 @@ for id in $old; do
|
||||
scripts/lb-wait.sh $((count + 1))
|
||||
new=$(docker compose ps -q "$service" | grep -v -F "$running")
|
||||
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
|
||||
docker stop -t "${STOP_TIMEOUT:-7}" "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
done
|
||||
scripts/lb-wait.sh "$count"
|
||||
wait
|
||||
echo "$(date -u +%T) done: $count replicas replaced"
|
||||
|
||||
+30
-7
@@ -2,17 +2,30 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
|
||||
undated() {
|
||||
python3 -c 'import json, sys
|
||||
def clean(value):
|
||||
if isinstance(value, dict):
|
||||
return {k: clean(v) for k, v in value.items() if not k.endswith("_date")}
|
||||
return [clean(v) for v in value] if isinstance(value, list) else value
|
||||
text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0
|
||||
while text[at:].strip():
|
||||
at += len(text[at:]) - len(text[at:].lstrip())
|
||||
value, at = decoder.raw_decode(text, at)
|
||||
print(json.dumps(clean(value), indent=2, ensure_ascii=False))'
|
||||
}
|
||||
snapshot() {
|
||||
cli sh -s <<'EOF' | grep -v '_date'
|
||||
cli sh -s <<'EOF' | undated
|
||||
ak=$(cat /run/secrets/s3_access_key_id)
|
||||
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
|
||||
rk=$(cat /run/secrets/s3_reconciler_access_key_id)
|
||||
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do
|
||||
rc $c --json
|
||||
done | sed "s/$ak/<api-key>/g"
|
||||
done | sed "s/$ak/<api-key>/g; s/$rk/<reconciler-key>/g"
|
||||
EOF
|
||||
}
|
||||
failed=0
|
||||
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
|
||||
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) == S3 privacy and bootstrap checks"
|
||||
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
|
||||
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
|
||||
before=$(snapshot)
|
||||
@@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same
|
||||
cli sh -s <<'EOF' || failed=1
|
||||
b=$S3_BUCKET
|
||||
key="attachments/s3-check-$(date +%s)-$$"
|
||||
spare="$key-reconciler"
|
||||
other="s3-check-$(date +%s)-$$"
|
||||
head -c 4096 /dev/urandom > /tmp/probe
|
||||
result=0
|
||||
expect() {
|
||||
want=$1; name=$2; shift 2
|
||||
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
|
||||
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi
|
||||
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
|
||||
}
|
||||
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
|
||||
expect allow "api: HeadObject" rc object stat "api/$b/$key"
|
||||
expect allow "api: GetObject" rc object show "api/$b/$key"
|
||||
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
|
||||
expect deny "api: ListBucket" rc object list "api/$b/attachments/"
|
||||
rc put /tmp/probe "api/$b/$spare" >/dev/null
|
||||
expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/"
|
||||
expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key"
|
||||
expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key"
|
||||
expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put"
|
||||
expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare"
|
||||
expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other"
|
||||
expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b"
|
||||
expect deny "reconciler: admin ListUsers" rc admin user list reconciler
|
||||
expect deny "api: CreateBucket" rc bucket create "api/$other"
|
||||
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
|
||||
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
|
||||
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
|
||||
expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
|
||||
expect deny "api: admin ListUsers" rc admin user list api
|
||||
rc bucket create --ignore-existing "root/$other" >/dev/null
|
||||
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
|
||||
|
||||
Executable
+524
@@ -0,0 +1,524 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import collections
|
||||
import contextlib
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from probe import ROOT, k6_metrics, lb_up, run_k6
|
||||
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
|
||||
PINNED = {
|
||||
"PG_CPUSET": "0-1",
|
||||
"LB_CPUSET": "2",
|
||||
"VALKEY_CPUSET": "3",
|
||||
"K6_CPUSET": "4-6",
|
||||
"K6_CPUS": "3",
|
||||
"K6_MEMORY": "1g",
|
||||
}
|
||||
API_FIRST_CPU = 7
|
||||
FIELDS = [
|
||||
"raw_dir",
|
||||
"variant",
|
||||
"config",
|
||||
"replicas",
|
||||
"cpus",
|
||||
"memory_mb",
|
||||
"workers",
|
||||
"pool",
|
||||
"maxconn",
|
||||
"api_cpuset",
|
||||
"rate",
|
||||
"repeat",
|
||||
"started",
|
||||
"db_mb",
|
||||
"messages",
|
||||
"k6_exit",
|
||||
"saturated",
|
||||
"requests",
|
||||
"rps",
|
||||
"ok_rps",
|
||||
"failed_ratio",
|
||||
"dropped_iterations",
|
||||
"p50_ms",
|
||||
"p95_ms",
|
||||
"p99_ms",
|
||||
"p95_post_ms",
|
||||
"p95_get_ms",
|
||||
"cpu_api",
|
||||
"cpu_postgres",
|
||||
"cpu_lb",
|
||||
"cpu_valkey",
|
||||
"cpu_k6",
|
||||
"mem_api_mb",
|
||||
"mem_postgres_mb",
|
||||
"mem_k6_mb",
|
||||
"pg_backends_max",
|
||||
"pg_active_avg",
|
||||
"pg_active_max",
|
||||
"pg_idle_avg",
|
||||
"pg_idle_tx_avg",
|
||||
"pg_lock_avg",
|
||||
"pg_lock_max",
|
||||
"pg_lwlock_avg",
|
||||
"pg_io_avg",
|
||||
"lb_scur_max",
|
||||
"lb_qcur_max",
|
||||
"lb_qtime_max_ms",
|
||||
"lb_rtime_avg_ms",
|
||||
"cpu_vm",
|
||||
"cpu_other",
|
||||
"gap_s",
|
||||
"wall_s",
|
||||
"disturbed",
|
||||
]
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
command = ["docker", "compose", *args]
|
||||
return subprocess.run(
|
||||
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
|
||||
).stdout
|
||||
|
||||
|
||||
def size(text):
|
||||
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
|
||||
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
|
||||
|
||||
|
||||
def rows(path, start, end):
|
||||
with path.open() as handle:
|
||||
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
|
||||
|
||||
|
||||
def mean(values):
|
||||
return round(statistics.mean(values), 2) if values else 0.0
|
||||
|
||||
|
||||
def docker_usage(path, project, start, end):
|
||||
ticks = collections.defaultdict(collections.Counter)
|
||||
memory = collections.Counter()
|
||||
for r in rows(path, start, end):
|
||||
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
|
||||
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
|
||||
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
|
||||
cpu = {
|
||||
name: mean([t[name] for t in ticks.values() if name in t])
|
||||
for name in ("api", "postgres", "lb", "valkey", "k6")
|
||||
}
|
||||
usage = {f"cpu_{name}": value for name, value in cpu.items()}
|
||||
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
|
||||
return usage
|
||||
|
||||
|
||||
def vm_usage(path, start, end):
|
||||
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
|
||||
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
|
||||
return 0.0
|
||||
first, last = samples[0], samples[-1]
|
||||
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
|
||||
|
||||
|
||||
def gap(path, start, end):
|
||||
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
|
||||
edges = [start, *[t for t in ticks if start <= t <= end], end]
|
||||
return max(b - a for a, b in zip(edges, edges[1:]))
|
||||
|
||||
|
||||
def vm_counters():
|
||||
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
|
||||
values = [int(v) for v in lines[0].split()[1:]]
|
||||
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
|
||||
return sum(values) - values[3] - values[4], sum(values), cpus
|
||||
|
||||
|
||||
def wait_quiet(args):
|
||||
deadline, calm = time.monotonic() + args.quiet_timeout, 0
|
||||
while calm < 3 and time.monotonic() < deadline:
|
||||
first = vm_counters()
|
||||
time.sleep(5)
|
||||
last = vm_counters()
|
||||
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
|
||||
calm = calm + 1 if busy < args.max_other else 0
|
||||
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
|
||||
|
||||
|
||||
def pg_usage(path, start, end):
|
||||
ticks = collections.defaultdict(collections.Counter)
|
||||
for r in rows(path, start, end):
|
||||
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
|
||||
tick = ticks[r["ts"]]
|
||||
tick["backends"] += count
|
||||
tick["active"] += count * (state == "active")
|
||||
tick["idle"] += count * (state == "idle")
|
||||
tick["idle_tx"] += count * state.startswith("idle in transaction")
|
||||
tick["lock"] += count * (wait == "Lock")
|
||||
tick["lwlock"] += count * (wait == "LWLock")
|
||||
tick["io"] += count * (wait == "IO")
|
||||
series = list(ticks.values())
|
||||
return {
|
||||
"pg_backends_max": max((t["backends"] for t in series), default=0),
|
||||
"pg_active_avg": mean([t["active"] for t in series]),
|
||||
"pg_active_max": max((t["active"] for t in series), default=0),
|
||||
"pg_idle_avg": mean([t["idle"] for t in series]),
|
||||
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
|
||||
"pg_lock_avg": mean([t["lock"] for t in series]),
|
||||
"pg_lock_max": max((t["lock"] for t in series), default=0),
|
||||
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
|
||||
"pg_io_avg": mean([t["io"] for t in series]),
|
||||
}
|
||||
|
||||
|
||||
def lb_usage(path, start, end):
|
||||
backend = [
|
||||
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
|
||||
]
|
||||
|
||||
def numbers(field):
|
||||
return [int(r[field] or 0) for r in backend]
|
||||
|
||||
return {
|
||||
"lb_scur_max": max(numbers("scur"), default=0),
|
||||
"lb_qcur_max": max(numbers("qcur"), default=0),
|
||||
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
|
||||
"lb_rtime_avg_ms": mean(numbers("rtime")),
|
||||
}
|
||||
|
||||
|
||||
def trend(path, name):
|
||||
metrics = json.loads(Path(path).read_text())["metrics"]
|
||||
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
|
||||
return round(found.get("values", {}).get("p(95)", 0.0), 2)
|
||||
|
||||
|
||||
def sampler(*command):
|
||||
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
|
||||
|
||||
|
||||
def stop(process):
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
process.wait()
|
||||
|
||||
|
||||
def wait_lb(expected, timeout=120):
|
||||
deadline, stable = time.monotonic() + timeout, 0
|
||||
while stable < 4:
|
||||
if time.monotonic() > deadline:
|
||||
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
|
||||
stable = stable + 1 if lb_up() == expected else 0
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def layout(config, args, variant):
|
||||
name, _, extra = variant.partition(":")
|
||||
replicas, cpus = (float(v) for v in config.split("x"))
|
||||
replicas = int(replicas)
|
||||
workers = args.workers or max(1, int(cpus))
|
||||
width = max(1, math.ceil(replicas * cpus))
|
||||
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
|
||||
memory = args.api_memory or args.api_memory_total // replicas
|
||||
settings = {
|
||||
"API_REPLICAS": str(replicas),
|
||||
"API_CPUS": f"{cpus:g}",
|
||||
"API_MEMORY": f"{memory}m",
|
||||
"WEB_CONCURRENCY": str(workers),
|
||||
"DB_POOL_MAX_SIZE": str(args.pool),
|
||||
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
|
||||
"API_CPUSET": cpuset if args.pin else "",
|
||||
}
|
||||
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
|
||||
base = {
|
||||
"variant": name,
|
||||
"config": config,
|
||||
"replicas": replicas,
|
||||
"cpus": cpus,
|
||||
"memory_mb": memory,
|
||||
"workers": int(settings["WEB_CONCURRENCY"]),
|
||||
"pool": int(settings["DB_POOL_MAX_SIZE"]),
|
||||
"maxconn": int(settings["API_SERVER_MAXCONN"]),
|
||||
"api_cpuset": settings["API_CPUSET"],
|
||||
}
|
||||
return base, settings
|
||||
|
||||
|
||||
def apply(base, settings, env, out):
|
||||
env = dict(env, **settings)
|
||||
for attempt in range(3):
|
||||
try:
|
||||
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
|
||||
break
|
||||
except subprocess.CalledProcessError as error:
|
||||
log = Path(out) / "compose-up-failed.txt"
|
||||
with log.open("a") as handle:
|
||||
stamp = time.strftime("%FT%TZ", time.gmtime())
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
handle.write(f"date: {stamp} commit: {commit}\n")
|
||||
handle.write(f"== {stamp} {base} attempt {attempt + 1}\n")
|
||||
handle.write(error.stdout + error.stderr + "\n")
|
||||
print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)}))
|
||||
if attempt == 2:
|
||||
raise
|
||||
time.sleep(30)
|
||||
wait_lb(base["replicas"])
|
||||
return env
|
||||
|
||||
|
||||
def deployment(env):
|
||||
ids = compose("ps", "-q", env=env).split()
|
||||
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "--format", template, *ids],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
containers = []
|
||||
for line in result.stdout.splitlines():
|
||||
name, image, config = line.split(" ", 2)
|
||||
limits = json.loads(config)
|
||||
containers.append(
|
||||
dict(
|
||||
name=json.loads(name),
|
||||
image=json.loads(image),
|
||||
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
|
||||
)
|
||||
)
|
||||
return containers
|
||||
|
||||
|
||||
def provenance(folder, args, env):
|
||||
stamp = time.strftime("%FT%TZ", time.gmtime())
|
||||
metadata = {
|
||||
"date": stamp,
|
||||
"commit": args.commit,
|
||||
"project": args.project,
|
||||
"command": sys.argv,
|
||||
"api_image": env.get("API_IMAGE", "compose default"),
|
||||
"warmup_seconds": args.warmup,
|
||||
"duration_seconds": args.duration,
|
||||
"containers": args.deployment,
|
||||
}
|
||||
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
|
||||
for path in folder.rglob("*"):
|
||||
if path.suffix in {".csv", ".txt"}:
|
||||
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
|
||||
elif path.name == "summary.json":
|
||||
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
|
||||
path.write_text(json.dumps(data) + "\n")
|
||||
|
||||
|
||||
def attempt_folder(out, base, rate, repeat):
|
||||
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
|
||||
attempt = 1
|
||||
while (Path(out) / f"{name}_attempt{attempt}").exists():
|
||||
attempt += 1
|
||||
folder = Path(out) / f"{name}_attempt{attempt}"
|
||||
folder.mkdir(parents=True)
|
||||
return folder
|
||||
|
||||
|
||||
def measure(folder, args, rate, env):
|
||||
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
|
||||
database = subprocess.run(
|
||||
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
|
||||
).stdout.split(",")
|
||||
samplers = [
|
||||
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
|
||||
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
|
||||
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
|
||||
]
|
||||
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
|
||||
began = time.time()
|
||||
try:
|
||||
code, started = run_k6(folder / "k6", load)
|
||||
finally:
|
||||
wall = time.time() - began
|
||||
for process in samplers:
|
||||
stop(process)
|
||||
subprocess.run(
|
||||
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
|
||||
)
|
||||
if started is None:
|
||||
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
|
||||
return code, started + args.warmup, database, wall
|
||||
|
||||
|
||||
def step(base, rate, repeat, args, env):
|
||||
folder = attempt_folder(args.out, base, rate, repeat)
|
||||
code, start, database, wall = measure(folder, args, rate, env)
|
||||
end = start + args.duration
|
||||
summary = folder / "k6/summary.json"
|
||||
metrics = k6_metrics(summary, args.duration)
|
||||
saturated = (
|
||||
metrics["failed_ratio"] > args.max_error_ratio
|
||||
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
|
||||
or metrics["p95_ms"] > args.slo_p95_ms
|
||||
)
|
||||
row = dict(
|
||||
base,
|
||||
raw_dir=folder.name,
|
||||
rate=rate,
|
||||
repeat=repeat,
|
||||
k6_exit=code,
|
||||
saturated=saturated,
|
||||
**metrics,
|
||||
)
|
||||
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
|
||||
row["db_mb"] = round(int(database[0]) / 2**20, 1)
|
||||
row["messages"] = int(database[1])
|
||||
row["p95_post_ms"] = trend(summary, "post_message")
|
||||
row["p95_get_ms"] = trend(summary, "get_history")
|
||||
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
|
||||
row.update(pg_usage(folder / "pg_activity.csv", start, end))
|
||||
row.update(lb_usage(folder / "lb_stats.csv", start, end))
|
||||
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
|
||||
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
|
||||
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
|
||||
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
|
||||
row["wall_s"] = round(wall, 1)
|
||||
row["disturbed"] = (
|
||||
row["cpu_other"] > args.max_other
|
||||
or row["gap_s"] > args.max_gap
|
||||
or wall > args.warmup + args.duration + args.max_overhead
|
||||
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
|
||||
or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6"))
|
||||
or not rows(folder / "pg_activity.csv", start, end)
|
||||
or not rows(folder / "lb_stats.csv", start, end)
|
||||
)
|
||||
provenance(folder, args, env)
|
||||
if code or not metrics["requests"]:
|
||||
raise SystemExit(f"invalid k6 run: {folder}")
|
||||
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
|
||||
return row
|
||||
|
||||
|
||||
def rates(args, base):
|
||||
if args.rates:
|
||||
return [int(r) for r in args.rates.split(",")]
|
||||
factor, top = (float(v) for v in args.search.split(":"))
|
||||
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
|
||||
while value <= top:
|
||||
values.append(int(round(value, -1)))
|
||||
value *= factor
|
||||
return values
|
||||
|
||||
|
||||
def arguments():
|
||||
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
|
||||
parser.add_argument("--out", required=True)
|
||||
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
|
||||
parser.add_argument(
|
||||
"--variant",
|
||||
dest="variants",
|
||||
action="append",
|
||||
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
|
||||
)
|
||||
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
|
||||
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
|
||||
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
|
||||
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
|
||||
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
|
||||
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
|
||||
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
|
||||
parser.add_argument("--rates", help="comma separated offered rates, rps")
|
||||
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
|
||||
parser.add_argument("--start-per-cpu", type=float, default=100.0)
|
||||
parser.add_argument("--repeats", type=int, default=3)
|
||||
parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat")
|
||||
parser.add_argument("--warmup", type=int, default=10)
|
||||
parser.add_argument("--duration", type=int, default=30)
|
||||
parser.add_argument("--pause", type=float, default=5.0)
|
||||
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
|
||||
parser.add_argument("--max-error-ratio", type=float, default=0.01)
|
||||
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
|
||||
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
|
||||
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
|
||||
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
|
||||
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
|
||||
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
|
||||
args = parser.parse_args()
|
||||
args.variants = args.variants or ["base"]
|
||||
if len(args.variants) > 1 and not args.rates:
|
||||
parser.error("several variants need explicit --rates")
|
||||
return args
|
||||
|
||||
|
||||
def block(args, output, config, variant, repeats):
|
||||
base, settings = layout(config, args, variant)
|
||||
current, streak = apply(base, settings, args.base_env, args.out), 0
|
||||
args.deployment = deployment(current)
|
||||
for rate in rates(args, base):
|
||||
results = []
|
||||
for repeat in repeats:
|
||||
for _ in range(args.retries + 1):
|
||||
row = step(base, rate, repeat, args, current)
|
||||
time.sleep(args.pause)
|
||||
if not row["disturbed"]:
|
||||
break
|
||||
args.rejected.writerow(row)
|
||||
wait_quiet(args)
|
||||
else:
|
||||
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
|
||||
results.append(row)
|
||||
output.writerows(results)
|
||||
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
|
||||
if streak == 2:
|
||||
break
|
||||
|
||||
|
||||
def series(args, output):
|
||||
every = range(args.first_repeat, args.first_repeat + args.repeats)
|
||||
configs = args.configs.split(",")
|
||||
if not args.rates:
|
||||
for config in configs:
|
||||
block(args, output, config, args.variants[0], every)
|
||||
return
|
||||
for repeat in every:
|
||||
for config in configs:
|
||||
for variant in args.variants:
|
||||
block(args, output, config, variant, [repeat])
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def table(path):
|
||||
fresh = not path.exists()
|
||||
with path.open("a", newline="", buffering=1) as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
|
||||
if fresh:
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
|
||||
writer.writeheader()
|
||||
yield writer
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments()
|
||||
out = Path(args.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
args.project = json.loads(compose("config", "--format", "json"))["name"]
|
||||
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
|
||||
try:
|
||||
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
|
||||
series(args, writer)
|
||||
finally:
|
||||
if not args.keep:
|
||||
compose("up", "-d", "--wait", "--wait-timeout", "180")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)"
|
||||
put s3_root_password "$(random)"
|
||||
put s3_access_key_id "api$(random | cut -c1-16)"
|
||||
put s3_secret_access_key "$(random)"
|
||||
put s3_reconciler_access_key_id "rec$(random | cut -c1-16)"
|
||||
put s3_reconciler_secret_access_key "$(random)"
|
||||
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
size="select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
|
||||
echo "$(date -u +%T) database bytes,messages: $(docker compose exec -T postgres psql -X -At -F, -U messenger -d messenger -c "$size")"
|
||||
docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q)
|
||||
+1
-1
@@ -6,4 +6,4 @@ ids=$(docker compose ps -q "${API_SERVICE:-api}")
|
||||
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
|
||||
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
|
||||
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
|
||||
docker stop -t "${STOP_TIMEOUT:-7}" "$target" >/dev/null
|
||||
|
||||
@@ -77,7 +77,9 @@ def main():
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.out
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
|
||||
if any(out.iterdir()):
|
||||
p.error("output directory must be empty; choose a new --out")
|
||||
commit = seed.run(["git", "rev-parse", "HEAD"]).strip()
|
||||
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
|
||||
lines = [
|
||||
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
|
||||
@@ -102,7 +104,12 @@ def main():
|
||||
link=0,
|
||||
ids_out=None,
|
||||
)
|
||||
report = {"upload_concurrency": slots, **seed.over_http(run)}
|
||||
report = {
|
||||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"commit": commit,
|
||||
"upload_concurrency": slots,
|
||||
**seed.over_http(run),
|
||||
}
|
||||
report["lb"] = balancer()
|
||||
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
|
||||
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
@@ -47,3 +47,14 @@ class CleanStaging(Command):
|
||||
class ReplayAttachment(Command):
|
||||
attachment_id: UUID
|
||||
operator: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReconcileStorage(Command):
|
||||
apply: bool
|
||||
grace: timedelta
|
||||
skew: timedelta
|
||||
deep: bool = False
|
||||
page_size: int = 1000
|
||||
deep_concurrency: int = 8
|
||||
samples: int = 20
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from contextlib import AbstractAsyncContextManager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Protocol, runtime_checkable
|
||||
@@ -40,6 +41,13 @@ class StagedObject:
|
||||
key: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StoredObject:
|
||||
key: str
|
||||
size: int
|
||||
modified: datetime
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ObjectStorage(Protocol):
|
||||
async def put(self, key: str, data: bytes, content_type: str) -> None: ...
|
||||
@@ -52,6 +60,10 @@ class ObjectStorage(Protocol):
|
||||
|
||||
async def delete(self, key: str) -> None: ...
|
||||
|
||||
def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]: ...
|
||||
|
||||
async def stat(self, key: str) -> StoredObject | None: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ImageInspector(Protocol):
|
||||
@@ -95,6 +107,14 @@ class AttachmentRepository(Protocol):
|
||||
|
||||
async def remove(self, attachment: Attachment) -> None: ...
|
||||
|
||||
async def page_after(self, key: str, limit: int) -> Sequence[Attachment]: ...
|
||||
|
||||
async def by_key(self, key: str) -> Attachment | None: ...
|
||||
|
||||
async def claim(self, attachment_id: UUID) -> Attachment | None: ...
|
||||
|
||||
async def mark(self, attachment: Attachment) -> None: ...
|
||||
|
||||
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: ...
|
||||
|
||||
async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ...
|
||||
@@ -153,3 +173,8 @@ class AttachmentDirectory(Protocol):
|
||||
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ...
|
||||
|
||||
async def failed_jobs(self, limit: int) -> list[FailedJob]: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SingleRun(Protocol):
|
||||
def hold(self) -> AbstractAsyncContextManager[bool]: ...
|
||||
|
||||
@@ -49,6 +49,9 @@ class OpenAttachment:
|
||||
if found is None:
|
||||
raise AttachmentNotFound("у вложения нет такого варианта")
|
||||
key, content_type = found.object_key, found.content_type
|
||||
if attachment.quarantine is not None:
|
||||
self._logger.warning("attachment.quarantined", attachment_id=str(attachment.id))
|
||||
raise AttachmentNotFound("вложение не найдено")
|
||||
try:
|
||||
content = await self._storage.open(str(key))
|
||||
except ObjectMissingError as error:
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
import asyncio
|
||||
from collections import Counter
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from hashlib import sha256
|
||||
from time import monotonic
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.attachments.application.commands import ReconcileStorage
|
||||
from messenger.attachments.application.errors import ObjectMissingError
|
||||
from messenger.attachments.application.handlers import drop
|
||||
from messenger.attachments.application.ports import (
|
||||
AttachmentsUnitOfWork,
|
||||
ObjectStorage,
|
||||
SingleRun,
|
||||
StoredObject,
|
||||
)
|
||||
from messenger.attachments.domain.models import (
|
||||
OBJECT_PREFIX,
|
||||
Attachment,
|
||||
AttachmentStatus,
|
||||
Quarantine,
|
||||
)
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.application.ports import Clock, EventPublisher, Logger
|
||||
|
||||
type UnitOfWorkFactory = Callable[[], AttachmentsUnitOfWork]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Reconciliation:
|
||||
apply: bool
|
||||
skipped: bool = False
|
||||
objects: int = 0
|
||||
rows: int = 0
|
||||
found: Counter[str] = field(default_factory=Counter)
|
||||
fixed: Counter[str] = field(default_factory=Counter)
|
||||
rechecked: Counter[str] = field(default_factory=Counter)
|
||||
requests: Counter[str] = field(default_factory=Counter)
|
||||
waited: dict[str, float] = field(default_factory=dict)
|
||||
samples: dict[str, list[str]] = field(default_factory=dict)
|
||||
hashed_bytes: int = 0
|
||||
errors: int = 0
|
||||
seconds: float = 0.0
|
||||
|
||||
|
||||
class StorageReconciler:
|
||||
__slots__ = ("_clock", "_logger", "_publisher", "_single", "_storage", "_uow")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
uow: UnitOfWorkFactory,
|
||||
storage: ObjectStorage,
|
||||
single: SingleRun,
|
||||
clock: Clock,
|
||||
*,
|
||||
logger: Logger,
|
||||
publisher: EventPublisher,
|
||||
) -> None:
|
||||
self._uow = uow
|
||||
self._storage = storage
|
||||
self._single = single
|
||||
self._clock = clock
|
||||
self._logger = logger
|
||||
self._publisher = publisher
|
||||
|
||||
async def __call__(self, command: ReconcileStorage) -> Reconciliation:
|
||||
report = Reconciliation(apply=command.apply)
|
||||
started = monotonic()
|
||||
async with self._single.hold() as held:
|
||||
report.skipped = not held
|
||||
if held:
|
||||
before = self._clock.now() - command.grace - command.skew
|
||||
await Pass(
|
||||
self._uow,
|
||||
self._storage,
|
||||
command,
|
||||
before=before,
|
||||
report=report,
|
||||
logger=self._logger,
|
||||
publisher=self._publisher,
|
||||
).run()
|
||||
report.seconds = round(monotonic() - started, 3)
|
||||
return report
|
||||
|
||||
|
||||
class Pass:
|
||||
__slots__ = (
|
||||
"_before",
|
||||
"_command",
|
||||
"_logger",
|
||||
"_publisher",
|
||||
"_report",
|
||||
"_slots",
|
||||
"_storage",
|
||||
"_uow",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
uow: UnitOfWorkFactory,
|
||||
storage: ObjectStorage,
|
||||
command: ReconcileStorage,
|
||||
*,
|
||||
before: datetime,
|
||||
report: Reconciliation,
|
||||
logger: Logger,
|
||||
publisher: EventPublisher,
|
||||
) -> None:
|
||||
self._uow = uow
|
||||
self._storage = storage
|
||||
self._command = command
|
||||
self._before = before
|
||||
self._report = report
|
||||
self._logger = logger
|
||||
self._publisher = publisher
|
||||
self._slots = asyncio.Semaphore(command.deep_concurrency)
|
||||
|
||||
async def run(self) -> None:
|
||||
objects = aiter(self._objects())
|
||||
item = await anext(objects, None)
|
||||
try:
|
||||
async with asyncio.TaskGroup() as checks:
|
||||
async for row in self._rows():
|
||||
while item is not None and item.key < row.object_key:
|
||||
await self._orphan(item)
|
||||
item = await anext(objects, None)
|
||||
matched = item if item is not None and item.key == row.object_key else None
|
||||
await self._row(row, matched, checks)
|
||||
if matched is not None:
|
||||
item = await anext(objects, None)
|
||||
while item is not None:
|
||||
await self._orphan(item)
|
||||
item = await anext(objects, None)
|
||||
except* Unavailable as failed:
|
||||
error = failed.exceptions[0]
|
||||
raise error from error.__cause__
|
||||
|
||||
async def _timed[ResultT](self, name: str, step: Awaitable[ResultT]) -> ResultT:
|
||||
started = monotonic()
|
||||
try:
|
||||
return await step
|
||||
finally:
|
||||
spent = self._report.waited.get(name, 0.0) + monotonic() - started
|
||||
self._report.waited[name] = spent
|
||||
|
||||
async def _objects(self) -> AsyncIterator[StoredObject]:
|
||||
pages = aiter(self._storage.listing(OBJECT_PREFIX, self._command.page_size))
|
||||
while (page := await self._timed("s3_list", anext(pages, None))) is not None:
|
||||
self._report.requests["s3_list"] += 1
|
||||
self._report.objects += len(page)
|
||||
for item in page:
|
||||
yield item
|
||||
|
||||
async def _rows(self) -> AsyncIterator[Attachment]:
|
||||
after, size = "", self._command.page_size
|
||||
while True:
|
||||
page = await self._timed("sql", self._page(after, size + 1))
|
||||
self._report.requests["sql"] += 1
|
||||
rows = page[:size]
|
||||
self._report.rows += len(rows)
|
||||
for row in rows:
|
||||
yield row
|
||||
if len(page) <= size:
|
||||
return
|
||||
after = rows[-1].object_key
|
||||
|
||||
async def _page(self, after: str, size: int) -> Sequence[Attachment]:
|
||||
uow = self._uow()
|
||||
async with uow.transaction():
|
||||
return await uow.attachments.page_after(after, size)
|
||||
|
||||
def _found(self, name: str, subject: str) -> None:
|
||||
self._report.found[name] += 1
|
||||
sample = self._report.samples.setdefault(name, [])
|
||||
if len(sample) < self._command.samples:
|
||||
sample.append(subject)
|
||||
|
||||
async def _safely(self, name: str, action: Awaitable[bool], subject: UUID | None) -> None:
|
||||
attachment_id = None if subject is None else str(subject)
|
||||
try:
|
||||
done = await action
|
||||
except Exception as error:
|
||||
self._report.errors += 1
|
||||
self._logger.warning(
|
||||
"reconcile.action_failed",
|
||||
reason=name,
|
||||
attachment_id=attachment_id,
|
||||
error=type(error).__name__,
|
||||
)
|
||||
return
|
||||
(self._report.fixed if done else self._report.rechecked)[name] += 1
|
||||
event = "reconcile.fixed" if done else "reconcile.rechecked"
|
||||
self._logger.info(event, reason=name, attachment_id=attachment_id)
|
||||
|
||||
async def _orphan(self, item: StoredObject) -> None:
|
||||
if item.modified >= self._before:
|
||||
self._found("orphan_fresh", item.key)
|
||||
return
|
||||
self._found("orphan_stale", item.key)
|
||||
if self._command.apply:
|
||||
await self._safely("orphan_stale", self._delete_orphan(item), None)
|
||||
|
||||
async def _delete_orphan(self, item: StoredObject) -> bool:
|
||||
uow = self._uow()
|
||||
async with uow.transaction():
|
||||
owner = await uow.attachments.by_key(item.key)
|
||||
self._report.requests["sql"] += 1
|
||||
if owner is not None:
|
||||
return False
|
||||
self._report.requests["s3_delete"] += 1
|
||||
await self._storage.delete(item.key)
|
||||
return True
|
||||
|
||||
async def _row(
|
||||
self, row: Attachment, item: StoredObject | None, checks: asyncio.TaskGroup
|
||||
) -> None:
|
||||
if row.status is AttachmentStatus.PENDING:
|
||||
await self._pending(row)
|
||||
elif item is None:
|
||||
await self._broken(row, Quarantine.MISSING)
|
||||
elif item.size != row.size_bytes:
|
||||
await self._broken(row, Quarantine.SIZE_MISMATCH)
|
||||
elif self._command.deep or row.quarantine is not None:
|
||||
await self._slots.acquire()
|
||||
checks.create_task(self._verify(row))
|
||||
|
||||
async def _verify(self, row: Attachment) -> None:
|
||||
try:
|
||||
digest = await self._digest(row.object_key)
|
||||
except Exception as error:
|
||||
self._report.errors += 1
|
||||
self._logger.warning(
|
||||
"reconcile.hash_failed", attachment_id=str(row.id), error=type(error).__name__
|
||||
)
|
||||
return
|
||||
finally:
|
||||
self._slots.release()
|
||||
if digest is None:
|
||||
await self._broken(row, Quarantine.MISSING)
|
||||
elif digest != row.sha256:
|
||||
await self._broken(row, Quarantine.SHA_MISMATCH)
|
||||
else:
|
||||
await self._healthy(row)
|
||||
|
||||
async def _digest(self, key: str) -> str | None:
|
||||
self._report.requests["s3_get"] += 1
|
||||
try:
|
||||
content = await self._storage.open(key)
|
||||
except ObjectMissingError:
|
||||
return None
|
||||
digest = sha256()
|
||||
async for chunk in content.chunks:
|
||||
digest.update(chunk)
|
||||
self._report.hashed_bytes += len(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
async def _broken(self, row: Attachment, reason: Quarantine) -> None:
|
||||
if row.quarantine is not None:
|
||||
self._found("quarantined", str(row.id))
|
||||
return
|
||||
self._found(reason.value, str(row.id))
|
||||
if self._command.apply:
|
||||
await self._safely(reason.value, self._isolate(row, reason), row.id)
|
||||
|
||||
async def _isolate(self, row: Attachment, reason: Quarantine) -> bool:
|
||||
if reason is not Quarantine.SHA_MISMATCH:
|
||||
self._report.requests["s3_head"] += 1
|
||||
found = await self._storage.stat(row.object_key)
|
||||
if found is not None and found.size == row.size_bytes:
|
||||
return False
|
||||
row.isolate(reason)
|
||||
await self._mark(row)
|
||||
return True
|
||||
|
||||
async def _healthy(self, row: Attachment) -> None:
|
||||
if row.quarantine is None:
|
||||
return
|
||||
self._found("healed", str(row.id))
|
||||
if self._command.apply:
|
||||
await self._safely("healed", self._release(row), row.id)
|
||||
|
||||
async def _release(self, row: Attachment) -> bool:
|
||||
row.release()
|
||||
await self._mark(row)
|
||||
return True
|
||||
|
||||
async def _mark(self, row: Attachment) -> None:
|
||||
uow = self._uow()
|
||||
async with uow.transaction():
|
||||
await uow.attachments.mark(row)
|
||||
await uow.commit()
|
||||
self._report.requests["sql"] += 1
|
||||
|
||||
async def _pending(self, row: Attachment) -> None:
|
||||
if row.created_at >= self._before:
|
||||
self._found("pending_fresh", str(row.id))
|
||||
return
|
||||
self._found("pending_stale", str(row.id))
|
||||
if self._command.apply:
|
||||
await self._safely("pending_stale", self._drop(row), row.id)
|
||||
|
||||
async def _drop(self, row: Attachment) -> bool:
|
||||
uow = self._uow()
|
||||
async with uow.transaction():
|
||||
current = await uow.attachments.claim(row.id)
|
||||
self._report.requests["sql"] += 1
|
||||
if (
|
||||
current is None
|
||||
or current.status is not AttachmentStatus.PENDING
|
||||
or current.created_at >= self._before
|
||||
):
|
||||
return False
|
||||
self._report.requests["s3_delete"] += 1
|
||||
await drop(uow, self._storage, current, "reconcile")
|
||||
await self._publisher.publish(uow.collect_new_events())
|
||||
return True
|
||||
@@ -8,6 +8,7 @@ from messenger.attachments.domain.values import Filename, ImageInfo
|
||||
from messenger.attachments.domain.variants import Variant
|
||||
from messenger.shared.domain.events import DomainEvent
|
||||
|
||||
OBJECT_PREFIX = "attachments/"
|
||||
STAGING_PREFIX = "staging/"
|
||||
ORIGINALS_PREFIX = "originals/"
|
||||
|
||||
@@ -24,6 +25,12 @@ def original_key(attachment_id: UUID) -> str:
|
||||
return f"{ORIGINALS_PREFIX}{attachment_id}"
|
||||
|
||||
|
||||
class Quarantine(StrEnum):
|
||||
MISSING = "missing"
|
||||
SIZE_MISMATCH = "size_mismatch"
|
||||
SHA_MISMATCH = "sha_mismatch"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Attachment:
|
||||
id: UUID
|
||||
@@ -41,6 +48,7 @@ class Attachment:
|
||||
published_version: int = 0
|
||||
error_code: str | None = None
|
||||
variants: tuple[Variant, ...] = ()
|
||||
quarantine: Quarantine | None = None
|
||||
_events: list[DomainEvent] = field(init=False, default_factory=list, repr=False, compare=False)
|
||||
|
||||
@classmethod
|
||||
@@ -70,6 +78,10 @@ class Attachment:
|
||||
staging_key=f"{STAGING_PREFIX}{attachment_id}",
|
||||
)
|
||||
|
||||
@property
|
||||
def object_key(self) -> str:
|
||||
return self.original_key or self.staging_key or f"{OBJECT_PREFIX}{self.id}"
|
||||
|
||||
@property
|
||||
def content_type(self) -> str:
|
||||
return self.image.kind.value
|
||||
@@ -98,6 +110,13 @@ class Attachment:
|
||||
AttachmentDiscarded(attachment_id=self.id, chat_id=self.chat_id, reason=reason)
|
||||
)
|
||||
|
||||
def isolate(self, reason: Quarantine) -> None:
|
||||
if self.status is AttachmentStatus.READY:
|
||||
self.quarantine = reason
|
||||
|
||||
def release(self) -> None:
|
||||
self.quarantine = None
|
||||
|
||||
def collect_events(self) -> list[DomainEvent]:
|
||||
events, self._events = self._events, []
|
||||
return events
|
||||
|
||||
@@ -74,7 +74,7 @@ async def object_storage(settings: AttachmentsSettings) -> AsyncIterator[ObjectS
|
||||
yield S3ObjectStorage(client, settings.s3_bucket)
|
||||
|
||||
|
||||
class AttachmentsProvider(Provider):
|
||||
class AttachmentStorageProvider(Provider):
|
||||
def __init__(self, settings: AttachmentsSettings) -> None:
|
||||
super().__init__()
|
||||
self._settings = settings
|
||||
@@ -88,6 +88,8 @@ class AttachmentsProvider(Provider):
|
||||
async with object_storage(settings) as storage:
|
||||
yield storage
|
||||
|
||||
|
||||
class AttachmentsProvider(AttachmentStorageProvider):
|
||||
@provide(scope=Scope.APP)
|
||||
def images(self, settings: AttachmentsSettings) -> ImageInspector:
|
||||
return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency)
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import resource
|
||||
import sys
|
||||
from collections.abc import Callable, Sequence
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from dishka import Scope, make_async_container, provide
|
||||
|
||||
from messenger.attachments.application.commands import ReconcileStorage
|
||||
from messenger.attachments.application.ports import ObjectStorage
|
||||
from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler
|
||||
from messenger.attachments.config import AttachmentsSettings
|
||||
from messenger.attachments.domain.events import AttachmentDiscarded
|
||||
from messenger.attachments.entrypoints.ioc import AttachmentStorageProvider
|
||||
from messenger.attachments.entrypoints.journal import LOG_FIELDS, AttachmentsJournal
|
||||
from messenger.attachments.infrastructure.postgres import (
|
||||
PostgresAttachmentsUnitOfWork,
|
||||
PostgresSingleRun,
|
||||
)
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.application.messagebus import EventBus
|
||||
from messenger.shared.application.ports import Clock, EventPublisher, Logger
|
||||
from messenger.shared.config import PlatformSettings
|
||||
from messenger.shared.entrypoints.ioc import PlatformProvider
|
||||
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
|
||||
from messenger.shared.infrastructure.postgres import Database
|
||||
|
||||
LOCK_KEY = 20_260_004
|
||||
CLASSES = (
|
||||
"orphan_stale",
|
||||
"orphan_fresh",
|
||||
"missing",
|
||||
"size_mismatch",
|
||||
"sha_mismatch",
|
||||
"pending_stale",
|
||||
"pending_fresh",
|
||||
"quarantined",
|
||||
"healed",
|
||||
)
|
||||
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale", "healed")
|
||||
REQUESTS = ("s3_list", "s3_head", "s3_get", "s3_delete", "sql")
|
||||
|
||||
|
||||
def bounded(low: int, high: int | None = None) -> Callable[[str], int]:
|
||||
def parse(text: str) -> int:
|
||||
value = int(text)
|
||||
if value < low or (high is not None and value > high):
|
||||
raise argparse.ArgumentTypeError(f"ожидается {low}..{high or '∞'}")
|
||||
return value
|
||||
|
||||
return parse
|
||||
|
||||
|
||||
def command_of(argv: Sequence[str] | None = None) -> ReconcileStorage:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="python -m messenger.attachments.entrypoints.reconcile",
|
||||
description="Сверка метаданных вложений в PostgreSQL с объектами bucket S3",
|
||||
)
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--dry-run", dest="apply", action="store_false", help="только отчёт")
|
||||
mode.add_argument("--apply", dest="apply", action="store_true", help="исправить найденное")
|
||||
parser.add_argument("--grace", type=bounded(60), default=3600, help="секунды, ≥ 60")
|
||||
parser.add_argument("--skew", type=bounded(0), default=60, help="запас на часы, секунды")
|
||||
parser.add_argument("--deep", action="store_true", help="сверить SHA-256 каждого объекта")
|
||||
parser.add_argument("--page-size", type=bounded(1, 1000), default=1000)
|
||||
parser.add_argument("--deep-concurrency", type=bounded(1, 48), default=8)
|
||||
parser.add_argument("--samples", type=bounded(0, 1000), default=20)
|
||||
args = parser.parse_args(argv)
|
||||
return ReconcileStorage(
|
||||
apply=args.apply,
|
||||
grace=timedelta(seconds=args.grace),
|
||||
skew=timedelta(seconds=args.skew),
|
||||
deep=args.deep,
|
||||
page_size=args.page_size,
|
||||
deep_concurrency=args.deep_concurrency,
|
||||
samples=args.samples,
|
||||
)
|
||||
|
||||
|
||||
class ReconcileProvider(AttachmentStorageProvider):
|
||||
@provide(scope=Scope.APP)
|
||||
def reconciler(
|
||||
self,
|
||||
database: Database,
|
||||
storage: ObjectStorage,
|
||||
clock: Clock,
|
||||
logger: Logger,
|
||||
publisher: EventPublisher,
|
||||
) -> StorageReconciler:
|
||||
return StorageReconciler(
|
||||
lambda: PostgresAttachmentsUnitOfWork(database),
|
||||
storage,
|
||||
PostgresSingleRun(database, LOCK_KEY),
|
||||
clock,
|
||||
logger=logger,
|
||||
publisher=publisher,
|
||||
)
|
||||
|
||||
|
||||
async def reconcile(
|
||||
command: ReconcileStorage,
|
||||
platform: PlatformSettings,
|
||||
settings: AttachmentsSettings,
|
||||
logger: Logger,
|
||||
) -> Reconciliation:
|
||||
container = make_async_container(
|
||||
PlatformProvider(platform, logger), ReconcileProvider(settings)
|
||||
)
|
||||
bus = await container.get(EventBus)
|
||||
bus.subscribe(AttachmentDiscarded, AttachmentsJournal(logger).on_attachment_discarded)
|
||||
bus.start()
|
||||
try:
|
||||
reconciler = await container.get(StorageReconciler)
|
||||
return await reconciler(command)
|
||||
finally:
|
||||
await bus.stop()
|
||||
await container.close()
|
||||
|
||||
|
||||
def failure(error: BaseException) -> str:
|
||||
if isinstance(error, BaseExceptionGroup):
|
||||
return failure(error.exceptions[0])
|
||||
if isinstance(error, Unavailable) and error.__cause__ is not None:
|
||||
return type(error.__cause__).__name__
|
||||
return type(error).__name__
|
||||
|
||||
|
||||
def report_of(command: ReconcileStorage, result: Reconciliation) -> dict[str, Any]:
|
||||
usage = resource.getrusage(resource.RUSAGE_SELF)
|
||||
return {
|
||||
"mode": "apply" if command.apply else "dry-run",
|
||||
"skipped": result.skipped,
|
||||
"grace_seconds": int(command.grace.total_seconds()),
|
||||
"skew_seconds": int(command.skew.total_seconds()),
|
||||
"deep": command.deep,
|
||||
"page_size": command.page_size,
|
||||
"scanned": {"objects": result.objects, "rows": result.rows},
|
||||
"found": {name: result.found[name] for name in CLASSES},
|
||||
"fixed": {name: result.fixed[name] for name in FIXABLE},
|
||||
"rechecked": dict(result.rechecked),
|
||||
"errors": result.errors,
|
||||
"samples": result.samples,
|
||||
"cost": {
|
||||
"seconds": result.seconds,
|
||||
"requests": {name: result.requests[name] for name in REQUESTS},
|
||||
"waited_seconds": {name: round(spent, 3) for name, spent in result.waited.items()},
|
||||
"hashed_bytes": result.hashed_bytes,
|
||||
"cpu_seconds": round(usage.ru_utime + usage.ru_stime, 3),
|
||||
"max_rss_mib": round(usage.ru_maxrss / 1024, 1),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> None:
|
||||
command = command_of(argv)
|
||||
platform = PlatformSettings(db_pool_max_size=command.deep_concurrency + 2)
|
||||
settings = AttachmentsSettings()
|
||||
logger = StructlogLogger(
|
||||
level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS, stream=sys.stderr
|
||||
)
|
||||
try:
|
||||
result = asyncio.run(reconcile(command, platform, settings, logger))
|
||||
except Exception as error:
|
||||
logger.error("reconcile.failed", error=failure(error))
|
||||
raise SystemExit(1) from None
|
||||
sys.stdout.write(json.dumps(report_of(command, result), ensure_ascii=False, indent=2) + "\n")
|
||||
raise SystemExit(1 if result.errors else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
@@ -8,7 +8,7 @@ from botocore.exceptions import BotoCoreError, ClientError
|
||||
from botocore.handlers import add_expect_header
|
||||
|
||||
from messenger.attachments.application.errors import ObjectMissingError
|
||||
from messenger.attachments.application.ports import ObjectContent
|
||||
from messenger.attachments.application.ports import ObjectContent, StoredObject
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
|
||||
CHUNK_BYTES = 64 * 1024
|
||||
@@ -98,6 +98,28 @@ class S3ObjectStorage:
|
||||
except FAILURES as error:
|
||||
raise Unavailable("S3 недоступно") from error
|
||||
|
||||
async def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]:
|
||||
pages = self._client.get_paginator("list_objects_v2").paginate(
|
||||
Bucket=self._bucket, Prefix=prefix, PaginationConfig={"PageSize": page_size}
|
||||
)
|
||||
try:
|
||||
async for page in pages:
|
||||
yield [
|
||||
StoredObject(item["Key"], item["Size"], item["LastModified"])
|
||||
for item in page.get("Contents", ())
|
||||
]
|
||||
except FAILURES as error:
|
||||
raise Unavailable("S3 недоступно") from error
|
||||
|
||||
async def stat(self, key: str) -> StoredObject | None:
|
||||
try:
|
||||
head = await self._client.head_object(Bucket=self._bucket, Key=key)
|
||||
except FAILURES as error:
|
||||
if missing(error):
|
||||
return None
|
||||
raise Unavailable("S3 недоступно") from error
|
||||
return StoredObject(key, head["ContentLength"], head["LastModified"])
|
||||
|
||||
async def _chunks(self, body: Any) -> AsyncIterator[bytes]:
|
||||
async with body as stream:
|
||||
async for chunk in stream.iter_chunks(CHUNK_BYTES):
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from typing import Any
|
||||
@@ -10,12 +9,13 @@ from uvicorn.server import Server
|
||||
from uvicorn_worker import UvicornWorker
|
||||
|
||||
from messenger.main import LOG_FIELDS
|
||||
from messenger.shared.entrypoints.health import DRAINING
|
||||
from messenger.shared.infrastructure.logging import configure
|
||||
from messenger.shared.infrastructure.logging import StructlogLogger, configure
|
||||
from messenger.shared.infrastructure.readiness import DRAINING
|
||||
|
||||
LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
|
||||
SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5"))
|
||||
SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "6"))
|
||||
DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2"))
|
||||
LOGGER = StructlogLogger(level=LOG_LEVEL, allowed_fields=LOG_FIELDS)
|
||||
|
||||
|
||||
class StructlogGunicornLogger(GunicornBaseLogger):
|
||||
@@ -31,7 +31,7 @@ class DrainingServer(Server):
|
||||
await super().main_loop()
|
||||
if not self.force_exit:
|
||||
DRAINING.set()
|
||||
logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS)
|
||||
LOGGER.info("service.draining", limit=DRAIN_SECONDS)
|
||||
await asyncio.sleep(DRAIN_SECONDS)
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ class MessengerWorker(UvicornWorker):
|
||||
"access_log": False,
|
||||
"server_header": False,
|
||||
"date_header": False,
|
||||
"timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)),
|
||||
"timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS) - 1),
|
||||
}
|
||||
|
||||
async def _serve(self) -> None:
|
||||
|
||||
@@ -4,7 +4,7 @@ from messenger.identity.application.commands import CloseSession, OpenSession, R
|
||||
from messenger.identity.application.ports import (
|
||||
IdentityUnitOfWork,
|
||||
PasswordHasher,
|
||||
SessionDirectory,
|
||||
SessionRevocations,
|
||||
)
|
||||
from messenger.identity.domain.events import SessionClosed, SessionOpened
|
||||
from messenger.identity.domain.models import Session, User
|
||||
@@ -62,15 +62,19 @@ class OpenSessionHandler:
|
||||
|
||||
|
||||
class CloseSessionHandler:
|
||||
__slots__ = ("_sessions", "_uow")
|
||||
__slots__ = ("_revocations", "_uow")
|
||||
|
||||
def __init__(self, uow: IdentityUnitOfWork, sessions: SessionDirectory) -> None:
|
||||
def __init__(self, uow: IdentityUnitOfWork, revocations: SessionRevocations) -> None:
|
||||
self._uow = uow
|
||||
self._sessions = sessions
|
||||
self._revocations = revocations
|
||||
|
||||
async def __call__(self, command: CloseSession) -> None:
|
||||
await self._sessions.invalidate(command.session)
|
||||
async with self._uow.transaction():
|
||||
await self._uow.sessions.remove(command.session.token_hash)
|
||||
self._uow.record(SessionClosed(user_id=command.session.user_id))
|
||||
await self._uow.commit()
|
||||
await self._revocations.revoke(command.session)
|
||||
try:
|
||||
async with self._uow.transaction():
|
||||
await self._uow.sessions.remove(command.session.token_hash)
|
||||
self._uow.record(SessionClosed(user_id=command.session.user_id))
|
||||
await self._uow.commit()
|
||||
except BaseException:
|
||||
await self._revocations.restore(command.session)
|
||||
raise
|
||||
|
||||
@@ -38,7 +38,12 @@ class UserDirectory(Protocol):
|
||||
class SessionDirectory(Protocol):
|
||||
async def find(self, token_hash: bytes, now: datetime) -> Session | None: ...
|
||||
|
||||
async def invalidate(self, session: Session) -> None: ...
|
||||
|
||||
@runtime_checkable
|
||||
class SessionRevocations(Protocol):
|
||||
async def revoke(self, session: Session) -> None: ...
|
||||
|
||||
async def restore(self, session: Session) -> None: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
||||
@@ -17,6 +17,7 @@ from messenger.identity.application.ports import (
|
||||
IdentityUnitOfWork,
|
||||
PasswordHasher,
|
||||
SessionDirectory,
|
||||
SessionRevocations,
|
||||
UserDirectory,
|
||||
)
|
||||
from messenger.identity.application.queries import Authenticate, GetCurrentUser, VerifyCredentials
|
||||
@@ -27,7 +28,7 @@ from messenger.identity.infrastructure.postgres import (
|
||||
PostgresSessionDirectory,
|
||||
PostgresUserDirectory,
|
||||
)
|
||||
from messenger.identity.infrastructure.valkey import CachedSessionDirectory
|
||||
from messenger.identity.infrastructure.valkey import CachedSessionDirectory, NoRevocations
|
||||
from messenger.shared.application.commands import Command
|
||||
from messenger.shared.application.messagebus import CommandHandler
|
||||
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
|
||||
@@ -76,6 +77,10 @@ class IdentityProvider(Provider):
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def revocations(self, sessions: SessionDirectory) -> SessionRevocations:
|
||||
return sessions if isinstance(sessions, CachedSessionDirectory) else NoRevocations()
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def verify(self, users: UserDirectory, hasher: PasswordHasher) -> VerifyCredentials:
|
||||
return VerifyCredentials(users, hasher)
|
||||
@@ -100,13 +105,13 @@ class IdentityProvider(Provider):
|
||||
clock: Clock,
|
||||
ids: IdGenerator,
|
||||
hasher: PasswordHasher,
|
||||
sessions: SessionDirectory,
|
||||
revocations: SessionRevocations,
|
||||
settings: IdentitySettings,
|
||||
) -> IdentityCommandBus:
|
||||
ttl = timedelta(seconds=settings.session_ttl_seconds)
|
||||
handlers: dict[type[Command], CommandHandler] = {
|
||||
RegisterUser: RegisterUserHandler(uow, clock, ids, hasher),
|
||||
OpenSession: OpenSessionHandler(uow, clock, ttl),
|
||||
CloseSession: CloseSessionHandler(uow, sessions),
|
||||
CloseSession: CloseSessionHandler(uow, revocations),
|
||||
}
|
||||
return IdentityCommandBus(handlers, uow, publisher)
|
||||
|
||||
@@ -1,85 +1,126 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from asyncpg import Record
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
LargeBinary,
|
||||
UniqueConstraint,
|
||||
delete,
|
||||
literal,
|
||||
select,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from messenger.identity.domain.errors import UsernameAlreadyTaken
|
||||
from messenger.identity.domain.models import Session, User
|
||||
from messenger.identity.domain.values import DisplayName, Username
|
||||
from messenger.shared.infrastructure.postgres import Connection, Database, PostgresUnitOfWork
|
||||
|
||||
INSERT_USER = """
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (username) DO NOTHING
|
||||
RETURNING true
|
||||
"""
|
||||
USER_BY_ID = """
|
||||
SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1
|
||||
"""
|
||||
USER_BY_USERNAME = """
|
||||
SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1
|
||||
"""
|
||||
INSERT_SESSION = """
|
||||
WITH expired AS (DELETE FROM sessions WHERE user_id = $2 AND expires_at <= $3)
|
||||
INSERT INTO sessions (token_hash, user_id, created_at, expires_at) VALUES ($1, $2, $3, $4)
|
||||
"""
|
||||
DELETE_SESSION = "DELETE FROM sessions WHERE token_hash = $1"
|
||||
ACTIVE_SESSION = """
|
||||
SELECT token_hash, user_id, created_at, expires_at
|
||||
FROM sessions
|
||||
WHERE token_hash = $1 AND expires_at > $2
|
||||
"""
|
||||
from messenger.shared.infrastructure.postgres import (
|
||||
Base,
|
||||
Database,
|
||||
DatabaseSession,
|
||||
PostgresUnitOfWork,
|
||||
)
|
||||
|
||||
|
||||
def user_of(row: Record | None) -> User | None:
|
||||
class UserRow(Base):
|
||||
__tablename__ = "users"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("username", name="users_username_key"),
|
||||
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="users_username_format"),
|
||||
CheckConstraint(
|
||||
"char_length(display_name) BETWEEN 1 AND 100", name="users_display_name_check"
|
||||
),
|
||||
CheckConstraint("password_hash LIKE '$argon2id$%'", name="users_password_hash_check"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
username: Mapped[str]
|
||||
display_name: Mapped[str]
|
||||
password_hash: Mapped[str]
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class SessionRow(Base):
|
||||
__tablename__ = "sessions"
|
||||
__table_args__ = (
|
||||
CheckConstraint("octet_length(token_hash) = 32", name="sessions_token_hash_check"),
|
||||
CheckConstraint("expires_at > created_at", name="sessions_check"),
|
||||
Index("sessions_user_idx", "user_id"),
|
||||
)
|
||||
|
||||
token_hash: Mapped[bytes] = mapped_column(LargeBinary, primary_key=True)
|
||||
user_id: Mapped[UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
def user_of(row: UserRow | None) -> User | None:
|
||||
if row is None:
|
||||
return None
|
||||
return User(
|
||||
id=row["id"],
|
||||
username=Username(row["username"]),
|
||||
display_name=DisplayName(row["display_name"]),
|
||||
created_at=row["created_at"],
|
||||
password_hash=row["password_hash"],
|
||||
id=row.id,
|
||||
username=Username(row.username),
|
||||
display_name=DisplayName(row.display_name),
|
||||
created_at=row.created_at,
|
||||
password_hash=row.password_hash,
|
||||
)
|
||||
|
||||
|
||||
class PostgresUserRepository:
|
||||
__slots__ = ("_connection",)
|
||||
__slots__ = ("_session",)
|
||||
|
||||
def __init__(self, connection: Connection) -> None:
|
||||
self._connection = connection
|
||||
def __init__(self, session: DatabaseSession) -> None:
|
||||
self._session = session
|
||||
|
||||
async def add(self, user: User) -> None:
|
||||
inserted = await self._connection.fetchval(
|
||||
INSERT_USER,
|
||||
user.id,
|
||||
user.username.value,
|
||||
user.display_name.value,
|
||||
user.password_hash,
|
||||
user.created_at,
|
||||
statement = (
|
||||
insert(UserRow)
|
||||
.values(
|
||||
id=user.id,
|
||||
username=user.username.value,
|
||||
display_name=user.display_name.value,
|
||||
password_hash=user.password_hash,
|
||||
created_at=user.created_at,
|
||||
)
|
||||
.on_conflict_do_nothing(index_elements=["username"])
|
||||
.returning(literal(True))
|
||||
)
|
||||
inserted = (await self._session.execute(statement)).scalar_one_or_none()
|
||||
if inserted is None:
|
||||
raise UsernameAlreadyTaken("username уже занят")
|
||||
|
||||
|
||||
class PostgresSessionRepository:
|
||||
__slots__ = ("_connection",)
|
||||
__slots__ = ("_session",)
|
||||
|
||||
def __init__(self, connection: Connection) -> None:
|
||||
self._connection = connection
|
||||
def __init__(self, session: DatabaseSession) -> None:
|
||||
self._session = session
|
||||
|
||||
async def add(self, session: Session) -> None:
|
||||
await self._connection.execute(
|
||||
INSERT_SESSION,
|
||||
session.token_hash,
|
||||
session.user_id,
|
||||
session.created_at,
|
||||
session.expires_at,
|
||||
expired = (
|
||||
delete(SessionRow)
|
||||
.where(
|
||||
SessionRow.user_id == session.user_id, SessionRow.expires_at <= session.created_at
|
||||
)
|
||||
.cte("expired")
|
||||
)
|
||||
await self._session.execute(
|
||||
insert(SessionRow)
|
||||
.values(
|
||||
token_hash=session.token_hash,
|
||||
user_id=session.user_id,
|
||||
created_at=session.created_at,
|
||||
expires_at=session.expires_at,
|
||||
)
|
||||
.add_cte(expired)
|
||||
)
|
||||
|
||||
async def remove(self, token_hash: bytes) -> None:
|
||||
await self._connection.execute(DELETE_SESSION, token_hash)
|
||||
await self._session.execute(delete(SessionRow).where(SessionRow.token_hash == token_hash))
|
||||
|
||||
|
||||
class PostgresIdentityUnitOfWork(PostgresUnitOfWork):
|
||||
@@ -87,11 +128,11 @@ class PostgresIdentityUnitOfWork(PostgresUnitOfWork):
|
||||
|
||||
@property
|
||||
def users(self) -> PostgresUserRepository:
|
||||
return PostgresUserRepository(self.connection)
|
||||
return PostgresUserRepository(self.session)
|
||||
|
||||
@property
|
||||
def sessions(self) -> PostgresSessionRepository:
|
||||
return PostgresSessionRepository(self.connection)
|
||||
return PostgresSessionRepository(self.session)
|
||||
|
||||
|
||||
class PostgresUserDirectory:
|
||||
@@ -101,12 +142,18 @@ class PostgresUserDirectory:
|
||||
self._database = database
|
||||
|
||||
async def find(self, user_id: UUID) -> User | None:
|
||||
async with self._database.connection() as connection:
|
||||
return user_of(await connection.fetchrow(USER_BY_ID, user_id))
|
||||
async with self._database.session() as session:
|
||||
row = (
|
||||
await session.execute(select(UserRow).where(UserRow.id == user_id))
|
||||
).scalar_one_or_none()
|
||||
return user_of(row)
|
||||
|
||||
async def find_by_username(self, username: str) -> User | None:
|
||||
async with self._database.connection() as connection:
|
||||
return user_of(await connection.fetchrow(USER_BY_USERNAME, username))
|
||||
async with self._database.session() as session:
|
||||
row = (
|
||||
await session.execute(select(UserRow).where(UserRow.username == username))
|
||||
).scalar_one_or_none()
|
||||
return user_of(row)
|
||||
|
||||
|
||||
class PostgresSessionDirectory:
|
||||
@@ -116,9 +163,20 @@ class PostgresSessionDirectory:
|
||||
self._database = database
|
||||
|
||||
async def find(self, token_hash: bytes, now: datetime) -> Session | None:
|
||||
async with self._database.connection() as connection:
|
||||
row = await connection.fetchrow(ACTIVE_SESSION, token_hash, now)
|
||||
return None if row is None else Session(**dict(row))
|
||||
|
||||
async def invalidate(self, session: Session) -> None:
|
||||
return None
|
||||
async with self._database.session() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(SessionRow).where(
|
||||
SessionRow.token_hash == token_hash,
|
||||
SessionRow.expires_at > now,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
return Session(
|
||||
token_hash=row.token_hash,
|
||||
user_id=row.user_id,
|
||||
created_at=row.created_at,
|
||||
expires_at=row.expires_at,
|
||||
)
|
||||
|
||||
@@ -37,6 +37,14 @@ def decode(token_hash: bytes, value: str) -> Session:
|
||||
)
|
||||
|
||||
|
||||
class NoRevocations:
|
||||
async def revoke(self, session: Session) -> None:
|
||||
return None
|
||||
|
||||
async def restore(self, session: Session) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class CachedSessionDirectory:
|
||||
__slots__ = ("_client", "_cooldown", "_limit", "_logger", "_origin", "_resume_at")
|
||||
|
||||
@@ -70,7 +78,7 @@ class CachedSessionDirectory:
|
||||
await self._quietly(self._client.set(key, encode(session), pxat=until, nx=True))
|
||||
return session
|
||||
|
||||
async def invalidate(self, session: Session) -> None:
|
||||
async def revoke(self, session: Session) -> None:
|
||||
if not self._closed():
|
||||
raise Unavailable("кэш сессий недоступен")
|
||||
try:
|
||||
@@ -81,6 +89,9 @@ class CachedSessionDirectory:
|
||||
self._trip(error)
|
||||
raise Unavailable("кэш сессий недоступен") from error
|
||||
|
||||
async def restore(self, session: Session) -> None:
|
||||
await self._quietly(self._client.delete(key_of(session.token_hash)))
|
||||
|
||||
async def _read(self, key: str) -> str | None:
|
||||
if not self._closed():
|
||||
return None
|
||||
|
||||
@@ -40,7 +40,7 @@ from messenger.shared.entrypoints.ioc import PlatformProvider
|
||||
from messenger.shared.entrypoints.pipeline import RequestPipeline
|
||||
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
|
||||
|
||||
TRANSPORT_FIELDS = frozenset({"dropped_total", "queue", "instance"})
|
||||
TRANSPORT_FIELDS = frozenset({"dropped_total"})
|
||||
|
||||
LOG_FIELDS = (
|
||||
PLATFORM_FIELDS | TRANSPORT_FIELDS | IDENTITY_FIELDS | MESSAGING_FIELDS | ATTACHMENT_FIELDS
|
||||
@@ -148,6 +148,7 @@ def create_app(
|
||||
guard=guard,
|
||||
instance=platform.instance_id,
|
||||
limits=((UPLOAD_PATH, attachments.max_upload_bytes),),
|
||||
min_rate=transport.min_transfer_rate_bytes,
|
||||
)
|
||||
errors.install(app, logger, ERROR_CODES, challenge_for, platform.instance_id)
|
||||
app.include_router(health.router)
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from messenger.messaging.application.commands import CreateChat, PostMessage
|
||||
from messenger.messaging.application.ports import AttachmentStates, MessagingUnitOfWork
|
||||
from messenger.messaging.domain.errors import AttachmentUnavailable, ChatNotFound
|
||||
from messenger.messaging.domain.models import Chat, Message, Receipt
|
||||
from messenger.messaging.application.ports import ChatDirectory, MessageLog, MessagingUnitOfWork
|
||||
from messenger.messaging.domain.errors import (
|
||||
ChatNotFound,
|
||||
InvalidAttachmentList,
|
||||
InvalidMessageText,
|
||||
)
|
||||
from messenger.messaging.domain.models import Chat, Message, attachment_list
|
||||
from messenger.messaging.domain.values import MessageText
|
||||
from messenger.shared.application.ports import Clock, IdGenerator
|
||||
|
||||
|
||||
@@ -30,44 +35,41 @@ class CreateChatHandler:
|
||||
|
||||
|
||||
class PostMessageHandler:
|
||||
__slots__ = ("_attachments", "_clock", "_ids", "_ttl", "_uow")
|
||||
__slots__ = ("_chats", "_clock", "_ids", "_log", "_ttl", "_uow")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
uow: MessagingUnitOfWork,
|
||||
attachments: AttachmentStates,
|
||||
log: MessageLog,
|
||||
chats: ChatDirectory,
|
||||
clock: Clock,
|
||||
ids: IdGenerator,
|
||||
*,
|
||||
ttl: timedelta,
|
||||
) -> None:
|
||||
self._uow = uow
|
||||
self._attachments = attachments
|
||||
self._log = log
|
||||
self._chats = chats
|
||||
self._clock = clock
|
||||
self._ids = ids
|
||||
self._ttl = ttl
|
||||
|
||||
async def __call__(self, command: PostMessage) -> Message:
|
||||
async with self._uow.transaction():
|
||||
chat = await self._uow.chats.get(command.chat_id)
|
||||
if chat is None:
|
||||
raise ChatNotFound("чат не найден")
|
||||
now = self._clock.now()
|
||||
message = chat.post(
|
||||
message_id=self._ids.new_id(),
|
||||
sender_id=command.sender_id,
|
||||
text=command.text,
|
||||
now=now,
|
||||
attachment_ids=command.attachment_ids,
|
||||
)
|
||||
known = await self._uow.chats.receipt(
|
||||
chat.id, command.sender_id, command.key, now - self._ttl
|
||||
)
|
||||
if known is not None:
|
||||
return known.replay(command.fingerprint)
|
||||
if message.attachment_ids and await self._attachments.unusable(
|
||||
chat.id, message.attachment_ids
|
||||
):
|
||||
raise AttachmentUnavailable("вложение не обработано и не может быть отправлено")
|
||||
await self._uow.chats.append(Receipt(command.key, command.fingerprint, message))
|
||||
await self._uow.commit()
|
||||
try:
|
||||
text = MessageText(command.text)
|
||||
attachments = attachment_list(text, command.attachment_ids)
|
||||
except InvalidMessageText, InvalidAttachmentList:
|
||||
if await self._chats.chat_for_member(command.chat_id, command.sender_id) is None:
|
||||
raise ChatNotFound("чат не найден") from None
|
||||
raise
|
||||
now = self._clock.now()
|
||||
draft = Message(
|
||||
self._ids.new_id(), command.chat_id, command.sender_id, text, now, attachments
|
||||
)
|
||||
receipt = await self._log.post(draft, command.key, command.fingerprint, now - self._ttl)
|
||||
if receipt is None:
|
||||
raise ChatNotFound("чат не найден")
|
||||
message = receipt.replay(command.fingerprint)
|
||||
if message.id == draft.id:
|
||||
self._uow.record(message.posted())
|
||||
return message
|
||||
|
||||
@@ -19,14 +19,13 @@ class CursorCodec(Protocol):
|
||||
class ChatRepository(Protocol):
|
||||
async def add(self, chat: Chat) -> None: ...
|
||||
|
||||
async def get(self, chat_id: UUID) -> Chat | None: ...
|
||||
|
||||
async def receipt(
|
||||
self, chat_id: UUID, sender_id: UUID, key: str, since: datetime
|
||||
@runtime_checkable
|
||||
class MessageLog(Protocol):
|
||||
async def post(
|
||||
self, message: Message, key: str, fingerprint: bytes, since: datetime
|
||||
) -> Receipt | None: ...
|
||||
|
||||
async def append(self, receipt: Receipt) -> None: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class MessagingUnitOfWork(UnitOfWork, Protocol):
|
||||
|
||||
@@ -43,7 +43,7 @@ class ListChats:
|
||||
|
||||
async def __call__(self, user_id: UUID, *, limit: int, cursor: str | None) -> Page[Chat]:
|
||||
scope = chats_scope(user_id)
|
||||
after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None
|
||||
after: HistoryKey | None = None if cursor is None else self._cursors.decode(scope, cursor)
|
||||
items, has_more = await self._chats.chats_of(user_id, after=after, limit=limit)
|
||||
next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None
|
||||
return Page(items=items, next_cursor=next_cursor)
|
||||
@@ -67,7 +67,7 @@ class ReadHistory:
|
||||
if await self._chats.chat_for_member(chat_id, user_id) is None:
|
||||
raise ChatNotFound("чат не найден")
|
||||
scope = messages_scope(chat_id)
|
||||
after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None
|
||||
after: HistoryKey | None = None if cursor is None else self._cursors.decode(scope, cursor)
|
||||
items, has_more = await self._chats.history(chat_id, after=after, limit=limit)
|
||||
next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None
|
||||
return Page(items=items, next_cursor=next_cursor)
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.messaging.domain.errors import (
|
||||
ChatNotFound,
|
||||
IdempotencyConflict,
|
||||
InvalidAttachmentList,
|
||||
InvalidChatData,
|
||||
@@ -20,8 +19,6 @@ from messenger.messaging.domain.values import (
|
||||
)
|
||||
from messenger.shared.domain.events import DomainEvent
|
||||
|
||||
TICK = timedelta(microseconds=1)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Message:
|
||||
@@ -36,6 +33,15 @@ class Message:
|
||||
def key(self) -> HistoryKey:
|
||||
return (self.created_at, self.id.int)
|
||||
|
||||
def posted(self) -> MessagePosted:
|
||||
return MessagePosted(
|
||||
message_id=self.id,
|
||||
chat_id=self.chat_id,
|
||||
sender_id=self.sender_id,
|
||||
text_length=len(self.text.value),
|
||||
created_at=self.created_at,
|
||||
)
|
||||
|
||||
|
||||
def attachment_list(text: MessageText, attachment_ids: Sequence[UUID]) -> tuple[UUID, ...]:
|
||||
if len(attachment_ids) > MESSAGE_ATTACHMENTS_MAX:
|
||||
@@ -62,15 +68,7 @@ class Receipt:
|
||||
|
||||
|
||||
class Chat:
|
||||
__slots__ = (
|
||||
"_events",
|
||||
"_members",
|
||||
"created_at",
|
||||
"id",
|
||||
"last_message_at",
|
||||
"member_ids",
|
||||
"title",
|
||||
)
|
||||
__slots__ = ("_events", "created_at", "id", "member_ids", "title")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -79,14 +77,11 @@ class Chat:
|
||||
title: ChatTitle,
|
||||
member_ids: Sequence[UUID],
|
||||
created_at: datetime,
|
||||
last_message_at: datetime | None = None,
|
||||
) -> None:
|
||||
self.id = chat_id
|
||||
self.title = title
|
||||
self.member_ids: tuple[UUID, ...] = tuple(member_ids)
|
||||
self.created_at = created_at
|
||||
self.last_message_at = last_message_at
|
||||
self._members: frozenset[UUID] = frozenset(member_ids)
|
||||
self._events: list[DomainEvent] = []
|
||||
|
||||
@classmethod
|
||||
@@ -131,47 +126,6 @@ class Chat:
|
||||
def record(self, event: DomainEvent) -> None:
|
||||
self._events.append(event)
|
||||
|
||||
def is_member(self, user_id: UUID) -> bool:
|
||||
return user_id in self._members
|
||||
|
||||
def require_member(self, user_id: UUID) -> None:
|
||||
if not self.is_member(user_id):
|
||||
raise ChatNotFound("чат не найден")
|
||||
|
||||
def post(
|
||||
self,
|
||||
*,
|
||||
message_id: UUID,
|
||||
sender_id: UUID,
|
||||
text: str,
|
||||
now: datetime,
|
||||
attachment_ids: Sequence[UUID] = (),
|
||||
) -> Message:
|
||||
self.require_member(sender_id)
|
||||
body = MessageText(text)
|
||||
attachments = attachment_list(body, attachment_ids)
|
||||
last = self.last_message_at
|
||||
created_at = now if last is None or now > last else last + TICK
|
||||
message = Message(
|
||||
id=message_id,
|
||||
chat_id=self.id,
|
||||
sender_id=sender_id,
|
||||
text=body,
|
||||
created_at=created_at,
|
||||
attachment_ids=attachments,
|
||||
)
|
||||
self.last_message_at = created_at
|
||||
self.record(
|
||||
MessagePosted(
|
||||
message_id=message.id,
|
||||
chat_id=self.id,
|
||||
sender_id=sender_id,
|
||||
text_length=len(message.text.value),
|
||||
created_at=created_at,
|
||||
)
|
||||
)
|
||||
return message
|
||||
|
||||
def collect_events(self) -> list[DomainEvent]:
|
||||
events, self._events = self._events, []
|
||||
return events
|
||||
|
||||
@@ -9,6 +9,7 @@ from messenger.messaging.application.ports import (
|
||||
AttachmentStates,
|
||||
ChatDirectory,
|
||||
CursorCodec,
|
||||
MessageLog,
|
||||
MessagingUnitOfWork,
|
||||
)
|
||||
from messenger.messaging.application.queries import GetChat, ListChats, ReadHistory
|
||||
@@ -16,6 +17,7 @@ from messenger.messaging.config import MessagingSettings
|
||||
from messenger.messaging.infrastructure.cursor import SignedCursorCodec
|
||||
from messenger.messaging.infrastructure.postgres import (
|
||||
PostgresChatDirectory,
|
||||
PostgresMessageLog,
|
||||
PostgresMessagingUnitOfWork,
|
||||
)
|
||||
from messenger.shared.application.commands import Command
|
||||
@@ -37,6 +39,10 @@ class MessagingProvider(Provider):
|
||||
def directory(self, database: Database) -> ChatDirectory:
|
||||
return PostgresChatDirectory(database)
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def log(self, database: Database, attachments: AttachmentStates) -> MessageLog:
|
||||
return PostgresMessageLog(database, attachments)
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def cursors(self, settings: MessagingSettings) -> CursorCodec:
|
||||
return SignedCursorCodec(settings.cursor_secret.get_secret_value().encode())
|
||||
@@ -61,7 +67,8 @@ class MessagingProvider(Provider):
|
||||
def command_bus(
|
||||
self,
|
||||
uow: MessagingUnitOfWork,
|
||||
attachments: AttachmentStates,
|
||||
log: MessageLog,
|
||||
directory: ChatDirectory,
|
||||
publisher: EventPublisher,
|
||||
clock: Clock,
|
||||
ids: IdGenerator,
|
||||
@@ -70,6 +77,6 @@ class MessagingProvider(Provider):
|
||||
ttl = timedelta(seconds=settings.idempotency_ttl_seconds)
|
||||
handlers: dict[type[Command], CommandHandler] = {
|
||||
CreateChat: CreateChatHandler(uow, clock, ids),
|
||||
PostMessage: PostMessageHandler(uow, attachments, clock, ids, ttl),
|
||||
PostMessage: PostMessageHandler(uow, log, directory, clock, ids, ttl=ttl),
|
||||
}
|
||||
return MessagingCommandBus(handlers, uow, publisher)
|
||||
|
||||
@@ -37,7 +37,7 @@ class CreateMessageRequest(BaseModel):
|
||||
|
||||
class PageQuery(BaseModel):
|
||||
limit: int = Field(default=50, ge=1, le=100)
|
||||
cursor: str | None = None
|
||||
cursor: str | None = Field(default=None, min_length=1)
|
||||
|
||||
|
||||
class ChatOut(BaseModel):
|
||||
|
||||
@@ -1,122 +1,266 @@
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime
|
||||
from dataclasses import replace
|
||||
from datetime import datetime, timedelta
|
||||
from typing import cast
|
||||
from uuid import UUID
|
||||
|
||||
from asyncpg import ForeignKeyViolationError, Record
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
ForeignKeyConstraint,
|
||||
Index,
|
||||
LargeBinary,
|
||||
RowMapping,
|
||||
Select,
|
||||
SmallInteger,
|
||||
Table,
|
||||
UniqueConstraint,
|
||||
bindparam,
|
||||
column,
|
||||
func,
|
||||
insert,
|
||||
lambda_stmt,
|
||||
literal,
|
||||
select,
|
||||
true,
|
||||
tuple_,
|
||||
update,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import ARRAY
|
||||
from sqlalchemy.dialects.postgresql import UUID as PGUUID
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.sql.elements import Label
|
||||
|
||||
from messenger.messaging.domain.errors import InvalidChatData, UnknownAttachment
|
||||
from messenger.messaging.application.ports import AttachmentStates
|
||||
from messenger.messaging.domain.errors import (
|
||||
AttachmentUnavailable,
|
||||
InvalidChatData,
|
||||
UnknownAttachment,
|
||||
)
|
||||
from messenger.messaging.domain.keys import HistoryKey
|
||||
from messenger.messaging.domain.models import Chat, Message, Receipt
|
||||
from messenger.messaging.domain.values import ChatTitle, MessageText
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.infrastructure.postgres import (
|
||||
Connection,
|
||||
Base,
|
||||
Database,
|
||||
DatabaseSession,
|
||||
EventSource,
|
||||
PostgresUnitOfWork,
|
||||
)
|
||||
|
||||
INSERT_CHAT = "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)"
|
||||
INSERT_MEMBERS = """
|
||||
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
|
||||
SELECT $1, member.user_id, member.position, $3
|
||||
FROM unnest($2::uuid[]) WITH ORDINALITY AS member (user_id, position)
|
||||
"""
|
||||
LOCK_CHAT = """
|
||||
SELECT c.id, c.title, c.created_at, c.last_message_at,
|
||||
ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids
|
||||
FROM chats c
|
||||
WHERE c.id = $1
|
||||
FOR UPDATE
|
||||
"""
|
||||
ATTACHMENT_FKEY = "message_attachments_attachment_fkey"
|
||||
RECEIPT = """
|
||||
SELECT k.fingerprint, m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.chat_id = $1 AND k.user_id = $2 AND k.key = $3 AND k.created_at > $4
|
||||
"""
|
||||
APPEND_MESSAGE = """
|
||||
WITH message AS (
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING id, chat_id, sender_id, created_at
|
||||
), links AS (
|
||||
INSERT INTO message_attachments (message_id, position, chat_id, attachment_id)
|
||||
SELECT message.id, link.position - 1, message.chat_id, link.attachment_id
|
||||
FROM message, unnest($8::uuid[]) WITH ORDINALITY AS link (attachment_id, position)
|
||||
), receipt AS (
|
||||
INSERT INTO idempotency_keys (chat_id, user_id, key, fingerprint, message_id, created_at)
|
||||
SELECT chat_id, sender_id, $6, $7, id, created_at FROM message
|
||||
ON CONFLICT (chat_id, user_id, key) DO UPDATE
|
||||
SET fingerprint = EXCLUDED.fingerprint,
|
||||
message_id = EXCLUDED.message_id,
|
||||
created_at = EXCLUDED.created_at
|
||||
)
|
||||
UPDATE chats SET last_message_at = message.created_at
|
||||
FROM message
|
||||
WHERE chats.id = message.chat_id
|
||||
"""
|
||||
CHAT_OF_MEMBER = """
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids
|
||||
FROM chat_members m JOIN chats c ON c.id = m.chat_id
|
||||
WHERE m.chat_id = $1 AND m.user_id = $2
|
||||
"""
|
||||
CHATS_FIRST = """
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids
|
||||
FROM chat_members m JOIN chats c ON c.id = m.chat_id
|
||||
WHERE m.user_id = $1
|
||||
ORDER BY m.chat_created_at, m.chat_id
|
||||
LIMIT $2
|
||||
"""
|
||||
CHATS_AFTER = """
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids
|
||||
FROM chat_members m JOIN chats c ON c.id = m.chat_id
|
||||
WHERE m.user_id = $1 AND (m.chat_created_at, m.chat_id) > ($2, $3)
|
||||
ORDER BY m.chat_created_at, m.chat_id
|
||||
LIMIT $4
|
||||
"""
|
||||
HISTORY_FIRST = """
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT $2
|
||||
"""
|
||||
HISTORY_AFTER = """
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3)
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT $4
|
||||
"""
|
||||
FOREIGN_KEY_VIOLATED = "23503"
|
||||
|
||||
|
||||
def chat_of(row: Record) -> Chat:
|
||||
class ChatRow(Base):
|
||||
__tablename__ = "chats"
|
||||
__table_args__ = (
|
||||
CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="chats_title_check"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
title: Mapped[str]
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
last_message_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class ChatMemberRow(Base):
|
||||
__tablename__ = "chat_members"
|
||||
__table_args__ = (
|
||||
CheckConstraint("position BETWEEN 1 AND 100", name="chat_members_position_check"),
|
||||
Index("chat_members_user_idx", "user_id", "chat_created_at", "chat_id"),
|
||||
)
|
||||
|
||||
chat_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
user_id: Mapped[UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
|
||||
position: Mapped[int] = mapped_column(SmallInteger)
|
||||
chat_created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class MessageRow(Base):
|
||||
__tablename__ = "messages"
|
||||
__table_args__ = (
|
||||
CheckConstraint("char_length(text) <= 4000", name="messages_text_check"),
|
||||
ForeignKeyConstraint(
|
||||
["chat_id", "sender_id"], ["chat_members.chat_id", "chat_members.user_id"]
|
||||
),
|
||||
Index("messages_history_idx", "chat_id", "created_at", "id"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
chat_id: Mapped[UUID]
|
||||
sender_id: Mapped[UUID]
|
||||
text: Mapped[str]
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class IdempotencyKeyRow(Base):
|
||||
__tablename__ = "idempotency_keys"
|
||||
__table_args__ = (
|
||||
CheckConstraint("key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="idempotency_keys_key_check"),
|
||||
CheckConstraint(
|
||||
"octet_length(fingerprint) = 32", name="idempotency_keys_fingerprint_check"
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["chat_id", "user_id"], ["chat_members.chat_id", "chat_members.user_id"]
|
||||
),
|
||||
)
|
||||
|
||||
chat_id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
key: Mapped[str] = mapped_column(primary_key=True)
|
||||
fingerprint: Mapped[bytes] = mapped_column(LargeBinary)
|
||||
message_id: Mapped[UUID] = mapped_column(ForeignKey("messages.id", ondelete="CASCADE"))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class MessageAttachmentRow(Base):
|
||||
__tablename__ = "message_attachments"
|
||||
__table_args__ = (
|
||||
CheckConstraint("position BETWEEN 0 AND 9", name="message_attachments_position_check"),
|
||||
UniqueConstraint("message_id", "attachment_id"),
|
||||
ForeignKeyConstraint(
|
||||
["attachment_id", "chat_id"],
|
||||
["attachments.id", "attachments.chat_id"],
|
||||
name="message_attachments_attachment_fkey",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index("message_attachments_attachment_idx", "attachment_id"),
|
||||
)
|
||||
|
||||
message_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
|
||||
chat_id: Mapped[UUID]
|
||||
attachment_id: Mapped[UUID]
|
||||
|
||||
|
||||
def post_message() -> Select:
|
||||
chats, members = cast(Table, ChatRow.__table__), cast(Table, ChatMemberRow.__table__)
|
||||
keys, messages = cast(Table, IdempotencyKeyRow.__table__), cast(Table, MessageRow.__table__)
|
||||
sender = bindparam("sender_id", type_=PGUUID)
|
||||
chat_id = bindparam("chat_id", type_=PGUUID)
|
||||
message_id = bindparam("message_id", type_=PGUUID)
|
||||
chat = (
|
||||
select(
|
||||
chats.c.id,
|
||||
func.greatest(
|
||||
bindparam("stamp", type_=DateTime(timezone=True)),
|
||||
chats.c.last_message_at + timedelta(microseconds=1),
|
||||
).label("stamp"),
|
||||
)
|
||||
.join(members, (members.c.chat_id == chats.c.id) & (members.c.user_id == sender))
|
||||
.where(chats.c.id == chat_id)
|
||||
.with_for_update(of=chats)
|
||||
.cte("chat")
|
||||
)
|
||||
receipt_insert = pg_insert(keys).from_select(
|
||||
["chat_id", "user_id", "key", "fingerprint", "message_id", "created_at"],
|
||||
select(
|
||||
chat.c.id,
|
||||
sender,
|
||||
bindparam("key"),
|
||||
bindparam("fingerprint", type_=LargeBinary),
|
||||
message_id,
|
||||
chat.c.stamp,
|
||||
),
|
||||
)
|
||||
receipt = (
|
||||
receipt_insert.on_conflict_do_update(
|
||||
index_elements=[keys.c.chat_id, keys.c.user_id, keys.c.key],
|
||||
set_={
|
||||
name: receipt_insert.excluded[name]
|
||||
for name in ("fingerprint", "message_id", "created_at")
|
||||
},
|
||||
where=keys.c.created_at <= bindparam("since", type_=DateTime(timezone=True)),
|
||||
)
|
||||
.returning(keys.c.created_at)
|
||||
.cte("receipt")
|
||||
)
|
||||
message = (
|
||||
insert(messages)
|
||||
.from_select(
|
||||
["id", "chat_id", "sender_id", "text", "created_at"],
|
||||
select(message_id, chat_id, sender, bindparam("text"), receipt.c.created_at),
|
||||
)
|
||||
.returning(messages.c.chat_id, messages.c.created_at)
|
||||
.cte("message")
|
||||
)
|
||||
attachments = (
|
||||
func.unnest(bindparam("attachment_ids", type_=ARRAY(PGUUID)))
|
||||
.table_valued(column("attachment_id", PGUUID), with_ordinality="position")
|
||||
.render_derived(name="attachment")
|
||||
)
|
||||
links = (
|
||||
insert(cast(Table, MessageAttachmentRow.__table__))
|
||||
.from_select(
|
||||
["message_id", "position", "chat_id", "attachment_id"],
|
||||
select(
|
||||
message_id, attachments.c.position - 1, chat_id, attachments.c.attachment_id
|
||||
).select_from(message.join(attachments, true())),
|
||||
)
|
||||
.cte("links")
|
||||
)
|
||||
touched = (
|
||||
update(chats)
|
||||
.where(chats.c.id == message.c.chat_id)
|
||||
.values(last_message_at=message.c.created_at)
|
||||
.cte("touched")
|
||||
)
|
||||
return select(
|
||||
select(chat.c.id).exists().label("member"),
|
||||
select(message.c.created_at).scalar_subquery().label("created_at"),
|
||||
).add_cte(links, touched)
|
||||
|
||||
|
||||
POST_MESSAGE = lambda_stmt(post_message, enable_tracking=False)
|
||||
|
||||
|
||||
def member_ids() -> Label[Sequence[UUID]]:
|
||||
return func.array(
|
||||
select(ChatMemberRow.user_id)
|
||||
.where(ChatMemberRow.chat_id == ChatRow.id)
|
||||
.order_by(ChatMemberRow.position)
|
||||
.correlate(ChatRow)
|
||||
.scalar_subquery(),
|
||||
type_=ARRAY(PGUUID(as_uuid=True)),
|
||||
).label("member_ids")
|
||||
|
||||
|
||||
def attachment_ids() -> Label[Sequence[UUID]]:
|
||||
return func.array(
|
||||
select(MessageAttachmentRow.attachment_id)
|
||||
.where(MessageAttachmentRow.message_id == MessageRow.id)
|
||||
.order_by(MessageAttachmentRow.position)
|
||||
.correlate(MessageRow)
|
||||
.scalar_subquery(),
|
||||
type_=ARRAY(PGUUID(as_uuid=True)),
|
||||
).label("attachment_ids")
|
||||
|
||||
|
||||
def chat_of(row: RowMapping) -> Chat:
|
||||
return Chat(
|
||||
chat_id=row["id"],
|
||||
title=ChatTitle(row["title"]),
|
||||
member_ids=row["member_ids"],
|
||||
created_at=row["created_at"],
|
||||
last_message_at=row.get("last_message_at"),
|
||||
)
|
||||
|
||||
|
||||
def message_of(row: Record) -> Message:
|
||||
def message_of(row: RowMapping) -> Message:
|
||||
return Message(
|
||||
id=row["id"],
|
||||
chat_id=row["chat_id"],
|
||||
sender_id=row["sender_id"],
|
||||
text=MessageText(row["text"]),
|
||||
created_at=row["created_at"],
|
||||
attachment_ids=tuple(row["attachment_ids"]),
|
||||
created_at=row["created_at"],
|
||||
)
|
||||
|
||||
|
||||
@@ -125,54 +269,110 @@ def page[ItemT](rows: Sequence[ItemT], limit: int) -> tuple[list[ItemT], bool]:
|
||||
|
||||
|
||||
class PostgresChatRepository:
|
||||
__slots__ = ("_connection", "_seen")
|
||||
__slots__ = ("_seen", "_session")
|
||||
|
||||
def __init__(self, connection: Connection, seen: list[EventSource]) -> None:
|
||||
self._connection = connection
|
||||
def __init__(self, session: DatabaseSession, seen: list[EventSource]) -> None:
|
||||
self._session = session
|
||||
self._seen = seen
|
||||
|
||||
async def add(self, chat: Chat) -> None:
|
||||
await self._connection.execute(INSERT_CHAT, chat.id, chat.title.value, chat.created_at)
|
||||
await self._session.execute(
|
||||
insert(ChatRow).values(id=chat.id, title=chat.title.value, created_at=chat.created_at)
|
||||
)
|
||||
try:
|
||||
await self._connection.execute(
|
||||
INSERT_MEMBERS, chat.id, list(chat.member_ids), chat.created_at
|
||||
await self._session.execute(
|
||||
insert(ChatMemberRow),
|
||||
[
|
||||
{
|
||||
"chat_id": chat.id,
|
||||
"user_id": member_id,
|
||||
"position": position,
|
||||
"chat_created_at": chat.created_at,
|
||||
}
|
||||
for position, member_id in enumerate(chat.member_ids, 1)
|
||||
],
|
||||
)
|
||||
except ForeignKeyViolationError as error:
|
||||
except IntegrityError as error:
|
||||
if getattr(error.orig, "sqlstate", None) != FOREIGN_KEY_VIOLATED:
|
||||
raise
|
||||
raise InvalidChatData("поле member_ids содержит неизвестного пользователя") from error
|
||||
self._seen.append(chat)
|
||||
|
||||
async def get(self, chat_id: UUID) -> Chat | None:
|
||||
row = await self._connection.fetchrow(LOCK_CHAT, chat_id)
|
||||
if row is None:
|
||||
return None
|
||||
chat = chat_of(row)
|
||||
self._seen.append(chat)
|
||||
return chat
|
||||
|
||||
async def receipt(
|
||||
self, chat_id: UUID, sender_id: UUID, key: str, since: datetime
|
||||
class PostgresMessageLog:
|
||||
__slots__ = ("_attachments", "_database")
|
||||
|
||||
def __init__(self, database: Database, attachments: AttachmentStates) -> None:
|
||||
self._database = database
|
||||
self._attachments = attachments
|
||||
|
||||
async def post(
|
||||
self, message: Message, key: str, fingerprint: bytes, since: datetime
|
||||
) -> Receipt | None:
|
||||
row = await self._connection.fetchrow(RECEIPT, chat_id, sender_id, key, since)
|
||||
return None if row is None else Receipt(key, row["fingerprint"], message_of(row))
|
||||
|
||||
async def append(self, receipt: Receipt) -> None:
|
||||
message = receipt.message
|
||||
try:
|
||||
await self._connection.execute(
|
||||
APPEND_MESSAGE,
|
||||
message.id,
|
||||
message.chat_id,
|
||||
message.sender_id,
|
||||
message.text.value,
|
||||
message.created_at,
|
||||
receipt.key,
|
||||
receipt.fingerprint,
|
||||
list(message.attachment_ids),
|
||||
chat_id, sender_id = message.chat_id, message.sender_id
|
||||
async with self._database.session() as session:
|
||||
try:
|
||||
row = (
|
||||
(
|
||||
await session.execute(
|
||||
POST_MESSAGE,
|
||||
{
|
||||
"message_id": message.id,
|
||||
"chat_id": chat_id,
|
||||
"sender_id": sender_id,
|
||||
"text": message.text.value,
|
||||
"stamp": message.created_at,
|
||||
"key": key,
|
||||
"fingerprint": fingerprint,
|
||||
"since": since,
|
||||
"attachment_ids": list(message.attachment_ids),
|
||||
},
|
||||
)
|
||||
)
|
||||
.mappings()
|
||||
.first()
|
||||
)
|
||||
except IntegrityError as error:
|
||||
cause = getattr(error.orig, "__cause__", None)
|
||||
if getattr(cause, "constraint_name", None) != "message_attachments_attachment_fkey":
|
||||
raise
|
||||
raise UnknownAttachment("вложение не найдено в этом чате") from error
|
||||
if row is None or not row["member"]:
|
||||
return None
|
||||
if row["created_at"] is not None:
|
||||
if message.attachment_ids and await self._attachments.unusable(
|
||||
chat_id, message.attachment_ids
|
||||
):
|
||||
raise AttachmentUnavailable("вложение не обработано и не может быть отправлено")
|
||||
await session.commit()
|
||||
return Receipt(key, fingerprint, replace(message, created_at=row["created_at"]))
|
||||
known = (
|
||||
(
|
||||
await session.execute(
|
||||
select(
|
||||
IdempotencyKeyRow.fingerprint,
|
||||
MessageRow.id,
|
||||
MessageRow.chat_id,
|
||||
MessageRow.sender_id,
|
||||
MessageRow.text,
|
||||
MessageRow.created_at,
|
||||
attachment_ids(),
|
||||
)
|
||||
.join(MessageRow, MessageRow.id == IdempotencyKeyRow.message_id)
|
||||
.where(
|
||||
IdempotencyKeyRow.chat_id == chat_id,
|
||||
IdempotencyKeyRow.user_id == sender_id,
|
||||
IdempotencyKeyRow.key == key,
|
||||
IdempotencyKeyRow.created_at > since,
|
||||
)
|
||||
)
|
||||
)
|
||||
.mappings()
|
||||
.first()
|
||||
)
|
||||
except ForeignKeyViolationError as error:
|
||||
if getattr(error, "constraint_name", None) != ATTACHMENT_FKEY:
|
||||
raise
|
||||
raise UnknownAttachment("вложение не найдено в этом чате") from error
|
||||
if known is None:
|
||||
raise Unavailable("ключ идемпотентности занят незавершённой записью")
|
||||
return Receipt(key, known["fingerprint"], message_of(known))
|
||||
|
||||
|
||||
class PostgresMessagingUnitOfWork(PostgresUnitOfWork):
|
||||
@@ -180,7 +380,7 @@ class PostgresMessagingUnitOfWork(PostgresUnitOfWork):
|
||||
|
||||
@property
|
||||
def chats(self) -> PostgresChatRepository:
|
||||
return PostgresChatRepository(self.connection, self.seen)
|
||||
return PostgresChatRepository(self.session, self.seen)
|
||||
|
||||
|
||||
class PostgresChatDirectory:
|
||||
@@ -190,30 +390,62 @@ class PostgresChatDirectory:
|
||||
self._database = database
|
||||
|
||||
async def chat_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None:
|
||||
async with self._database.connection() as connection:
|
||||
row = await connection.fetchrow(CHAT_OF_MEMBER, chat_id, user_id)
|
||||
async with self._database.session() as session:
|
||||
row = (
|
||||
(
|
||||
await session.execute(
|
||||
select(ChatRow.id, ChatRow.title, ChatRow.created_at, member_ids())
|
||||
.select_from(ChatMemberRow)
|
||||
.join(ChatRow, ChatRow.id == ChatMemberRow.chat_id)
|
||||
.where(ChatMemberRow.chat_id == chat_id, ChatMemberRow.user_id == user_id)
|
||||
)
|
||||
)
|
||||
.mappings()
|
||||
.first()
|
||||
)
|
||||
return None if row is None else chat_of(row)
|
||||
|
||||
async def chats_of(
|
||||
self, user_id: UUID, *, after: HistoryKey | None, limit: int
|
||||
) -> tuple[list[Chat], bool]:
|
||||
async with self._database.connection() as connection:
|
||||
if after is None:
|
||||
rows = await connection.fetch(CHATS_FIRST, user_id, limit + 1)
|
||||
else:
|
||||
rows = await connection.fetch(
|
||||
CHATS_AFTER, user_id, after[0], UUID(int=after[1]), limit + 1
|
||||
)
|
||||
statement = (
|
||||
select(ChatRow.id, ChatRow.title, ChatRow.created_at, member_ids())
|
||||
.select_from(ChatMemberRow)
|
||||
.join(ChatRow, ChatRow.id == ChatMemberRow.chat_id)
|
||||
.where(ChatMemberRow.user_id == user_id)
|
||||
.order_by(ChatMemberRow.chat_created_at, ChatMemberRow.chat_id)
|
||||
.limit(limit + 1)
|
||||
)
|
||||
if after is not None:
|
||||
statement = statement.where(
|
||||
tuple_(ChatMemberRow.chat_created_at, ChatMemberRow.chat_id)
|
||||
> tuple_(literal(after[0]), literal(UUID(int=after[1])))
|
||||
)
|
||||
async with self._database.session() as session:
|
||||
rows = (await session.execute(statement)).mappings().all()
|
||||
return page([chat_of(row) for row in rows], limit)
|
||||
|
||||
async def history(
|
||||
self, chat_id: UUID, *, after: HistoryKey | None, limit: int
|
||||
) -> tuple[list[Message], bool]:
|
||||
async with self._database.connection() as connection:
|
||||
if after is None:
|
||||
rows = await connection.fetch(HISTORY_FIRST, chat_id, limit + 1)
|
||||
else:
|
||||
rows = await connection.fetch(
|
||||
HISTORY_AFTER, chat_id, after[0], UUID(int=after[1]), limit + 1
|
||||
)
|
||||
statement = (
|
||||
select(
|
||||
MessageRow.id,
|
||||
MessageRow.chat_id,
|
||||
MessageRow.sender_id,
|
||||
MessageRow.text,
|
||||
MessageRow.created_at,
|
||||
attachment_ids(),
|
||||
)
|
||||
.where(MessageRow.chat_id == chat_id)
|
||||
.order_by(MessageRow.created_at, MessageRow.id)
|
||||
.limit(limit + 1)
|
||||
)
|
||||
if after is not None:
|
||||
statement = statement.where(
|
||||
tuple_(MessageRow.created_at, MessageRow.id)
|
||||
> tuple_(literal(after[0]), literal(UUID(int=after[1])))
|
||||
)
|
||||
async with self._database.session() as session:
|
||||
rows = (await session.execute(statement)).mappings().all()
|
||||
return page([message_of(row) for row in rows], limit)
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
from importlib import resources
|
||||
|
||||
import asyncpg
|
||||
|
||||
from messenger.shared.config import PlatformSettings
|
||||
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
|
||||
|
||||
LOCK_KEY = 20_260_002
|
||||
JOURNAL = """
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
checksum text NOT NULL,
|
||||
applied_at timestamptz NOT NULL DEFAULT now()
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
class MigrationChangedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def migrations() -> list[tuple[str, str]]:
|
||||
folder = resources.files("messenger") / "migrations"
|
||||
found = (item for item in folder.iterdir() if item.name.endswith(".sql"))
|
||||
return sorted((item.name.removesuffix(".sql"), item.read_text()) for item in found)
|
||||
|
||||
|
||||
async def migrate(dsn: str) -> list[str]:
|
||||
connection = await asyncpg.connect(dsn, timeout=10)
|
||||
try:
|
||||
await connection.execute("SELECT pg_advisory_lock($1)", LOCK_KEY)
|
||||
await connection.execute(JOURNAL)
|
||||
applied = dict(await connection.fetch("SELECT version, checksum FROM schema_migrations"))
|
||||
done: list[str] = []
|
||||
for version, sql in migrations():
|
||||
checksum = hashlib.sha256(sql.encode()).hexdigest()
|
||||
if version in applied:
|
||||
if applied[version] != checksum:
|
||||
raise MigrationChangedError(version)
|
||||
continue
|
||||
async with connection.transaction():
|
||||
await connection.execute(sql)
|
||||
await connection.execute(
|
||||
"INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)",
|
||||
version,
|
||||
checksum,
|
||||
)
|
||||
done.append(version)
|
||||
return done
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
settings = PlatformSettings()
|
||||
logger = StructlogLogger(level=settings.log_level, allowed_fields=PLATFORM_FIELDS | {"version"})
|
||||
try:
|
||||
done = asyncio.run(migrate(settings.database_url.get_secret_value()))
|
||||
except MigrationChangedError as error:
|
||||
logger.error("migration.changed", version=str(error))
|
||||
raise SystemExit(1) from None
|
||||
for version in done:
|
||||
logger.info("migration.applied", version=version)
|
||||
logger.info("migration.completed", reason="up to date" if not done else "applied")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -37,28 +37,21 @@ class CommandBus:
|
||||
|
||||
|
||||
class EventBus:
|
||||
__slots__ = ("_deferred", "_dropped", "_logger", "_projections", "_queue", "_worker")
|
||||
__slots__ = ("_deferred", "_dropped", "_logger", "_queue", "_worker")
|
||||
|
||||
def __init__(self, logger: Logger, *, max_queue: int = 10_000) -> None:
|
||||
self._logger = logger
|
||||
self._queue: asyncio.Queue[DomainEvent] = asyncio.Queue(maxsize=max_queue)
|
||||
self._projections: dict[type[DomainEvent], list[Any]] = {}
|
||||
self._deferred: dict[type[DomainEvent], list[Any]] = {}
|
||||
self._worker: asyncio.Task[None] | None = None
|
||||
self._dropped = 0
|
||||
|
||||
def project[EventT: DomainEvent](
|
||||
self, event_type: type[EventT], handler: EventHandler[EventT]
|
||||
) -> None:
|
||||
self._projections.setdefault(event_type, []).append(handler)
|
||||
|
||||
def subscribe[EventT: DomainEvent](
|
||||
self, event_type: type[EventT], handler: EventHandler[EventT]
|
||||
) -> None:
|
||||
self._deferred.setdefault(event_type, []).append(handler)
|
||||
|
||||
async def emit(self, event: DomainEvent) -> bool:
|
||||
await self._run(self._projections.get(type(event), []), event, "projection")
|
||||
if not self._deferred.get(type(event)):
|
||||
return True
|
||||
try:
|
||||
@@ -96,11 +89,11 @@ class EventBus:
|
||||
while True:
|
||||
event = await self._queue.get()
|
||||
try:
|
||||
await self._run(self._deferred.get(type(event), []), event, "subscriber")
|
||||
await self._run(self._deferred.get(type(event), []), event)
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
|
||||
async def _run(self, handlers: Sequence[Any], event: DomainEvent, kind: str) -> None:
|
||||
async def _run(self, handlers: Sequence[Any], event: DomainEvent) -> None:
|
||||
for handler in handlers:
|
||||
try:
|
||||
await handler(event)
|
||||
@@ -108,7 +101,6 @@ class EventBus:
|
||||
self._logger.error(
|
||||
"event.handler_failed",
|
||||
event_name=event.name,
|
||||
reason=kind,
|
||||
error=type(error).__name__,
|
||||
)
|
||||
|
||||
|
||||
@@ -26,6 +26,11 @@ class Logger(Protocol):
|
||||
def error(self, event: str, /, **fields: Any) -> None: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Readiness(Protocol):
|
||||
async def check(self) -> None: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class UnitOfWork(Protocol):
|
||||
def transaction(self) -> AbstractAsyncContextManager[None]: ...
|
||||
|
||||
@@ -9,7 +9,7 @@ from messenger.shared.entrypoints.errors import Unauthenticated
|
||||
async def current_actor(request: Request) -> UUID:
|
||||
actor = getattr(request.state, "actor", None)
|
||||
if not isinstance(actor, UUID):
|
||||
raise Unauthenticated("заголовок идентификации обязателен и должен быть действительным")
|
||||
raise Unauthenticated("сессия не подтверждена")
|
||||
return actor
|
||||
|
||||
|
||||
|
||||
@@ -7,4 +7,4 @@ class TransportSettings(Settings):
|
||||
docs: bool = False
|
||||
max_body_bytes: int = Field(default=1_048_576, ge=4_096)
|
||||
request_timeout_seconds: float = Field(default=8.0, gt=0, le=300)
|
||||
shutdown_timeout_seconds: int = Field(default=5, ge=1, le=10)
|
||||
min_transfer_rate_bytes: int = Field(default=131_072, ge=1_024)
|
||||
|
||||
@@ -46,7 +46,7 @@ class Unauthenticated(Exception):
|
||||
|
||||
def classify(error: DomainError, codes: ErrorCodes) -> tuple[str, int]:
|
||||
for kind in type(error).__mro__:
|
||||
known = codes.get(kind) # type: ignore[arg-type]
|
||||
known = codes.get(kind)
|
||||
if known is not None:
|
||||
return known
|
||||
return "internal_error", status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
from threading import Event
|
||||
|
||||
from dishka.integrations.fastapi import FromDishka, inject
|
||||
from fastapi import APIRouter
|
||||
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.application.ports import Readiness
|
||||
from messenger.shared.entrypoints.schemas import HealthOut
|
||||
from messenger.shared.infrastructure.postgres import Database
|
||||
|
||||
router = APIRouter(tags=["health"])
|
||||
DRAINING = Event()
|
||||
|
||||
|
||||
@router.get("/health/live")
|
||||
@@ -18,8 +14,6 @@ async def live() -> HealthOut:
|
||||
|
||||
@router.get("/health/ready")
|
||||
@inject
|
||||
async def ready(database: FromDishka[Database]) -> HealthOut:
|
||||
if DRAINING.is_set():
|
||||
raise Unavailable("Реплика выходит из ротации")
|
||||
await database.ping()
|
||||
async def ready(readiness: FromDishka[Readiness]) -> HealthOut:
|
||||
await readiness.check()
|
||||
return HealthOut()
|
||||
|
||||
@@ -3,11 +3,12 @@ from collections.abc import AsyncIterator
|
||||
from dishka import Provider, Scope, provide
|
||||
|
||||
from messenger.shared.application.messagebus import EventBus, InMemoryEventPublisher
|
||||
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
|
||||
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger, Readiness
|
||||
from messenger.shared.config import PlatformSettings
|
||||
from messenger.shared.infrastructure.clock import MonotonicUtcClock
|
||||
from messenger.shared.infrastructure.ids import Uuid7Generator
|
||||
from messenger.shared.infrastructure.postgres import Database
|
||||
from messenger.shared.infrastructure.readiness import DatabaseReadiness
|
||||
|
||||
|
||||
class PlatformProvider(Provider):
|
||||
@@ -45,6 +46,10 @@ class PlatformProvider(Provider):
|
||||
yield database
|
||||
await database.close()
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def readiness(self, database: Database) -> Readiness:
|
||||
return DatabaseReadiness(database)
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def event_bus(self, logger: Logger, settings: PlatformSettings) -> EventBus:
|
||||
return EventBus(logger, max_queue=settings.event_queue_max)
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from uuid import uuid4
|
||||
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
@@ -57,8 +58,44 @@ async def send_error(
|
||||
await send({"type": "http.response.body", "body": body})
|
||||
|
||||
|
||||
class Pace:
|
||||
__slots__ = ("_deadline", "_moved", "_opened", "_rate", "_timeout")
|
||||
|
||||
def __init__(self, deadline: asyncio.Timeout, timeout: float, rate: float) -> None:
|
||||
self._deadline = deadline
|
||||
self._timeout = timeout
|
||||
self._rate = rate
|
||||
self._opened = asyncio.get_running_loop().time()
|
||||
self._moved = 0
|
||||
|
||||
def around(self, receive: Receive, send: Send) -> tuple[Receive, Send]:
|
||||
async def paced_receive() -> Message:
|
||||
return self._count(await receive())
|
||||
|
||||
async def paced_send(message: Message) -> None:
|
||||
await send(self._count(message))
|
||||
|
||||
return paced_receive, paced_send
|
||||
|
||||
def _count(self, message: Message) -> Message:
|
||||
if size := len(message.get("body", b"")):
|
||||
self._moved += size
|
||||
moved = min(asyncio.get_running_loop().time(), self._opened + self._moved / self._rate)
|
||||
self._deadline.reschedule(moved + self._timeout)
|
||||
return message
|
||||
|
||||
|
||||
class RequestPipeline:
|
||||
__slots__ = ("_app", "_guard", "_instance", "_limits", "_logger", "_max_bytes", "_timeout")
|
||||
__slots__ = (
|
||||
"_app",
|
||||
"_guard",
|
||||
"_instance",
|
||||
"_limits",
|
||||
"_logger",
|
||||
"_max_bytes",
|
||||
"_min_rate",
|
||||
"_timeout",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -69,12 +106,14 @@ class RequestPipeline:
|
||||
guard: Guard,
|
||||
instance: str,
|
||||
limits: BodyLimits = (),
|
||||
min_rate: float = 131_072,
|
||||
) -> None:
|
||||
self._app = app
|
||||
self._logger = logger
|
||||
self._max_bytes = max_bytes
|
||||
self._limits = limits
|
||||
self._timeout = timeout
|
||||
self._min_rate = min_rate
|
||||
self._guard = guard
|
||||
self._instance = instance.encode()
|
||||
|
||||
@@ -85,9 +124,7 @@ class RequestPipeline:
|
||||
|
||||
incoming = header(scope, HEADER_BYTES)
|
||||
request_id = incoming[:MAX_REQUEST_ID] if incoming else uuid4().hex
|
||||
state = scope.setdefault("state", {})
|
||||
state["request_id"] = request_id
|
||||
state["actor"] = None
|
||||
scope.setdefault("state", {}).update(request_id=request_id, actor=None)
|
||||
started = time.perf_counter()
|
||||
status_holder = [500]
|
||||
answered = [False]
|
||||
@@ -106,18 +143,23 @@ class RequestPipeline:
|
||||
message = {**message, "headers": headers}
|
||||
await send(message)
|
||||
|
||||
declared = header(scope, b"content-length")
|
||||
if declared.isdigit() and int(declared) > self._limit(scope["path"]):
|
||||
await send_error(wrapped_send, 413, "payload_too_large", request_id)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
return
|
||||
limit = self._limit(scope["path"])
|
||||
received = [0]
|
||||
|
||||
async def limited_receive() -> Message:
|
||||
message = await receive()
|
||||
received[0] += len(message.get("body", b""))
|
||||
if received[0] > limit:
|
||||
raise HTTPException(413)
|
||||
return message
|
||||
|
||||
rejection: Rejection | None = None
|
||||
try:
|
||||
async with asyncio.timeout(self._timeout):
|
||||
rejection = await self._guard(scope)
|
||||
async with asyncio.timeout(self._timeout) as deadline:
|
||||
rejection = await self._guard(scope) or self._oversized(scope, limit)
|
||||
if rejection is None:
|
||||
await self._app(scope, receive, wrapped_send)
|
||||
pace = Pace(deadline, self._timeout, self._min_rate)
|
||||
await self._app(scope, *pace.around(limited_receive, wrapped_send))
|
||||
except Unavailable as error:
|
||||
self._logger.warning(
|
||||
"dependency.unavailable",
|
||||
@@ -126,18 +168,21 @@ class RequestPipeline:
|
||||
request_id=request_id,
|
||||
)
|
||||
rejection = Rejection(503, "unavailable")
|
||||
except TimeoutError:
|
||||
except (TimeoutError, asyncio.CancelledError) as error:
|
||||
cancelled = isinstance(error, asyncio.CancelledError)
|
||||
self._logger.warning(
|
||||
"request.timeout",
|
||||
"request.cancelled" if cancelled else "request.timeout",
|
||||
method=scope["method"],
|
||||
route=self._route(scope),
|
||||
request_id=request_id,
|
||||
limit=self._timeout,
|
||||
limit=None if cancelled else self._timeout,
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 3),
|
||||
)
|
||||
if not answered[0]:
|
||||
await send_error(wrapped_send, 503, "unavailable", request_id)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
if cancelled:
|
||||
raise
|
||||
return
|
||||
except Exception:
|
||||
self._logger.error(
|
||||
@@ -160,6 +205,12 @@ class RequestPipeline:
|
||||
return limit
|
||||
return self._max_bytes
|
||||
|
||||
def _oversized(self, scope: Scope, limit: int) -> Rejection | None:
|
||||
declared = header(scope, b"content-length")
|
||||
if declared.isdigit() and int(declared) > limit:
|
||||
return Rejection(413, "payload_too_large")
|
||||
return None
|
||||
|
||||
def _route(self, scope: Scope) -> str:
|
||||
route = scope.get("route")
|
||||
return getattr(route, "path", scope["path"])
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
from typing import Any, TextIO
|
||||
|
||||
import orjson
|
||||
import structlog
|
||||
@@ -51,7 +51,11 @@ def field_filter(allowed: frozenset[str]) -> Processor:
|
||||
return drop_unlisted_fields
|
||||
|
||||
|
||||
def configure(level: str = "INFO", allowed_fields: frozenset[str] = PLATFORM_FIELDS) -> None:
|
||||
def configure(
|
||||
level: str = "INFO",
|
||||
allowed_fields: frozenset[str] = PLATFORM_FIELDS,
|
||||
stream: TextIO | None = None,
|
||||
) -> None:
|
||||
shared = [
|
||||
structlog.contextvars.merge_contextvars,
|
||||
structlog.processors.add_log_level,
|
||||
@@ -63,7 +67,7 @@ def configure(level: str = "INFO", allowed_fields: frozenset[str] = PLATFORM_FIE
|
||||
wrapper_class=structlog.stdlib.BoundLogger,
|
||||
cache_logger_on_first_use=False,
|
||||
)
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler = logging.StreamHandler(stream or sys.stdout)
|
||||
handler.setFormatter(
|
||||
structlog.stdlib.ProcessorFormatter(
|
||||
foreign_pre_chain=shared,
|
||||
@@ -99,8 +103,9 @@ class StructlogLogger:
|
||||
name: str = "messenger",
|
||||
level: str = "INFO",
|
||||
allowed_fields: frozenset[str] = PLATFORM_FIELDS,
|
||||
stream: TextIO | None = None,
|
||||
) -> None:
|
||||
configure(level, allowed_fields)
|
||||
configure(level, allowed_fields, stream)
|
||||
self._logger = structlog.get_logger(name)
|
||||
|
||||
def info(self, event: str, /, **fields: Any) -> None:
|
||||
|
||||
@@ -1,32 +1,45 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any, Protocol
|
||||
from typing import ClassVar, Protocol
|
||||
|
||||
import asyncpg
|
||||
from sqlalchemy import Text, make_url, select
|
||||
from sqlalchemy.exc import DBAPIError, IntegrityError, InterfaceError, OperationalError
|
||||
from sqlalchemy.exc import TimeoutError as PoolTimeout
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.domain.events import DomainEvent
|
||||
|
||||
FAILURES = (
|
||||
OSError,
|
||||
asyncpg.InterfaceError,
|
||||
asyncpg.PostgresConnectionError,
|
||||
asyncpg.exceptions.OperatorInterventionError,
|
||||
asyncpg.exceptions.InsufficientResourcesError,
|
||||
asyncpg.exceptions.LockNotAvailableError,
|
||||
)
|
||||
FAILURES = (OSError, InterfaceError, OperationalError, PoolTimeout, TimeoutError)
|
||||
|
||||
PING_TIMEOUT_SECONDS = 0.4
|
||||
|
||||
type Connection = asyncpg.Connection
|
||||
type DatabaseSession = AsyncSession
|
||||
|
||||
|
||||
class EventSource(Protocol):
|
||||
def collect_events(self) -> list[DomainEvent]: ...
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
type_annotation_map: ClassVar = {str: Text}
|
||||
|
||||
|
||||
def unavailable(error: BaseException) -> bool:
|
||||
if isinstance(error, IntegrityError):
|
||||
return False
|
||||
if isinstance(error, FAILURES):
|
||||
return True
|
||||
if isinstance(error, DBAPIError) and error.connection_invalidated:
|
||||
return True
|
||||
state = getattr(error.orig if isinstance(error, DBAPIError) else error, "sqlstate", None)
|
||||
return isinstance(state, str) and (state.startswith(("08", "53", "57")) or state == "55P03")
|
||||
|
||||
|
||||
class Database:
|
||||
__slots__ = ("_acquire_timeout", "_pool")
|
||||
__slots__ = ("_engine", "_sessions")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -37,63 +50,71 @@ class Database:
|
||||
acquire_timeout: float,
|
||||
command_timeout: float,
|
||||
) -> None:
|
||||
self._acquire_timeout = acquire_timeout
|
||||
self._pool = asyncpg.create_pool(
|
||||
dsn,
|
||||
min_size=0,
|
||||
max_size=max_size,
|
||||
timeout=connect_timeout,
|
||||
command_timeout=command_timeout,
|
||||
max_inactive_connection_lifetime=60,
|
||||
server_settings={
|
||||
"statement_timeout": str(int(command_timeout * 1000)),
|
||||
"lock_timeout": str(int(command_timeout * 1000)),
|
||||
"idle_in_transaction_session_timeout": "10000",
|
||||
self._engine = create_async_engine(
|
||||
make_url(dsn).set(drivername="postgresql+asyncpg"),
|
||||
pool_size=max_size,
|
||||
max_overflow=0,
|
||||
pool_timeout=acquire_timeout,
|
||||
pool_recycle=60,
|
||||
pool_pre_ping=True,
|
||||
hide_parameters=True,
|
||||
connect_args={
|
||||
"timeout": connect_timeout,
|
||||
"command_timeout": command_timeout,
|
||||
"server_settings": {
|
||||
"statement_timeout": str(int(command_timeout * 1000)),
|
||||
"lock_timeout": str(int(command_timeout * 1000)),
|
||||
"idle_in_transaction_session_timeout": "10000",
|
||||
},
|
||||
},
|
||||
)
|
||||
self._sessions = async_sessionmaker(self._engine, expire_on_commit=False)
|
||||
|
||||
async def open(self) -> None:
|
||||
await self._pool
|
||||
pass
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._pool.close()
|
||||
await self._engine.dispose()
|
||||
|
||||
@asynccontextmanager
|
||||
async def connection(self, wait: float | None = None) -> AsyncIterator[Connection]:
|
||||
async def session(self) -> AsyncIterator[DatabaseSession]:
|
||||
try:
|
||||
async with self._pool.acquire(timeout=wait or self._acquire_timeout) as connection:
|
||||
yield connection
|
||||
except FAILURES as error:
|
||||
async with self._sessions() as session:
|
||||
yield session
|
||||
except Exception as error:
|
||||
if not unavailable(error):
|
||||
raise
|
||||
raise Unavailable("PostgreSQL недоступен") from error
|
||||
|
||||
async def ping(self) -> None:
|
||||
async with self.connection(PING_TIMEOUT_SECONDS) as connection:
|
||||
await connection.fetchval("SELECT 1", timeout=PING_TIMEOUT_SECONDS)
|
||||
try:
|
||||
async with asyncio.timeout(PING_TIMEOUT_SECONDS):
|
||||
async with self.session() as session:
|
||||
await session.execute(select(1))
|
||||
except TimeoutError as error:
|
||||
raise Unavailable("PostgreSQL недоступен") from error
|
||||
|
||||
|
||||
class PostgresUnitOfWork:
|
||||
__slots__ = ("_committed", "_connection", "_database", "_events", "_transaction", "seen")
|
||||
__slots__ = ("_committed", "_database", "_events", "_session", "seen")
|
||||
|
||||
def __init__(self, database: Database) -> None:
|
||||
self._database = database
|
||||
self._connection: Connection | None = None
|
||||
self._transaction: Any = None
|
||||
self._session: DatabaseSession | None = None
|
||||
self._committed = False
|
||||
self._events: list[DomainEvent] = []
|
||||
self.seen: list[EventSource] = []
|
||||
|
||||
@property
|
||||
def connection(self) -> Connection:
|
||||
if self._connection is None:
|
||||
def session(self) -> DatabaseSession:
|
||||
if self._session is None:
|
||||
raise RuntimeError("репозиторий используется вне транзакции")
|
||||
return self._connection
|
||||
return self._session
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self) -> AsyncIterator[None]:
|
||||
async with self._database.connection() as connection:
|
||||
self._connection = connection
|
||||
self._transaction = connection.transaction()
|
||||
await self._transaction.start()
|
||||
async with self._database.session() as session:
|
||||
self._session = session
|
||||
self._committed = False
|
||||
mark = len(self._events)
|
||||
try:
|
||||
@@ -101,12 +122,11 @@ class PostgresUnitOfWork:
|
||||
finally:
|
||||
if not self._committed:
|
||||
del self._events[mark:]
|
||||
await self._transaction.rollback()
|
||||
self.seen.clear()
|
||||
self._connection = None
|
||||
self._session = None
|
||||
|
||||
async def commit(self) -> None:
|
||||
await self._transaction.commit()
|
||||
await self.session.commit()
|
||||
self._committed = True
|
||||
for source in self.seen:
|
||||
self._events.extend(source.collect_events())
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
from threading import Event
|
||||
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.infrastructure.postgres import Database
|
||||
|
||||
DRAINING = Event()
|
||||
|
||||
|
||||
class DatabaseReadiness:
|
||||
__slots__ = ("_database", "_draining")
|
||||
|
||||
def __init__(self, database: Database, draining: Event = DRAINING) -> None:
|
||||
self._database = database
|
||||
self._draining = draining
|
||||
|
||||
async def check(self) -> None:
|
||||
if self._draining.is_set():
|
||||
raise Unavailable("Реплика выходит из ротации")
|
||||
await self._database.ping()
|
||||
@@ -17,7 +17,7 @@ from dishka import Provider, Scope, make_async_container, provide
|
||||
from dishka_faststream import setup_dishka
|
||||
from faststream.rabbit import Channel, RabbitBroker
|
||||
|
||||
from conftest import ROOT, STARTUP_SECONDS, container, image, published
|
||||
from conftest import ROOT, STARTUP_SECONDS, container, free_port, image, published
|
||||
from fakes import NullLogger
|
||||
from messenger.attachments.application.bus import AttachmentsCommandBus
|
||||
from messenger.attachments.application.commands import ProcessJob
|
||||
@@ -90,7 +90,7 @@ def rabbitmq() -> Iterator[dict[str, Any]]:
|
||||
"100:101",
|
||||
*volumes,
|
||||
"-p",
|
||||
"127.0.0.1::5672",
|
||||
f"127.0.0.1:{free_port()}:5672",
|
||||
image(RABBITMQ_IMAGE),
|
||||
)
|
||||
try:
|
||||
|
||||
@@ -9,7 +9,7 @@ from uuid import uuid4
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from conftest import STARTUP_SECONDS, container, image, published
|
||||
from conftest import STARTUP_SECONDS, container, free_port, image, published
|
||||
from messenger.attachments.application.errors import ObjectMissingError
|
||||
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
@@ -23,7 +23,7 @@ BUCKET = "messenger-attachments"
|
||||
def rustfs() -> Iterator[str]:
|
||||
name = f"mtusi-lab5-test-rustfs-{uuid4().hex[:8]}"
|
||||
keys = ["-e", f"RUSTFS_ACCESS_KEY={ROOT_KEY}", "-e", f"RUSTFS_SECRET_KEY={ROOT_SECRET}"]
|
||||
container(name, *keys, "-p", "127.0.0.1::9000", image(RUSTFS_IMAGE))
|
||||
container(name, *keys, "-p", f"127.0.0.1:{free_port()}:9000", image(RUSTFS_IMAGE))
|
||||
try:
|
||||
endpoint = f"http://127.0.0.1:{published(name, 9000)}"
|
||||
deadline = time.monotonic() + STARTUP_SECONDS
|
||||
@@ -71,6 +71,27 @@ async def test_round_trip_of_the_largest_object(rustfs: str) -> None:
|
||||
await objects.delete(key)
|
||||
|
||||
|
||||
async def test_listing_is_paged_in_byte_order_and_stat_reads_size(rustfs: str) -> None:
|
||||
prefix = f"reconcile-{uuid4().hex}/"
|
||||
keys = [f"{prefix}{name}" for name in ("b", "a", "Z", "a-1", "é")]
|
||||
async with storage(rustfs) as objects:
|
||||
for size, key in enumerate(keys, start=1):
|
||||
await objects.put(key, b"x" * size, "image/png")
|
||||
pages = [page async for page in objects.listing(prefix, 2)]
|
||||
assert [len(page) for page in pages] == [2, 2, 1]
|
||||
listed = [item.key for page in pages for item in page]
|
||||
assert listed == sorted(keys, key=str.encode)
|
||||
sizes = {item.key: item.size for page in pages for item in page}
|
||||
assert sizes == {key: size for size, key in enumerate(keys, start=1)}
|
||||
found = await objects.stat(keys[0])
|
||||
assert found is not None
|
||||
assert found.size == 1
|
||||
assert await objects.stat(f"{prefix}absent") is None
|
||||
for key in keys:
|
||||
await objects.delete(key)
|
||||
assert not [item async for page in objects.listing(prefix, 2) for item in page]
|
||||
|
||||
|
||||
async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None:
|
||||
started = time.monotonic()
|
||||
async with storage_without_bucket("http://127.0.0.1:1") as objects:
|
||||
@@ -78,7 +99,11 @@ async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None:
|
||||
await objects.put("attachments/x", b"x", "image/png")
|
||||
with pytest.raises(Unavailable):
|
||||
await objects.open("attachments/x")
|
||||
assert time.monotonic() - started < 8
|
||||
with pytest.raises(Unavailable):
|
||||
await objects.stat("attachments/x")
|
||||
with pytest.raises(Unavailable):
|
||||
await anext(aiter(objects.listing("attachments/", 10)))
|
||||
assert time.monotonic() - started < 12
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
from uuid import uuid4
|
||||
import time
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from httpx import AsyncClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from conftest import actor, keyed, make_chat, register
|
||||
from conftest import actor, keyed, make_chat, register, sql_connection
|
||||
from messenger.messaging.infrastructure.postgres import ChatRow
|
||||
|
||||
|
||||
async def test_outsider_gets_404_on_chat(client: AsyncClient) -> None:
|
||||
@@ -82,3 +85,36 @@ async def test_403_is_never_used(client: AsyncClient) -> None:
|
||||
]
|
||||
for path in paths:
|
||||
assert (await client.get(path, headers=actor(eve))).status_code != 403
|
||||
|
||||
|
||||
async def test_request_shape_is_checked_before_access(client: AsyncClient) -> None:
|
||||
alice = await register(client, "alice")
|
||||
eve = await register(client, "eve")
|
||||
chat = await make_chat(client, alice)
|
||||
for chat_id in (chat["id"], uuid4()):
|
||||
path = f"/api/v1/chats/{chat_id}/messages"
|
||||
unkeyed = await client.post(path, json={"text": "чужое"}, headers=actor(eve))
|
||||
extra = await client.post(path, json={"text": "x", "a": 1}, headers=keyed(eve))
|
||||
assert unkeyed.status_code == extra.status_code == 400
|
||||
assert unkeyed.json()["error"]["code"] == extra.json()["error"]["code"] == "invalid_request"
|
||||
blank = await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages", json={"text": " "}, headers=keyed(eve)
|
||||
)
|
||||
assert blank.status_code == 404
|
||||
|
||||
|
||||
async def test_outsider_does_not_wait_for_members_lock(client: AsyncClient) -> None:
|
||||
alice = await register(client, "alice")
|
||||
eve = await register(client, "eve")
|
||||
chat = await make_chat(client, alice)
|
||||
async with sql_connection() as holder:
|
||||
await holder.execute(
|
||||
select(ChatRow.id).where(ChatRow.id == UUID(chat["id"])).with_for_update()
|
||||
)
|
||||
started = time.perf_counter()
|
||||
response = await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages", json={"text": "чужое"}, headers=keyed(eve)
|
||||
)
|
||||
elapsed = time.perf_counter() - started
|
||||
assert response.status_code == 404
|
||||
assert elapsed < 0.5
|
||||
|
||||
@@ -9,7 +9,7 @@ import pytest
|
||||
from httpx import AsyncClient, Response
|
||||
from PIL import Image
|
||||
|
||||
from conftest import actor, make_chat, register, resolve, scalar
|
||||
from conftest import actor, make_chat, register, resolve, running_app, scalar
|
||||
from fakes import MemoryObjectStorage
|
||||
from messenger.attachments.application.ports import ObjectStorage
|
||||
from pictures import GIF, JPEG, SAMPLE, WEBP, png
|
||||
@@ -140,6 +140,18 @@ async def test_access_is_checked_before_content(client: AsyncClient) -> None:
|
||||
assert anonymous.headers["www-authenticate"] == 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
async def test_token_is_checked_before_the_declared_size() -> None:
|
||||
async with running_app(max_upload_bytes=4096) as http:
|
||||
alice, _, _, chat = await trio(http)
|
||||
oversized = png(16, 16, size=8192)
|
||||
anonymous = await http.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", content=oversized, headers=PNG
|
||||
)
|
||||
assert anonymous.status_code == 401
|
||||
assert anonymous.headers["www-authenticate"] == 'Bearer realm="messenger"'
|
||||
assert (await upload(http, alice, chat, oversized)).status_code == 413
|
||||
|
||||
|
||||
async def test_exact_limit_is_accepted(client: AsyncClient) -> None:
|
||||
alice, _, _, chat = await trio(client)
|
||||
response = await upload(client, alice, chat, png(16, 16, size=LIMIT))
|
||||
|
||||
@@ -33,20 +33,6 @@ def test_identity_does_not_know_about_messaging() -> None:
|
||||
assert not {k: v for k, v in offenders.items() if v}
|
||||
|
||||
|
||||
def test_messaging_touches_identity_only_through_events() -> None:
|
||||
allowed = f"{IDENTITY}.domain.events"
|
||||
offenders: dict[str, list[str]] = {}
|
||||
for path in modules_under("messaging"):
|
||||
borrowed = [
|
||||
module
|
||||
for module in imports_of(path)
|
||||
if module.startswith(IDENTITY) and module != allowed
|
||||
]
|
||||
if borrowed:
|
||||
offenders[str(path.relative_to(SOURCE))] = borrowed
|
||||
assert not offenders
|
||||
|
||||
|
||||
def test_attachments_and_other_zones_do_not_import_each_other() -> None:
|
||||
offenders: dict[str, list[str]] = {}
|
||||
for zone, foreign in (
|
||||
@@ -125,6 +111,7 @@ def test_shared_core_holds_only_platform() -> None:
|
||||
"ids.py",
|
||||
"logging.py",
|
||||
"postgres.py",
|
||||
"readiness.py",
|
||||
}
|
||||
|
||||
|
||||
@@ -267,6 +254,16 @@ def sql_tables(zone: str, tables: tuple[str, ...]) -> set[str]:
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||||
words.update(re.findall(r"\b(?:FROM|JOIN|INTO|UPDATE)\s+(\w+)", node.value))
|
||||
if (
|
||||
isinstance(node, ast.Assign)
|
||||
and any(
|
||||
isinstance(target, ast.Name) and target.id == "__tablename__"
|
||||
for target in node.targets
|
||||
)
|
||||
and isinstance(node.value, ast.Constant)
|
||||
and isinstance(node.value.value, str)
|
||||
):
|
||||
words.add(node.value.value)
|
||||
return words & set(tables)
|
||||
|
||||
|
||||
@@ -287,3 +284,11 @@ def test_messaging_does_not_import_identity_at_all() -> None:
|
||||
for path in modules_under("messaging")
|
||||
}
|
||||
assert not {key: value for key, value in borrowed.items() if value}
|
||||
|
||||
|
||||
def test_sqlalchemy_stays_in_infrastructure_and_asyncpg_is_only_a_driver() -> None:
|
||||
for path in SOURCE.rglob("*.py"):
|
||||
modules = {module.split(".")[0] for module in imports_of(path)}
|
||||
assert "asyncpg" not in modules
|
||||
if "infrastructure" not in path.parts:
|
||||
assert "sqlalchemy" not in modules
|
||||
|
||||
@@ -2,13 +2,18 @@ import asyncio
|
||||
import time
|
||||
from datetime import datetime
|
||||
from hashlib import sha256
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from redis.asyncio import Redis
|
||||
from sqlalchemy import select
|
||||
|
||||
from conftest import actor, bearer, register, resolve, running_app, scalar
|
||||
from conftest import actor, bearer, keyed, make_chat, register, resolve, running_app, scalar
|
||||
from messenger.identity.application.ports import SessionDirectory
|
||||
from messenger.identity.domain.models import Session
|
||||
from messenger.identity.infrastructure.postgres import PostgresSessionRepository, SessionRow
|
||||
from messenger.identity.infrastructure.valkey import CachedSessionDirectory
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
|
||||
BEARER = 'Bearer realm="messenger"'
|
||||
|
||||
@@ -31,8 +36,9 @@ async def test_second_check_is_served_from_cache(valkey: dict[str, str]) -> None
|
||||
assert (await cache.info("stats"))["keyspace_hits"] == hits + 1
|
||||
key = key_of(token_of(alice))
|
||||
stored = str(await cache.get(key))
|
||||
sql = "SELECT expires_at FROM sessions WHERE user_id = $1::uuid"
|
||||
expires_at: datetime = await scalar(sql, alice["id"])
|
||||
expires_at: datetime = await scalar(
|
||||
select(SessionRow.expires_at).where(SessionRow.user_id == UUID(alice["id"]))
|
||||
)
|
||||
assert alice["id"] in stored and token_of(alice) not in stored
|
||||
assert await cache.pexpiretime(key) <= int(expires_at.timestamp() * 1000)
|
||||
await cache.aclose()
|
||||
@@ -53,6 +59,29 @@ async def test_logout_writes_tombstone_before_database(valkey: dict[str, str]) -
|
||||
await cache.aclose()
|
||||
|
||||
|
||||
async def test_logout_rejected_by_database_changes_nothing(
|
||||
valkey: dict[str, str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def broken(self: PostgresSessionRepository, token_hash: bytes) -> None:
|
||||
raise Unavailable("PostgreSQL недоступен")
|
||||
|
||||
cache = Redis.from_url(valkey["url"])
|
||||
logout = "/api/v1/auth/sessions/current"
|
||||
async with running_app(cache_url=valkey["url"]) as http:
|
||||
alice = await register(http, "alice")
|
||||
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(PostgresSessionRepository, "remove", broken)
|
||||
failed = await http.delete(logout, headers=actor(alice))
|
||||
assert failed.status_code == 503
|
||||
assert failed.json()["error"]["code"] == "unavailable"
|
||||
assert await cache.get(key_of(token_of(alice))) is None
|
||||
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200
|
||||
assert (await http.delete(logout, headers=actor(alice))).status_code == 204
|
||||
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 401
|
||||
await cache.aclose()
|
||||
|
||||
|
||||
async def test_fill_racing_with_logout_does_not_revive_token(valkey: dict[str, str]) -> None:
|
||||
async with running_app(cache_url=valkey["url"]) as http:
|
||||
alice = await register(http, "alice")
|
||||
@@ -69,9 +98,6 @@ async def test_fill_racing_with_logout_does_not_revive_token(valkey: dict[str, s
|
||||
await released.wait()
|
||||
return stale[-1]
|
||||
|
||||
async def invalidate(self, session: Session) -> None:
|
||||
await origin.invalidate(session)
|
||||
|
||||
directory._origin = SlowOrigin()
|
||||
reader = asyncio.create_task(http.get("/api/v1/users/me", headers=actor(alice)))
|
||||
await entered.wait()
|
||||
@@ -106,6 +132,10 @@ async def test_unreachable_cache_is_bypassed_without_anonymous_access() -> None:
|
||||
stranger = await http.get("/api/v1/users/me", headers=bearer("x" * 43))
|
||||
elapsed = time.perf_counter() - started
|
||||
ready = await http.get("/health/ready")
|
||||
chat = await make_chat(http, alice)
|
||||
posted = await http.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages", json={"text": "без кэша"}, headers=keyed(alice)
|
||||
)
|
||||
logout = await http.delete("/api/v1/auth/sessions/current", headers=actor(alice))
|
||||
still = await http.get("/api/v1/users/me", headers=actor(alice))
|
||||
assert me.json() == alice
|
||||
@@ -113,6 +143,7 @@ async def test_unreachable_cache_is_bypassed_without_anonymous_access() -> None:
|
||||
assert stranger.headers["WWW-Authenticate"] == BEARER
|
||||
assert elapsed < 1
|
||||
assert ready.status_code == 200
|
||||
assert posted.status_code == 201
|
||||
assert logout.status_code == 503
|
||||
assert logout.json()["error"]["code"] == "unavailable"
|
||||
assert still.status_code == 200
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from httpx import AsyncClient
|
||||
|
||||
from conftest import PASSWORD, actor, keyed, make_chat, register
|
||||
from messenger.shared.infrastructure.clock import MonotonicUtcClock
|
||||
|
||||
CLIENTS = 10
|
||||
PER_CLIENT = 10
|
||||
@@ -60,6 +63,22 @@ async def test_hundred_messages_from_ten_clients(client: AsyncClient) -> None:
|
||||
assert len(set(stamps)) == TOTAL, "created_at внутри чата обязан строго возрастать"
|
||||
|
||||
|
||||
async def test_history_follows_the_clock_but_never_moves_back(
|
||||
monkeypatch: pytest.MonkeyPatch, client: AsyncClient
|
||||
) -> None:
|
||||
alice = await register(client, "alice")
|
||||
chat = await make_chat(client, alice)
|
||||
base = datetime.now(UTC) + timedelta(minutes=10)
|
||||
stamps = []
|
||||
for moment in (base, base - timedelta(minutes=5), base + timedelta(seconds=5)):
|
||||
monkeypatch.setattr(MonotonicUtcClock, "now", lambda _self, moment=moment: moment)
|
||||
response = await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages", json={"text": "часы"}, headers=keyed(alice)
|
||||
)
|
||||
stamps.append(datetime.fromisoformat(response.json()["created_at"]))
|
||||
assert stamps == [base, base + timedelta(microseconds=1), base + timedelta(seconds=5)]
|
||||
|
||||
|
||||
async def test_parallel_registration_keeps_username_unique(client: AsyncClient) -> None:
|
||||
body = {"username": f"race_{uuid4().hex[:12]}", "display_name": "Гонка", "password": PASSWORD}
|
||||
responses = await asyncio.gather(*(client.post("/api/v1/users", json=body) for _ in range(20)))
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user