Forward the lab 4 foundation and migrate the lab 5 pipeline to SQLAlchemy

This commit is contained in:
Artyom
2026-11-06 09:12:25 +03:00
parent ae9f60fe1f
commit 1217da5f5e
134 changed files with 8243 additions and 1409 deletions
+2
View File
@@ -42,3 +42,5 @@ compose*.yaml
lab.json
requirements-dev.txt
.claude
.verification/
+16 -3
View File
@@ -27,8 +27,8 @@ STAGING_RETENTION_SECONDS=86400
# LB queue per replica, drain window before closing the listener on SIGTERM, PostgreSQL
# limits, load generator limits (profile bench), cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone:
# only the load balancer is published.
LB_HTTP_PORT=8080
LB_STATS_PORT=8404
LB_HTTP_PORT=18085
LB_STATS_PORT=25085
API_CPUS=1
API_MEMORY=512m
WEB_CONCURRENCY=2
@@ -58,7 +58,7 @@ MALLOC_MMAP_THRESHOLD_=131072
# Solution-specific for lab5: RabbitMQ management UI on 127.0.0.1, backoff base of a transient
# retry (doubles per attempt up to 60 s), relay poll and sweep intervals, worker limits and the
# PostgreSQL pool of relay and worker. FAULT_POINT is set only by infra/compose.faults.yaml.
BROKER_MANAGEMENT_PORT=15672
BROKER_MANAGEMENT_PORT=25086
JOB_BACKOFF_BASE_SECONDS=2
OUTBOX_POLL_SECONDS=1
SWEEP_INTERVAL_SECONDS=5
@@ -71,3 +71,16 @@ PIPELINE_DB_POOL_MAX_SIZE=4
RENDER_TIMEOUT_SECONDS=30
RENDER_MEMORY_MB=448
RENDER_TASKS_PER_CHILD=100
# Minimal client transfer rate, bytes/s: every body byte moved in either direction extends the
# request deadline by 1/rate, so slow but steady uploads and downloads outlive REQUEST_TIMEOUT_SECONDS.
MIN_TRANSFER_RATE_BYTES=131072
# Solution-specific for lab4 track 2: limits of the one-off reconciler (profile reconcile), its S3
# read timeout for listing pages, optional CPU pinning of the measured services (empty = no pinning)
# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for the 100k point).
RECONCILE_CPUS=1
RECONCILE_MEMORY=256m
RECONCILE_S3_READ_TIMEOUT_SECONDS=10
RECONCILE_CPUSET=
S3_CPUSET=
PG_CPUSET=
S3_CPUS=1
+14 -1
View File
@@ -59,6 +59,9 @@ jobs:
- name: Несколько реплик на одном токене и гонка одного ключа
run: make lb-check
- name: Повтор неподтверждённой записи после потерянного ответа и убитой реплики
run: make retry-check
- name: Остановка реплики под запросами, затем возврат
run: |
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
@@ -84,7 +87,17 @@ jobs:
- name: Процессы работают не от root
run: |
docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}'
for id in $(docker compose ps -q); do
docker top "$id" -eo pid,uid,comm | awk 'NR>1 && $2==0 {print; bad=1} END {exit bad}' || exit 1
done
- name: Одноразовые сервисы не от root
run: |
for service in migrate s3-init; do
user=$(docker inspect -f '{{.Config.User}}' "$(docker compose ps -aq "$service")")
echo "$service: $user"
case "${user%%:*}" in ""|0|root) exit 1 ;; esac
done
- name: Корректная остановка relay и worker по SIGTERM
run: |
+2
View File
@@ -38,3 +38,5 @@ coverage/
# Локальные настройки агента
.claude/settings.local.json
.verification/
+4 -1
View File
@@ -27,7 +27,7 @@ ENV PYTHONUNBUFFERED=1 \
WEB_CONCURRENCY=2 \
REQUEST_TIMEOUT_SECONDS=8 \
DRAIN_SECONDS=2 \
SHUTDOWN_TIMEOUT_SECONDS=5
SHUTDOWN_TIMEOUT_SECONDS=6
RUN groupadd --system --gid 10001 messenger \
&& useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
@@ -36,6 +36,9 @@ COPY --from=builder --chown=root:root /opt/venv /opt/venv
WORKDIR /app
COPY --chown=root:root alembic.ini ./
COPY --chown=root:root alembic ./alembic
USER 10001:10001
EXPOSE 8080
+63 -16
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://$(shell docker compose port lb 8080 2>/dev/null || echo localhost:8080)
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -8,13 +8,14 @@ VENV ?= .venv
UV ?= uv
COMPOSE ?= docker compose
RATE ?= 100
API_IMAGE ?= mtusi-messenger:lab5
FROM_REV ?=
SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py scripts/make_images.py scripts/pipeline_suite.py \
scripts/pipeline_evidence.py
SHIFT ?= 0
SCRIPTS_PY = scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
SCENARIO ?= failover-upload
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
PYTEST_ARGS ?=
.PHONY: help check validate test load test-persistence test-failover \
install lint format typecheck student-test \
@@ -23,7 +24,11 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
limits detection failover-each rolling-update lab4-level4 \
scenario s3-check attachments-evidence upload-faults upload-memory \
images queues topology worker-stop worker-start broker-restart suite-preflight level3 \
pipeline-evidence level4 replay failed dlq
pipeline-evidence level4 replay failed dlq \
faults-evidence retry-check reconcile reconcile-demo reconcile-cost load-evidence \
clean-clone-evidence overload-leftovers
.NOTPARALLEL:
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -36,10 +41,23 @@ help:
@echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/"
@echo "lab5: queues, topology, worker-stop/worker-start, broker-restart, images, suite-preflight, level3, level4"
@echo "lab5 оператор: failed (реестр), replay ID=<attachment_id>, dlq (просмотр DLQ без извлечения)"
@echo "retry-check: потерянный ответ на живой и на убитой реплике, повтор тем же ключом"
@echo "lab4-level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps"
@echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence"
@echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда"
@echo "upload-memory ARGS='--out evidence/upload-memory/runN': пик памяти реплик при 10/20/40 загрузках 1 и 10 MiB"
@echo "load-evidence: tests/load.py ×3 клиентом-контейнером в сети edge стенда → evidence/load.txt, evidence/load/"
@echo "overload-leftovers: после upload-memory — намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt"
@echo "clean-clone-evidence: блок «Запуск из чистого clone» в свежем клоне рядом → evidence/clean-clone.txt"
@echo "reconcile ARGS='--apply --deep': сверка PostgreSQL ↔ S3 (без ARGS — dry-run); трек 2:"
@echo "reconcile-demo: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/"
@echo "reconcile-cost: стоимость обхода 1k/10k/30k/100k объектов × 3 прогона на чистом стенде → evidence/reconcile/cost/"
@echo "BASE_URL=$(BASE_URL) (порт lb из docker compose); failover-evidence SHIFT=1 — сдвиг round robin на вторую реплику"
check:
$(PYTHON) scripts/check_contract.py
validate: PYTHON := $(if $(wildcard $(VENV)/bin/python),$(VENV)/bin/python,$(PYTHON))
validate:
$(PYTHON) scripts/validate_openapi.py
@@ -63,17 +81,17 @@ install:
lint:
$(VENV)/bin/ruff check src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ruff format --check src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ty check src student-tests
$(VENV)/bin/ty check src student-tests alembic
format:
$(VENV)/bin/ruff check --fix src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ruff format src student-tests $(SCRIPTS_PY)
typecheck:
$(VENV)/bin/ty check src student-tests
$(VENV)/bin/ty check src student-tests alembic
student-test:
$(VENV)/bin/pytest
$(VENV)/bin/pytest $(PYTEST_ARGS)
secrets:
sh scripts/secrets.sh
@@ -82,13 +100,14 @@ secrets:
up: secrets
$(COMPOSE) build
$(COMPOSE) up -d --wait
scripts/lb-wait.sh
down:
$(COMPOSE) down
$(COMPOSE) --profile bench --profile tools --profile reconcile down
reset-data:
@echo "Удаляются volumes с учебными данными стенда"
$(COMPOSE) down -v --remove-orphans
$(COMPOSE) --profile bench --profile tools --profile reconcile down -v --remove-orphans
migrate:
$(COMPOSE) run --rm migrate
@@ -105,19 +124,23 @@ restore:
lb-check:
$(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json
retry-check:
$(PYTHON) scripts/probe.py retry --out evidence/retry.json
limits:
$(PYTHON) scripts/limits.py --out evidence/limits.txt
detection:
sh scripts/detection.sh evidence/detection
rm -rf evidence/detection
for run in 1 2 3; do sh scripts/detection.sh evidence/detection/run$$run || exit 1; done
failover-each:
RATE=$(RATE) sh scripts/failover-each.sh evidence/failover-each
rolling-update:
ifneq ($(FROM_REV),)
git archive "$(FROM_REV)" | docker build -q -t "$(API_IMAGE)" -
$(COMPOSE) up -d --no-deps --force-recreate --wait api
git archive "$(FROM_REV)" | docker build -q -t "mtusi-messenger:$(FROM_REV)" -
API_IMAGE="mtusi-messenger:$(FROM_REV)" $(COMPOSE) up -d --no-deps --force-recreate --wait api
scripts/lb-wait.sh
endif
$(COMPOSE) build -q api
@@ -144,10 +167,28 @@ upload-faults:
status=$$?; cat evidence/upload-faults.txt; exit $$status
upload-memory:
$(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)"
$(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)" $(ARGS)
load-evidence:
sh scripts/load-evidence.sh
overload-leftovers:
sh scripts/overload-leftovers.sh
clean-clone-evidence:
sh scripts/clean-clone.sh
reconcile:
$(COMPOSE) run --rm --no-deps reconciler $(ARGS)
reconcile-demo:
$(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)" $(ARGS)
reconcile-cost:
$(PYTHON) scripts/reconcile_track.py cost $(ARGS)
failover-evidence:
sh scripts/failover-evidence.sh "$(BASE_URL)"
sh scripts/failover-evidence.sh "$(BASE_URL)" "$(SHIFT)"
stack-evidence:
sh scripts/evidence-stack.sh
@@ -166,6 +207,12 @@ cache-evidence:
test-faults:
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack
faults-evidence:
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; echo "\$$ make test-faults"; \
$(MAKE) --no-print-directory test-faults 2>&1; } | sed 's#$(CURDIR)#<repo>#g' > evidence/faults.txt
cat evidence/faults.txt
grep -qE '^=+ [0-9]+ passed in' evidence/faults.txt
seed:
$(PSQL) -f - < scripts/sql/seed-history.sql
+4
View File
@@ -0,0 +1,4 @@
[alembic]
script_location = %(here)s/alembic
prepend_sys_path = %(here)s/src %(here)s/alembic
path_separator = space
+102
View File
@@ -0,0 +1,102 @@
import asyncio
from alembic import context
from alembic.script import ScriptDirectory
from legacy import adopt_legacy
from sqlalchemy import Connection, func, make_url, pool, select
from sqlalchemy.ext.asyncio import create_async_engine
from messenger.attachments.infrastructure.postgres import (
AttachmentRow,
JobEventRow,
JobRow,
OutboxRow,
VariantRow,
)
from messenger.identity.infrastructure.postgres import SessionRow, UserRow
from messenger.messaging.infrastructure.postgres import (
ChatMemberRow,
ChatRow,
IdempotencyKeyRow,
MessageAttachmentRow,
MessageRow,
)
from messenger.shared.config import PlatformSettings
LOCK_KEY = 20_260_002
MODELS = (
UserRow,
SessionRow,
ChatRow,
ChatMemberRow,
MessageRow,
IdempotencyKeyRow,
AttachmentRow,
JobRow,
OutboxRow,
JobEventRow,
VariantRow,
MessageAttachmentRow,
)
target_metadata = UserRow.metadata
config = context.config
def include_object(_obj, name, _type, _reflected, _compare_to):
return name != "schema_migrations"
def run_migrations(connection: Connection) -> None:
with connection.engine.connect() as lock:
lock.execute(select(func.pg_advisory_lock(LOCK_KEY)))
lock.commit()
try:
with connection.begin():
adopt_legacy(connection, ScriptDirectory.from_config(config))
context.configure(
connection=connection,
target_metadata=target_metadata,
transaction_per_migration=True,
compare_type=True,
include_object=include_object,
)
with context.begin_transaction():
context.run_migrations()
finally:
lock.execute(select(func.pg_advisory_unlock(LOCK_KEY)))
lock.commit()
async def run_async_migrations() -> None:
dsn = config.attributes.get("database_url")
if dsn is None:
dsn = PlatformSettings().database_url.get_secret_value()
engine = create_async_engine(
make_url(dsn).set(drivername="postgresql+asyncpg"),
poolclass=pool.NullPool,
hide_parameters=True,
connect_args={
"timeout": 10,
"server_settings": {"lock_timeout": "10000", "statement_timeout": "60000"},
},
)
try:
async with engine.connect() as connection:
await connection.run_sync(run_migrations)
finally:
await engine.dispose()
if context.is_offline_mode():
context.configure(
dialect_name="postgresql",
target_metadata=target_metadata,
literal_binds=True,
transaction_per_migration=True,
)
with context.begin_transaction():
context.run_migrations()
elif config.attributes.get("connection") is not None:
run_migrations(config.attributes["connection"])
else:
asyncio.run(run_async_migrations())
+94
View File
@@ -0,0 +1,94 @@
from uuid import uuid4
from alembic.migration import MigrationContext
from alembic.operations import Operations
from alembic.script import ScriptDirectory
from sqlalchemy import Connection, MetaData, Table, func, inspect, select, text
from sqlalchemy.schema import CreateSchema
LEGACY = {
"0001_init": "9a13618f078ae9056431391e2e5197a31b981ae1fff047a9b3421ac750d3fd0d",
"0002_idempotency_keys": "d9159a35553249a1e51af5f2e4e7c39b014e97593c49748e13f3799839063dd0",
"0003_attachments": "c4417f560538a083efba1c67f660147dc94532fcbb2fa91f3f3e8258a0f01f2e",
"0004_jobs": "c3887fa7a886ae8153a216aa43fd4c3c6190a2074e8ce20c3e77f29bfb01550e",
"0005_failed_registry": "02b80061a08a6929feb092148c453ad970e3d001901c651b5a73b90b1e40f145",
"0006_attachment_quarantine": (
"541f5399d87fac3584c4102b4e0bf92c23cd372cc20a7d56dad03e771773202c"
),
}
SNAPSHOT = text("""
WITH relations AS (
SELECT c.* FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = :schema AND c.relkind IN ('r', 'p')
AND c.relname NOT IN ('schema_migrations', 'alembic_version')
)
SELECT 'table' AS kind, c.relname AS name,
concat_ws('|', c.relkind, c.relpersistence,
c.relrowsecurity, c.relforcerowsecurity) AS definition
FROM relations c
UNION ALL
SELECT 'column', c.relname || '.' || a.attname,
concat_ws('|', a.attnum, format_type(a.atttypid, a.atttypmod), a.attnotnull,
a.attidentity, a.attgenerated, a.attcollation::regcollation,
pg_get_expr(d.adbin, d.adrelid))
FROM relations c JOIN pg_attribute a ON a.attrelid = c.oid
LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum
WHERE a.attnum > 0 AND NOT a.attisdropped
UNION ALL
SELECT 'constraint', c.relname || '.' || k.conname,
concat_ws('|', pg_get_constraintdef(k.oid), k.convalidated)
FROM relations c JOIN pg_constraint k ON k.conrelid = c.oid
UNION ALL
SELECT 'index', c.relname || '.' || x.relname,
concat_ws('|', pg_get_indexdef(i.indexrelid), i.indisvalid, i.indisready)
FROM relations c JOIN pg_index i ON i.indrelid = c.oid
JOIN pg_class x ON x.oid = i.indexrelid
UNION ALL
SELECT 'trigger', c.relname || '.' || t.tgname,
concat_ws('|', pg_get_triggerdef(t.oid), t.tgenabled)
FROM relations c JOIN pg_trigger t ON t.tgrelid = c.oid WHERE NOT t.tgisinternal
ORDER BY kind, name
""")
def snapshot(connection: Connection, schema: str) -> list[tuple[str, str, str]]:
return [
(kind, name, definition.replace(f"{schema}.", ""))
for kind, name, definition in connection.execute(SNAPSHOT, {"schema": schema})
]
def adopt_legacy(connection: Connection, scripts: ScriptDirectory) -> None:
migration = MigrationContext.configure(connection)
if migration.get_current_heads():
return
tables = set(inspect(connection).get_table_names(schema="public"))
if tables <= {"alembic_version"}:
return
if "schema_migrations" not in tables:
raise RuntimeError("Existing schema has no legacy migration journal")
journal = Table("schema_migrations", MetaData(), schema="public", autoload_with=connection)
applied = dict(connection.execute(select(journal.c.version, journal.c.checksum)).tuples().all())
expected = dict(list(LEGACY.items())[: len(applied)])
if applied != expected:
raise RuntimeError("Unknown or changed legacy migrations")
if not applied:
if tables - {"schema_migrations", "alembic_version"}:
raise RuntimeError("Existing schema has an empty migration journal")
return
revision = list(expected)[-1]
actual = snapshot(connection, "public")
reference = f"alembic_verify_{uuid4().hex}"
savepoint = connection.begin_nested()
try:
connection.execute(CreateSchema(reference))
connection.execute(select(func.set_config("search_path", reference, True)))
with Operations.context(MigrationContext.configure(connection)):
for script in reversed(list(scripts.walk_revisions(base="base", head=revision))):
script.module.upgrade()
wanted = snapshot(connection, reference)
finally:
savepoint.rollback()
if actual != wanted:
raise RuntimeError("Legacy schema differs from the recorded migrations")
migration.stamp(scripts, revision)
+16
View File
@@ -0,0 +1,16 @@
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
+87
View File
@@ -0,0 +1,87 @@
import sqlalchemy as sa
from alembic import op
revision = "0001_init"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"users",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("username", sa.Text(), nullable=False),
sa.Column("display_name", sa.Text(), nullable=False),
sa.Column("password_hash", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("password_hash LIKE '$argon2id$%'", name="users_password_hash_check"),
sa.CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="users_username_format"),
sa.CheckConstraint(
"char_length(display_name) BETWEEN 1 AND 100", name="users_display_name_check"
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("username", name="users_username_key"),
)
op.create_table(
"sessions",
sa.Column("token_hash", sa.LargeBinary(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("expires_at > created_at", name="sessions_check"),
sa.CheckConstraint("octet_length(token_hash) = 32", name="sessions_token_hash_check"),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("token_hash"),
)
op.create_index("sessions_user_idx", "sessions", ["user_id"], unique=False)
op.create_table(
"chats",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("title", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("last_message_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="chats_title_check"),
sa.PrimaryKeyConstraint("id"),
)
op.create_table(
"chat_members",
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("position", sa.SmallInteger(), nullable=False),
sa.Column("chat_created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("position BETWEEN 1 AND 100", name="chat_members_position_check"),
sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(
["user_id"],
["users.id"],
),
sa.PrimaryKeyConstraint("chat_id", "user_id"),
)
op.create_index(
"chat_members_user_idx",
"chat_members",
["user_id", "chat_created_at", "chat_id"],
unique=False,
)
op.create_table(
"messages",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("sender_id", sa.Uuid(), nullable=False),
sa.Column("text", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("char_length(text) BETWEEN 1 AND 4000", name="messages_text_check"),
sa.ForeignKeyConstraint(
["chat_id", "sender_id"],
["chat_members.chat_id", "chat_members.user_id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"messages_history_idx", "messages", ["chat_id", "created_at", "id"], unique=False
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete application data")
+33
View File
@@ -0,0 +1,33 @@
import sqlalchemy as sa
from alembic import op
revision = "0002_idempotency_keys"
down_revision = "0001_init"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"idempotency_keys",
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("key", sa.Text(), nullable=False),
sa.Column("fingerprint", sa.LargeBinary(), nullable=False),
sa.Column("message_id", sa.Uuid(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="idempotency_keys_key_check"),
sa.CheckConstraint(
"octet_length(fingerprint) = 32", name="idempotency_keys_fingerprint_check"
),
sa.ForeignKeyConstraint(
["chat_id", "user_id"],
["chat_members.chat_id", "chat_members.user_id"],
),
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("chat_id", "user_id", "key"),
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete application data")
+77
View File
@@ -0,0 +1,77 @@
import sqlalchemy as sa
from alembic import op
revision = "0003_attachments"
down_revision = "0002_idempotency_keys"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.drop_constraint("messages_text_check", "messages", type_="check")
op.create_check_constraint("messages_text_check", "messages", "char_length(text) <= 4000")
op.create_table(
"attachments",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("owner_id", sa.Uuid(), nullable=False),
sa.Column("filename", sa.Text(), nullable=False),
sa.Column("content_type", sa.Text(), nullable=False),
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
sa.Column("sha256", sa.Text(), nullable=False),
sa.Column("width", sa.Integer(), nullable=False),
sa.Column("height", sa.Integer(), nullable=False),
sa.Column("object_key", sa.Text(collation="C"), nullable=False),
sa.Column("status", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("object_key", name="attachments_object_key_key"),
sa.UniqueConstraint("id", "chat_id", name="attachments_chat_key"),
sa.ForeignKeyConstraint(
["chat_id", "owner_id"],
["chat_members.chat_id", "chat_members.user_id"],
name="attachments_owner_fkey",
),
sa.CheckConstraint(
"char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_check"
),
sa.CheckConstraint(
"content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_check"
),
sa.CheckConstraint(
"size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_check"
),
sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachments_sha256_check"),
sa.CheckConstraint("width > 0", name="attachments_width_check"),
sa.CheckConstraint("height > 0", name="attachments_height_check"),
sa.CheckConstraint("status IN ('pending', 'ready')", name="attachments_status_check"),
)
op.create_index(
"attachments_pending_idx",
"attachments",
["created_at"],
postgresql_where=sa.text("status = 'pending'"),
)
op.create_table(
"message_attachments",
sa.Column("message_id", sa.Uuid(), nullable=False),
sa.Column("position", sa.SmallInteger(), nullable=False),
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("attachment_id", sa.Uuid(), nullable=False),
sa.PrimaryKeyConstraint("message_id", "position"),
sa.UniqueConstraint("message_id", "attachment_id"),
sa.CheckConstraint("position BETWEEN 0 AND 9", name="message_attachments_position_check"),
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(
["attachment_id", "chat_id"],
["attachments.id", "attachments.chat_id"],
name="message_attachments_attachment_fkey",
ondelete="RESTRICT",
),
)
op.create_index("message_attachments_attachment_idx", "message_attachments", ["attachment_id"])
def downgrade() -> None:
raise RuntimeError("Downgrade would delete application data")
+146
View File
@@ -0,0 +1,146 @@
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision = "0004_jobs"
down_revision = "0003_attachments"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.drop_constraint("attachments_status_check", "attachments", type_="check")
op.create_check_constraint(
"attachments_status_check",
"attachments",
"status IN ('pending', 'queued', 'processing', 'ready', 'failed')",
)
for name in ("staging_key", "original_key"):
op.add_column("attachments", sa.Column(name, sa.Text(collation="C"), nullable=True))
op.create_unique_constraint(f"attachments_{name}_key", "attachments", [name])
op.add_column(
"attachments",
sa.Column("published_version", sa.Integer(), nullable=False, server_default="0"),
)
op.create_check_constraint(
"attachments_published_version_check", "attachments", "published_version >= 0"
)
op.add_column("attachments", sa.Column("error_code", sa.Text(), nullable=True))
op.create_check_constraint(
"attachments_error_code_check",
"attachments",
"error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')",
)
op.alter_column("attachments", "object_key", nullable=True)
attachments = sa.table(
"attachments",
sa.column("status"),
sa.column("object_key"),
sa.column("original_key"),
sa.column("staging_key"),
sa.column("published_version"),
)
op.execute(
attachments.update()
.where(attachments.c.status == "ready")
.values(original_key=attachments.c.object_key, published_version=1)
)
op.execute(
attachments.update()
.where(attachments.c.status == "pending")
.values(staging_key=attachments.c.object_key)
)
op.create_index(
"attachments_staged_idx",
"attachments",
["updated_at"],
postgresql_where=sa.column("staging_key").is_not(None),
)
op.create_table(
"attachment_variants",
sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False),
sa.Column("pipeline_version", sa.Integer(), nullable=False),
sa.Column("name", sa.Text(), nullable=False),
sa.Column("object_key", sa.Text(collation="C"), nullable=False),
sa.Column("content_type", sa.Text(), nullable=False),
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
sa.Column("width", sa.Integer(), nullable=False),
sa.Column("height", sa.Integer(), nullable=False),
sa.Column("sha256", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("attachment_id", "pipeline_version", "name"),
sa.UniqueConstraint("object_key", name="attachment_variants_object_key_key"),
sa.CheckConstraint(
"pipeline_version >= 1", name="attachment_variants_pipeline_version_check"
),
sa.CheckConstraint(
"name IN ('thumbnail', 'crop', 'optimized')", name="attachment_variants_name_check"
),
sa.CheckConstraint(
"content_type IN ('image/jpeg', 'image/png', 'image/webp')",
name="attachment_variants_content_type_check",
),
sa.CheckConstraint("size_bytes >= 1", name="attachment_variants_size_bytes_check"),
sa.CheckConstraint("width >= 1", name="attachment_variants_width_check"),
sa.CheckConstraint("height >= 1", name="attachment_variants_height_check"),
sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachment_variants_sha256_check"),
)
op.create_table(
"jobs",
sa.Column("job_id", sa.Uuid(), primary_key=True),
sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False),
sa.Column("pipeline_version", sa.Integer(), nullable=False),
sa.Column("status", sa.Text(), nullable=False),
sa.Column("attempt", sa.Integer(), nullable=False),
sa.Column("claims", sa.Integer(), nullable=False, server_default="0"),
sa.Column("lease_token", sa.Uuid(), nullable=True),
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("attachment_id", "pipeline_version", name="jobs_effect_key"),
sa.CheckConstraint("pipeline_version >= 1", name="jobs_pipeline_version_check"),
sa.CheckConstraint(
"status IN ('queued', 'processing', 'done', 'failed', 'superseded')",
name="jobs_status_check",
),
sa.CheckConstraint("attempt >= 1", name="jobs_attempt_check"),
sa.CheckConstraint("claims >= 0", name="jobs_claims_check"),
)
op.create_index(
"jobs_lease_idx",
"jobs",
["lease_until"],
postgresql_where=sa.column("status") == "processing",
)
op.create_table(
"outbox",
sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True),
sa.Column("job_id", sa.Uuid(), sa.ForeignKey("jobs.job_id"), nullable=False),
sa.Column("payload", postgresql.JSONB(), nullable=False),
sa.Column("available_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("published_at", sa.DateTime(timezone=True), nullable=True),
)
op.create_index(
"outbox_pending_idx",
"outbox",
["available_at", "id"],
postgresql_where=sa.column("published_at").is_(None),
)
op.create_index("outbox_job_idx", "outbox", ["job_id"])
op.create_table(
"job_events",
sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True),
sa.Column("job_id", sa.Uuid(), nullable=False),
sa.Column("attachment_id", sa.Uuid(), nullable=False),
sa.Column("kind", sa.Text(), nullable=False),
sa.Column("attempt", sa.Integer(), nullable=True),
sa.Column("detail", postgresql.JSONB(), nullable=False, server_default="{}"),
sa.Column("at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
op.create_index("job_events_attachment_idx", "job_events", ["attachment_id", "id"])
def downgrade() -> None:
raise RuntimeError("Downgrade would delete pipeline data")
+20
View File
@@ -0,0 +1,20 @@
import sqlalchemy as sa
from alembic import op
revision = "0005_failed_registry"
down_revision = "0004_jobs"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_index(
"jobs_failed_idx",
"jobs",
[sa.column("updated_at").desc()],
postgresql_where=sa.column("status") == "failed",
)
def downgrade() -> None:
op.drop_index("jobs_failed_idx", table_name="jobs")
@@ -0,0 +1,28 @@
import sqlalchemy as sa
from alembic import op
revision = "0006_attachment_quarantine"
down_revision = "0005_failed_registry"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("attachments", sa.Column("quarantine_reason", sa.Text(), nullable=True))
op.create_check_constraint(
"attachments_quarantine_reason_check",
"attachments",
"quarantine_reason IN ('missing', 'size_mismatch', 'sha_mismatch')",
)
op.add_column(
"attachments", sa.Column("quarantined_at", sa.DateTime(timezone=True), nullable=True)
)
op.create_check_constraint(
"attachments_quarantine_check",
"attachments",
"(quarantine_reason IS NULL) = (quarantined_at IS NULL)",
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete quarantine data")
+58 -12
View File
@@ -1,5 +1,5 @@
x-app: &app
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:${COMPOSE_PROJECT_NAME}}
networks: [backend]
x-s3-env: &s3-env
@@ -19,7 +19,7 @@ x-s3-env: &s3-env
UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100}
x-pipeline: &pipeline
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:${COMPOSE_PROJECT_NAME}}
pull_policy: never
restart: unless-stopped
init: true
@@ -64,10 +64,17 @@ x-s3-client: &s3-client
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
RC_CONFIG_DIR: /tmp/rc
user: "65534:65534"
volumes: ["./infra/s3:/s3:ro"]
secrets:
[s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key,
s3_worker_access_key_id, s3_worker_secret_access_key]
- s3_root_user
- s3_root_password
- s3_access_key_id
- s3_secret_access_key
- s3_reconciler_access_key_id
- s3_reconciler_secret_access_key
- s3_worker_access_key_id
- s3_worker_secret_access_key
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
restart: "no"
@@ -83,15 +90,16 @@ services:
API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64}
volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
ports:
- "127.0.0.1:${LB_HTTP_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
- "127.0.0.1:${LB_HTTP_PORT:-18085}:8080"
- "127.0.0.1:${LB_STATS_PORT:-25085}:8404"
networks: [edge, backend]
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
interval: 2s
timeout: 2s
retries: 3
start_period: 10s
start_period: 60s
start_interval: 1s
sysctls:
net.ipv4.ip_local_port_range: "10240 65535"
net.ipv4.tcp_tw_reuse: "1"
@@ -117,10 +125,11 @@ services:
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
MIN_TRANSFER_RATE_BYTES: ${MIN_TRANSFER_RATE_BYTES:-131072}
DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1}
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2}
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-6}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
MALLOC_MMAP_THRESHOLD_: ${MALLOC_MMAP_THRESHOLD_:-131072}
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
@@ -130,7 +139,7 @@ services:
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
start_interval: 200ms
stop_grace_period: 8s
restart: unless-stopped
read_only: true
@@ -153,7 +162,8 @@ services:
migrate:
<<: *app
pull_policy: never
command: ["python", "-m", "messenger.migrate"]
init: true
command: ["alembic", "upgrade", "head"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
LOG_LEVEL: ${LOG_LEVEL:-INFO}
@@ -192,12 +202,14 @@ services:
secrets: [postgres_password]
volumes: [pgdata:/var/lib/postgresql/data]
shm_size: 256m
cpuset: ${PG_CPUSET:-}
healthcheck:
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
start_interval: 200ms
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
@@ -245,6 +257,7 @@ services:
RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn}
secrets: [s3_root_user, s3_root_password]
volumes: [s3data:/data]
cpuset: ${S3_CPUSET:-}
read_only: true
tmpfs: ["/tmp:size=64m"]
cap_drop: [ALL]
@@ -260,7 +273,36 @@ services:
networks: [backend]
deploy:
resources:
limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"}
limits: {cpus: "${S3_CPUS:-1}", memory: "${S3_MEMORY:-512m}"}
reconciler:
<<: *app
pull_policy: never
profiles: [reconcile]
init: true
entrypoint: ["python", "-m", "messenger.attachments.entrypoints.reconcile"]
command: ["--dry-run"]
environment:
<<: *s3-env
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
S3_READ_TIMEOUT_SECONDS: ${RECONCILE_S3_READ_TIMEOUT_SECONDS:-10}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key]
healthcheck:
disable: true
restart: "no"
read_only: true
cpuset: ${RECONCILE_CPUSET:-}
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
deploy:
resources:
limits: {cpus: "${RECONCILE_CPUS:-1}", memory: "${RECONCILE_MEMORY:-256m}"}
s3-init:
<<: *s3-client
@@ -284,7 +326,7 @@ services:
security_opt: ["no-new-privileges:true"]
cap_drop: [ALL]
ports:
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672"
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-25086}:15672"
networks: [backend, edge]
volumes:
- broker-data:/var/lib/rabbitmq
@@ -387,6 +429,10 @@ secrets:
file: ./secrets/broker_url_api
broker_url_worker:
file: ./secrets/broker_url_worker
s3_reconciler_access_key_id:
file: ./secrets/s3_reconciler_access_key_id
s3_reconciler_secret_access_key:
file: ./secrets/s3_reconciler_secret_access_key
volumes:
pgdata:
+1 -13
View File
@@ -4,20 +4,8 @@
{
"Sid": "MessengerApiObjects",
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},
{
"Sid": "MessengerApiBucket",
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+2 -1
View File
@@ -4,7 +4,8 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
endpoint="${S3_ENDPOINT:-http://s3:9000}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}"
export RC_HOST_root RC_HOST_api
RC_HOST_reconciler="${endpoint%%://*}://$(secret s3_reconciler_access_key_id):$(secret s3_reconciler_secret_access_key)@${endpoint#*://}"
export RC_HOST_root RC_HOST_api RC_HOST_reconciler
rc alias set anon "$endpoint" --anonymous >/dev/null
case "${1:-}" in
sh|wget) exec "$@" ;;
+12 -13
View File
@@ -4,23 +4,22 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
bucket="${S3_BUCKET:-messenger-attachments}"
endpoint="${S3_ENDPOINT:-http://s3:9000}"
policy="${S3_API_POLICY:-messenger-api}"
reconciler="${S3_RECONCILER_POLICY:-messenger-reconciler}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
export RC_HOST_root
api_key=$(secret s3_access_key_id)
grant() {
sed "s/messenger-attachments/$bucket/g" "/s3/$2" > "/tmp/$2"
rc admin policy create root "$1" "/tmp/$2" >/dev/null
rc admin user add root "$(secret "$3")" "$(secret "$4")" >/dev/null
rc admin policy attach root "$1" --user "$(secret "$3")" >/dev/null
rc admin user info root "$(secret "$3")" --json | grep -q "\"$1\""
}
rc ready root --timeout 10 >/dev/null
rc bucket create --ignore-existing "root/$bucket" >/dev/null
rc bucket anonymous set private "root/$bucket" >/dev/null
sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json
rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null
rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null
rc admin policy attach root "$policy" --user "$api_key" >/dev/null
worker_key=$(secret s3_worker_access_key_id)
sed "s/messenger-attachments/$bucket/g" /s3/worker-policy.json > /tmp/worker-policy.json
rc admin policy create root messenger-worker /tmp/worker-policy.json >/dev/null
rc admin user add root "$worker_key" "$(secret s3_worker_secret_access_key)" >/dev/null
rc admin policy attach root messenger-worker --user "$worker_key" >/dev/null
grant "$policy" api-policy.json s3_access_key_id s3_secret_access_key
grant "$reconciler" reconciler-policy.json s3_reconciler_access_key_id s3_reconciler_secret_access_key
grant messenger-worker worker-policy.json s3_worker_access_key_id s3_worker_secret_access_key
rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null
rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"'
rc admin user info root "$api_key" --json | grep -q "\"$policy\""
rc admin user info root "$worker_key" --json | grep -q '"messenger-worker"'
echo "s3-init: bucket=$bucket anonymous=private policy=$policy worker=messenger-worker lifecycle=abort-incomplete-uploads"
echo "s3-init: bucket=$bucket anonymous=private policies=$policy,$reconciler lifecycle=abort-incomplete-uploads"
+17
View File
@@ -0,0 +1,17 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "MessengerReconcilerObjects",
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},
{
"Sid": "MessengerReconcilerListing",
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
@@ -20,7 +20,7 @@
- Несовместимость тестов ready/queued → адаптировать собственные тесты к текущему API, публичные не менять.
- Регрессия транзакций → проверить rollback, одновременные claim, fencing, retry и outbox существующими тестами.
- Дрейф legacy-схемы → отказ adoption вместо угадывания версии, тесты populated migration и повторного запуска.
- Сверка ограничена legacy-префиксом; pipeline владеет staging/originals/variants. Это ограничение передачи основы фиксируется для отдельного отчёта.
- Сверка ограничена legacy-префиксом; pipeline владеет staging/originals/variants. Это ограничение передачи основы фиксируется для отдельного отчёта. Унаследованный `reconcile_track.py demo` также рассчитан на синхронный upload ЛР4 (`201`) и legacy-ключи; он не подтверждает критерии ЛР5 и в этой фазе не запускается.
## Migration Plan
@@ -0,0 +1,7 @@
[
{
"severity": "minor",
"file": "scripts/reconcile_track.py:396",
"problem": "Унаследованный демонстратор сверки ЛР4 ожидает upload 201 и legacy-ключи attachments/. Для API ЛР5 (202 queued) не адаптирован; не использовать как доказательство ЛР5. Сама сверка legacy-объектов и её изоляция от pipeline проверены."
}
]
@@ -13,7 +13,7 @@
### New Capabilities
Нет.
- `lab4-storage-reconciler`: унаследованная ручная сверка объектов прежнего префикса `attachments/`.
### Modified Capabilities
@@ -19,7 +19,7 @@
- **WHEN** БД содержит проверенный SQL-журнал ЛР5, вложения, задания, outbox и варианты
- **THEN** переход сохраняет строки, ограничения, состояния и ссылки на объекты
#### Scenario: Изменённая миграция или схема
#### Scenario: Изменённая миграция
- **WHEN** контрольная сумма SQL-журнала или фактическая схема не совпадает с ожидаемой
- **THEN** принятие истории завершается ошибкой без изменения данных и отметки версии
@@ -0,0 +1,37 @@
## Purpose
Сохраняет совместимость перенесённой из ЛР4 сверки PostgreSQL и S3 с ранее записанными объектами и ограничивает её область, чтобы не вмешиваться в асинхронную обработку ЛР5. Основание — сохранение предыдущего решения при переносе; не является заявкой на грейд ЛР5.
## ADDED Requirements
### Requirement: Ручная сверка legacy-объектов
Одноразовый reconciler SHALL по умолчанию работать в dry-run над прежним префиксом `attachments/`, выдавая JSON со счётчиками расхождений. Apply SHALL сохранять свежие объекты и повторно проверять ссылку в БД перед удалением старого orphan. Одновременный второй проход SHALL завершаться с признаком skipped.
#### Scenario: Dry-run без изменения данных
- **WHEN** оператор запускает reconciler без apply для legacy-объектов
- **THEN** он возвращает JSON и не меняет строки или байты
#### Scenario: Повторная проверка защищает объект
- **WHEN** после листинга старого orphan появляется строка с его ключом
- **THEN** apply сохраняет объект
### Requirement: Карантин сохраняет данные
При подтверждённом расхождении готового legacy-вложения apply SHALL поместить метаданные в карантин без удаления строки и объекта. GET `/api/v1/attachments/{id}/content` SHALL возвращать 404 с ошибкой not_found для карантина, метаданные остаются доступны участнику. Снятие карантина SHALL требовать совпадения SHA-256 даже без deep.
#### Scenario: Возвращены другие байты той же длины
- **WHEN** объект в карантине заменён байтами прежнего размера, но другого SHA-256
- **THEN** карантин остаётся и скачивание не разрешается
### Requirement: Разделение со staging и pipeline
Перенесённая сверка SHALL NOT обходить или удалять объекты из `staging/`, `originals/` и `variants/`. Этими префиксами управляет pipeline ЛР5 и его существующая уборка staging.
#### Scenario: Готовое вложение ЛР5 вне legacy-префикса
- **WHEN** у нового вложения оригинал и варианты опубликованы pipeline
- **THEN** legacy-сверка не считает их orphan и не изменяет их
@@ -1,16 +1,16 @@
## 1. Перенос основы обязательного минимума
- [ ] 1.1 Перенести платформу и identity из immutable snapshot; сохранить границы зон.
- [ ] 1.2 Перенести messaging и собственные проверки идемпотентности; сохранить порт проверки failed-вложений.
- [ ] 1.3 Перенести attachments и перевести jobs/outbox/variants на SQLAlchemy, сохранив fencing и транзакции.
- [x] 1.1 Перенести платформу и identity из immutable snapshot; сохранить границы зон.
- [x] 1.2 Перенести messaging и собственные проверки идемпотентности; сохранить порт проверки failed-вложений.
- [x] 1.3 Перенести attachments и перевести jobs/outbox/variants на SQLAlchemy, сохранив fencing и транзакции.
## 2. Миграции и стенд
- [ ] 2.1 Перенести Alembic, добавить историю ЛР5 и проверяемое принятие legacy-схемы с данными.
- [ ] 2.2 Перенести зависимости, Docker/Compose, собственные скрипты и CI с сохранением worker/relay.
- [ ] 2.3 Адаптировать собственные тесты и проверить make check/lint/student-test.
- [x] 2.1 Перенести Alembic, добавить историю ЛР5 и проверяемое принятие legacy-схемы с данными.
- [x] 2.2 Перенести зависимости, Docker/Compose, собственные скрипты и CI с сохранением worker/relay.
- [x] 2.3 Адаптировать собственные тесты и проверить make check/lint/student-test.
## 3. Завершение
- [ ] 3.1 Проверить smoke через LB 18085 на своём стенде; остановить стенд без удаления volumes.
- [x] 3.1 Проверить smoke через LB 18085 на своём стенде; остановить стенд без удаления volumes.
- [ ] 3.2 Зафиксировать перенос кода, синхронизировать спецификации и архивировать change.
@@ -0,0 +1,21 @@
# Проверка переноса основы
Источник: immutable snapshot `d7478726a85791c21e065d58f610735a9e687433`, diff от `d2744bc`. Исходный HEAD ЛР5: `64d5c67`. Эта фаза не реализует и не подтверждает грейды 3/4/5.
| Команда | Фактический результат |
| --- | --- |
| `make lint` | ruff check/format, ty: PASS |
| `make check` | PASS: структура skeleton, ссылки и синтаксис Python |
| `make validate` | PASS: OpenAPI 3.1 и JSON Schema |
| `NO_PROXY=localhost,127.0.0.1,::1 make student-test PYTEST_ARGS='--tb=short'` | 494 passed, 1 skipped, 19 deselected; 141.49 s |
| `NO_PROXY=localhost,127.0.0.1,::1 .venv/bin/pytest student-tests/api/test_reconciler.py -k pipeline --tb=short` | 1 passed, 10 deselected; 2.49 s |
| `COMPOSE_PROJECT_NAME=mtusi2026lab5 LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 NO_PROXY=localhost,127.0.0.1,::1 make up` | Собран образ, API ×2, worker, relay и хранилища healthy; migrate и s3-init exit 0 |
| `NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab5 LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 make test BASE_URL=http://127.0.0.1:18085` | PASS: lab5 public API scenario |
| `COMPOSE_PROJECT_NAME=mtusi2026lab5 LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 make down` | Контейнеры и сети своего стенда остановлены/удалены; постоянные volumes сохранены |
| `git diff --exit-code 64d5c67 -- contracts tests README.md COURSE.md REPORT.md evidence AGENTS.md` | exit 0, защищённые файлы неизменны |
Миграционные тесты включают populated legacy-истории 1–6, сохранность jobs/outbox/job_events/variants, дрейф схемы и контрольных сумм, два одновременных migrate, rollback DDL и совпадение ORM metadata с БД. Тесты pipeline проверяют аренды, fencing, дубли, retry, replay и окна отказа. На существующем volume стенда получена версия `0006_attachment_quarantine`; сохранены ready/failed-вложения и done/failed-задания.
Пропущен только Linux-specific RLIMIT_AS на macOS. 19 stack-тестов не входят в штатный student-test; в этой фазе из них не заявляется результат. Перед успешным полным прогоном исправлены несовместимые с ЛР5 вспомогательные тесты: DML без возвращаемых строк и ожидание 201 вместо 202. Первый RustFS-прогон без NO_PROXY не прошёл readiness; с исключением localhost из прокси полный прогон прошёл.
Ограничение для отдельного REPORT: findings.json, severity minor — старый демонстратор сверки рассчитан на ЛР4. Legacy-сверка намеренно ограничена attachments/; staging/originals/variants обслуживает pipeline ЛР5. REPORT и evidence этой фазой не менялись. Публикации, Telegram, нагрузочных измерений и фоновых агентов не было.
+3 -3
View File
@@ -40,7 +40,7 @@ context: |
Python 3.14 (аннотации вычисляются лениво по PEP 649, `TYPE_CHECKING` и
`from __future__ import annotations` не используются), FastAPI, gunicorn с несколькими
`UvicornWorker` (пакет `uvicorn-worker`), dishka (DI, интеграции FastAPI и FastStream),
pydantic v2 + pydantic-settings, structlog, asyncpg (параметризованный SQL без ORM),
pydantic v2 + pydantic-settings, structlog, SQLAlchemy 2 async (драйвер asyncpg), Alembic,
redis-py (`redis.asyncio`) для Valkey, argon2-cffi, aiobotocore (S3 path-style, SigV4,
таймауты, checksum `when_required`), Pillow (проверка в API, варианты в worker), FastStream
0.7 с `faststream[rabbit]` поверх aio-pika (consumer worker с `AckPolicy.MANUAL` и prefetch,
@@ -68,7 +68,7 @@ context: |
main.py сборка HTTP-приложения: собирает вклады зон, связывает зоны портами
relay.py сборка процесса relay: outbox → брокер, возврат просроченных аренд
worker.py сборка процесса worker: FastStream-подписчик очереди заданий
migrate.py одноразовая точка входа: применяет SQL-миграции под advisory lock
alembic/ одноразовые миграции под advisory lock с проверкой legacy-схемы
identity/ пользователи, пароли и сессии
messaging/ чаты, история и идемпотентная отправка, ссылки сообщения на вложения
attachments/ вложения: приём и staging, задания и outbox, обработка и варианты,
@@ -159,7 +159,7 @@ operations:
- "Логирование — только через порт `Logger`; второй логгер в приложении не заводится. Object key, ключи S3 и содержимое файлов в журнал не пишутся."
- "Код приложения — в `src/`, конфигурация LB, нагрузки и S3 — в `infra/`, собственные тесты — в `student-tests/`, обезличенные доказательства — в `evidence/`."
- "Перед коммитом прогоняй `make lint`, `make student-test` и `make check`."
- "SQL только параметризованный (`$1`, `$2`); значения из запроса никогда не склеиваются в текст запроса."
- "SQL через SQLAlchemy ORM/Core с параметрами; миграции Alembic; значения запроса не склеиваются в SQL."
- "Тело загрузки читается потоком со счётчиком до лимита; изображение не декодируется целиком до проверки размера по заголовку; тяжёлая проверка идёт в потоке под ограничением параллелизма."
- "API не обрабатывает изображения синхронно и не подключается к брокеру: задание уходит только через outbox; `202` возвращает снимок записанного объекта."
- "Worker и relay не объявляют очереди и exchange: топология и права — в `infra/rabbitmq/`; `ack` — строго после `COMMIT`, сообщение брокера — ровно поля `jobs.schema.json`."
+10 -3
View File
@@ -13,6 +13,8 @@ dependencies = [
"gunicorn>=26.2.0",
"orjson>=3.12.0",
"asyncpg>=0.32.0",
"sqlalchemy[asyncio]==2.0.45",
"alembic==1.16.5",
"argon2-cffi>=25.1.0",
"uvicorn-worker>=0.4.0",
"redis>=8.1.0",
@@ -29,6 +31,7 @@ dev = [
"httpx>=0.28.0",
"ruff>=0.14.0",
"ty>=0.0.1a21",
"openapi-spec-validator==0.7.2",
]
[build-system]
@@ -82,6 +85,7 @@ ignore = [
[tool.ruff.lint.per-file-ignores]
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "ARG002", "T20", "TID251"]
"tools/*.py" = ["T20"]
"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"]
"scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"]
"scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"]
@@ -91,10 +95,10 @@ ignore = [
"src/messenger/*/entrypoints/**" = ["TID251", "N818", "PLR0917", "S105"]
"src/messenger/shared/config.py" = ["ARG002", "ARG003"]
"src/messenger/main.py" = ["TID251"]
"src/messenger/migrate.py" = ["TID251"]
"src/messenger/relay.py" = ["TID251"]
"src/messenger/worker.py" = ["TID251"]
"src/messenger/admin.py" = ["TID251"]
"alembic/**" = ["TID251"]
"src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"]
[tool.ruff.lint.flake8-tidy-imports.banned-api]
@@ -107,7 +111,10 @@ ignore = [
known-first-party = ["messenger"]
[tool.ty.src]
include = ["src", "student-tests"]
include = ["src", "student-tests", "alembic"]
[tool.ty.environment]
extra-paths = ["alembic"]
[tool.ty.rules]
unresolved-import = "error"
@@ -118,7 +125,7 @@ possibly-missing-attribute = "warn"
testpaths = ["student-tests"]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
addopts = "-q --strict-markers -m 'not stack'"
addopts = "--strict-markers -m 'not stack'"
markers = [
"stack: black-box проверки против поднятого compose-стенда (make test-faults)",
]
+73
View File
@@ -0,0 +1,73 @@
import argparse
import json
import os
import shlex
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SETTINGS = (
"COMPOSE_PROJECT_NAME",
"COMPOSE_FILE",
"LB_HTTP_PORT",
"LB_STATS_PORT",
"BASE_URL",
"EVIDENCE_DIR",
"RECONCILE_EVIDENCE_DIR",
"NO_PROXY",
"S3_CPUS",
"S3_MEMORY",
"UPLOAD_GRACE_SECONDS",
"API_IMAGE",
"API_REPLICAS",
"WEB_CONCURRENCY",
"MALLOC_MMAP_THRESHOLD_",
)
def main():
parser = argparse.ArgumentParser(description="Capture a command without overwriting evidence")
parser.add_argument("output", type=Path)
parser.add_argument("command", nargs=argparse.REMAINDER)
args = parser.parse_args()
command = args.command[1:] if args.command[:1] == ["--"] else args.command
if not command:
parser.error("command is required")
args.output.parent.mkdir(parents=True, exist_ok=True)
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
started = time.monotonic()
with args.output.open("x") as output:
def write(line):
line = line.replace(str(ROOT), "<repo>")
output.write(line)
output.flush()
sys.stdout.write(line)
sys.stdout.flush()
write(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit: {commit}\n")
write(
"environment: "
+ json.dumps({k: os.environ[k] for k in SETTINGS if k in os.environ})
+ "\n"
)
write("$ " + shlex.join(command) + "\n")
with subprocess.Popen(
command,
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
errors="replace",
) as process:
for line in process.stdout:
write(line)
code = process.wait()
write(f"exit_code: {code} elapsed_seconds: {time.monotonic() - started:.3f}\n")
return code
if __name__ == "__main__":
sys.exit(main())
+53
View File
@@ -0,0 +1,53 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
repo=$PWD
project="hlpipe-lab5-clean-$(date -u +%Y%m%d%H%M%S)"
dir="$repo/.verification/$project"
out="$repo/${EVIDENCE_DIR:-evidence}/clean-clone.txt"
sha=$(git rev-parse HEAD)
[ ! -e "$out" ] || { echo "Choose a new EVIDENCE_DIR" >&2; exit 2; }
mkdir -p "$(dirname "$dir")" "$(dirname "$out")"
exec 3>&1
exec > "$out" 2>&1
cleanup() {
status=$?
trap - EXIT HUP INT TERM
if [ -d "$dir/.git" ]; then
cd "$dir"
docker compose --profile bench --profile tools --profile reconcile down || status=1
docker volume ls --filter "label=com.docker.compose.project=$project"
fi
echo "exit_code: $status; volumes retained"
sed "s#$repo#<repo>#g" "$out" >&3
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' HUP INT TERM
step() { echo "\$ $*"; "$@"; }
export COMPOSE_PROJECT_NAME="$project" COMPOSE_FILE=compose.yaml
export LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 BASE_URL=http://127.0.0.1:18085
export NO_PROXY=localhost,127.0.0.1,::1
unset API_IMAGE
echo "date: $(date -u +%FT%TZ) commit: $sha project: $project"
echo "HTTP=$LB_HTTP_PORT stats=$LB_STATS_PORT"
step git clone --quiet --no-hardlinks "$repo" "$dir"
cd "$dir"
step git checkout --quiet --detach "$sha"
step make secrets
step docker compose --env-file .env config --quiet
step make up
step docker compose ps -a --format '{{.Service}} {{.State}} {{.Status}} {{.Ports}}'
step make test
step make test-persistence ACTION_SCRIPT=scripts/restart.sh
step make test-failover ACTION_SCRIPT=scripts/stop-one.sh
step docker compose ps -a --format '{{.Service}} {{.Name}} {{.State}}' api
step make restore
step make scenario SCENARIO=failover-upload
step make restore
step make scenario SCENARIO=s3-outage
step sh scripts/s3-check.sh
step make reconcile
step make migrate
step make down
step docker volume ls --filter "label=com.docker.compose.project=$project"
+1 -1
View File
@@ -8,7 +8,7 @@ replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
detect() { python3 scripts/probe.py detect "$@"; }
failed=0
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
-- docker stop -t "${STOP_TIMEOUT:-7}" "$(replica)" || failed=1
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
-- docker kill "$(replica)" || failed=1
hung=$(replica)
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
out=${1:?usage: docker-stats.sh <csv>}
vm=$(dirname "$out")/vm_stat.csv
cd "$(dirname "$0")/.."
project=$(docker compose config | sed -n 's/^name: //p')
echo 'ts,name,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids' > "$out"
echo 'ts,busy,total,cpus' > "$vm"
while :; do
ts=$(date +%s)
docker compose exec -T valkey cat /proc/stat 2>/dev/null | awk -v ts="$ts" '/^cpu /{b=$2+$3+$4+$7+$8+$9; t=b+$5+$6} /^cpu[0-9]/{n++} END{if (t) print ts "," b "," t "," n}' >> "$vm" || true
ids=$(docker ps -q --filter "label=com.docker.compose.project=$project")
[ -z "$ids" ] || docker stats --no-stream --format "$ts,{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true
sleep "${INTERVAL:-1}"
done
+5 -3
View File
@@ -1,12 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; }
value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; }
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
bucket=${S3_BUCKET:-messenger-attachments}
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'"
docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}'
for table in attachments message_attachments; do
@@ -36,5 +38,5 @@ bucket=${S3_BUCKET:-messenger-attachments}
docker compose config --format json | python3 -c 'import json,sys
for name, service in sorted(json.load(sys.stdin)["services"].items()):
print(name, sorted(s["source"] for s in service.get("secrets", [])))'
} > evidence/storage-layout.txt
cat evidence/storage-layout.txt
} > "$out"/storage-layout.txt
cat "$out"/storage-layout.txt
+19 -8
View File
@@ -11,14 +11,25 @@ cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
cli dbsize
cli "info stats" | grep -E '^keyspace_(hits|misses)'
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
hash=${key#session:}
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
echo "\$ количество отметок отзыва и закэшированных сессий"
revoked=0; cached=0
echo "\$ valkey-cli --scan --pattern 'session:*' и вид значения (хеш сокращён, значения не печатаются)"
revoked=0; cached=0; key=""; stored=""
for k in $(cli "--scan --pattern session:* --count 1000"); do
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
if [ "$(cli "get $k")" = "-" ]; then
revoked=$((revoked + 1)); printf '%.16s… отметка отзыва "-"\n' "$k"; continue
fi
cached=$((cached + 1)); printf '%.16s… сессия\n' "$k"
if [ -z "$key" ]; then
stored=$(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('${k#session:}', 'hex')")
if [ -n "$stored" ]; then key=$k; fi
fi
done
echo "tombstones: $revoked, cached sessions: $cached"
if [ "$revoked" -eq 0 ] || [ -z "$key" ]; then
echo "нужны закэшированная сессия из БД и отметка отзыва: сначала make test" >&2
exit 1
fi
expires=$(cli "pexpiretime $key")
echo "\$ срок ключа кэша против срока сессии для $(printf '%.16s' "$key")…"
echo "valkey pexpiretime, ms: $expires"
echo "postgres expires_at, ms: $stored"
if [ "$expires" -le "$stored" ]; then echo "ключ не переживает сессию: да"; else echo "ключ не переживает сессию: НЕТ"; exit 1; fi
+7 -5
View File
@@ -1,9 +1,11 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
base=${1:-http://$(docker compose port lb 8080)}
stats=http://$(docker compose port lb 8404)/stats\;csv
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
{
header
echo "\$ docker compose --env-file .env config --quiet && echo valid"
@@ -25,7 +27,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
} > evidence/stack.txt
} > "$out"/stack.txt
{
header
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
@@ -38,7 +40,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)"
echo "packages (uv.lock):"
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
for name in fastapi uvicorn uvicorn-worker gunicorn sqlalchemy alembic asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
faststream aio-pika aiormq dishka-faststream; do
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
done
@@ -47,5 +49,5 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
done | sort -u
} > evidence/environment.txt
cat evidence/stack.txt evidence/environment.txt
} > "$out"/environment.txt
cat "$out"/stack.txt "$out"/environment.txt
+16 -13
View File
@@ -1,20 +1,22 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM alembic_version) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, username = 'seed_author' AS seed_stub, count(*) AS users, min(length(password_hash)) AS min_length, max(length(password_hash)) AS max_length FROM users GROUP BY 1, 2 ORDER BY 2"
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
{
header
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments alembic_version; do
echo "\$ psql -c '\\d+ $table'"
psql "\\d+ $table"
done
echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'"
psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations"
} > evidence/schema.txt
echo "\$ psql -c 'SELECT version_num FROM alembic_version'"
psql "SELECT version_num FROM alembic_version"
} > "$out"/schema.txt
{
header
echo "\$ psql -c \"$counts\""
@@ -23,14 +25,14 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
docker compose run --rm migrate 2>/dev/null
echo "\$ psql -c \"$counts\""
psql "$counts"
} > evidence/migrations.txt
} > "$out"/migrations.txt
{
header
echo "\$ psql -c \"$kdf\""
psql "$kdf"
echo "\$ psql -c \"$tokens\""
psql "$tokens"
} > evidence/kdf.txt
} > "$out"/kdf.txt
{
header
echo "\$ git ls-files secrets .env backups volumes"
@@ -38,7 +40,8 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
echo "\$ docker volume ls --filter name=mtusi2026lab5"
docker volume ls --filter name=mtusi2026lab5
} > evidence/secrets.txt
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
project=$(docker compose config --format json | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])')
echo "\$ docker volume ls --filter label=com.docker.compose.project=$project"
docker volume ls --filter "label=com.docker.compose.project=$project"
} > "$out"/secrets.txt
echo ""$out"/{schema,migrations,kdf,secrets}.txt обновлены"
+1 -1
View File
@@ -10,7 +10,7 @@ failed=0
set --
for host in $hosts; do
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
-- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1
scripts/restore.sh
set -- "$@" "$out/$host/result.json"
done
+11 -6
View File
@@ -1,28 +1,33 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
replicas() {
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
}
tmp=evidence/failover.tmp
url=${1:-http://localhost:8080}
shift=0
while [ "$shift" -lt "${2:-0}" ]; do curl -fsS -o /dev/null "$url/health/live"; shift=$((shift + 1)); done
tmp="$out"/failover.tmp
since=$(date -u +%FT%TZ)
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
replicas
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$url"
echo "== after: api replicas"
replicas
} > "$tmp" 2>&1
stopped=$(replicas | awk '$2 == "exited" {print $1}')
{
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|service.draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
echo "\$ scripts/restore.sh"
scripts/restore.sh
echo "== after restore"
replicas
} >> "$tmp" 2>&1
mv "$tmp" "evidence/failover-$stopped.txt"
cat "evidence/failover-$stopped.txt"
mv "$tmp" "$out/failover-$stopped.txt"
cat "$out/failover-$stopped.txt"
+34 -2
View File
@@ -23,6 +23,19 @@ def compose(*args, stdin=None):
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
def containers(service):
ids = compose("ps", "-aq", service).stdout.split()
fields = ("hostname", "status", "exit_code", "started_at", "finished_at")
template = (
"{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}"
" {{.State.StartedAt}} {{.State.FinishedAt}}"
)
out = subprocess.run(
["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False
)
return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()]
def upload(c, actor, chat, expected=202):
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
schema = "Attachment" if expected == 202 else None
@@ -90,6 +103,10 @@ def failover_upload(c, args):
recovered = time.monotonic() - started
served = got["headers"].get("X-Instance-Id")
check(served != target, "download served by the stopped replica")
during = containers(args.service)
states = {item["hostname"]: item["status"] for item in during}
check(states.get(target) == "exited", f"target {target} is not exited: {states}")
check(states.get(served) == "running", f"neighbour {served} is not running: {states}")
digest = hashlib.sha256(got["bytes"]).hexdigest()
check(
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
@@ -104,7 +121,9 @@ def failover_upload(c, args):
"sha256_equal": True,
"hook_seconds": round(hook_seconds, 3),
"download_ok_after_seconds": round(recovered, 3),
"replicas_at_download": during,
"restored": args.restore,
"replicas_after_restore": containers(args.service) if args.restore else None,
}
@@ -124,6 +143,7 @@ def s3_outage(c, args):
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
}
stopped = containers(args.s3_service)
finally:
started = time.monotonic()
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
@@ -131,8 +151,11 @@ def s3_outage(c, args):
got = wait(lambda: content(c, bob, att), 30)
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
expected = {"upload": 503, "download": 503, "text_message": 201}
check(all(during[k]["status"] == v for k, v in expected.items()), f"S3 outage: {during}")
check(all(during[k]["seconds"] < 6 for k in ("upload", "download")), "S3 timeout exceeded")
return {
"scenario": "s3-outage",
"s3_container_during_outage": stopped,
"during_outage": during,
"expected": expected,
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
@@ -180,6 +203,8 @@ def abort_upload(c, args):
)
rows = out.stdout.split()
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
check(out.returncode == 0, "cannot inspect aborted upload")
check(counts.get("ready", 0) == 0, "aborted upload became ready")
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
return {
"scenario": "abort-upload",
@@ -216,10 +241,17 @@ def main():
"s3-outage": s3_outage,
"abort-upload": abort_upload,
}[args.scenario]
commit = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, check=False
)
stamp = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"commit": commit.stdout.decode().strip(),
}
try:
report = {"ok": True, **run(c, args)}
report = {"ok": True, **stamp, **run(c, args)}
except Failure as exc:
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
out=${1:?usage: lb-stats.sh <csv>}
cd "$(dirname "$0")/.."
url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv}
curl -fsS "$url" | sed -n '1s/^# /ts,/p' > "$out"
while :; do
ts=$(date +%s)
curl -fsS "$url" | grep -E '^(public|api),' | sed "s/^/$ts,/" >> "$out" || true
sleep "${INTERVAL:-1}"
done
+29 -5
View File
@@ -9,12 +9,20 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CLIENT_SECRETS = [
"s3_access_key_id",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
"s3_worker_access_key_id",
"s3_worker_secret_access_key",
"s3_root_password",
"s3_root_user",
"s3_secret_access_key",
"s3_worker_access_key_id",
"s3_worker_secret_access_key",
]
RECONCILER_SECRETS = [
"database_url",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
]
OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli")
POLICY = {
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
"api": {
@@ -29,6 +37,11 @@ POLICY = {
"networks": ["backend"],
},
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
"reconciler": {
"secrets": RECONCILER_SECRETS,
"volumes": [],
"networks": ["backend"],
},
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
@@ -94,6 +107,9 @@ def declared(name, service, networks, problems):
problems.append(f"{name}: publishes {ports} beyond loopback")
if name == "api" and not service.get("read_only"):
problems.append("api: root filesystem is writable")
user = str(service.get("user", ""))
if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"):
problems.append(f"{name}: runs as root")
internal = [n for n in nets if networks.get(n, {}).get("internal")]
return [
name,
@@ -103,6 +119,7 @@ def declared(name, service, networks, problems):
",".join(volumes) or "-",
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
str(service.get("read_only", False)).lower(),
user or "image",
]
@@ -149,6 +166,8 @@ def main():
"bench",
"--profile",
"tools",
"--profile",
"reconcile",
"config",
"--format",
"json",
@@ -162,9 +181,14 @@ def main():
if not any(row[0] == "api" for row in containers):
problems.append("no running api replicas to inspect")
lines = [
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
"$ docker compose --profile bench --profile tools config --format json (declared)",
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} "
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
" (declared)",
*table(
["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"],
services,
),
"",
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
*table(["service", "hostname", "cpus", "memory", "published"], containers),
+62
View File
@@ -0,0 +1,62 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
runs="${RUNS:-3}"
pause="${PAUSE:-15}"
project=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$(docker compose ps -q lb)")
network="${project}_edge"
image=$(sed -n 's/^FROM \(python:[^ ]*\) AS runtime$/\1/p' Dockerfile)
name="$project-load-client"
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out/load"
[ -z "$(ls -A "$out/load")" ] || { echo "Choose an empty EVIDENCE_DIR/load" >&2; exit 2; }
client() {
docker run --rm --name "$name" --network "$network" --cpus 2 --memory 512m \
--sysctl net.ipv4.tcp_tw_reuse=1 -v "$PWD/tests:/w/tests:ro" -v "$PWD/contracts:/w/contracts:ro" \
-v "$PWD/lab.json:/w/lab.json:ro" -v "$PWD/$out/load:/out" "$image" python /w/tests/load.py "$@"
}
handled() {
docker compose logs --no-log-prefix --since "$1" api | python3 -c 'import collections, json, sys
seen = collections.Counter()
for line in sys.stdin:
if line.startswith("{"):
event = json.loads(line)
if event.get("event") == "request.handled":
seen[(event["status"], event["method"], event["route"])] += 1
for (status, method, route), count in sorted(seen.items()):
print(status, method, route, count)'
}
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) network: $network"
echo "# make load-evidence (scripts/load-evidence.sh): закрытый контур tests/load.py, 64 потока, 8 с, все записи"
echo "# прогона в один новый чат, $runs повтора с паузой $pause с; клиент — одноразовый контейнер в сети <проект>_edge"
echo "# стенда (2 CPU, 512 MiB, базовый образ API по digest, tests/, contracts/ и lab.json только для чтения,"
echo "# tcp_tw_reuse=1), адрес http://lb:8080. На 4-й секунде — docker stats; после прогона — request.handled реплик."
echo "\$ docker ps --format '{{.Names}}' | grep -v '^$project-' # чужие контейнеры на VM во время серии"
docker ps --format '{{.Names}}' | grep -v "^$project-" || true
echo "\$ client() { docker run --rm --name $name --network $network --cpus 2 --memory 512m --sysctl net.ipv4.tcp_tw_reuse=1 -v \"\$PWD/tests:/w/tests:ro\" -v \"\$PWD/contracts:/w/contracts:ro\" -v \"\$PWD/lab.json:/w/lab.json:ro\" -v \"\$PWD/$out/load:/out\" $image python /w/tests/load.py \"\$@\"; }"
run=1
while [ "$run" -le "$runs" ]; do
since=$(date -u +%FT%TZ)
echo "\$ client --base-url http://lb:8080 --duration 8 --concurrency 64 --output /out/run$run.json"
client --base-url http://lb:8080 --duration 8 --concurrency 64 --output "/out/run$run.json" > "$work/out" 2>&1 &
pid=$!
sleep 4
stats=$(docker stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}}' $(docker compose ps -q) "$name" | sed "s/^$project-//" | sort)
if wait "$pid"; then code=0; else code=$?; fi
echo "exit $code"
cat "$work/out"
echo "\$ docker stats --no-stream # 4-я секунда прогона: контейнер CPU% память"
echo "$stats"
echo "\$ docker compose logs --since $since api | request.handled: статус метод маршрут число"
handled "$since"
[ "$code" -eq 0 ] || { exit "$code"; }
[ "$run" -lt "$runs" ] && sleep "$pause"
run=$((run + 1))
done
echo "\$ docker compose exec -T postgres psql -c 'SELECT count(*) FROM messages'"
docker compose exec -T postgres psql -U messenger -d messenger -qAt -c 'SELECT count(*) FROM messages'
} > "$out/load.txt"
cat "$out/load.txt"
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
limit="${WAIT_SECONDS:-1500}"
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
sql() { docker compose exec -T postgres psql -U messenger -d messenger -X "$@"; }
pending() { sql -tAc "SELECT count(*) FROM attachments WHERE status = 'pending'"; }
statuses="SELECT status, count(*), min(created_at), max(created_at) FROM attachments GROUP BY status ORDER BY status"
since=$(date -u +%FT%TZ)
{
echo "date: $since commit: $(git rev-parse HEAD)"
echo "# make overload-leftovers (scripts/overload-leftovers.sh) после серии памяти: невидимые намерения от 503 по"
echo "# таймауту видны сверке свежими, уборка API удаляет их после UPLOAD_GRACE_SECONDS; стенд тот же"
echo "\$ make reconcile # dry-run"
docker compose run --rm --no-deps -T reconciler 2>/dev/null
echo "\$ psql -c \"$statuses\""
sql -c "$statuses"
echo "\$ until [ \"\$(psql -tAc \"SELECT count(*) FROM attachments WHERE status = 'pending'\")\" = 0 ]; do sleep 10; done"
started=$(date +%s)
until [ "$(pending)" = 0 ] || [ $(( $(date +%s) - started )) -gt "$limit" ]; do sleep 10; done
echo "pending = $(pending) в $(date -u +%FT%TZ)"
[ "$(pending)" = 0 ] || { echo "FAIL: pending cleanup timed out"; exit 1; }
echo "\$ docker compose logs --since $since api | audit.attachment_discarded и attachments.swept"
docker compose logs --no-log-prefix --since "$since" api | python3 -c 'import collections, json, sys
reasons, swept, times = collections.Counter(), 0, []
for line in sys.stdin:
if not line.startswith("{"):
continue
event = json.loads(line)
if event.get("event") == "audit.attachment_discarded":
reasons[event["reason"]] += 1
times.append(event["ts"][:19] + "Z")
elif event.get("event") == "attachments.swept":
swept += event["count"]
print(dict(reasons), "swept", swept, "first", min(times, default="-"), "last", max(times, default="-"))'
echo "\$ psql -c \"$statuses\""
sql -c "$statuses"
echo "\$ make reconcile # dry-run после уборки"
docker compose run --rm --no-deps -T reconciler 2>/dev/null
} > "$out/overload-leftovers.txt"
cat "$out/overload-leftovers.txt"
+1 -1
View File
@@ -6,7 +6,7 @@ started() {
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
done | sort
}
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
echo "== before: service hostname StartedAt status"
started
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
mode=${1:?usage: pg-stats.sh activity <csv> | statements <csv> | size | reset}
cd "$(dirname "$0")/.."
psql() {
docker compose exec -T postgres sh -c 'exec psql -X -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "${POSTGRES_DB:-$POSTGRES_USER}" "$@"' psql "$@"
}
case $mode in
reset)
psql -c 'set client_min_messages = warning' -c 'create extension if not exists pg_stat_statements' -c 'select pg_stat_statements_reset()' >/dev/null
;;
size)
psql -At -F, -c "select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
;;
statements)
psql -c "copy (select calls, round(total_exec_time::numeric, 3) as total_ms, round(mean_exec_time::numeric, 3) as mean_ms, rows, shared_blks_hit, shared_blks_read, left(regexp_replace(query, '\s+', ' ', 'g'), 200) as query from pg_stat_statements where dbid = (select oid from pg_database where datname = current_database()) and query not like '%pg_stat%' order by total_exec_time desc limit ${TOP:-20}) to stdout with csv header" > "${2:?csv}"
;;
activity)
echo 'ts,state,wait_event_type,wait_event,backends,max_connections' > "${2:?csv}"
while :; do
ts=$(date +%s)
psql -At -F, -c "select coalesce(state, ''), coalesce(wait_event_type, ''), coalesce(wait_event, ''), count(*), current_setting('max_connections') from pg_stat_activity where backend_type = 'client backend' and pid <> pg_backend_pid() group by 1, 2, 3" | sed "s/^/$ts,/" >> "$2" || true
sleep "${INTERVAL:-1}"
done
;;
*)
echo "unknown mode $mode" >&2
exit 2
;;
esac
+163 -7
View File
@@ -10,12 +10,16 @@ import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from datetime import datetime
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
PSQL += ["-X", "-q", "-At"]
@functools.cache
@@ -207,16 +211,112 @@ def check(args):
return 0 if passed else 1
def output(*command):
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return done.stdout + done.stderr
def replicas():
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
def answered(container, request_id):
for line in output("docker", "logs", container).splitlines():
if request_id in line and "request.handled" in line:
return json.loads(line).get("status")
return None
def lost(alice, bob, chat, kill, args):
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
request_id = "lost-" + key[:12]
lock = subprocess.Popen(
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
)
lock.stdin.write(
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
)
lock.stdin.flush()
lock.stdout.readline()
sent = time.time()
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
path = f"/api/v1/chats/{chat}/messages"
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
waiting = set(output(*PSQL, "-c", sql).split())
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
if kill and holder[0]:
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
lock.stdin.close()
lock.wait()
retried = time.time()
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
while status != 201 and attempts < 20:
time.sleep(0.2)
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
copies = [m for m in page["items"] if m["text"] == text]
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
report = {
"variant": "kill" if kill else "timeout",
"lost_attempt": {
"status": first,
"client_timeout_s": args.timeout,
"chat_lock_held_s": args.hold,
"replica": holder[1],
"replica_killed": bool(kill and holder[0]),
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
},
"retry": {
"after_s": round(retried - sent, 3),
"status": status,
"attempts": attempts,
"replica": instance,
},
"copies_in_history": len(copies),
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
"written_by": None
if stamp is None
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
}
if kill and holder[0]:
subprocess.call(
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
report["pass"] = (
first == 0
and holder[0] is not None
and status == 201
and report["copies_in_history"] == 1
and report["same_message"]
and (kill or report["lost_attempt"]["replica_answered"] == 201)
)
return report
def retry(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
passed = all(v["pass"] for v in variants)
emit({"variants": variants, "pass": passed}, args.out)
return 0 if passed else 1
def record(stop, alice, chat):
probes, trace = [], []
probes, trace, confirmed = [], [], []
def prober():
key = uuid.uuid4().hex
while not stop.is_set():
started = time.time()
status, instance, _ = send(alice, chat, "probe " + key, key)
status, instance, body = send(alice, chat, "probe " + key, key)
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
if status == 201:
confirmed.append(body)
key = uuid.uuid4().hex
time.sleep(0.1)
@@ -228,7 +328,38 @@ def record(stop, alice, chat):
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
for thread in threads:
thread.start()
return probes, trace, threads
return probes, trace, threads, confirmed
def history_check(reader, chat, confirmed):
items, cursors = [], set()
cursor = None
for _ in range(len(confirmed) // 100 + 2):
path = f"/api/v1/chats/{chat}/messages?limit=100"
if cursor:
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
status, _, page = call("GET", path, headers=reader["auth"])
if status != 200 or not isinstance(page, dict):
break
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"])
return {
"confirmed": len(confirmed),
"stored": len(items),
"exact_match": bool(confirmed) and same,
"unique": len({m["id"] for m in items}) == len(items),
}
if cursor in cursors:
break
cursors.add(cursor)
return {
"confirmed": len(confirmed),
"stored": len(items),
"exact_match": False,
"unique": False,
}
def write_csv(path, header, rows):
@@ -241,6 +372,10 @@ def summarize(args, probes, trace, action, expected):
t0, end = action["start"], action["end"]
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0]
recovery = (
next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0
)
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
outage = [t for t in down if not back or t < back[0]] or [end]
@@ -256,13 +391,15 @@ def summarize(args, probes, trace, action, expected):
"action_finished_before_load_end": end < action["load_end"],
"lb_up_before": expected,
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
"lb_samples_missing": sum(up is None for _, up in trace),
"detection_seconds": round(down[0] - t0, 3) if down else None,
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
"probe": {
"attempts": len(probes),
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
"first_failure_s": round(failures[0], 3) if failures else None,
"recovery_s": round(failures[-1], 3) if failures else 0.0,
"first_success_s": round(successes[0][1], 3) if successes else None,
"recovery_s": round(recovery, 3) if recovery is not None else None,
"instances_before": dict(before),
"instances_during_outage": dict(during),
"instances_after": dict(after),
@@ -275,7 +412,13 @@ def verdict(args, result):
checks = {
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
"load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0,
"lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0,
"write_recovery_ok": all(
probe[k] is not None and probe[k] <= args.max_recovery
for k in ("first_success_s", "recovery_s")
),
"history_preserved": result["history"]["exact_match"] and result["history"]["unique"],
}
if args.expect == "dip":
checks["detected"] = result["detection_seconds"] is not None
@@ -296,7 +439,8 @@ def load(args):
expected = lb_up() or 0
alice, bob = actor("alice"), actor("bob")
stop, action = threading.Event(), {}
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
chat = chat_of(alice, bob)
probes, trace, threads, confirmed = record(stop, alice, chat)
def act():
time.sleep(args.warmup + args.lead)
@@ -327,6 +471,7 @@ def load(args):
)
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
result = summarize(args, probes, trace, action, expected)
result["history"] = history_check(bob, chat, confirmed)
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
result["k6"] = dict(
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
@@ -344,10 +489,13 @@ def detect(args):
started = time.time()
process = subprocess.Popen(args.command, cwd=str(ROOT))
detected = acted = None
missing = 0
while time.time() - started < args.timeout and (detected is None or acted is None):
if acted is None and process.poll() is not None:
acted = time.time() - started
if detected is None and (lb_up() or 0) <= until:
up = lb_up()
missing += up is None
if detected is None and up is not None and up <= until:
detected = time.time() - started
time.sleep(0.05)
code = process.wait()
@@ -362,6 +510,7 @@ def detect(args):
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
checks = {
"action_ok": code == 0,
"lb_observed": before >= 2 and missing == 0,
"detected_in_time": detected is not None and detected <= args.max + args.slack,
"lb_answer_ok": status == args.lb_status,
}
@@ -374,6 +523,7 @@ def detect(args):
"action_exit": code,
"action_seconds": round(acted, 3),
"lb_up_before": before,
"lb_samples_missing": missing,
"lb_up_counted_as_detected": until,
"detection_seconds": None if detected is None else round(detected, 3),
"declared_max_seconds": args.max,
@@ -413,6 +563,10 @@ def main():
under.add_argument("--max-failed-ratio", type=float, default=0.01)
under.add_argument("--max-recovery", type=float, default=10.0)
under.add_argument("command", nargs=argparse.REMAINDER)
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
again.add_argument("--out", help="JSON file, written only after a complete report")
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
@@ -426,6 +580,8 @@ def main():
args = parser.parse_args()
if args.mode == "check":
return check(args)
if args.mode == "retry":
return retry(args)
args.command = [c for c in args.command if c != "--"]
if not args.command:
parser.error(f"{args.mode} needs an action command after --")
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
kind="${1:?usage: reconcile-inject.sh orphan|missing|corrupt|bitflip|stale-pending|pending-object|multipart [count]}"
count="${2:-1}"
age="${INJECT_AGE_SECONDS:-7200}"
prefix="${S3_PREFIX:-attachments/}"
sql() { docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -qAt -F ' ' -U "${PGUSER:-messenger}" -d "${PGDATABASE:-messenger}" -c "$1"; }
s3() { docker compose run --rm --no-deps -T -e ROWS="$1" s3-cli sh -c "$2"; }
ready() { sql "SELECT id, object_key, size_bytes FROM attachments WHERE status = 'ready' ${INJECT_IDS:+AND id = ANY('{$INJECT_IDS}'::uuid[])} ORDER BY random() LIMIT $count"; }
pending() {
sql "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at)
SELECT g.id, s.chat_id, s.owner_id, 'injected.png', 'image/png', 2048, repeat('0', 64), 1, 1, '$prefix' || g.id, 'pending',
clock_timestamp() - make_interval(secs => $age), clock_timestamp() - make_interval(secs => $age)
FROM (SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1) s,
(SELECT gen_random_uuid() AS id FROM generate_series(1, $count)) g
RETURNING id, object_key"
}
fresh() { sql "SELECT g, '$prefix' || g FROM (SELECT gen_random_uuid() AS g FROM generate_series(1, $count)) s"; }
put_random='printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; head -c "${size:-2048}" /dev/urandom | rc pipe "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done'
case "$kind" in
orphan) rows=$(fresh); s3 "$rows" "$put_random" | sed "s/^/orphan /" ;;
missing) rows=$(ready); s3 "$rows" 'printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; rc object remove --force "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' | sed "s/^/missing /" ;;
corrupt) rows=$(ready | awk '{print $1, $2, int($3 / 2) + 1}'); s3 "$rows" "$put_random" | sed "s/^/corrupt /" ;;
bitflip) rows=$(ready); s3 "$rows" "$put_random" | sed "s/^/bitflip /" ;;
stale-pending) pending | sed "s/^/stale-pending /" ;;
pending-object) rows=$(pending); s3 "$rows" "$put_random" | sed "s/^/pending-object /" ;;
multipart)
fresh | while read -r _ key; do
docker compose exec -T -e KEY="$key" -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" s3 sh -c 'u="http://127.0.0.1:9000/$S3_BUCKET/$KEY"; a="$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"; s="aws:amz:${RUSTFS_REGION:-us-east-1}:s3"
upload=$(curl -fsS --aws-sigv4 "$s" --user "$a" -X POST "$u?uploads" | sed -n "s:.*<UploadId>\(.*\)</UploadId>.*:\1:p")
head -c 4096 /dev/urandom | curl -fsS --aws-sigv4 "$s" --user "$a" -X PUT --data-binary @- "$u?partNumber=1&uploadId=$upload" -o /dev/null' </dev/null
echo "multipart - $key"
done
;;
*) echo "unknown kind: $kind" >&2; exit 2 ;;
esac
+199
View File
@@ -0,0 +1,199 @@
#!/usr/bin/env python3
import argparse
import datetime as dt
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SQL = (
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256,"
" floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint,"
" EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id),"
" (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL"
" FROM attachments a ORDER BY a.object_key"
)
HASH = (
'while read -r k; do printf "%s %s\\n" "$k" '
'"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done'
)
CLASSES = (
"orphan_stale",
"orphan_fresh",
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"pending_stale_with_object",
"pending_stale_without_object",
"pending_fresh",
"quarantined",
"incomplete_stale",
"incomplete_fresh",
"lost_ready",
)
TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"}
BROKEN = (
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"quarantined",
)
def timed(cmd, stdin=None):
started = time.monotonic()
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
if out.returncode:
raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}")
return out.stdout, round(time.monotonic() - started, 3)
def s3(*args, stdin=None):
return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin)
def age(stamp, now):
return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp()
def load(bucket, prefix, deep):
listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}")
uploads, uploads_seconds = s3(
"object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}"
)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"]
rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL])
objects = {}
for item in json.loads(listing).get("items", []):
key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"]
objects[key] = item
rows = {}
for line in rows_raw.splitlines():
rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t")
rows[key] = {
"id": rid,
"status": status,
"size": int(size),
"sha256": sha,
"age": int(row_age),
"referenced": referenced == "t",
"quarantined": quarantined == "t",
}
hashes, deep_seconds = {}, 0.0
if deep:
present = "".join(
f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects
)
out, deep_seconds = s3("sh", "-c", HASH, stdin=present)
hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
seconds = {
"s3_list": list_seconds,
"s3_incomplete": uploads_seconds,
"pg_query": pg_seconds,
"deep_hash": deep_seconds,
}
return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds
def row_class(key, row, objects, hashes, grace):
if row["quarantined"]:
return "quarantined"
if row["status"] == "pending":
stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object"
return "pending_fresh" if row["age"] <= grace else stale
if key not in objects:
return "missing_referenced" if row["referenced"] else "missing_unreferenced"
if objects[key]["size_bytes"] != row["size"]:
return "size_mismatch"
return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None
def classify(objects, rows, hashes, uploads, grace):
now = time.time()
classes = {name: [] for name in CLASSES}
for key, item in objects.items():
if key not in rows:
classes[
"orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh"
].append(key)
for key, row in rows.items():
name = row_class(key, row, objects, hashes, grace)
if name:
classes[name].append(key if name.startswith("orphan") else row["id"])
for item in uploads:
classes[
"incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh"
].append(item["key"])
return classes
def main():
p = argparse.ArgumentParser(
description="Independent PG <-> S3 invariant check for lab4 attachments"
)
p.add_argument(
"--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600"))
)
p.add_argument(
"--deep", action="store_true", help="stream every present object and compare sha256"
)
p.add_argument(
"--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments"
)
p.add_argument(
"--expect-ready",
metavar="FILE",
help="fail if any attachment from FILE is no longer healthy",
)
p.add_argument("--limit", type=int, default=20, help="max ids per class in the report")
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
started = time.monotonic()
prefix = os.environ.get("S3_PREFIX", "attachments/")
objects, rows, hashes, uploads, seconds = load(
os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep
)
classes = classify(objects, rows, hashes, uploads, args.grace)
bad = {i for c in BROKEN for i in classes[c]}
healthy = {
r["id"]: r["sha256"]
for k, r in rows.items()
if r["status"] == "ready" and k in objects and r["id"] not in bad
}
if args.expect_ready:
expected = json.loads(Path(args.expect_ready).read_text())
classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha)
if args.save_ready:
Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True))
violations = [c for c in CLASSES if classes[c] and c not in TOLERATED]
seconds["total"] = round(time.monotonic() - started, 3)
report = {
"checked_at": dt.datetime.now(dt.timezone.utc)
.isoformat(timespec="seconds")
.replace("+00:00", "Z"),
"grace_seconds": args.grace,
"deep": args.deep,
"objects": len(objects),
"rows": len(rows),
"ready_healthy": len(healthy),
"counts": {c: len(v) for c, v in classes.items()},
"samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v},
"seconds": seconds,
"violations": violations,
"ok": not violations,
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()
+823
View File
@@ -0,0 +1,823 @@
#!/usr/bin/env python3
import argparse
import http.client
import json
import os
import re
import statistics
import struct
import subprocess
import sys
import threading
import time
import uuid
import zlib
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
GRACE = 60
FLAGS = ("--grace", str(GRACE), "--skew", "0")
INTENT_AGE = 300
LOCK_KEY = 20260004
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale")
USER_HZ = 100
BUSY = (0, 1, 2, 5, 6, 7)
def run(cmd, env=None, stdin=None):
started = time.monotonic()
done = subprocess.run(
cmd,
cwd=ROOT,
input=stdin,
capture_output=True,
check=False,
env=dict(os.environ, **(env or {})),
)
out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace")
return done.returncode, out, err, round(time.monotonic() - started, 3)
def must(cmd, env=None, stdin=None):
code, out, err, _ = run(cmd, env, stdin)
if code:
raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}")
return out
def compose(*args, env=None, stdin=None):
return must(["docker", "compose", *args], env, stdin)
def psql(sql):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-v",
"ON_ERROR_STOP=1",
"-qAt",
"-U",
"messenger",
"-d",
"messenger",
"-c",
sql,
).strip()
def reconcile(*flags, env=None):
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags]
code, out, err, wall = run(cmd, env)
if code not in (0, 1) or not out.strip():
raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}")
report = json.loads(out)
audit = [json.loads(line) for line in err.splitlines() if '"audit.' in line]
report.update(exit_code=code, wall_seconds=wall, audit=audit)
return report
def oracle(*flags):
code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags])
if not out.strip():
raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}")
return json.loads(out)
def inject(kind, count, ids=(), age=None):
env = {"INJECT_IDS": ",".join(ids)} if ids else {}
if age is not None:
env["INJECT_AGE_SECONDS"] = str(age)
out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env)
return [line.split()[1] for line in out.splitlines() if line.strip()]
def head():
return must(["git", "rev-parse", "HEAD"]).strip()
def evidence_root():
return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
def save(name, data, *, stamp=True):
path = evidence_root() / name
path.parent.mkdir(parents=True, exist_ok=True)
stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data}
path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n")
return path
class Checks:
def __init__(self):
self.items = []
def __call__(self, name, ok, **detail):
self.items.append(dict(name=name, ok=bool(ok), **detail))
sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n")
@property
def ok(self):
return all(item["ok"] for item in self.items)
def chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
def tiny_png():
raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8))
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw))
+ chunk(b"IEND", b"")
)
def pick(found, names):
return {name: found.get(name, 0) for name in names}
def faults(c, check, pre):
alice, bob, _, chat = c.bootstrap()
ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)]
linked = ids[:6]
for part in (linked[:3], linked[3:]):
c.message(alice, chat, "", attachments=part)
targets = {
"missing": [linked[0], linked[1], ids[6]],
"corrupt": [linked[2], ids[7], ids[8]],
"bitflip": [linked[3], ids[9], ids[10]],
}
damaged = {i for group in targets.values() for i in group}
ready = evidence_root() / "ready-before.json"
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
save("ready-untouched.json", untouched, stamp=False)
started = time.monotonic()
injected = {
"orphan_stale": inject("orphan", 3),
"missing": inject("missing", 3, targets["missing"]),
"size_mismatch": inject("corrupt", 3, targets["corrupt"]),
"sha_mismatch": inject("bitflip", 3, targets["bitflip"]),
"pending_stale": inject("stale-pending", 3, age=INTENT_AGE)
+ inject("pending-object", 3, age=INTENT_AGE),
}
time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started)))
injected["orphan_fresh"] = inject("orphan", 2)
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
expected = {name: len(keys) for name, keys in injected.items()}
expected.update(quarantined=pre["counts"]["quarantined"])
before = oracle(
"--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json")
)
dry = reconcile("--dry-run", "--deep", *FLAGS)
save("dry-run.json", dry)
after_dry = oracle("--grace", str(GRACE), "--deep")
named = {k: sorted(s.rsplit("/", 1)[-1] for s in dry["samples"].get(k, [])) for k in injected}
check(
"dry-run находит каждый внедрённый класс, и найдено ровно внедрённое",
pick(dry["found"], expected) == expected
and named == {name: sorted(ids) for name, ids in injected.items()},
expected=expected,
found=dry["found"],
)
check(
"dry-run ничего не меняет",
not any(dry["fixed"].values())
and after_dry["counts"] == before["counts"]
and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]),
objects=[before["objects"], after_dry["objects"]],
rows=[before["rows"], after_dry["rows"]],
)
first = reconcile("--apply", "--deep", *FLAGS)
save("apply-1.json", first)
discarded = sorted(e["attachment_id"] for e in first["audit"] if e["reason"] == "reconcile")
check(
"apply исправляет ровно найденное dry-run, удаление намерений — в журнале аудита",
pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE)
and not first["errors"]
and discarded == sorted(injected["pending_stale"]),
dry_run=pick(dry["found"], FIXABLE),
fixed=pick(first["fixed"], FIXABLE),
audit_discarded=len(discarded),
)
second = reconcile("--apply", "--deep", *FLAGS)
save("apply-2.json", second)
check(
"повторный apply — 0 действий",
not any(second["fixed"].values()),
fixed=second["fixed"],
found=second["found"],
)
after = oracle(
"--grace",
str(GRACE),
"--deep",
"--expect-ready",
str(evidence_root() / "ready-untouched.json"),
"--output",
str(evidence_root() / "check-after.json"),
)
tolerated = ("orphan_fresh", "pending_fresh")
check(
"независимая проверка после apply: нарушений нет, готовые не потеряны",
after["ok"]
and not after["counts"]["lost_ready"]
and pick(after["counts"], tolerated) == pick(before["counts"], tolerated)
and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9,
counts=after["counts"],
)
marks = dict(
line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split()
)
links = psql("SELECT attachment_id FROM message_attachments").split()
kept = {i: marks.get(i) for i in sorted(damaged)}
check(
"строки с расхождениями и ссылки сообщений сохранены",
all(kept.values()) and sum(i in linked for i in links) == len(linked),
quarantine=kept,
message_links=sum(i in linked for i in links),
)
meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment")
blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob)
intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob)
check(
"карантин: метаданные 200, /content 404; нетронутое скачивается",
meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE,
)
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
for attachment, data in ((linked[0], FIXTURE), (linked[1], os.urandom(len(FIXTURE)))):
compose(
"run",
"--rm",
"--no-deps",
"-T",
"s3-cli",
"pipe",
f"root/{bucket}/attachments/{attachment}",
stdin=data,
)
healed = reconcile("--apply", *FLAGS)
save("healed.json", healed)
restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob)
foreign = c.call("GET", f"/api/v1/attachments/{linked[1]}/content", 404, actor=bob)
check(
"вернули те же байты — карантин снят; другие байты той же длины — остался",
healed["fixed"]["healed"] == 1
and healed["samples"].get("healed") == [linked[0]]
and linked[1] in healed["samples"].get("quarantined", [])
and restored["bytes"] == FIXTURE
and foreign["status"] == 404,
fixed=healed["fixed"],
s3_get=healed["cost"]["requests"]["s3_get"],
)
return {"injected": injected, "targets": targets, "linked": linked}
def single(check):
holder = subprocess.Popen(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-c",
f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)",
],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
time.sleep(3)
busy = reconcile("--apply", *FLAGS)
finally:
holder.wait()
save("lock-busy.json", busy)
check(
"второй одновременный проход пропущен",
busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()),
)
def healthy(service, seconds=90):
container = compose("ps", "-q", service).strip()
probe = ["docker", "inspect", "-f", "{{.State.Health.Status}}", container]
deadline = time.monotonic() + seconds
while must(probe).strip() != "healthy":
if time.monotonic() > deadline:
raise SystemExit(f"{service} is not healthy after {seconds} s")
time.sleep(1)
def interrupt(check):
name = f"hlpipe-lab5-reconcile-sigterm-{os.getpid()}"
compose("pause", "s3")
try:
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "--name", name, "reconciler"]
runner = subprocess.Popen(
[*cmd, "--apply", *FLAGS],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
time.sleep(4)
state = must(["docker", "inspect", "-f", "{{.State.Status}} {{.HostConfig.Init}}", name])
started = time.monotonic()
must(["docker", "stop", name])
stopped = round(time.monotonic() - started, 2)
code = runner.wait(timeout=60)
finally:
compose("unpause", "s3")
healthy("s3")
after = reconcile("--dry-run", *FLAGS)
status, init = state.split()
report = {
"s3": "paused",
"state_before_stop": status,
"init": init,
"stop_seconds": stopped,
"exit_code": code,
"next_pass_skipped": after["skipped"],
}
save("sigterm.json", report)
check(
"SIGTERM: идущий проход останавливается за секунды, блокировка освобождена",
status == "running" and code == 143 and stopped < 5 and not after["skipped"],
**report,
)
def outage(check):
before = oracle("--grace", str(GRACE))
compose("stop", "s3")
try:
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", "--apply"]
code, out, err, wall = run([*cmd, *FLAGS])
finally:
compose("up", "-d", "--wait", "--no-deps", "s3")
after = oracle("--grace", str(GRACE))
log = [line for line in err.splitlines() if "reconcile." in line]
same = all(after[k] == before[k] for k in ("objects", "rows", "counts"))
save("s3-down.json", {"exit_code": code, "wall_seconds": wall, "stdout": out, "log": log})
check(
"S3 остановлен: код 1 за ограниченное время, состояние не изменилось",
code == 1 and wall < 30 and not out.strip() and same,
exit_code=code,
wall_seconds=wall,
)
def uploader(base, token, chat, *, deadline, results, lock):
host = urlsplit(base).netloc
conn = http.client.HTTPConnection(host, timeout=30)
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
sent = 0
while time.monotonic() < deadline:
path = f"/api/v1/chats/{chat}/attachments?filename=load.png"
try:
conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"})
response = conn.getresponse()
status, data = response.status, response.read()
if status == 201 and sent % 10 == 0:
body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]})
extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}
conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra})
linked = conn.getresponse()
linked.read()
status = 201 if linked.status == 201 else -linked.status
except (OSError, http.client.HTTPException):
conn.close()
conn = http.client.HTTPConnection(host, timeout=30)
status, data = 0, b""
with lock:
results.append((status, json.loads(data) if status == 201 else None))
sent += 1
time.sleep(0.05)
def under_load(c, check, base, seconds, threads):
alice, _, _, chat = c.bootstrap()
ready = evidence_root() / "ready-load.json"
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
leftover = {
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
"pending_stale": left["pending_fresh"]
+ left["pending_stale_with_object"]
+ left["pending_stale_without_object"],
}
orphans = inject("orphan", 3)
intents = inject("stale-pending", 3, age=INTENT_AGE)
results, lock = [], threading.Lock()
deadline = time.monotonic() + seconds
workers = [
threading.Thread(
target=uploader,
args=(base, alice["token"], chat["id"]),
kwargs={"deadline": deadline, "results": results, "lock": lock},
)
for _ in range(threads)
]
for worker in workers:
worker.start()
runs = []
while time.monotonic() < deadline:
runs.append(reconcile("--apply", *FLAGS))
for worker in workers:
worker.join()
runs.append(reconcile("--apply", *FLAGS))
created = {att["id"]: att["sha256"] for status, att in results if att}
expected = dict(json.loads(ready.read_text()), **created)
save("ready-load.json", expected, stamp=False)
after = oracle(
"--grace",
str(GRACE),
"--deep",
"--expect-ready",
str(ready),
"--output",
str(evidence_root() / "check-under-load.json"),
)
statuses = {}
for status, _ in results:
statuses[str(status)] = statuses.get(str(status), 0) + 1
fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE}
rechecked = {}
for r in runs:
for name, count in r["rechecked"].items():
rechecked[name] = rechecked.get(name, 0) + count
seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")}
summary = {
"seconds": seconds,
"upload_threads": threads,
"uploads": statuses,
"created": len(created),
"linked_every": 10,
"reconciler_runs": len(runs),
"run_seconds": [r["cost"]["seconds"] for r in runs],
"fixed_total": fixed,
"fresh_seen_total": seen,
"rechecked_total": rechecked,
"errors": sum(r["errors"] for r in runs),
"skipped": sum(1 for r in runs if r["skipped"]),
"injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)},
"left_from_faults": leftover,
"check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")},
}
save("under-load.json", summary)
check(
"под нагрузкой: исправлены только внедрённые, ошибок нет",
fixed
== {
**dict.fromkeys(FIXABLE, 0),
"orphan_stale": leftover["orphan_stale"] + len(orphans),
"pending_stale": leftover["pending_stale"] + len(intents),
}
and not summary["errors"]
and not summary["skipped"],
fixed=fixed,
runs=len(runs),
)
check(
"под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0",
after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"},
uploads=statuses,
lost_ready=after["counts"]["lost_ready"],
)
def demo(args):
evidence_root().mkdir(parents=True, exist_ok=False)
c = client.Client(args.base_url, timeout=30)
check = Checks()
pre = oracle("--grace", str(GRACE))
dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"}
if dirty:
raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply")
started = time.monotonic()
result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()}
result["faults"] = faults(c, check, pre)
single(check)
interrupt(check)
outage(check)
under_load(c, check, args.base_url, args.load_seconds, args.load_threads)
result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok)
save("demo.json", result)
print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False))
sys.exit(0 if check.ok else 1)
def pins(args):
cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip())
sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory}
if args.no_pin or cpus < 8:
return sized
return dict(
sized,
S3_CPUSET=f"{cpus - 5}-{cpus - 4}",
PG_CPUSET=f"{cpus - 3}-{cpus - 2}",
RECONCILE_CPUSET=str(cpus - 1),
)
def cpu_set(env):
cpus = set()
for name in ("S3_CPUSET", "PG_CPUSET", "RECONCILE_CPUSET"):
for part in filter(None, env.get(name, "").split(",")):
low, _, high = part.partition("-")
cpus.update(range(int(low), int(high or low) + 1))
return cpus
def limits():
ids = compose("ps", "-q", "s3", "postgres").split()
fmt = (
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}"
" cpuset={{.HostConfig.CpusetCpus}}"
)
return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines()
def host(title):
stats = must(
[
"docker",
"stats",
"--no-stream",
"--all",
"--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}",
]
)
running = must(["docker", "ps", "--format", "{{.Names}} {{.Image}} {{.Status}}"])
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
load = " ".join(f"{v:.2f}" for v in os.getloadavg())
return (
f"== {title} date: {stamp} commit: {head()} host loadavg: {load}\n"
f"$ docker stats --no-stream --all\n{stats}\n$ docker ps\n{running}\n"
)
def usage(text):
return int(re.search(r"^usage_usec (\d+)$", text, re.M).group(1)) / 1e6
def sample():
vm = compose(
"exec", "-T", "postgres", "cat", "/proc/uptime", "/proc/stat", "/sys/fs/cgroup/cpu.stat"
)
s3 = compose("exec", "-T", "s3", "cat", "/sys/fs/cgroup/cpu.stat")
busy = {}
for line in vm.splitlines()[1:]:
name, *values = line.split() or [""]
if name.startswith("cpu") and name[3:].isdigit():
busy[int(name[3:])] = sum(int(values[i]) for i in BUSY) / USER_HZ
return float(vm.split()[0]), busy, usage(vm) + usage(s3)
def screen(first, last, own, pinned, args, *, started=True):
window = max(last[0] - first[0], 0.01)
busy = {cpu: spent - first[1].get(cpu, 0.0) for cpu, spent in last[1].items()}
ours = last[2] - first[2] + own
other = sum(busy.values()) - ours - args.start_other * started
on_pinned = sum(busy.get(cpu, 0.0) for cpu in pinned) - ours - args.start_pinned * started
other, on_pinned = max(0.0, other / window), max(0.0, on_pinned / window) if pinned else 0.0
return {
"window_seconds": round(window, 2),
"ours_cores": round(ours / window, 2),
"other_cores": round(other, 2),
"other_on_pinned_cores": round(on_pinned, 2),
"host_loadavg_1m": round(os.getloadavg()[0], 2),
"disturbed": other > args.max_other or on_pinned > args.max_pinned,
}
def wait_quiet(args, pinned):
deadline, calm = time.monotonic() + args.quiet_timeout, 0
while calm < 3 and time.monotonic() < deadline:
first = sample()
time.sleep(5)
quiet = screen(first, sample(), 0.0, pinned, args, started=False)
calm = 0 if quiet["disturbed"] else calm + 1
sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n")
return calm == 3
def measure(flags, env, args, pinned):
rejected = []
for attempt in range(args.retries + 1):
first = sample()
report = reconcile(*flags, "--samples", "3", env=env)
report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args)
if not report["screen"]["disturbed"]:
return report, rejected
rejected.append(report)
if attempt < args.retries and not wait_quiet(args, pinned):
break
return None, rejected
def seed(count, env):
if count <= 0:
return {"added": 0}
env = dict(env, API_CPUS="4", API_MEMORY="512m")
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
out = compose(
"run",
"--rm",
"--no-deps",
"-T",
"-v",
tools,
"--entrypoint",
"python",
"api",
"/tools/seed_storage.py",
"--count",
str(count),
"--concurrency",
"4",
env=env,
)
records = [json.loads(line) for line in out.strip().splitlines()]
return {**records[-1], "progress": records[:-1]}
def median(runs, path):
values = []
for r in runs:
value = r
for key in path:
value = value[key]
values.append(value)
return statistics.median(values)
def summarize(results, rejected):
rows = []
for (n, mode), runs in sorted(results.items()):
seconds = median(runs, ("cost", "seconds"))
rows.append(
{
"objects": n,
"mode": mode,
"runs": len(runs),
"seconds_median": seconds,
"seconds_all": [r["cost"]["seconds"] for r in runs],
"wall_seconds_median": median(runs, ("wall_seconds",)),
"us_per_object": round(seconds / n * 1e6, 1),
"cpu_seconds_median": median(runs, ("cost", "cpu_seconds")),
"s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")),
"sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")),
"requests": runs[0]["cost"]["requests"],
"max_rss_mib_median": median(runs, ("cost", "max_rss_mib")),
"other_cores_max": max(r["screen"]["other_cores"] for r in runs),
"other_on_pinned_cores_max": max(
r["screen"]["other_on_pinned_cores"] for r in runs
),
"disturbed_kept": sum(r["screen"]["disturbed"] for r in runs),
"rejected": rejected.get((n, mode), 0),
"scanned": runs[0]["scanned"],
"found_nonzero": {k: v for k, v in runs[0]["found"].items() if v},
}
)
return rows
def table(rows):
head = (
"objects mode runs t_med,s wall,s us/obj cpu,s list,s sql,s lists sqls"
" gets rss,MiB other pinned rejected"
)
lines = [head]
for r in rows:
q = r["requests"]
lines.append(
f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} "
f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} "
f"{r['cpu_seconds_median']:>5.2f} {r['s3_list_wait_median']:>6.3f} "
f"{r['sql_wait_median']:>5.3f} {q['s3_list']:>5} {q['sql']:>4} {q['s3_get']:>5} "
f"{r['max_rss_mib_median']:>7.1f} {r['other_cores_max']:>5.2f} "
f"{r['other_on_pinned_cores_max']:>6.2f} {r['rejected']:>8}"
)
return "\n".join(lines) + "\n"
def cost(args):
out = evidence_root() / "cost"
out.mkdir(parents=True, exist_ok=False)
(out / "disturbed").mkdir()
rows = int(psql("SELECT count(*) FROM attachments"))
if rows > args.sizes[0]:
raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project")
snapshot = out / "host-docker-stats.txt"
snapshot.write_text(host("before the series"))
env = pins(args)
pinned = cpu_set(env)
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env)
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []}
plan["screen"] = {
"max_other_cores": args.max_other,
"max_other_on_pinned_cores": args.max_pinned,
"start_allowance_core_seconds": {"vm": args.start_other, "pinned": args.start_pinned},
"retries": args.retries,
}
results, rejected = {}, {}
try:
for n in args.sizes:
plan["seeds"].append(dict(seed(n - rows, env), target=n))
rows = n
time.sleep(args.settle)
save("cost/plan.json", plan)
if not wait_quiet(args, pinned):
raise SystemExit("no quiet measurement window before timeout")
modes = [("shallow", ["--dry-run"])]
if n <= args.deep_max:
modes.append(("deep", ["--dry-run", "--deep"]))
if n == args.page_compare:
modes.append(("page100", ["--dry-run", "--page-size", "100"]))
for mode, flags in modes:
for attempt in range(1, args.repeats + 1):
report, dropped = measure(flags, env, args, pinned)
name = f"n{n}-{mode}-{attempt}"
for number, bad in enumerate(dropped, 1):
save(f"cost/disturbed/{name}-try{number}.json", bad)
rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped)
if report is None:
raise SystemExit(f"{name}: no clean run after retries; rejected preserved")
if (
report["exit_code"]
or report["errors"]
or report["skipped"]
or report["scanned"] != {"objects": n, "rows": n}
or any(report["found"].values())
):
save(f"cost/invalid/{name}.json", report)
raise SystemExit(f"{name}: inconsistent dataset or failed pass")
save(f"cost/{name}.json", report)
results.setdefault((n, mode), []).append(report)
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n")
finally:
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
with snapshot.open("a") as handle:
handle.write("\n" + host("after the series"))
summary = summarize(results, rejected)
save("cost/summary.json", {**plan, "results": summary})
(out / "summary.txt").write_text(
f"date: {stamp} commit: {plan['commit']} pins: {env}\n" + table(summary)
)
sys.stdout.write(table(summary))
def main():
p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass")
sub = p.add_subparsers(dest="command", required=True)
d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load")
d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
d.add_argument("--load-seconds", type=int, default=75)
d.add_argument("--load-threads", type=int, default=3)
k = sub.add_parser("cost", help="cost of a pass for N objects, repeated")
k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 30000, 100000])
k.add_argument("--repeats", type=int, default=3)
k.add_argument("--deep-max", type=int, default=100000)
k.add_argument("--page-compare", type=int, default=10000)
k.add_argument("--settle", type=float, default=10)
k.add_argument("--s3-memory", default="3g")
k.add_argument("--no-pin", action="store_true")
k.add_argument("--max-other", type=float, default=2.0, help="foreign CPU in the VM, cores")
k.add_argument("--max-pinned", type=float, default=0.25, help="foreign CPU on pinned cores")
k.add_argument("--start-other", type=float, default=1.5, help="container start, core-s")
k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores")
k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run")
k.add_argument("--quiet-timeout", type=float, default=900.0)
for command in (d, k):
command.add_argument(
"--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
)
args = p.parse_args()
os.environ["RECONCILE_EVIDENCE_DIR"] = args.out
if args.command == "demo":
demo(args)
else:
cost(args)
if __name__ == "__main__":
main()
+4 -2
View File
@@ -2,7 +2,8 @@
set -eu
cd "$(dirname "$0")/.."
service=${API_SERVICE:-api}
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; }
scripts/stand-usage.sh &
echo "$(date -u +%T) migrate once before the first replacement"
docker compose run --rm --no-deps -T migrate 2>&1
old=$(docker compose ps -q "$service")
@@ -13,8 +14,9 @@ for id in $old; do
scripts/lb-wait.sh $((count + 1))
new=$(docker compose ps -q "$service" | grep -v -F "$running")
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
docker stop -t "${STOP_TIMEOUT:-7}" "$id" >/dev/null
docker rm "$id" >/dev/null
done
scripts/lb-wait.sh "$count"
wait
echo "$(date -u +%T) done: $count replicas replaced"
+30 -7
View File
@@ -2,17 +2,30 @@
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
undated() {
python3 -c 'import json, sys
def clean(value):
if isinstance(value, dict):
return {k: clean(v) for k, v in value.items() if not k.endswith("_date")}
return [clean(v) for v in value] if isinstance(value, list) else value
text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0
while text[at:].strip():
at += len(text[at:]) - len(text[at:].lstrip())
value, at = decoder.raw_decode(text, at)
print(json.dumps(clean(value), indent=2, ensure_ascii=False))'
}
snapshot() {
cli sh -s <<'EOF' | grep -v '_date'
cli sh -s <<'EOF' | undated
ak=$(cat /run/secrets/s3_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
rk=$(cat /run/secrets/s3_reconciler_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do
rc $c --json
done | sed "s/$ak/<api-key>/g"
done | sed "s/$ak/<api-key>/g; s/$rk/<reconciler-key>/g"
EOF
}
failed=0
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) == S3 privacy and bootstrap checks"
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
before=$(snapshot)
@@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same
cli sh -s <<'EOF' || failed=1
b=$S3_BUCKET
key="attachments/s3-check-$(date +%s)-$$"
spare="$key-reconciler"
other="s3-check-$(date +%s)-$$"
head -c 4096 /dev/urandom > /tmp/probe
result=0
expect() {
want=$1; name=$2; shift 2
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
}
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
expect allow "api: HeadObject" rc object stat "api/$b/$key"
expect allow "api: GetObject" rc object show "api/$b/$key"
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
expect deny "api: ListBucket" rc object list "api/$b/attachments/"
rc put /tmp/probe "api/$b/$spare" >/dev/null
expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/"
expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key"
expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key"
expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put"
expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare"
expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other"
expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b"
expect deny "reconciler: admin ListUsers" rc admin user list reconciler
expect deny "api: CreateBucket" rc bucket create "api/$other"
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: admin ListUsers" rc admin user list api
rc bucket create --ignore-existing "root/$other" >/dev/null
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
+524
View File
@@ -0,0 +1,524 @@
#!/usr/bin/env python3
import argparse
import collections
import contextlib
import csv
import json
import math
import os
import re
import signal
import statistics
import subprocess
import sys
import time
from pathlib import Path
from probe import ROOT, k6_metrics, lb_up, run_k6
SCRIPTS = ROOT / "scripts"
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
PINNED = {
"PG_CPUSET": "0-1",
"LB_CPUSET": "2",
"VALKEY_CPUSET": "3",
"K6_CPUSET": "4-6",
"K6_CPUS": "3",
"K6_MEMORY": "1g",
}
API_FIRST_CPU = 7
FIELDS = [
"raw_dir",
"variant",
"config",
"replicas",
"cpus",
"memory_mb",
"workers",
"pool",
"maxconn",
"api_cpuset",
"rate",
"repeat",
"started",
"db_mb",
"messages",
"k6_exit",
"saturated",
"requests",
"rps",
"ok_rps",
"failed_ratio",
"dropped_iterations",
"p50_ms",
"p95_ms",
"p99_ms",
"p95_post_ms",
"p95_get_ms",
"cpu_api",
"cpu_postgres",
"cpu_lb",
"cpu_valkey",
"cpu_k6",
"mem_api_mb",
"mem_postgres_mb",
"mem_k6_mb",
"pg_backends_max",
"pg_active_avg",
"pg_active_max",
"pg_idle_avg",
"pg_idle_tx_avg",
"pg_lock_avg",
"pg_lock_max",
"pg_lwlock_avg",
"pg_io_avg",
"lb_scur_max",
"lb_qcur_max",
"lb_qtime_max_ms",
"lb_rtime_avg_ms",
"cpu_vm",
"cpu_other",
"gap_s",
"wall_s",
"disturbed",
]
def compose(*args, env=None):
command = ["docker", "compose", *args]
return subprocess.run(
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
).stdout
def size(text):
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
def rows(path, start, end):
with path.open() as handle:
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
def mean(values):
return round(statistics.mean(values), 2) if values else 0.0
def docker_usage(path, project, start, end):
ticks = collections.defaultdict(collections.Counter)
memory = collections.Counter()
for r in rows(path, start, end):
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
cpu = {
name: mean([t[name] for t in ticks.values() if name in t])
for name in ("api", "postgres", "lb", "valkey", "k6")
}
usage = {f"cpu_{name}": value for name, value in cpu.items()}
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
return usage
def vm_usage(path, start, end):
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
return 0.0
first, last = samples[0], samples[-1]
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
def gap(path, start, end):
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
edges = [start, *[t for t in ticks if start <= t <= end], end]
return max(b - a for a, b in zip(edges, edges[1:]))
def vm_counters():
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
values = [int(v) for v in lines[0].split()[1:]]
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
return sum(values) - values[3] - values[4], sum(values), cpus
def wait_quiet(args):
deadline, calm = time.monotonic() + args.quiet_timeout, 0
while calm < 3 and time.monotonic() < deadline:
first = vm_counters()
time.sleep(5)
last = vm_counters()
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
calm = calm + 1 if busy < args.max_other else 0
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
def pg_usage(path, start, end):
ticks = collections.defaultdict(collections.Counter)
for r in rows(path, start, end):
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
tick = ticks[r["ts"]]
tick["backends"] += count
tick["active"] += count * (state == "active")
tick["idle"] += count * (state == "idle")
tick["idle_tx"] += count * state.startswith("idle in transaction")
tick["lock"] += count * (wait == "Lock")
tick["lwlock"] += count * (wait == "LWLock")
tick["io"] += count * (wait == "IO")
series = list(ticks.values())
return {
"pg_backends_max": max((t["backends"] for t in series), default=0),
"pg_active_avg": mean([t["active"] for t in series]),
"pg_active_max": max((t["active"] for t in series), default=0),
"pg_idle_avg": mean([t["idle"] for t in series]),
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
"pg_lock_avg": mean([t["lock"] for t in series]),
"pg_lock_max": max((t["lock"] for t in series), default=0),
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
"pg_io_avg": mean([t["io"] for t in series]),
}
def lb_usage(path, start, end):
backend = [
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
]
def numbers(field):
return [int(r[field] or 0) for r in backend]
return {
"lb_scur_max": max(numbers("scur"), default=0),
"lb_qcur_max": max(numbers("qcur"), default=0),
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
"lb_rtime_avg_ms": mean(numbers("rtime")),
}
def trend(path, name):
metrics = json.loads(Path(path).read_text())["metrics"]
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
return round(found.get("values", {}).get("p(95)", 0.0), 2)
def sampler(*command):
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
def stop(process):
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
process.wait()
def wait_lb(expected, timeout=120):
deadline, stable = time.monotonic() + timeout, 0
while stable < 4:
if time.monotonic() > deadline:
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
stable = stable + 1 if lb_up() == expected else 0
time.sleep(0.5)
def layout(config, args, variant):
name, _, extra = variant.partition(":")
replicas, cpus = (float(v) for v in config.split("x"))
replicas = int(replicas)
workers = args.workers or max(1, int(cpus))
width = max(1, math.ceil(replicas * cpus))
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
memory = args.api_memory or args.api_memory_total // replicas
settings = {
"API_REPLICAS": str(replicas),
"API_CPUS": f"{cpus:g}",
"API_MEMORY": f"{memory}m",
"WEB_CONCURRENCY": str(workers),
"DB_POOL_MAX_SIZE": str(args.pool),
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
"API_CPUSET": cpuset if args.pin else "",
}
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
base = {
"variant": name,
"config": config,
"replicas": replicas,
"cpus": cpus,
"memory_mb": memory,
"workers": int(settings["WEB_CONCURRENCY"]),
"pool": int(settings["DB_POOL_MAX_SIZE"]),
"maxconn": int(settings["API_SERVER_MAXCONN"]),
"api_cpuset": settings["API_CPUSET"],
}
return base, settings
def apply(base, settings, env, out):
env = dict(env, **settings)
for attempt in range(3):
try:
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
break
except subprocess.CalledProcessError as error:
log = Path(out) / "compose-up-failed.txt"
with log.open("a") as handle:
stamp = time.strftime("%FT%TZ", time.gmtime())
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"date: {stamp} commit: {commit}\n")
handle.write(f"== {stamp} {base} attempt {attempt + 1}\n")
handle.write(error.stdout + error.stderr + "\n")
print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)}))
if attempt == 2:
raise
time.sleep(30)
wait_lb(base["replicas"])
return env
def deployment(env):
ids = compose("ps", "-q", env=env).split()
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
result = subprocess.run(
["docker", "inspect", "--format", template, *ids],
check=True,
capture_output=True,
text=True,
)
containers = []
for line in result.stdout.splitlines():
name, image, config = line.split(" ", 2)
limits = json.loads(config)
containers.append(
dict(
name=json.loads(name),
image=json.loads(image),
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
)
)
return containers
def provenance(folder, args, env):
stamp = time.strftime("%FT%TZ", time.gmtime())
metadata = {
"date": stamp,
"commit": args.commit,
"project": args.project,
"command": sys.argv,
"api_image": env.get("API_IMAGE", "compose default"),
"warmup_seconds": args.warmup,
"duration_seconds": args.duration,
"containers": args.deployment,
}
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
for path in folder.rglob("*"):
if path.suffix in {".csv", ".txt"}:
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
elif path.name == "summary.json":
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
path.write_text(json.dumps(data) + "\n")
def attempt_folder(out, base, rate, repeat):
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
attempt = 1
while (Path(out) / f"{name}_attempt{attempt}").exists():
attempt += 1
folder = Path(out) / f"{name}_attempt{attempt}"
folder.mkdir(parents=True)
return folder
def measure(folder, args, rate, env):
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
database = subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
).stdout.split(",")
samplers = [
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
]
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
began = time.time()
try:
code, started = run_k6(folder / "k6", load)
finally:
wall = time.time() - began
for process in samplers:
stop(process)
subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
)
if started is None:
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
return code, started + args.warmup, database, wall
def step(base, rate, repeat, args, env):
folder = attempt_folder(args.out, base, rate, repeat)
code, start, database, wall = measure(folder, args, rate, env)
end = start + args.duration
summary = folder / "k6/summary.json"
metrics = k6_metrics(summary, args.duration)
saturated = (
metrics["failed_ratio"] > args.max_error_ratio
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
or metrics["p95_ms"] > args.slo_p95_ms
)
row = dict(
base,
raw_dir=folder.name,
rate=rate,
repeat=repeat,
k6_exit=code,
saturated=saturated,
**metrics,
)
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
row["db_mb"] = round(int(database[0]) / 2**20, 1)
row["messages"] = int(database[1])
row["p95_post_ms"] = trend(summary, "post_message")
row["p95_get_ms"] = trend(summary, "get_history")
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
row.update(pg_usage(folder / "pg_activity.csv", start, end))
row.update(lb_usage(folder / "lb_stats.csv", start, end))
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
row["wall_s"] = round(wall, 1)
row["disturbed"] = (
row["cpu_other"] > args.max_other
or row["gap_s"] > args.max_gap
or wall > args.warmup + args.duration + args.max_overhead
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6"))
or not rows(folder / "pg_activity.csv", start, end)
or not rows(folder / "lb_stats.csv", start, end)
)
provenance(folder, args, env)
if code or not metrics["requests"]:
raise SystemExit(f"invalid k6 run: {folder}")
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
return row
def rates(args, base):
if args.rates:
return [int(r) for r in args.rates.split(",")]
factor, top = (float(v) for v in args.search.split(":"))
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
while value <= top:
values.append(int(round(value, -1)))
value *= factor
return values
def arguments():
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
parser.add_argument("--out", required=True)
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
parser.add_argument(
"--variant",
dest="variants",
action="append",
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
)
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
parser.add_argument("--rates", help="comma separated offered rates, rps")
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
parser.add_argument("--start-per-cpu", type=float, default=100.0)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat")
parser.add_argument("--warmup", type=int, default=10)
parser.add_argument("--duration", type=int, default=30)
parser.add_argument("--pause", type=float, default=5.0)
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
parser.add_argument("--max-error-ratio", type=float, default=0.01)
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
args = parser.parse_args()
args.variants = args.variants or ["base"]
if len(args.variants) > 1 and not args.rates:
parser.error("several variants need explicit --rates")
return args
def block(args, output, config, variant, repeats):
base, settings = layout(config, args, variant)
current, streak = apply(base, settings, args.base_env, args.out), 0
args.deployment = deployment(current)
for rate in rates(args, base):
results = []
for repeat in repeats:
for _ in range(args.retries + 1):
row = step(base, rate, repeat, args, current)
time.sleep(args.pause)
if not row["disturbed"]:
break
args.rejected.writerow(row)
wait_quiet(args)
else:
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
results.append(row)
output.writerows(results)
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
if streak == 2:
break
def series(args, output):
every = range(args.first_repeat, args.first_repeat + args.repeats)
configs = args.configs.split(",")
if not args.rates:
for config in configs:
block(args, output, config, args.variants[0], every)
return
for repeat in every:
for config in configs:
for variant in args.variants:
block(args, output, config, variant, [repeat])
@contextlib.contextmanager
def table(path):
fresh = not path.exists()
with path.open("a", newline="", buffering=1) as handle:
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
if fresh:
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
writer.writeheader()
yield writer
def main():
args = arguments()
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
args.project = json.loads(compose("config", "--format", "json"))["name"]
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
try:
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
series(args, writer)
finally:
if not args.keep:
compose("up", "-d", "--wait", "--wait-timeout", "180")
return 0
if __name__ == "__main__":
sys.exit(main())
+2
View File
@@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)"
put s3_root_password "$(random)"
put s3_access_key_id "api$(random | cut -c1-16)"
put s3_secret_access_key "$(random)"
put s3_reconciler_access_key_id "rec$(random | cut -c1-16)"
put s3_reconciler_secret_access_key "$(random)"
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
size="select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
echo "$(date -u +%T) database bytes,messages: $(docker compose exec -T postgres psql -X -At -F, -U messenger -d messenger -c "$size")"
docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q)
+1 -1
View File
@@ -6,4 +6,4 @@ ids=$(docker compose ps -q "${API_SERVICE:-api}")
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
docker stop -t "${STOP_TIMEOUT:-7}" "$target" >/dev/null
+9 -2
View File
@@ -77,7 +77,9 @@ def main():
args = p.parse_args()
out = ROOT / args.out
out.mkdir(parents=True, exist_ok=True)
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
if any(out.iterdir()):
p.error("output directory must be empty; choose a new --out")
commit = seed.run(["git", "rev-parse", "HEAD"]).strip()
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
lines = [
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
@@ -102,7 +104,12 @@ def main():
link=0,
ids_out=None,
)
report = {"upload_concurrency": slots, **seed.over_http(run)}
report = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"commit": commit,
"upload_concurrency": slots,
**seed.over_http(run),
}
report["lb"] = balancer()
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
@@ -47,3 +47,14 @@ class CleanStaging(Command):
class ReplayAttachment(Command):
attachment_id: UUID
operator: str
@dataclass(frozen=True, slots=True)
class ReconcileStorage(Command):
apply: bool
grace: timedelta
skew: timedelta
deep: bool = False
page_size: int = 1000
deep_concurrency: int = 8
samples: int = 20
@@ -1,4 +1,5 @@
from collections.abc import AsyncIterator, Sequence
from contextlib import AbstractAsyncContextManager
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Protocol, runtime_checkable
@@ -40,6 +41,13 @@ class StagedObject:
key: str
@dataclass(frozen=True, slots=True)
class StoredObject:
key: str
size: int
modified: datetime
@runtime_checkable
class ObjectStorage(Protocol):
async def put(self, key: str, data: bytes, content_type: str) -> None: ...
@@ -52,6 +60,10 @@ class ObjectStorage(Protocol):
async def delete(self, key: str) -> None: ...
def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]: ...
async def stat(self, key: str) -> StoredObject | None: ...
@runtime_checkable
class ImageInspector(Protocol):
@@ -95,6 +107,14 @@ class AttachmentRepository(Protocol):
async def remove(self, attachment: Attachment) -> None: ...
async def page_after(self, key: str, limit: int) -> Sequence[Attachment]: ...
async def by_key(self, key: str) -> Attachment | None: ...
async def claim(self, attachment_id: UUID) -> Attachment | None: ...
async def mark(self, attachment: Attachment) -> None: ...
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: ...
async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ...
@@ -153,3 +173,8 @@ class AttachmentDirectory(Protocol):
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ...
async def failed_jobs(self, limit: int) -> list[FailedJob]: ...
@runtime_checkable
class SingleRun(Protocol):
def hold(self) -> AbstractAsyncContextManager[bool]: ...
@@ -49,6 +49,9 @@ class OpenAttachment:
if found is None:
raise AttachmentNotFound("у вложения нет такого варианта")
key, content_type = found.object_key, found.content_type
if attachment.quarantine is not None:
self._logger.warning("attachment.quarantined", attachment_id=str(attachment.id))
raise AttachmentNotFound("вложение не найдено")
try:
content = await self._storage.open(str(key))
except ObjectMissingError as error:
@@ -0,0 +1,318 @@
import asyncio
from collections import Counter
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
from dataclasses import dataclass, field
from datetime import datetime
from hashlib import sha256
from time import monotonic
from uuid import UUID
from messenger.attachments.application.commands import ReconcileStorage
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.application.handlers import drop
from messenger.attachments.application.ports import (
AttachmentsUnitOfWork,
ObjectStorage,
SingleRun,
StoredObject,
)
from messenger.attachments.domain.models import (
OBJECT_PREFIX,
Attachment,
AttachmentStatus,
Quarantine,
)
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import Clock, EventPublisher, Logger
type UnitOfWorkFactory = Callable[[], AttachmentsUnitOfWork]
@dataclass(slots=True)
class Reconciliation:
apply: bool
skipped: bool = False
objects: int = 0
rows: int = 0
found: Counter[str] = field(default_factory=Counter)
fixed: Counter[str] = field(default_factory=Counter)
rechecked: Counter[str] = field(default_factory=Counter)
requests: Counter[str] = field(default_factory=Counter)
waited: dict[str, float] = field(default_factory=dict)
samples: dict[str, list[str]] = field(default_factory=dict)
hashed_bytes: int = 0
errors: int = 0
seconds: float = 0.0
class StorageReconciler:
__slots__ = ("_clock", "_logger", "_publisher", "_single", "_storage", "_uow")
def __init__(
self,
uow: UnitOfWorkFactory,
storage: ObjectStorage,
single: SingleRun,
clock: Clock,
*,
logger: Logger,
publisher: EventPublisher,
) -> None:
self._uow = uow
self._storage = storage
self._single = single
self._clock = clock
self._logger = logger
self._publisher = publisher
async def __call__(self, command: ReconcileStorage) -> Reconciliation:
report = Reconciliation(apply=command.apply)
started = monotonic()
async with self._single.hold() as held:
report.skipped = not held
if held:
before = self._clock.now() - command.grace - command.skew
await Pass(
self._uow,
self._storage,
command,
before=before,
report=report,
logger=self._logger,
publisher=self._publisher,
).run()
report.seconds = round(monotonic() - started, 3)
return report
class Pass:
__slots__ = (
"_before",
"_command",
"_logger",
"_publisher",
"_report",
"_slots",
"_storage",
"_uow",
)
def __init__(
self,
uow: UnitOfWorkFactory,
storage: ObjectStorage,
command: ReconcileStorage,
*,
before: datetime,
report: Reconciliation,
logger: Logger,
publisher: EventPublisher,
) -> None:
self._uow = uow
self._storage = storage
self._command = command
self._before = before
self._report = report
self._logger = logger
self._publisher = publisher
self._slots = asyncio.Semaphore(command.deep_concurrency)
async def run(self) -> None:
objects = aiter(self._objects())
item = await anext(objects, None)
try:
async with asyncio.TaskGroup() as checks:
async for row in self._rows():
while item is not None and item.key < row.object_key:
await self._orphan(item)
item = await anext(objects, None)
matched = item if item is not None and item.key == row.object_key else None
await self._row(row, matched, checks)
if matched is not None:
item = await anext(objects, None)
while item is not None:
await self._orphan(item)
item = await anext(objects, None)
except* Unavailable as failed:
error = failed.exceptions[0]
raise error from error.__cause__
async def _timed[ResultT](self, name: str, step: Awaitable[ResultT]) -> ResultT:
started = monotonic()
try:
return await step
finally:
spent = self._report.waited.get(name, 0.0) + monotonic() - started
self._report.waited[name] = spent
async def _objects(self) -> AsyncIterator[StoredObject]:
pages = aiter(self._storage.listing(OBJECT_PREFIX, self._command.page_size))
while (page := await self._timed("s3_list", anext(pages, None))) is not None:
self._report.requests["s3_list"] += 1
self._report.objects += len(page)
for item in page:
yield item
async def _rows(self) -> AsyncIterator[Attachment]:
after, size = "", self._command.page_size
while True:
page = await self._timed("sql", self._page(after, size + 1))
self._report.requests["sql"] += 1
rows = page[:size]
self._report.rows += len(rows)
for row in rows:
yield row
if len(page) <= size:
return
after = rows[-1].object_key
async def _page(self, after: str, size: int) -> Sequence[Attachment]:
uow = self._uow()
async with uow.transaction():
return await uow.attachments.page_after(after, size)
def _found(self, name: str, subject: str) -> None:
self._report.found[name] += 1
sample = self._report.samples.setdefault(name, [])
if len(sample) < self._command.samples:
sample.append(subject)
async def _safely(self, name: str, action: Awaitable[bool], subject: UUID | None) -> None:
attachment_id = None if subject is None else str(subject)
try:
done = await action
except Exception as error:
self._report.errors += 1
self._logger.warning(
"reconcile.action_failed",
reason=name,
attachment_id=attachment_id,
error=type(error).__name__,
)
return
(self._report.fixed if done else self._report.rechecked)[name] += 1
event = "reconcile.fixed" if done else "reconcile.rechecked"
self._logger.info(event, reason=name, attachment_id=attachment_id)
async def _orphan(self, item: StoredObject) -> None:
if item.modified >= self._before:
self._found("orphan_fresh", item.key)
return
self._found("orphan_stale", item.key)
if self._command.apply:
await self._safely("orphan_stale", self._delete_orphan(item), None)
async def _delete_orphan(self, item: StoredObject) -> bool:
uow = self._uow()
async with uow.transaction():
owner = await uow.attachments.by_key(item.key)
self._report.requests["sql"] += 1
if owner is not None:
return False
self._report.requests["s3_delete"] += 1
await self._storage.delete(item.key)
return True
async def _row(
self, row: Attachment, item: StoredObject | None, checks: asyncio.TaskGroup
) -> None:
if row.status is AttachmentStatus.PENDING:
await self._pending(row)
elif item is None:
await self._broken(row, Quarantine.MISSING)
elif item.size != row.size_bytes:
await self._broken(row, Quarantine.SIZE_MISMATCH)
elif self._command.deep or row.quarantine is not None:
await self._slots.acquire()
checks.create_task(self._verify(row))
async def _verify(self, row: Attachment) -> None:
try:
digest = await self._digest(row.object_key)
except Exception as error:
self._report.errors += 1
self._logger.warning(
"reconcile.hash_failed", attachment_id=str(row.id), error=type(error).__name__
)
return
finally:
self._slots.release()
if digest is None:
await self._broken(row, Quarantine.MISSING)
elif digest != row.sha256:
await self._broken(row, Quarantine.SHA_MISMATCH)
else:
await self._healthy(row)
async def _digest(self, key: str) -> str | None:
self._report.requests["s3_get"] += 1
try:
content = await self._storage.open(key)
except ObjectMissingError:
return None
digest = sha256()
async for chunk in content.chunks:
digest.update(chunk)
self._report.hashed_bytes += len(chunk)
return digest.hexdigest()
async def _broken(self, row: Attachment, reason: Quarantine) -> None:
if row.quarantine is not None:
self._found("quarantined", str(row.id))
return
self._found(reason.value, str(row.id))
if self._command.apply:
await self._safely(reason.value, self._isolate(row, reason), row.id)
async def _isolate(self, row: Attachment, reason: Quarantine) -> bool:
if reason is not Quarantine.SHA_MISMATCH:
self._report.requests["s3_head"] += 1
found = await self._storage.stat(row.object_key)
if found is not None and found.size == row.size_bytes:
return False
row.isolate(reason)
await self._mark(row)
return True
async def _healthy(self, row: Attachment) -> None:
if row.quarantine is None:
return
self._found("healed", str(row.id))
if self._command.apply:
await self._safely("healed", self._release(row), row.id)
async def _release(self, row: Attachment) -> bool:
row.release()
await self._mark(row)
return True
async def _mark(self, row: Attachment) -> None:
uow = self._uow()
async with uow.transaction():
await uow.attachments.mark(row)
await uow.commit()
self._report.requests["sql"] += 1
async def _pending(self, row: Attachment) -> None:
if row.created_at >= self._before:
self._found("pending_fresh", str(row.id))
return
self._found("pending_stale", str(row.id))
if self._command.apply:
await self._safely("pending_stale", self._drop(row), row.id)
async def _drop(self, row: Attachment) -> bool:
uow = self._uow()
async with uow.transaction():
current = await uow.attachments.claim(row.id)
self._report.requests["sql"] += 1
if (
current is None
or current.status is not AttachmentStatus.PENDING
or current.created_at >= self._before
):
return False
self._report.requests["s3_delete"] += 1
await drop(uow, self._storage, current, "reconcile")
await self._publisher.publish(uow.collect_new_events())
return True
@@ -8,6 +8,7 @@ from messenger.attachments.domain.values import Filename, ImageInfo
from messenger.attachments.domain.variants import Variant
from messenger.shared.domain.events import DomainEvent
OBJECT_PREFIX = "attachments/"
STAGING_PREFIX = "staging/"
ORIGINALS_PREFIX = "originals/"
@@ -24,6 +25,12 @@ def original_key(attachment_id: UUID) -> str:
return f"{ORIGINALS_PREFIX}{attachment_id}"
class Quarantine(StrEnum):
MISSING = "missing"
SIZE_MISMATCH = "size_mismatch"
SHA_MISMATCH = "sha_mismatch"
@dataclass(slots=True)
class Attachment:
id: UUID
@@ -41,6 +48,7 @@ class Attachment:
published_version: int = 0
error_code: str | None = None
variants: tuple[Variant, ...] = ()
quarantine: Quarantine | None = None
_events: list[DomainEvent] = field(init=False, default_factory=list, repr=False, compare=False)
@classmethod
@@ -70,6 +78,10 @@ class Attachment:
staging_key=f"{STAGING_PREFIX}{attachment_id}",
)
@property
def object_key(self) -> str:
return self.original_key or self.staging_key or f"{OBJECT_PREFIX}{self.id}"
@property
def content_type(self) -> str:
return self.image.kind.value
@@ -98,6 +110,13 @@ class Attachment:
AttachmentDiscarded(attachment_id=self.id, chat_id=self.chat_id, reason=reason)
)
def isolate(self, reason: Quarantine) -> None:
if self.status is AttachmentStatus.READY:
self.quarantine = reason
def release(self) -> None:
self.quarantine = None
def collect_events(self) -> list[DomainEvent]:
events, self._events = self._events, []
return events
+3 -1
View File
@@ -74,7 +74,7 @@ async def object_storage(settings: AttachmentsSettings) -> AsyncIterator[ObjectS
yield S3ObjectStorage(client, settings.s3_bucket)
class AttachmentsProvider(Provider):
class AttachmentStorageProvider(Provider):
def __init__(self, settings: AttachmentsSettings) -> None:
super().__init__()
self._settings = settings
@@ -88,6 +88,8 @@ class AttachmentsProvider(Provider):
async with object_storage(settings) as storage:
yield storage
class AttachmentsProvider(AttachmentStorageProvider):
@provide(scope=Scope.APP)
def images(self, settings: AttachmentsSettings) -> ImageInspector:
return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency)
@@ -0,0 +1,174 @@
import argparse
import asyncio
import json
import resource
import sys
from collections.abc import Callable, Sequence
from datetime import timedelta
from typing import Any
from dishka import Scope, make_async_container, provide
from messenger.attachments.application.commands import ReconcileStorage
from messenger.attachments.application.ports import ObjectStorage
from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler
from messenger.attachments.config import AttachmentsSettings
from messenger.attachments.domain.events import AttachmentDiscarded
from messenger.attachments.entrypoints.ioc import AttachmentStorageProvider
from messenger.attachments.entrypoints.journal import LOG_FIELDS, AttachmentsJournal
from messenger.attachments.infrastructure.postgres import (
PostgresAttachmentsUnitOfWork,
PostgresSingleRun,
)
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.messagebus import EventBus
from messenger.shared.application.ports import Clock, EventPublisher, Logger
from messenger.shared.config import PlatformSettings
from messenger.shared.entrypoints.ioc import PlatformProvider
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
from messenger.shared.infrastructure.postgres import Database
LOCK_KEY = 20_260_004
CLASSES = (
"orphan_stale",
"orphan_fresh",
"missing",
"size_mismatch",
"sha_mismatch",
"pending_stale",
"pending_fresh",
"quarantined",
"healed",
)
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale", "healed")
REQUESTS = ("s3_list", "s3_head", "s3_get", "s3_delete", "sql")
def bounded(low: int, high: int | None = None) -> Callable[[str], int]:
def parse(text: str) -> int:
value = int(text)
if value < low or (high is not None and value > high):
raise argparse.ArgumentTypeError(f"ожидается {low}..{high or '∞'}")
return value
return parse
def command_of(argv: Sequence[str] | None = None) -> ReconcileStorage:
parser = argparse.ArgumentParser(
prog="python -m messenger.attachments.entrypoints.reconcile",
description="Сверка метаданных вложений в PostgreSQL с объектами bucket S3",
)
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--dry-run", dest="apply", action="store_false", help="только отчёт")
mode.add_argument("--apply", dest="apply", action="store_true", help="исправить найденное")
parser.add_argument("--grace", type=bounded(60), default=3600, help="секунды, ≥ 60")
parser.add_argument("--skew", type=bounded(0), default=60, help="запас на часы, секунды")
parser.add_argument("--deep", action="store_true", help="сверить SHA-256 каждого объекта")
parser.add_argument("--page-size", type=bounded(1, 1000), default=1000)
parser.add_argument("--deep-concurrency", type=bounded(1, 48), default=8)
parser.add_argument("--samples", type=bounded(0, 1000), default=20)
args = parser.parse_args(argv)
return ReconcileStorage(
apply=args.apply,
grace=timedelta(seconds=args.grace),
skew=timedelta(seconds=args.skew),
deep=args.deep,
page_size=args.page_size,
deep_concurrency=args.deep_concurrency,
samples=args.samples,
)
class ReconcileProvider(AttachmentStorageProvider):
@provide(scope=Scope.APP)
def reconciler(
self,
database: Database,
storage: ObjectStorage,
clock: Clock,
logger: Logger,
publisher: EventPublisher,
) -> StorageReconciler:
return StorageReconciler(
lambda: PostgresAttachmentsUnitOfWork(database),
storage,
PostgresSingleRun(database, LOCK_KEY),
clock,
logger=logger,
publisher=publisher,
)
async def reconcile(
command: ReconcileStorage,
platform: PlatformSettings,
settings: AttachmentsSettings,
logger: Logger,
) -> Reconciliation:
container = make_async_container(
PlatformProvider(platform, logger), ReconcileProvider(settings)
)
bus = await container.get(EventBus)
bus.subscribe(AttachmentDiscarded, AttachmentsJournal(logger).on_attachment_discarded)
bus.start()
try:
reconciler = await container.get(StorageReconciler)
return await reconciler(command)
finally:
await bus.stop()
await container.close()
def failure(error: BaseException) -> str:
if isinstance(error, BaseExceptionGroup):
return failure(error.exceptions[0])
if isinstance(error, Unavailable) and error.__cause__ is not None:
return type(error.__cause__).__name__
return type(error).__name__
def report_of(command: ReconcileStorage, result: Reconciliation) -> dict[str, Any]:
usage = resource.getrusage(resource.RUSAGE_SELF)
return {
"mode": "apply" if command.apply else "dry-run",
"skipped": result.skipped,
"grace_seconds": int(command.grace.total_seconds()),
"skew_seconds": int(command.skew.total_seconds()),
"deep": command.deep,
"page_size": command.page_size,
"scanned": {"objects": result.objects, "rows": result.rows},
"found": {name: result.found[name] for name in CLASSES},
"fixed": {name: result.fixed[name] for name in FIXABLE},
"rechecked": dict(result.rechecked),
"errors": result.errors,
"samples": result.samples,
"cost": {
"seconds": result.seconds,
"requests": {name: result.requests[name] for name in REQUESTS},
"waited_seconds": {name: round(spent, 3) for name, spent in result.waited.items()},
"hashed_bytes": result.hashed_bytes,
"cpu_seconds": round(usage.ru_utime + usage.ru_stime, 3),
"max_rss_mib": round(usage.ru_maxrss / 1024, 1),
},
}
def main(argv: Sequence[str] | None = None) -> None:
command = command_of(argv)
platform = PlatformSettings(db_pool_max_size=command.deep_concurrency + 2)
settings = AttachmentsSettings()
logger = StructlogLogger(
level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS, stream=sys.stderr
)
try:
result = asyncio.run(reconcile(command, platform, settings, logger))
except Exception as error:
logger.error("reconcile.failed", error=failure(error))
raise SystemExit(1) from None
sys.stdout.write(json.dumps(report_of(command, result), ensure_ascii=False, indent=2) + "\n")
raise SystemExit(1 if result.errors else 0)
if __name__ == "__main__":
main()
File diff suppressed because it is too large Load Diff
+24 -2
View File
@@ -1,4 +1,4 @@
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Sequence
from contextlib import asynccontextmanager
from typing import Any
@@ -8,7 +8,7 @@ from botocore.exceptions import BotoCoreError, ClientError
from botocore.handlers import add_expect_header
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.application.ports import ObjectContent
from messenger.attachments.application.ports import ObjectContent, StoredObject
from messenger.shared.application.errors import Unavailable
CHUNK_BYTES = 64 * 1024
@@ -98,6 +98,28 @@ class S3ObjectStorage:
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]:
pages = self._client.get_paginator("list_objects_v2").paginate(
Bucket=self._bucket, Prefix=prefix, PaginationConfig={"PageSize": page_size}
)
try:
async for page in pages:
yield [
StoredObject(item["Key"], item["Size"], item["LastModified"])
for item in page.get("Contents", ())
]
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def stat(self, key: str) -> StoredObject | None:
try:
head = await self._client.head_object(Bucket=self._bucket, Key=key)
except FAILURES as error:
if missing(error):
return None
raise Unavailable("S3 недоступно") from error
return StoredObject(key, head["ContentLength"], head["LastModified"])
async def _chunks(self, body: Any) -> AsyncIterator[bytes]:
async with body as stream:
async for chunk in stream.iter_chunks(CHUNK_BYTES):
+6 -6
View File
@@ -1,5 +1,4 @@
import asyncio
import logging
import os
import sys
from typing import Any
@@ -10,12 +9,13 @@ from uvicorn.server import Server
from uvicorn_worker import UvicornWorker
from messenger.main import LOG_FIELDS
from messenger.shared.entrypoints.health import DRAINING
from messenger.shared.infrastructure.logging import configure
from messenger.shared.infrastructure.logging import StructlogLogger, configure
from messenger.shared.infrastructure.readiness import DRAINING
LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5"))
SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "6"))
DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2"))
LOGGER = StructlogLogger(level=LOG_LEVEL, allowed_fields=LOG_FIELDS)
class StructlogGunicornLogger(GunicornBaseLogger):
@@ -31,7 +31,7 @@ class DrainingServer(Server):
await super().main_loop()
if not self.force_exit:
DRAINING.set()
logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS)
LOGGER.info("service.draining", limit=DRAIN_SECONDS)
await asyncio.sleep(DRAIN_SECONDS)
@@ -40,7 +40,7 @@ class MessengerWorker(UvicornWorker):
"access_log": False,
"server_header": False,
"date_header": False,
"timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)),
"timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS) - 1),
}
async def _serve(self) -> None:
+13 -9
View File
@@ -4,7 +4,7 @@ from messenger.identity.application.commands import CloseSession, OpenSession, R
from messenger.identity.application.ports import (
IdentityUnitOfWork,
PasswordHasher,
SessionDirectory,
SessionRevocations,
)
from messenger.identity.domain.events import SessionClosed, SessionOpened
from messenger.identity.domain.models import Session, User
@@ -62,15 +62,19 @@ class OpenSessionHandler:
class CloseSessionHandler:
__slots__ = ("_sessions", "_uow")
__slots__ = ("_revocations", "_uow")
def __init__(self, uow: IdentityUnitOfWork, sessions: SessionDirectory) -> None:
def __init__(self, uow: IdentityUnitOfWork, revocations: SessionRevocations) -> None:
self._uow = uow
self._sessions = sessions
self._revocations = revocations
async def __call__(self, command: CloseSession) -> None:
await self._sessions.invalidate(command.session)
async with self._uow.transaction():
await self._uow.sessions.remove(command.session.token_hash)
self._uow.record(SessionClosed(user_id=command.session.user_id))
await self._uow.commit()
await self._revocations.revoke(command.session)
try:
async with self._uow.transaction():
await self._uow.sessions.remove(command.session.token_hash)
self._uow.record(SessionClosed(user_id=command.session.user_id))
await self._uow.commit()
except BaseException:
await self._revocations.restore(command.session)
raise
+6 -1
View File
@@ -38,7 +38,12 @@ class UserDirectory(Protocol):
class SessionDirectory(Protocol):
async def find(self, token_hash: bytes, now: datetime) -> Session | None: ...
async def invalidate(self, session: Session) -> None: ...
@runtime_checkable
class SessionRevocations(Protocol):
async def revoke(self, session: Session) -> None: ...
async def restore(self, session: Session) -> None: ...
@runtime_checkable
+8 -3
View File
@@ -17,6 +17,7 @@ from messenger.identity.application.ports import (
IdentityUnitOfWork,
PasswordHasher,
SessionDirectory,
SessionRevocations,
UserDirectory,
)
from messenger.identity.application.queries import Authenticate, GetCurrentUser, VerifyCredentials
@@ -27,7 +28,7 @@ from messenger.identity.infrastructure.postgres import (
PostgresSessionDirectory,
PostgresUserDirectory,
)
from messenger.identity.infrastructure.valkey import CachedSessionDirectory
from messenger.identity.infrastructure.valkey import CachedSessionDirectory, NoRevocations
from messenger.shared.application.commands import Command
from messenger.shared.application.messagebus import CommandHandler
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
@@ -76,6 +77,10 @@ class IdentityProvider(Provider):
finally:
await client.aclose()
@provide(scope=Scope.APP)
def revocations(self, sessions: SessionDirectory) -> SessionRevocations:
return sessions if isinstance(sessions, CachedSessionDirectory) else NoRevocations()
@provide(scope=Scope.APP)
def verify(self, users: UserDirectory, hasher: PasswordHasher) -> VerifyCredentials:
return VerifyCredentials(users, hasher)
@@ -100,13 +105,13 @@ class IdentityProvider(Provider):
clock: Clock,
ids: IdGenerator,
hasher: PasswordHasher,
sessions: SessionDirectory,
revocations: SessionRevocations,
settings: IdentitySettings,
) -> IdentityCommandBus:
ttl = timedelta(seconds=settings.session_ttl_seconds)
handlers: dict[type[Command], CommandHandler] = {
RegisterUser: RegisterUserHandler(uow, clock, ids, hasher),
OpenSession: OpenSessionHandler(uow, clock, ttl),
CloseSession: CloseSessionHandler(uow, sessions),
CloseSession: CloseSessionHandler(uow, revocations),
}
return IdentityCommandBus(handlers, uow, publisher)
+121 -63
View File
@@ -1,85 +1,126 @@
from datetime import datetime
from uuid import UUID
from asyncpg import Record
from sqlalchemy import (
CheckConstraint,
DateTime,
ForeignKey,
Index,
LargeBinary,
UniqueConstraint,
delete,
literal,
select,
)
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.orm import Mapped, mapped_column
from messenger.identity.domain.errors import UsernameAlreadyTaken
from messenger.identity.domain.models import Session, User
from messenger.identity.domain.values import DisplayName, Username
from messenger.shared.infrastructure.postgres import Connection, Database, PostgresUnitOfWork
INSERT_USER = """
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (username) DO NOTHING
RETURNING true
"""
USER_BY_ID = """
SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1
"""
USER_BY_USERNAME = """
SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1
"""
INSERT_SESSION = """
WITH expired AS (DELETE FROM sessions WHERE user_id = $2 AND expires_at <= $3)
INSERT INTO sessions (token_hash, user_id, created_at, expires_at) VALUES ($1, $2, $3, $4)
"""
DELETE_SESSION = "DELETE FROM sessions WHERE token_hash = $1"
ACTIVE_SESSION = """
SELECT token_hash, user_id, created_at, expires_at
FROM sessions
WHERE token_hash = $1 AND expires_at > $2
"""
from messenger.shared.infrastructure.postgres import (
Base,
Database,
DatabaseSession,
PostgresUnitOfWork,
)
def user_of(row: Record | None) -> User | None:
class UserRow(Base):
__tablename__ = "users"
__table_args__ = (
UniqueConstraint("username", name="users_username_key"),
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="users_username_format"),
CheckConstraint(
"char_length(display_name) BETWEEN 1 AND 100", name="users_display_name_check"
),
CheckConstraint("password_hash LIKE '$argon2id$%'", name="users_password_hash_check"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
username: Mapped[str]
display_name: Mapped[str]
password_hash: Mapped[str]
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class SessionRow(Base):
__tablename__ = "sessions"
__table_args__ = (
CheckConstraint("octet_length(token_hash) = 32", name="sessions_token_hash_check"),
CheckConstraint("expires_at > created_at", name="sessions_check"),
Index("sessions_user_idx", "user_id"),
)
token_hash: Mapped[bytes] = mapped_column(LargeBinary, primary_key=True)
user_id: Mapped[UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
def user_of(row: UserRow | None) -> User | None:
if row is None:
return None
return User(
id=row["id"],
username=Username(row["username"]),
display_name=DisplayName(row["display_name"]),
created_at=row["created_at"],
password_hash=row["password_hash"],
id=row.id,
username=Username(row.username),
display_name=DisplayName(row.display_name),
created_at=row.created_at,
password_hash=row.password_hash,
)
class PostgresUserRepository:
__slots__ = ("_connection",)
__slots__ = ("_session",)
def __init__(self, connection: Connection) -> None:
self._connection = connection
def __init__(self, session: DatabaseSession) -> None:
self._session = session
async def add(self, user: User) -> None:
inserted = await self._connection.fetchval(
INSERT_USER,
user.id,
user.username.value,
user.display_name.value,
user.password_hash,
user.created_at,
statement = (
insert(UserRow)
.values(
id=user.id,
username=user.username.value,
display_name=user.display_name.value,
password_hash=user.password_hash,
created_at=user.created_at,
)
.on_conflict_do_nothing(index_elements=["username"])
.returning(literal(True))
)
inserted = (await self._session.execute(statement)).scalar_one_or_none()
if inserted is None:
raise UsernameAlreadyTaken("username уже занят")
class PostgresSessionRepository:
__slots__ = ("_connection",)
__slots__ = ("_session",)
def __init__(self, connection: Connection) -> None:
self._connection = connection
def __init__(self, session: DatabaseSession) -> None:
self._session = session
async def add(self, session: Session) -> None:
await self._connection.execute(
INSERT_SESSION,
session.token_hash,
session.user_id,
session.created_at,
session.expires_at,
expired = (
delete(SessionRow)
.where(
SessionRow.user_id == session.user_id, SessionRow.expires_at <= session.created_at
)
.cte("expired")
)
await self._session.execute(
insert(SessionRow)
.values(
token_hash=session.token_hash,
user_id=session.user_id,
created_at=session.created_at,
expires_at=session.expires_at,
)
.add_cte(expired)
)
async def remove(self, token_hash: bytes) -> None:
await self._connection.execute(DELETE_SESSION, token_hash)
await self._session.execute(delete(SessionRow).where(SessionRow.token_hash == token_hash))
class PostgresIdentityUnitOfWork(PostgresUnitOfWork):
@@ -87,11 +128,11 @@ class PostgresIdentityUnitOfWork(PostgresUnitOfWork):
@property
def users(self) -> PostgresUserRepository:
return PostgresUserRepository(self.connection)
return PostgresUserRepository(self.session)
@property
def sessions(self) -> PostgresSessionRepository:
return PostgresSessionRepository(self.connection)
return PostgresSessionRepository(self.session)
class PostgresUserDirectory:
@@ -101,12 +142,18 @@ class PostgresUserDirectory:
self._database = database
async def find(self, user_id: UUID) -> User | None:
async with self._database.connection() as connection:
return user_of(await connection.fetchrow(USER_BY_ID, user_id))
async with self._database.session() as session:
row = (
await session.execute(select(UserRow).where(UserRow.id == user_id))
).scalar_one_or_none()
return user_of(row)
async def find_by_username(self, username: str) -> User | None:
async with self._database.connection() as connection:
return user_of(await connection.fetchrow(USER_BY_USERNAME, username))
async with self._database.session() as session:
row = (
await session.execute(select(UserRow).where(UserRow.username == username))
).scalar_one_or_none()
return user_of(row)
class PostgresSessionDirectory:
@@ -116,9 +163,20 @@ class PostgresSessionDirectory:
self._database = database
async def find(self, token_hash: bytes, now: datetime) -> Session | None:
async with self._database.connection() as connection:
row = await connection.fetchrow(ACTIVE_SESSION, token_hash, now)
return None if row is None else Session(**dict(row))
async def invalidate(self, session: Session) -> None:
return None
async with self._database.session() as session:
row = (
await session.execute(
select(SessionRow).where(
SessionRow.token_hash == token_hash,
SessionRow.expires_at > now,
)
)
).scalar_one_or_none()
if row is None:
return None
return Session(
token_hash=row.token_hash,
user_id=row.user_id,
created_at=row.created_at,
expires_at=row.expires_at,
)
@@ -37,6 +37,14 @@ def decode(token_hash: bytes, value: str) -> Session:
)
class NoRevocations:
async def revoke(self, session: Session) -> None:
return None
async def restore(self, session: Session) -> None:
return None
class CachedSessionDirectory:
__slots__ = ("_client", "_cooldown", "_limit", "_logger", "_origin", "_resume_at")
@@ -70,7 +78,7 @@ class CachedSessionDirectory:
await self._quietly(self._client.set(key, encode(session), pxat=until, nx=True))
return session
async def invalidate(self, session: Session) -> None:
async def revoke(self, session: Session) -> None:
if not self._closed():
raise Unavailable("кэш сессий недоступен")
try:
@@ -81,6 +89,9 @@ class CachedSessionDirectory:
self._trip(error)
raise Unavailable("кэш сессий недоступен") from error
async def restore(self, session: Session) -> None:
await self._quietly(self._client.delete(key_of(session.token_hash)))
async def _read(self, key: str) -> str | None:
if not self._closed():
return None
+2 -1
View File
@@ -40,7 +40,7 @@ from messenger.shared.entrypoints.ioc import PlatformProvider
from messenger.shared.entrypoints.pipeline import RequestPipeline
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
TRANSPORT_FIELDS = frozenset({"dropped_total", "queue", "instance"})
TRANSPORT_FIELDS = frozenset({"dropped_total"})
LOG_FIELDS = (
PLATFORM_FIELDS | TRANSPORT_FIELDS | IDENTITY_FIELDS | MESSAGING_FIELDS | ATTACHMENT_FIELDS
@@ -148,6 +148,7 @@ def create_app(
guard=guard,
instance=platform.instance_id,
limits=((UPLOAD_PATH, attachments.max_upload_bytes),),
min_rate=transport.min_transfer_rate_bytes,
)
errors.install(app, logger, ERROR_CODES, challenge_for, platform.instance_id)
app.include_router(health.router)
+31 -29
View File
@@ -1,9 +1,14 @@
from datetime import timedelta
from messenger.messaging.application.commands import CreateChat, PostMessage
from messenger.messaging.application.ports import AttachmentStates, MessagingUnitOfWork
from messenger.messaging.domain.errors import AttachmentUnavailable, ChatNotFound
from messenger.messaging.domain.models import Chat, Message, Receipt
from messenger.messaging.application.ports import ChatDirectory, MessageLog, MessagingUnitOfWork
from messenger.messaging.domain.errors import (
ChatNotFound,
InvalidAttachmentList,
InvalidMessageText,
)
from messenger.messaging.domain.models import Chat, Message, attachment_list
from messenger.messaging.domain.values import MessageText
from messenger.shared.application.ports import Clock, IdGenerator
@@ -30,44 +35,41 @@ class CreateChatHandler:
class PostMessageHandler:
__slots__ = ("_attachments", "_clock", "_ids", "_ttl", "_uow")
__slots__ = ("_chats", "_clock", "_ids", "_log", "_ttl", "_uow")
def __init__(
self,
uow: MessagingUnitOfWork,
attachments: AttachmentStates,
log: MessageLog,
chats: ChatDirectory,
clock: Clock,
ids: IdGenerator,
*,
ttl: timedelta,
) -> None:
self._uow = uow
self._attachments = attachments
self._log = log
self._chats = chats
self._clock = clock
self._ids = ids
self._ttl = ttl
async def __call__(self, command: PostMessage) -> Message:
async with self._uow.transaction():
chat = await self._uow.chats.get(command.chat_id)
if chat is None:
raise ChatNotFound("чат не найден")
now = self._clock.now()
message = chat.post(
message_id=self._ids.new_id(),
sender_id=command.sender_id,
text=command.text,
now=now,
attachment_ids=command.attachment_ids,
)
known = await self._uow.chats.receipt(
chat.id, command.sender_id, command.key, now - self._ttl
)
if known is not None:
return known.replay(command.fingerprint)
if message.attachment_ids and await self._attachments.unusable(
chat.id, message.attachment_ids
):
raise AttachmentUnavailable("вложение не обработано и не может быть отправлено")
await self._uow.chats.append(Receipt(command.key, command.fingerprint, message))
await self._uow.commit()
try:
text = MessageText(command.text)
attachments = attachment_list(text, command.attachment_ids)
except InvalidMessageText, InvalidAttachmentList:
if await self._chats.chat_for_member(command.chat_id, command.sender_id) is None:
raise ChatNotFound("чат не найден") from None
raise
now = self._clock.now()
draft = Message(
self._ids.new_id(), command.chat_id, command.sender_id, text, now, attachments
)
receipt = await self._log.post(draft, command.key, command.fingerprint, now - self._ttl)
if receipt is None:
raise ChatNotFound("чат не найден")
message = receipt.replay(command.fingerprint)
if message.id == draft.id:
self._uow.record(message.posted())
return message
+4 -5
View File
@@ -19,14 +19,13 @@ class CursorCodec(Protocol):
class ChatRepository(Protocol):
async def add(self, chat: Chat) -> None: ...
async def get(self, chat_id: UUID) -> Chat | None: ...
async def receipt(
self, chat_id: UUID, sender_id: UUID, key: str, since: datetime
@runtime_checkable
class MessageLog(Protocol):
async def post(
self, message: Message, key: str, fingerprint: bytes, since: datetime
) -> Receipt | None: ...
async def append(self, receipt: Receipt) -> None: ...
@runtime_checkable
class MessagingUnitOfWork(UnitOfWork, Protocol):
@@ -43,7 +43,7 @@ class ListChats:
async def __call__(self, user_id: UUID, *, limit: int, cursor: str | None) -> Page[Chat]:
scope = chats_scope(user_id)
after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None
after: HistoryKey | None = None if cursor is None else self._cursors.decode(scope, cursor)
items, has_more = await self._chats.chats_of(user_id, after=after, limit=limit)
next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None
return Page(items=items, next_cursor=next_cursor)
@@ -67,7 +67,7 @@ class ReadHistory:
if await self._chats.chat_for_member(chat_id, user_id) is None:
raise ChatNotFound("чат не найден")
scope = messages_scope(chat_id)
after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None
after: HistoryKey | None = None if cursor is None else self._cursors.decode(scope, cursor)
items, has_more = await self._chats.history(chat_id, after=after, limit=limit)
next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None
return Page(items=items, next_cursor=next_cursor)
+11 -57
View File
@@ -1,10 +1,9 @@
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta
from datetime import datetime
from uuid import UUID
from messenger.messaging.domain.errors import (
ChatNotFound,
IdempotencyConflict,
InvalidAttachmentList,
InvalidChatData,
@@ -20,8 +19,6 @@ from messenger.messaging.domain.values import (
)
from messenger.shared.domain.events import DomainEvent
TICK = timedelta(microseconds=1)
@dataclass(frozen=True, slots=True)
class Message:
@@ -36,6 +33,15 @@ class Message:
def key(self) -> HistoryKey:
return (self.created_at, self.id.int)
def posted(self) -> MessagePosted:
return MessagePosted(
message_id=self.id,
chat_id=self.chat_id,
sender_id=self.sender_id,
text_length=len(self.text.value),
created_at=self.created_at,
)
def attachment_list(text: MessageText, attachment_ids: Sequence[UUID]) -> tuple[UUID, ...]:
if len(attachment_ids) > MESSAGE_ATTACHMENTS_MAX:
@@ -62,15 +68,7 @@ class Receipt:
class Chat:
__slots__ = (
"_events",
"_members",
"created_at",
"id",
"last_message_at",
"member_ids",
"title",
)
__slots__ = ("_events", "created_at", "id", "member_ids", "title")
def __init__(
self,
@@ -79,14 +77,11 @@ class Chat:
title: ChatTitle,
member_ids: Sequence[UUID],
created_at: datetime,
last_message_at: datetime | None = None,
) -> None:
self.id = chat_id
self.title = title
self.member_ids: tuple[UUID, ...] = tuple(member_ids)
self.created_at = created_at
self.last_message_at = last_message_at
self._members: frozenset[UUID] = frozenset(member_ids)
self._events: list[DomainEvent] = []
@classmethod
@@ -131,47 +126,6 @@ class Chat:
def record(self, event: DomainEvent) -> None:
self._events.append(event)
def is_member(self, user_id: UUID) -> bool:
return user_id in self._members
def require_member(self, user_id: UUID) -> None:
if not self.is_member(user_id):
raise ChatNotFound("чат не найден")
def post(
self,
*,
message_id: UUID,
sender_id: UUID,
text: str,
now: datetime,
attachment_ids: Sequence[UUID] = (),
) -> Message:
self.require_member(sender_id)
body = MessageText(text)
attachments = attachment_list(body, attachment_ids)
last = self.last_message_at
created_at = now if last is None or now > last else last + TICK
message = Message(
id=message_id,
chat_id=self.id,
sender_id=sender_id,
text=body,
created_at=created_at,
attachment_ids=attachments,
)
self.last_message_at = created_at
self.record(
MessagePosted(
message_id=message.id,
chat_id=self.id,
sender_id=sender_id,
text_length=len(message.text.value),
created_at=created_at,
)
)
return message
def collect_events(self) -> list[DomainEvent]:
events, self._events = self._events, []
return events
+9 -2
View File
@@ -9,6 +9,7 @@ from messenger.messaging.application.ports import (
AttachmentStates,
ChatDirectory,
CursorCodec,
MessageLog,
MessagingUnitOfWork,
)
from messenger.messaging.application.queries import GetChat, ListChats, ReadHistory
@@ -16,6 +17,7 @@ from messenger.messaging.config import MessagingSettings
from messenger.messaging.infrastructure.cursor import SignedCursorCodec
from messenger.messaging.infrastructure.postgres import (
PostgresChatDirectory,
PostgresMessageLog,
PostgresMessagingUnitOfWork,
)
from messenger.shared.application.commands import Command
@@ -37,6 +39,10 @@ class MessagingProvider(Provider):
def directory(self, database: Database) -> ChatDirectory:
return PostgresChatDirectory(database)
@provide(scope=Scope.APP)
def log(self, database: Database, attachments: AttachmentStates) -> MessageLog:
return PostgresMessageLog(database, attachments)
@provide(scope=Scope.APP)
def cursors(self, settings: MessagingSettings) -> CursorCodec:
return SignedCursorCodec(settings.cursor_secret.get_secret_value().encode())
@@ -61,7 +67,8 @@ class MessagingProvider(Provider):
def command_bus(
self,
uow: MessagingUnitOfWork,
attachments: AttachmentStates,
log: MessageLog,
directory: ChatDirectory,
publisher: EventPublisher,
clock: Clock,
ids: IdGenerator,
@@ -70,6 +77,6 @@ class MessagingProvider(Provider):
ttl = timedelta(seconds=settings.idempotency_ttl_seconds)
handlers: dict[type[Command], CommandHandler] = {
CreateChat: CreateChatHandler(uow, clock, ids),
PostMessage: PostMessageHandler(uow, attachments, clock, ids, ttl),
PostMessage: PostMessageHandler(uow, log, directory, clock, ids, ttl=ttl),
}
return MessagingCommandBus(handlers, uow, publisher)
@@ -37,7 +37,7 @@ class CreateMessageRequest(BaseModel):
class PageQuery(BaseModel):
limit: int = Field(default=50, ge=1, le=100)
cursor: str | None = None
cursor: str | None = Field(default=None, min_length=1)
class ChatOut(BaseModel):
+375 -143
View File
@@ -1,122 +1,266 @@
from collections.abc import Sequence
from datetime import datetime
from dataclasses import replace
from datetime import datetime, timedelta
from typing import cast
from uuid import UUID
from asyncpg import ForeignKeyViolationError, Record
from sqlalchemy import (
CheckConstraint,
DateTime,
ForeignKey,
ForeignKeyConstraint,
Index,
LargeBinary,
RowMapping,
Select,
SmallInteger,
Table,
UniqueConstraint,
bindparam,
column,
func,
insert,
lambda_stmt,
literal,
select,
true,
tuple_,
update,
)
from sqlalchemy.dialects.postgresql import ARRAY
from sqlalchemy.dialects.postgresql import UUID as PGUUID
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.sql.elements import Label
from messenger.messaging.domain.errors import InvalidChatData, UnknownAttachment
from messenger.messaging.application.ports import AttachmentStates
from messenger.messaging.domain.errors import (
AttachmentUnavailable,
InvalidChatData,
UnknownAttachment,
)
from messenger.messaging.domain.keys import HistoryKey
from messenger.messaging.domain.models import Chat, Message, Receipt
from messenger.messaging.domain.values import ChatTitle, MessageText
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.postgres import (
Connection,
Base,
Database,
DatabaseSession,
EventSource,
PostgresUnitOfWork,
)
INSERT_CHAT = "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)"
INSERT_MEMBERS = """
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT $1, member.user_id, member.position, $3
FROM unnest($2::uuid[]) WITH ORDINALITY AS member (user_id, position)
"""
LOCK_CHAT = """
SELECT c.id, c.title, c.created_at, c.last_message_at,
ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids
FROM chats c
WHERE c.id = $1
FOR UPDATE
"""
ATTACHMENT_FKEY = "message_attachments_attachment_fkey"
RECEIPT = """
SELECT k.fingerprint, m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT a.attachment_id FROM message_attachments a
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.chat_id = $1 AND k.user_id = $2 AND k.key = $3 AND k.created_at > $4
"""
APPEND_MESSAGE = """
WITH message AS (
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES ($1, $2, $3, $4, $5)
RETURNING id, chat_id, sender_id, created_at
), links AS (
INSERT INTO message_attachments (message_id, position, chat_id, attachment_id)
SELECT message.id, link.position - 1, message.chat_id, link.attachment_id
FROM message, unnest($8::uuid[]) WITH ORDINALITY AS link (attachment_id, position)
), receipt AS (
INSERT INTO idempotency_keys (chat_id, user_id, key, fingerprint, message_id, created_at)
SELECT chat_id, sender_id, $6, $7, id, created_at FROM message
ON CONFLICT (chat_id, user_id, key) DO UPDATE
SET fingerprint = EXCLUDED.fingerprint,
message_id = EXCLUDED.message_id,
created_at = EXCLUDED.created_at
)
UPDATE chats SET last_message_at = message.created_at
FROM message
WHERE chats.id = message.chat_id
"""
CHAT_OF_MEMBER = """
SELECT c.id, c.title, c.created_at,
ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids
FROM chat_members m JOIN chats c ON c.id = m.chat_id
WHERE m.chat_id = $1 AND m.user_id = $2
"""
CHATS_FIRST = """
SELECT c.id, c.title, c.created_at,
ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids
FROM chat_members m JOIN chats c ON c.id = m.chat_id
WHERE m.user_id = $1
ORDER BY m.chat_created_at, m.chat_id
LIMIT $2
"""
CHATS_AFTER = """
SELECT c.id, c.title, c.created_at,
ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids
FROM chat_members m JOIN chats c ON c.id = m.chat_id
WHERE m.user_id = $1 AND (m.chat_created_at, m.chat_id) > ($2, $3)
ORDER BY m.chat_created_at, m.chat_id
LIMIT $4
"""
HISTORY_FIRST = """
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT a.attachment_id FROM message_attachments a
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids
FROM messages m
WHERE m.chat_id = $1
ORDER BY m.created_at, m.id
LIMIT $2
"""
HISTORY_AFTER = """
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT a.attachment_id FROM message_attachments a
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids
FROM messages m
WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3)
ORDER BY m.created_at, m.id
LIMIT $4
"""
FOREIGN_KEY_VIOLATED = "23503"
def chat_of(row: Record) -> Chat:
class ChatRow(Base):
__tablename__ = "chats"
__table_args__ = (
CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="chats_title_check"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
title: Mapped[str]
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
last_message_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class ChatMemberRow(Base):
__tablename__ = "chat_members"
__table_args__ = (
CheckConstraint("position BETWEEN 1 AND 100", name="chat_members_position_check"),
Index("chat_members_user_idx", "user_id", "chat_created_at", "chat_id"),
)
chat_id: Mapped[UUID] = mapped_column(
ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
)
user_id: Mapped[UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
position: Mapped[int] = mapped_column(SmallInteger)
chat_created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class MessageRow(Base):
__tablename__ = "messages"
__table_args__ = (
CheckConstraint("char_length(text) <= 4000", name="messages_text_check"),
ForeignKeyConstraint(
["chat_id", "sender_id"], ["chat_members.chat_id", "chat_members.user_id"]
),
Index("messages_history_idx", "chat_id", "created_at", "id"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
chat_id: Mapped[UUID]
sender_id: Mapped[UUID]
text: Mapped[str]
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class IdempotencyKeyRow(Base):
__tablename__ = "idempotency_keys"
__table_args__ = (
CheckConstraint("key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="idempotency_keys_key_check"),
CheckConstraint(
"octet_length(fingerprint) = 32", name="idempotency_keys_fingerprint_check"
),
ForeignKeyConstraint(
["chat_id", "user_id"], ["chat_members.chat_id", "chat_members.user_id"]
),
)
chat_id: Mapped[UUID] = mapped_column(primary_key=True)
user_id: Mapped[UUID] = mapped_column(primary_key=True)
key: Mapped[str] = mapped_column(primary_key=True)
fingerprint: Mapped[bytes] = mapped_column(LargeBinary)
message_id: Mapped[UUID] = mapped_column(ForeignKey("messages.id", ondelete="CASCADE"))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class MessageAttachmentRow(Base):
__tablename__ = "message_attachments"
__table_args__ = (
CheckConstraint("position BETWEEN 0 AND 9", name="message_attachments_position_check"),
UniqueConstraint("message_id", "attachment_id"),
ForeignKeyConstraint(
["attachment_id", "chat_id"],
["attachments.id", "attachments.chat_id"],
name="message_attachments_attachment_fkey",
ondelete="RESTRICT",
),
Index("message_attachments_attachment_idx", "attachment_id"),
)
message_id: Mapped[UUID] = mapped_column(
ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
)
position: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
chat_id: Mapped[UUID]
attachment_id: Mapped[UUID]
def post_message() -> Select:
chats, members = cast(Table, ChatRow.__table__), cast(Table, ChatMemberRow.__table__)
keys, messages = cast(Table, IdempotencyKeyRow.__table__), cast(Table, MessageRow.__table__)
sender = bindparam("sender_id", type_=PGUUID)
chat_id = bindparam("chat_id", type_=PGUUID)
message_id = bindparam("message_id", type_=PGUUID)
chat = (
select(
chats.c.id,
func.greatest(
bindparam("stamp", type_=DateTime(timezone=True)),
chats.c.last_message_at + timedelta(microseconds=1),
).label("stamp"),
)
.join(members, (members.c.chat_id == chats.c.id) & (members.c.user_id == sender))
.where(chats.c.id == chat_id)
.with_for_update(of=chats)
.cte("chat")
)
receipt_insert = pg_insert(keys).from_select(
["chat_id", "user_id", "key", "fingerprint", "message_id", "created_at"],
select(
chat.c.id,
sender,
bindparam("key"),
bindparam("fingerprint", type_=LargeBinary),
message_id,
chat.c.stamp,
),
)
receipt = (
receipt_insert.on_conflict_do_update(
index_elements=[keys.c.chat_id, keys.c.user_id, keys.c.key],
set_={
name: receipt_insert.excluded[name]
for name in ("fingerprint", "message_id", "created_at")
},
where=keys.c.created_at <= bindparam("since", type_=DateTime(timezone=True)),
)
.returning(keys.c.created_at)
.cte("receipt")
)
message = (
insert(messages)
.from_select(
["id", "chat_id", "sender_id", "text", "created_at"],
select(message_id, chat_id, sender, bindparam("text"), receipt.c.created_at),
)
.returning(messages.c.chat_id, messages.c.created_at)
.cte("message")
)
attachments = (
func.unnest(bindparam("attachment_ids", type_=ARRAY(PGUUID)))
.table_valued(column("attachment_id", PGUUID), with_ordinality="position")
.render_derived(name="attachment")
)
links = (
insert(cast(Table, MessageAttachmentRow.__table__))
.from_select(
["message_id", "position", "chat_id", "attachment_id"],
select(
message_id, attachments.c.position - 1, chat_id, attachments.c.attachment_id
).select_from(message.join(attachments, true())),
)
.cte("links")
)
touched = (
update(chats)
.where(chats.c.id == message.c.chat_id)
.values(last_message_at=message.c.created_at)
.cte("touched")
)
return select(
select(chat.c.id).exists().label("member"),
select(message.c.created_at).scalar_subquery().label("created_at"),
).add_cte(links, touched)
POST_MESSAGE = lambda_stmt(post_message, enable_tracking=False)
def member_ids() -> Label[Sequence[UUID]]:
return func.array(
select(ChatMemberRow.user_id)
.where(ChatMemberRow.chat_id == ChatRow.id)
.order_by(ChatMemberRow.position)
.correlate(ChatRow)
.scalar_subquery(),
type_=ARRAY(PGUUID(as_uuid=True)),
).label("member_ids")
def attachment_ids() -> Label[Sequence[UUID]]:
return func.array(
select(MessageAttachmentRow.attachment_id)
.where(MessageAttachmentRow.message_id == MessageRow.id)
.order_by(MessageAttachmentRow.position)
.correlate(MessageRow)
.scalar_subquery(),
type_=ARRAY(PGUUID(as_uuid=True)),
).label("attachment_ids")
def chat_of(row: RowMapping) -> Chat:
return Chat(
chat_id=row["id"],
title=ChatTitle(row["title"]),
member_ids=row["member_ids"],
created_at=row["created_at"],
last_message_at=row.get("last_message_at"),
)
def message_of(row: Record) -> Message:
def message_of(row: RowMapping) -> Message:
return Message(
id=row["id"],
chat_id=row["chat_id"],
sender_id=row["sender_id"],
text=MessageText(row["text"]),
created_at=row["created_at"],
attachment_ids=tuple(row["attachment_ids"]),
created_at=row["created_at"],
)
@@ -125,54 +269,110 @@ def page[ItemT](rows: Sequence[ItemT], limit: int) -> tuple[list[ItemT], bool]:
class PostgresChatRepository:
__slots__ = ("_connection", "_seen")
__slots__ = ("_seen", "_session")
def __init__(self, connection: Connection, seen: list[EventSource]) -> None:
self._connection = connection
def __init__(self, session: DatabaseSession, seen: list[EventSource]) -> None:
self._session = session
self._seen = seen
async def add(self, chat: Chat) -> None:
await self._connection.execute(INSERT_CHAT, chat.id, chat.title.value, chat.created_at)
await self._session.execute(
insert(ChatRow).values(id=chat.id, title=chat.title.value, created_at=chat.created_at)
)
try:
await self._connection.execute(
INSERT_MEMBERS, chat.id, list(chat.member_ids), chat.created_at
await self._session.execute(
insert(ChatMemberRow),
[
{
"chat_id": chat.id,
"user_id": member_id,
"position": position,
"chat_created_at": chat.created_at,
}
for position, member_id in enumerate(chat.member_ids, 1)
],
)
except ForeignKeyViolationError as error:
except IntegrityError as error:
if getattr(error.orig, "sqlstate", None) != FOREIGN_KEY_VIOLATED:
raise
raise InvalidChatData("поле member_ids содержит неизвестного пользователя") from error
self._seen.append(chat)
async def get(self, chat_id: UUID) -> Chat | None:
row = await self._connection.fetchrow(LOCK_CHAT, chat_id)
if row is None:
return None
chat = chat_of(row)
self._seen.append(chat)
return chat
async def receipt(
self, chat_id: UUID, sender_id: UUID, key: str, since: datetime
class PostgresMessageLog:
__slots__ = ("_attachments", "_database")
def __init__(self, database: Database, attachments: AttachmentStates) -> None:
self._database = database
self._attachments = attachments
async def post(
self, message: Message, key: str, fingerprint: bytes, since: datetime
) -> Receipt | None:
row = await self._connection.fetchrow(RECEIPT, chat_id, sender_id, key, since)
return None if row is None else Receipt(key, row["fingerprint"], message_of(row))
async def append(self, receipt: Receipt) -> None:
message = receipt.message
try:
await self._connection.execute(
APPEND_MESSAGE,
message.id,
message.chat_id,
message.sender_id,
message.text.value,
message.created_at,
receipt.key,
receipt.fingerprint,
list(message.attachment_ids),
chat_id, sender_id = message.chat_id, message.sender_id
async with self._database.session() as session:
try:
row = (
(
await session.execute(
POST_MESSAGE,
{
"message_id": message.id,
"chat_id": chat_id,
"sender_id": sender_id,
"text": message.text.value,
"stamp": message.created_at,
"key": key,
"fingerprint": fingerprint,
"since": since,
"attachment_ids": list(message.attachment_ids),
},
)
)
.mappings()
.first()
)
except IntegrityError as error:
cause = getattr(error.orig, "__cause__", None)
if getattr(cause, "constraint_name", None) != "message_attachments_attachment_fkey":
raise
raise UnknownAttachment("вложение не найдено в этом чате") from error
if row is None or not row["member"]:
return None
if row["created_at"] is not None:
if message.attachment_ids and await self._attachments.unusable(
chat_id, message.attachment_ids
):
raise AttachmentUnavailable("вложение не обработано и не может быть отправлено")
await session.commit()
return Receipt(key, fingerprint, replace(message, created_at=row["created_at"]))
known = (
(
await session.execute(
select(
IdempotencyKeyRow.fingerprint,
MessageRow.id,
MessageRow.chat_id,
MessageRow.sender_id,
MessageRow.text,
MessageRow.created_at,
attachment_ids(),
)
.join(MessageRow, MessageRow.id == IdempotencyKeyRow.message_id)
.where(
IdempotencyKeyRow.chat_id == chat_id,
IdempotencyKeyRow.user_id == sender_id,
IdempotencyKeyRow.key == key,
IdempotencyKeyRow.created_at > since,
)
)
)
.mappings()
.first()
)
except ForeignKeyViolationError as error:
if getattr(error, "constraint_name", None) != ATTACHMENT_FKEY:
raise
raise UnknownAttachment("вложение не найдено в этом чате") from error
if known is None:
raise Unavailable("ключ идемпотентности занят незавершённой записью")
return Receipt(key, known["fingerprint"], message_of(known))
class PostgresMessagingUnitOfWork(PostgresUnitOfWork):
@@ -180,7 +380,7 @@ class PostgresMessagingUnitOfWork(PostgresUnitOfWork):
@property
def chats(self) -> PostgresChatRepository:
return PostgresChatRepository(self.connection, self.seen)
return PostgresChatRepository(self.session, self.seen)
class PostgresChatDirectory:
@@ -190,30 +390,62 @@ class PostgresChatDirectory:
self._database = database
async def chat_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None:
async with self._database.connection() as connection:
row = await connection.fetchrow(CHAT_OF_MEMBER, chat_id, user_id)
async with self._database.session() as session:
row = (
(
await session.execute(
select(ChatRow.id, ChatRow.title, ChatRow.created_at, member_ids())
.select_from(ChatMemberRow)
.join(ChatRow, ChatRow.id == ChatMemberRow.chat_id)
.where(ChatMemberRow.chat_id == chat_id, ChatMemberRow.user_id == user_id)
)
)
.mappings()
.first()
)
return None if row is None else chat_of(row)
async def chats_of(
self, user_id: UUID, *, after: HistoryKey | None, limit: int
) -> tuple[list[Chat], bool]:
async with self._database.connection() as connection:
if after is None:
rows = await connection.fetch(CHATS_FIRST, user_id, limit + 1)
else:
rows = await connection.fetch(
CHATS_AFTER, user_id, after[0], UUID(int=after[1]), limit + 1
)
statement = (
select(ChatRow.id, ChatRow.title, ChatRow.created_at, member_ids())
.select_from(ChatMemberRow)
.join(ChatRow, ChatRow.id == ChatMemberRow.chat_id)
.where(ChatMemberRow.user_id == user_id)
.order_by(ChatMemberRow.chat_created_at, ChatMemberRow.chat_id)
.limit(limit + 1)
)
if after is not None:
statement = statement.where(
tuple_(ChatMemberRow.chat_created_at, ChatMemberRow.chat_id)
> tuple_(literal(after[0]), literal(UUID(int=after[1])))
)
async with self._database.session() as session:
rows = (await session.execute(statement)).mappings().all()
return page([chat_of(row) for row in rows], limit)
async def history(
self, chat_id: UUID, *, after: HistoryKey | None, limit: int
) -> tuple[list[Message], bool]:
async with self._database.connection() as connection:
if after is None:
rows = await connection.fetch(HISTORY_FIRST, chat_id, limit + 1)
else:
rows = await connection.fetch(
HISTORY_AFTER, chat_id, after[0], UUID(int=after[1]), limit + 1
)
statement = (
select(
MessageRow.id,
MessageRow.chat_id,
MessageRow.sender_id,
MessageRow.text,
MessageRow.created_at,
attachment_ids(),
)
.where(MessageRow.chat_id == chat_id)
.order_by(MessageRow.created_at, MessageRow.id)
.limit(limit + 1)
)
if after is not None:
statement = statement.where(
tuple_(MessageRow.created_at, MessageRow.id)
> tuple_(literal(after[0]), literal(UUID(int=after[1])))
)
async with self._database.session() as session:
rows = (await session.execute(statement)).mappings().all()
return page([message_of(row) for row in rows], limit)
-70
View File
@@ -1,70 +0,0 @@
import asyncio
import hashlib
from importlib import resources
import asyncpg
from messenger.shared.config import PlatformSettings
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
LOCK_KEY = 20_260_002
JOURNAL = """
CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
checksum text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT now()
)
"""
class MigrationChangedError(Exception):
pass
def migrations() -> list[tuple[str, str]]:
folder = resources.files("messenger") / "migrations"
found = (item for item in folder.iterdir() if item.name.endswith(".sql"))
return sorted((item.name.removesuffix(".sql"), item.read_text()) for item in found)
async def migrate(dsn: str) -> list[str]:
connection = await asyncpg.connect(dsn, timeout=10)
try:
await connection.execute("SELECT pg_advisory_lock($1)", LOCK_KEY)
await connection.execute(JOURNAL)
applied = dict(await connection.fetch("SELECT version, checksum FROM schema_migrations"))
done: list[str] = []
for version, sql in migrations():
checksum = hashlib.sha256(sql.encode()).hexdigest()
if version in applied:
if applied[version] != checksum:
raise MigrationChangedError(version)
continue
async with connection.transaction():
await connection.execute(sql)
await connection.execute(
"INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)",
version,
checksum,
)
done.append(version)
return done
finally:
await connection.close()
def main() -> None:
settings = PlatformSettings()
logger = StructlogLogger(level=settings.log_level, allowed_fields=PLATFORM_FIELDS | {"version"})
try:
done = asyncio.run(migrate(settings.database_url.get_secret_value()))
except MigrationChangedError as error:
logger.error("migration.changed", version=str(error))
raise SystemExit(1) from None
for version in done:
logger.info("migration.applied", version=version)
logger.info("migration.completed", reason="up to date" if not done else "applied")
if __name__ == "__main__":
main()
+3 -11
View File
@@ -37,28 +37,21 @@ class CommandBus:
class EventBus:
__slots__ = ("_deferred", "_dropped", "_logger", "_projections", "_queue", "_worker")
__slots__ = ("_deferred", "_dropped", "_logger", "_queue", "_worker")
def __init__(self, logger: Logger, *, max_queue: int = 10_000) -> None:
self._logger = logger
self._queue: asyncio.Queue[DomainEvent] = asyncio.Queue(maxsize=max_queue)
self._projections: dict[type[DomainEvent], list[Any]] = {}
self._deferred: dict[type[DomainEvent], list[Any]] = {}
self._worker: asyncio.Task[None] | None = None
self._dropped = 0
def project[EventT: DomainEvent](
self, event_type: type[EventT], handler: EventHandler[EventT]
) -> None:
self._projections.setdefault(event_type, []).append(handler)
def subscribe[EventT: DomainEvent](
self, event_type: type[EventT], handler: EventHandler[EventT]
) -> None:
self._deferred.setdefault(event_type, []).append(handler)
async def emit(self, event: DomainEvent) -> bool:
await self._run(self._projections.get(type(event), []), event, "projection")
if not self._deferred.get(type(event)):
return True
try:
@@ -96,11 +89,11 @@ class EventBus:
while True:
event = await self._queue.get()
try:
await self._run(self._deferred.get(type(event), []), event, "subscriber")
await self._run(self._deferred.get(type(event), []), event)
finally:
self._queue.task_done()
async def _run(self, handlers: Sequence[Any], event: DomainEvent, kind: str) -> None:
async def _run(self, handlers: Sequence[Any], event: DomainEvent) -> None:
for handler in handlers:
try:
await handler(event)
@@ -108,7 +101,6 @@ class EventBus:
self._logger.error(
"event.handler_failed",
event_name=event.name,
reason=kind,
error=type(error).__name__,
)
@@ -26,6 +26,11 @@ class Logger(Protocol):
def error(self, event: str, /, **fields: Any) -> None: ...
@runtime_checkable
class Readiness(Protocol):
async def check(self) -> None: ...
@runtime_checkable
class UnitOfWork(Protocol):
def transaction(self) -> AbstractAsyncContextManager[None]: ...
+1 -1
View File
@@ -9,7 +9,7 @@ from messenger.shared.entrypoints.errors import Unauthenticated
async def current_actor(request: Request) -> UUID:
actor = getattr(request.state, "actor", None)
if not isinstance(actor, UUID):
raise Unauthenticated("заголовок идентификации обязателен и должен быть действительным")
raise Unauthenticated("сессия не подтверждена")
return actor
+1 -1
View File
@@ -7,4 +7,4 @@ class TransportSettings(Settings):
docs: bool = False
max_body_bytes: int = Field(default=1_048_576, ge=4_096)
request_timeout_seconds: float = Field(default=8.0, gt=0, le=300)
shutdown_timeout_seconds: int = Field(default=5, ge=1, le=10)
min_transfer_rate_bytes: int = Field(default=131_072, ge=1_024)
+1 -1
View File
@@ -46,7 +46,7 @@ class Unauthenticated(Exception):
def classify(error: DomainError, codes: ErrorCodes) -> tuple[str, int]:
for kind in type(error).__mro__:
known = codes.get(kind) # type: ignore[arg-type]
known = codes.get(kind)
if known is not None:
return known
return "internal_error", status.HTTP_500_INTERNAL_SERVER_ERROR
+3 -9
View File
@@ -1,14 +1,10 @@
from threading import Event
from dishka.integrations.fastapi import FromDishka, inject
from fastapi import APIRouter
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import Readiness
from messenger.shared.entrypoints.schemas import HealthOut
from messenger.shared.infrastructure.postgres import Database
router = APIRouter(tags=["health"])
DRAINING = Event()
@router.get("/health/live")
@@ -18,8 +14,6 @@ async def live() -> HealthOut:
@router.get("/health/ready")
@inject
async def ready(database: FromDishka[Database]) -> HealthOut:
if DRAINING.is_set():
raise Unavailable("Реплика выходит из ротации")
await database.ping()
async def ready(readiness: FromDishka[Readiness]) -> HealthOut:
await readiness.check()
return HealthOut()
+6 -1
View File
@@ -3,11 +3,12 @@ from collections.abc import AsyncIterator
from dishka import Provider, Scope, provide
from messenger.shared.application.messagebus import EventBus, InMemoryEventPublisher
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger, Readiness
from messenger.shared.config import PlatformSettings
from messenger.shared.infrastructure.clock import MonotonicUtcClock
from messenger.shared.infrastructure.ids import Uuid7Generator
from messenger.shared.infrastructure.postgres import Database
from messenger.shared.infrastructure.readiness import DatabaseReadiness
class PlatformProvider(Provider):
@@ -45,6 +46,10 @@ class PlatformProvider(Provider):
yield database
await database.close()
@provide(scope=Scope.APP)
def readiness(self, database: Database) -> Readiness:
return DatabaseReadiness(database)
@provide(scope=Scope.APP)
def event_bus(self, logger: Logger, settings: PlatformSettings) -> EventBus:
return EventBus(logger, max_queue=settings.event_queue_max)
+66 -15
View File
@@ -5,6 +5,7 @@ from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from uuid import uuid4
from starlette.exceptions import HTTPException
from starlette.types import ASGIApp, Message, Receive, Scope, Send
from messenger.shared.application.errors import Unavailable
@@ -57,8 +58,44 @@ async def send_error(
await send({"type": "http.response.body", "body": body})
class Pace:
__slots__ = ("_deadline", "_moved", "_opened", "_rate", "_timeout")
def __init__(self, deadline: asyncio.Timeout, timeout: float, rate: float) -> None:
self._deadline = deadline
self._timeout = timeout
self._rate = rate
self._opened = asyncio.get_running_loop().time()
self._moved = 0
def around(self, receive: Receive, send: Send) -> tuple[Receive, Send]:
async def paced_receive() -> Message:
return self._count(await receive())
async def paced_send(message: Message) -> None:
await send(self._count(message))
return paced_receive, paced_send
def _count(self, message: Message) -> Message:
if size := len(message.get("body", b"")):
self._moved += size
moved = min(asyncio.get_running_loop().time(), self._opened + self._moved / self._rate)
self._deadline.reschedule(moved + self._timeout)
return message
class RequestPipeline:
__slots__ = ("_app", "_guard", "_instance", "_limits", "_logger", "_max_bytes", "_timeout")
__slots__ = (
"_app",
"_guard",
"_instance",
"_limits",
"_logger",
"_max_bytes",
"_min_rate",
"_timeout",
)
def __init__(
self,
@@ -69,12 +106,14 @@ class RequestPipeline:
guard: Guard,
instance: str,
limits: BodyLimits = (),
min_rate: float = 131_072,
) -> None:
self._app = app
self._logger = logger
self._max_bytes = max_bytes
self._limits = limits
self._timeout = timeout
self._min_rate = min_rate
self._guard = guard
self._instance = instance.encode()
@@ -85,9 +124,7 @@ class RequestPipeline:
incoming = header(scope, HEADER_BYTES)
request_id = incoming[:MAX_REQUEST_ID] if incoming else uuid4().hex
state = scope.setdefault("state", {})
state["request_id"] = request_id
state["actor"] = None
scope.setdefault("state", {}).update(request_id=request_id, actor=None)
started = time.perf_counter()
status_holder = [500]
answered = [False]
@@ -106,18 +143,23 @@ class RequestPipeline:
message = {**message, "headers": headers}
await send(message)
declared = header(scope, b"content-length")
if declared.isdigit() and int(declared) > self._limit(scope["path"]):
await send_error(wrapped_send, 413, "payload_too_large", request_id)
self._log(scope, status_holder[0], request_id, started)
return
limit = self._limit(scope["path"])
received = [0]
async def limited_receive() -> Message:
message = await receive()
received[0] += len(message.get("body", b""))
if received[0] > limit:
raise HTTPException(413)
return message
rejection: Rejection | None = None
try:
async with asyncio.timeout(self._timeout):
rejection = await self._guard(scope)
async with asyncio.timeout(self._timeout) as deadline:
rejection = await self._guard(scope) or self._oversized(scope, limit)
if rejection is None:
await self._app(scope, receive, wrapped_send)
pace = Pace(deadline, self._timeout, self._min_rate)
await self._app(scope, *pace.around(limited_receive, wrapped_send))
except Unavailable as error:
self._logger.warning(
"dependency.unavailable",
@@ -126,18 +168,21 @@ class RequestPipeline:
request_id=request_id,
)
rejection = Rejection(503, "unavailable")
except TimeoutError:
except (TimeoutError, asyncio.CancelledError) as error:
cancelled = isinstance(error, asyncio.CancelledError)
self._logger.warning(
"request.timeout",
"request.cancelled" if cancelled else "request.timeout",
method=scope["method"],
route=self._route(scope),
request_id=request_id,
limit=self._timeout,
limit=None if cancelled else self._timeout,
duration_ms=round((time.perf_counter() - started) * 1000, 3),
)
if not answered[0]:
await send_error(wrapped_send, 503, "unavailable", request_id)
self._log(scope, status_holder[0], request_id, started)
if cancelled:
raise
return
except Exception:
self._logger.error(
@@ -160,6 +205,12 @@ class RequestPipeline:
return limit
return self._max_bytes
def _oversized(self, scope: Scope, limit: int) -> Rejection | None:
declared = header(scope, b"content-length")
if declared.isdigit() and int(declared) > limit:
return Rejection(413, "payload_too_large")
return None
def _route(self, scope: Scope) -> str:
route = scope.get("route")
return getattr(route, "path", scope["path"])
@@ -1,6 +1,6 @@
import logging
import sys
from typing import Any
from typing import Any, TextIO
import orjson
import structlog
@@ -51,7 +51,11 @@ def field_filter(allowed: frozenset[str]) -> Processor:
return drop_unlisted_fields
def configure(level: str = "INFO", allowed_fields: frozenset[str] = PLATFORM_FIELDS) -> None:
def configure(
level: str = "INFO",
allowed_fields: frozenset[str] = PLATFORM_FIELDS,
stream: TextIO | None = None,
) -> None:
shared = [
structlog.contextvars.merge_contextvars,
structlog.processors.add_log_level,
@@ -63,7 +67,7 @@ def configure(level: str = "INFO", allowed_fields: frozenset[str] = PLATFORM_FIE
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=False,
)
handler = logging.StreamHandler(sys.stdout)
handler = logging.StreamHandler(stream or sys.stdout)
handler.setFormatter(
structlog.stdlib.ProcessorFormatter(
foreign_pre_chain=shared,
@@ -99,8 +103,9 @@ class StructlogLogger:
name: str = "messenger",
level: str = "INFO",
allowed_fields: frozenset[str] = PLATFORM_FIELDS,
stream: TextIO | None = None,
) -> None:
configure(level, allowed_fields)
configure(level, allowed_fields, stream)
self._logger = structlog.get_logger(name)
def info(self, event: str, /, **fields: Any) -> None:
+65 -45
View File
@@ -1,32 +1,45 @@
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from typing import Any, Protocol
from typing import ClassVar, Protocol
import asyncpg
from sqlalchemy import Text, make_url, select
from sqlalchemy.exc import DBAPIError, IntegrityError, InterfaceError, OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeout
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase
from messenger.shared.application.errors import Unavailable
from messenger.shared.domain.events import DomainEvent
FAILURES = (
OSError,
asyncpg.InterfaceError,
asyncpg.PostgresConnectionError,
asyncpg.exceptions.OperatorInterventionError,
asyncpg.exceptions.InsufficientResourcesError,
asyncpg.exceptions.LockNotAvailableError,
)
FAILURES = (OSError, InterfaceError, OperationalError, PoolTimeout, TimeoutError)
PING_TIMEOUT_SECONDS = 0.4
type Connection = asyncpg.Connection
type DatabaseSession = AsyncSession
class EventSource(Protocol):
def collect_events(self) -> list[DomainEvent]: ...
class Base(DeclarativeBase):
type_annotation_map: ClassVar = {str: Text}
def unavailable(error: BaseException) -> bool:
if isinstance(error, IntegrityError):
return False
if isinstance(error, FAILURES):
return True
if isinstance(error, DBAPIError) and error.connection_invalidated:
return True
state = getattr(error.orig if isinstance(error, DBAPIError) else error, "sqlstate", None)
return isinstance(state, str) and (state.startswith(("08", "53", "57")) or state == "55P03")
class Database:
__slots__ = ("_acquire_timeout", "_pool")
__slots__ = ("_engine", "_sessions")
def __init__(
self,
@@ -37,63 +50,71 @@ class Database:
acquire_timeout: float,
command_timeout: float,
) -> None:
self._acquire_timeout = acquire_timeout
self._pool = asyncpg.create_pool(
dsn,
min_size=0,
max_size=max_size,
timeout=connect_timeout,
command_timeout=command_timeout,
max_inactive_connection_lifetime=60,
server_settings={
"statement_timeout": str(int(command_timeout * 1000)),
"lock_timeout": str(int(command_timeout * 1000)),
"idle_in_transaction_session_timeout": "10000",
self._engine = create_async_engine(
make_url(dsn).set(drivername="postgresql+asyncpg"),
pool_size=max_size,
max_overflow=0,
pool_timeout=acquire_timeout,
pool_recycle=60,
pool_pre_ping=True,
hide_parameters=True,
connect_args={
"timeout": connect_timeout,
"command_timeout": command_timeout,
"server_settings": {
"statement_timeout": str(int(command_timeout * 1000)),
"lock_timeout": str(int(command_timeout * 1000)),
"idle_in_transaction_session_timeout": "10000",
},
},
)
self._sessions = async_sessionmaker(self._engine, expire_on_commit=False)
async def open(self) -> None:
await self._pool
pass
async def close(self) -> None:
await self._pool.close()
await self._engine.dispose()
@asynccontextmanager
async def connection(self, wait: float | None = None) -> AsyncIterator[Connection]:
async def session(self) -> AsyncIterator[DatabaseSession]:
try:
async with self._pool.acquire(timeout=wait or self._acquire_timeout) as connection:
yield connection
except FAILURES as error:
async with self._sessions() as session:
yield session
except Exception as error:
if not unavailable(error):
raise
raise Unavailable("PostgreSQL недоступен") from error
async def ping(self) -> None:
async with self.connection(PING_TIMEOUT_SECONDS) as connection:
await connection.fetchval("SELECT 1", timeout=PING_TIMEOUT_SECONDS)
try:
async with asyncio.timeout(PING_TIMEOUT_SECONDS):
async with self.session() as session:
await session.execute(select(1))
except TimeoutError as error:
raise Unavailable("PostgreSQL недоступен") from error
class PostgresUnitOfWork:
__slots__ = ("_committed", "_connection", "_database", "_events", "_transaction", "seen")
__slots__ = ("_committed", "_database", "_events", "_session", "seen")
def __init__(self, database: Database) -> None:
self._database = database
self._connection: Connection | None = None
self._transaction: Any = None
self._session: DatabaseSession | None = None
self._committed = False
self._events: list[DomainEvent] = []
self.seen: list[EventSource] = []
@property
def connection(self) -> Connection:
if self._connection is None:
def session(self) -> DatabaseSession:
if self._session is None:
raise RuntimeError("репозиторий используется вне транзакции")
return self._connection
return self._session
@asynccontextmanager
async def transaction(self) -> AsyncIterator[None]:
async with self._database.connection() as connection:
self._connection = connection
self._transaction = connection.transaction()
await self._transaction.start()
async with self._database.session() as session:
self._session = session
self._committed = False
mark = len(self._events)
try:
@@ -101,12 +122,11 @@ class PostgresUnitOfWork:
finally:
if not self._committed:
del self._events[mark:]
await self._transaction.rollback()
self.seen.clear()
self._connection = None
self._session = None
async def commit(self) -> None:
await self._transaction.commit()
await self.session.commit()
self._committed = True
for source in self.seen:
self._events.extend(source.collect_events())
@@ -0,0 +1,19 @@
from threading import Event
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.postgres import Database
DRAINING = Event()
class DatabaseReadiness:
__slots__ = ("_database", "_draining")
def __init__(self, database: Database, draining: Event = DRAINING) -> None:
self._database = database
self._draining = draining
async def check(self) -> None:
if self._draining.is_set():
raise Unavailable("Реплика выходит из ротации")
await self._database.ping()
+2 -2
View File
@@ -17,7 +17,7 @@ from dishka import Provider, Scope, make_async_container, provide
from dishka_faststream import setup_dishka
from faststream.rabbit import Channel, RabbitBroker
from conftest import ROOT, STARTUP_SECONDS, container, image, published
from conftest import ROOT, STARTUP_SECONDS, container, free_port, image, published
from fakes import NullLogger
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import ProcessJob
@@ -90,7 +90,7 @@ def rabbitmq() -> Iterator[dict[str, Any]]:
"100:101",
*volumes,
"-p",
"127.0.0.1::5672",
f"127.0.0.1:{free_port()}:5672",
image(RABBITMQ_IMAGE),
)
try:
+28 -3
View File
@@ -9,7 +9,7 @@ from uuid import uuid4
import httpx
import pytest
from conftest import STARTUP_SECONDS, container, image, published
from conftest import STARTUP_SECONDS, container, free_port, image, published
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
from messenger.shared.application.errors import Unavailable
@@ -23,7 +23,7 @@ BUCKET = "messenger-attachments"
def rustfs() -> Iterator[str]:
name = f"mtusi-lab5-test-rustfs-{uuid4().hex[:8]}"
keys = ["-e", f"RUSTFS_ACCESS_KEY={ROOT_KEY}", "-e", f"RUSTFS_SECRET_KEY={ROOT_SECRET}"]
container(name, *keys, "-p", "127.0.0.1::9000", image(RUSTFS_IMAGE))
container(name, *keys, "-p", f"127.0.0.1:{free_port()}:9000", image(RUSTFS_IMAGE))
try:
endpoint = f"http://127.0.0.1:{published(name, 9000)}"
deadline = time.monotonic() + STARTUP_SECONDS
@@ -71,6 +71,27 @@ async def test_round_trip_of_the_largest_object(rustfs: str) -> None:
await objects.delete(key)
async def test_listing_is_paged_in_byte_order_and_stat_reads_size(rustfs: str) -> None:
prefix = f"reconcile-{uuid4().hex}/"
keys = [f"{prefix}{name}" for name in ("b", "a", "Z", "a-1", "é")]
async with storage(rustfs) as objects:
for size, key in enumerate(keys, start=1):
await objects.put(key, b"x" * size, "image/png")
pages = [page async for page in objects.listing(prefix, 2)]
assert [len(page) for page in pages] == [2, 2, 1]
listed = [item.key for page in pages for item in page]
assert listed == sorted(keys, key=str.encode)
sizes = {item.key: item.size for page in pages for item in page}
assert sizes == {key: size for size, key in enumerate(keys, start=1)}
found = await objects.stat(keys[0])
assert found is not None
assert found.size == 1
assert await objects.stat(f"{prefix}absent") is None
for key in keys:
await objects.delete(key)
assert not [item async for page in objects.listing(prefix, 2) for item in page]
async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None:
started = time.monotonic()
async with storage_without_bucket("http://127.0.0.1:1") as objects:
@@ -78,7 +99,11 @@ async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None:
await objects.put("attachments/x", b"x", "image/png")
with pytest.raises(Unavailable):
await objects.open("attachments/x")
assert time.monotonic() - started < 8
with pytest.raises(Unavailable):
await objects.stat("attachments/x")
with pytest.raises(Unavailable):
await anext(aiter(objects.listing("attachments/", 10)))
assert time.monotonic() - started < 12
@asynccontextmanager
+38 -2
View File
@@ -1,8 +1,11 @@
from uuid import uuid4
import time
from uuid import UUID, uuid4
from httpx import AsyncClient
from sqlalchemy import select
from conftest import actor, keyed, make_chat, register
from conftest import actor, keyed, make_chat, register, sql_connection
from messenger.messaging.infrastructure.postgres import ChatRow
async def test_outsider_gets_404_on_chat(client: AsyncClient) -> None:
@@ -82,3 +85,36 @@ async def test_403_is_never_used(client: AsyncClient) -> None:
]
for path in paths:
assert (await client.get(path, headers=actor(eve))).status_code != 403
async def test_request_shape_is_checked_before_access(client: AsyncClient) -> None:
alice = await register(client, "alice")
eve = await register(client, "eve")
chat = await make_chat(client, alice)
for chat_id in (chat["id"], uuid4()):
path = f"/api/v1/chats/{chat_id}/messages"
unkeyed = await client.post(path, json={"text": "чужое"}, headers=actor(eve))
extra = await client.post(path, json={"text": "x", "a": 1}, headers=keyed(eve))
assert unkeyed.status_code == extra.status_code == 400
assert unkeyed.json()["error"]["code"] == extra.json()["error"]["code"] == "invalid_request"
blank = await client.post(
f"/api/v1/chats/{chat['id']}/messages", json={"text": " "}, headers=keyed(eve)
)
assert blank.status_code == 404
async def test_outsider_does_not_wait_for_members_lock(client: AsyncClient) -> None:
alice = await register(client, "alice")
eve = await register(client, "eve")
chat = await make_chat(client, alice)
async with sql_connection() as holder:
await holder.execute(
select(ChatRow.id).where(ChatRow.id == UUID(chat["id"])).with_for_update()
)
started = time.perf_counter()
response = await client.post(
f"/api/v1/chats/{chat['id']}/messages", json={"text": "чужое"}, headers=keyed(eve)
)
elapsed = time.perf_counter() - started
assert response.status_code == 404
assert elapsed < 0.5
+13 -1
View File
@@ -9,7 +9,7 @@ import pytest
from httpx import AsyncClient, Response
from PIL import Image
from conftest import actor, make_chat, register, resolve, scalar
from conftest import actor, make_chat, register, resolve, running_app, scalar
from fakes import MemoryObjectStorage
from messenger.attachments.application.ports import ObjectStorage
from pictures import GIF, JPEG, SAMPLE, WEBP, png
@@ -140,6 +140,18 @@ async def test_access_is_checked_before_content(client: AsyncClient) -> None:
assert anonymous.headers["www-authenticate"] == 'Bearer realm="messenger"'
async def test_token_is_checked_before_the_declared_size() -> None:
async with running_app(max_upload_bytes=4096) as http:
alice, _, _, chat = await trio(http)
oversized = png(16, 16, size=8192)
anonymous = await http.post(
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", content=oversized, headers=PNG
)
assert anonymous.status_code == 401
assert anonymous.headers["www-authenticate"] == 'Bearer realm="messenger"'
assert (await upload(http, alice, chat, oversized)).status_code == 413
async def test_exact_limit_is_accepted(client: AsyncClient) -> None:
alice, _, _, chat = await trio(client)
response = await upload(client, alice, chat, png(16, 16, size=LIMIT))
+19 -14
View File
@@ -33,20 +33,6 @@ def test_identity_does_not_know_about_messaging() -> None:
assert not {k: v for k, v in offenders.items() if v}
def test_messaging_touches_identity_only_through_events() -> None:
allowed = f"{IDENTITY}.domain.events"
offenders: dict[str, list[str]] = {}
for path in modules_under("messaging"):
borrowed = [
module
for module in imports_of(path)
if module.startswith(IDENTITY) and module != allowed
]
if borrowed:
offenders[str(path.relative_to(SOURCE))] = borrowed
assert not offenders
def test_attachments_and_other_zones_do_not_import_each_other() -> None:
offenders: dict[str, list[str]] = {}
for zone, foreign in (
@@ -125,6 +111,7 @@ def test_shared_core_holds_only_platform() -> None:
"ids.py",
"logging.py",
"postgres.py",
"readiness.py",
}
@@ -267,6 +254,16 @@ def sql_tables(zone: str, tables: tuple[str, ...]) -> set[str]:
for node in ast.walk(tree):
if isinstance(node, ast.Constant) and isinstance(node.value, str):
words.update(re.findall(r"\b(?:FROM|JOIN|INTO|UPDATE)\s+(\w+)", node.value))
if (
isinstance(node, ast.Assign)
and any(
isinstance(target, ast.Name) and target.id == "__tablename__"
for target in node.targets
)
and isinstance(node.value, ast.Constant)
and isinstance(node.value.value, str)
):
words.add(node.value.value)
return words & set(tables)
@@ -287,3 +284,11 @@ def test_messaging_does_not_import_identity_at_all() -> None:
for path in modules_under("messaging")
}
assert not {key: value for key, value in borrowed.items() if value}
def test_sqlalchemy_stays_in_infrastructure_and_asyncpg_is_only_a_driver() -> None:
for path in SOURCE.rglob("*.py"):
modules = {module.split(".")[0] for module in imports_of(path)}
assert "asyncpg" not in modules
if "infrastructure" not in path.parts:
assert "sqlalchemy" not in modules
+37 -6
View File
@@ -2,13 +2,18 @@ import asyncio
import time
from datetime import datetime
from hashlib import sha256
from uuid import UUID
import pytest
from redis.asyncio import Redis
from sqlalchemy import select
from conftest import actor, bearer, register, resolve, running_app, scalar
from conftest import actor, bearer, keyed, make_chat, register, resolve, running_app, scalar
from messenger.identity.application.ports import SessionDirectory
from messenger.identity.domain.models import Session
from messenger.identity.infrastructure.postgres import PostgresSessionRepository, SessionRow
from messenger.identity.infrastructure.valkey import CachedSessionDirectory
from messenger.shared.application.errors import Unavailable
BEARER = 'Bearer realm="messenger"'
@@ -31,8 +36,9 @@ async def test_second_check_is_served_from_cache(valkey: dict[str, str]) -> None
assert (await cache.info("stats"))["keyspace_hits"] == hits + 1
key = key_of(token_of(alice))
stored = str(await cache.get(key))
sql = "SELECT expires_at FROM sessions WHERE user_id = $1::uuid"
expires_at: datetime = await scalar(sql, alice["id"])
expires_at: datetime = await scalar(
select(SessionRow.expires_at).where(SessionRow.user_id == UUID(alice["id"]))
)
assert alice["id"] in stored and token_of(alice) not in stored
assert await cache.pexpiretime(key) <= int(expires_at.timestamp() * 1000)
await cache.aclose()
@@ -53,6 +59,29 @@ async def test_logout_writes_tombstone_before_database(valkey: dict[str, str]) -
await cache.aclose()
async def test_logout_rejected_by_database_changes_nothing(
valkey: dict[str, str], monkeypatch: pytest.MonkeyPatch
) -> None:
async def broken(self: PostgresSessionRepository, token_hash: bytes) -> None:
raise Unavailable("PostgreSQL недоступен")
cache = Redis.from_url(valkey["url"])
logout = "/api/v1/auth/sessions/current"
async with running_app(cache_url=valkey["url"]) as http:
alice = await register(http, "alice")
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200
with monkeypatch.context() as patch:
patch.setattr(PostgresSessionRepository, "remove", broken)
failed = await http.delete(logout, headers=actor(alice))
assert failed.status_code == 503
assert failed.json()["error"]["code"] == "unavailable"
assert await cache.get(key_of(token_of(alice))) is None
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200
assert (await http.delete(logout, headers=actor(alice))).status_code == 204
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 401
await cache.aclose()
async def test_fill_racing_with_logout_does_not_revive_token(valkey: dict[str, str]) -> None:
async with running_app(cache_url=valkey["url"]) as http:
alice = await register(http, "alice")
@@ -69,9 +98,6 @@ async def test_fill_racing_with_logout_does_not_revive_token(valkey: dict[str, s
await released.wait()
return stale[-1]
async def invalidate(self, session: Session) -> None:
await origin.invalidate(session)
directory._origin = SlowOrigin()
reader = asyncio.create_task(http.get("/api/v1/users/me", headers=actor(alice)))
await entered.wait()
@@ -106,6 +132,10 @@ async def test_unreachable_cache_is_bypassed_without_anonymous_access() -> None:
stranger = await http.get("/api/v1/users/me", headers=bearer("x" * 43))
elapsed = time.perf_counter() - started
ready = await http.get("/health/ready")
chat = await make_chat(http, alice)
posted = await http.post(
f"/api/v1/chats/{chat['id']}/messages", json={"text": "без кэша"}, headers=keyed(alice)
)
logout = await http.delete("/api/v1/auth/sessions/current", headers=actor(alice))
still = await http.get("/api/v1/users/me", headers=actor(alice))
assert me.json() == alice
@@ -113,6 +143,7 @@ async def test_unreachable_cache_is_bypassed_without_anonymous_access() -> None:
assert stranger.headers["WWW-Authenticate"] == BEARER
assert elapsed < 1
assert ready.status_code == 200
assert posted.status_code == 201
assert logout.status_code == 503
assert logout.json()["error"]["code"] == "unavailable"
assert still.status_code == 200
+19
View File
@@ -1,10 +1,13 @@
import asyncio
from datetime import UTC, datetime, timedelta
from urllib.parse import quote
from uuid import uuid4
import pytest
from httpx import AsyncClient
from conftest import PASSWORD, actor, keyed, make_chat, register
from messenger.shared.infrastructure.clock import MonotonicUtcClock
CLIENTS = 10
PER_CLIENT = 10
@@ -60,6 +63,22 @@ async def test_hundred_messages_from_ten_clients(client: AsyncClient) -> None:
assert len(set(stamps)) == TOTAL, "created_at внутри чата обязан строго возрастать"
async def test_history_follows_the_clock_but_never_moves_back(
monkeypatch: pytest.MonkeyPatch, client: AsyncClient
) -> None:
alice = await register(client, "alice")
chat = await make_chat(client, alice)
base = datetime.now(UTC) + timedelta(minutes=10)
stamps = []
for moment in (base, base - timedelta(minutes=5), base + timedelta(seconds=5)):
monkeypatch.setattr(MonotonicUtcClock, "now", lambda _self, moment=moment: moment)
response = await client.post(
f"/api/v1/chats/{chat['id']}/messages", json={"text": "часы"}, headers=keyed(alice)
)
stamps.append(datetime.fromisoformat(response.json()["created_at"]))
assert stamps == [base, base + timedelta(microseconds=1), base + timedelta(seconds=5)]
async def test_parallel_registration_keeps_username_unique(client: AsyncClient) -> None:
body = {"username": f"race_{uuid4().hex[:12]}", "display_name": "Гонка", "password": PASSWORD}
responses = await asyncio.gather(*(client.post("/api/v1/users", json=body) for _ in range(20)))

Some files were not shown because too many files have changed in this diff Show More