Forward the lab 5 foundation while preserving lab 6 telemetry

Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files.

Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
This commit is contained in:
Artyom
2026-11-06 12:29:35 +03:00
parent dc9256baf9
commit 12db77133f
161 changed files with 11160 additions and 1467 deletions
+2
View File
@@ -42,3 +42,5 @@ compose*.yaml
lab.json
requirements-dev.txt
.claude
.verification/
+31 -3
View File
@@ -32,8 +32,8 @@ OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# LB queue per replica, drain window before closing the listener on SIGTERM, PostgreSQL
# limits, load generator limits (profile bench), cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone:
# only the load balancer is published.
LB_HTTP_PORT=8080
LB_STATS_PORT=8404
LB_HTTP_PORT=18086
LB_STATS_PORT=26085
API_CPUS=1
API_MEMORY=512m
WEB_CONCURRENCY=2
@@ -63,7 +63,7 @@ MALLOC_MMAP_THRESHOLD_=131072
# Solution-specific for lab5: RabbitMQ management UI on 127.0.0.1, backoff base of a transient
# retry (doubles per attempt up to 60 s), relay poll and sweep intervals, worker limits and the
# PostgreSQL pool of relay and worker. FAULT_POINT is set only by infra/compose.faults.yaml.
BROKER_MANAGEMENT_PORT=15672
BROKER_MANAGEMENT_PORT=26086
JOB_BACKOFF_BASE_SECONDS=2
OUTBOX_POLL_SECONDS=1
SWEEP_INTERVAL_SECONDS=5
@@ -76,3 +76,31 @@ PIPELINE_DB_POOL_MAX_SIZE=4
RENDER_TIMEOUT_SECONDS=30
RENDER_MEMORY_MB=448
RENDER_TASKS_PER_CHILD=100
# Solution-specific for lab6: host ports of Prometheus, Loki, Tempo and Grafana (127.0.0.1 only),
# Prometheus retention, OTLP export timeout in seconds, metric export interval in ms, SDK queue
# sizes, the bounded flush on shutdown; OTEL_SDK_DISABLED=true turns the SDK and instrumentations off.
# relay and worker get OTEL_SERVICE_NAME messenger-relay and messenger-worker in compose.yaml.
PROMETHEUS_PORT=26090
LOKI_PORT=26310
TEMPO_PORT=26320
GRAFANA_PORT=26300
PROMETHEUS_RETENTION=7d
OTEL_SDK_DISABLED=false
OTEL_EXPORTER_OTLP_TIMEOUT=2
OTEL_METRIC_EXPORT_INTERVAL=10000
OTEL_BSP_MAX_QUEUE_SIZE=2048
OTEL_BLRP_MAX_QUEUE_SIZE=2048
TELEMETRY_SHUTDOWN_SECONDS=2
# Minimal client transfer rate, bytes/s: every body byte moved in either direction extends the
# request deadline by 1/rate, so slow but steady uploads and downloads outlive REQUEST_TIMEOUT_SECONDS.
MIN_TRANSFER_RATE_BYTES=131072
# Solution-specific for lab4 track 2: limits of the one-off reconciler (profile reconcile), its S3
# read timeout for listing pages, optional CPU pinning of the measured services (empty = no pinning)
# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for the 100k point).
RECONCILE_CPUS=1
RECONCILE_MEMORY=256m
RECONCILE_S3_READ_TIMEOUT_SECONDS=10
RECONCILE_CPUSET=
S3_CPUSET=
PG_CPUSET=
S3_CPUS=1
+14 -1
View File
@@ -59,6 +59,9 @@ jobs:
- name: Несколько реплик на одном токене и гонка одного ключа
run: make lb-check
- name: Повтор неподтверждённой записи после потерянного ответа и убитой реплики
run: make retry-check
- name: Остановка реплики под запросами, затем возврат
run: |
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
@@ -84,7 +87,17 @@ jobs:
- name: Процессы работают не от root
run: |
docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}'
for id in $(docker compose ps -q); do
docker top "$id" -eo pid,uid,comm | awk 'NR>1 && $2==0 {print; bad=1} END {exit bad}' || exit 1
done
- name: Одноразовые сервисы не от root
run: |
for service in migrate s3-init; do
user=$(docker inspect -f '{{.Config.User}}' "$(docker compose ps -aq "$service")")
echo "$service: $user"
case "${user%%:*}" in ""|0|root) exit 1 ;; esac
done
- name: Корректная остановка relay и worker по SIGTERM
run: |
+2
View File
@@ -38,3 +38,5 @@ coverage/
# Локальные настройки агента
.claude/settings.local.json
.verification/
+4 -1
View File
@@ -27,7 +27,7 @@ ENV PYTHONUNBUFFERED=1 \
WEB_CONCURRENCY=2 \
REQUEST_TIMEOUT_SECONDS=8 \
DRAIN_SECONDS=2 \
SHUTDOWN_TIMEOUT_SECONDS=5
SHUTDOWN_TIMEOUT_SECONDS=6
RUN groupadd --system --gid 10001 messenger \
&& useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
@@ -36,6 +36,9 @@ COPY --from=builder --chown=root:root /opt/venv /opt/venv
WORKDIR /app
COPY --chown=root:root alembic.ini ./
COPY --chown=root:root alembic ./alembic
USER 10001:10001
EXPOSE 8080
+96 -16
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://$(shell docker compose port lb 8080 2>/dev/null || echo localhost:8080)
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -8,13 +8,20 @@ VENV ?= .venv
UV ?= uv
COMPOSE ?= docker compose
RATE ?= 100
API_IMAGE ?= mtusi-messenger:lab6
FROM_REV ?=
SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py scripts/make_images.py scripts/pipeline_suite.py \
scripts/pipeline_evidence.py
SHIFT ?= 0
SCRIPTS_PY = scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
SCENARIO ?= failover-upload
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
OBS_CHECK = $(PYTHON) scripts/obs_check.py
APP_METRICS = {job=~"messenger-.*"}
TRACK2_OUTPUT ?= evidence/track2.json
TRACK2_SCENARIOS = smoke_pipeline,duplicate_after_ready,crash_commit_publish,crash_publish_confirm,crash_after_s3_before_commit,crash_after_commit_before_ack,lease_expiry_fencing,crash_before_side_effect,worker_kill_mid_task,duplicate_in_flight,reorder_attempts,reorder_versions,stale_after_new_version
PIPELINE_VERSION ?= 2
PYTEST_ARGS ?=
.PHONY: help check validate test load test-persistence test-failover \
install lint format typecheck student-test \
@@ -23,7 +30,12 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
limits detection failover-each rolling-update lab4-level4 \
scenario s3-check attachments-evidence upload-faults upload-memory \
images queues topology worker-stop worker-start broker-restart suite-preflight level3 \
pipeline-evidence level4 replay failed dlq
pipeline-evidence level4 replay failed dlq \
obs-validate telemetry-check \
faults-evidence retry-check reconcile reconcile-demo reconcile-cost load-evidence \
clean-clone-evidence overload-leftovers
.NOTPARALLEL:
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -36,10 +48,25 @@ help:
@echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/"
@echo "lab5: queues, topology, worker-stop/worker-start, broker-restart, images, suite-preflight, level3, level4"
@echo "lab5 оператор: failed (реестр), replay ID=<attachment_id>, dlq (просмотр DLQ без извлечения)"
@echo "lab6: obs-validate (конфиги Collector, Prometheus, Loki, Tempo, dashboards); telemetry-check → evidence/*.json"
@echo "retry-check: потерянный ответ на живой и на убитой реплике, повтор тем же ключом"
@echo "lab4-level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps"
@echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence"
@echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда"
@echo "upload-memory ARGS='--out evidence/upload-memory/runN': пик памяти реплик при 10/20/40 загрузках 1 и 10 MiB"
@echo "load-evidence: tests/load.py ×3 клиентом-контейнером в сети edge стенда → evidence/load.txt, evidence/load/"
@echo "overload-leftovers: после upload-memory — намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt"
@echo "clean-clone-evidence: блок «Запуск из чистого clone» в свежем клоне рядом → evidence/clean-clone.txt"
@echo "reconcile ARGS='--apply --deep': сверка PostgreSQL ↔ S3 (без ARGS — dry-run); трек 2:"
@echo "reconcile-demo: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/"
@echo "reconcile-cost: стоимость обхода 1k/10k/30k/100k объектов × 3 прогона на чистом стенде → evidence/reconcile/cost/"
@echo "BASE_URL=$(BASE_URL) (порт lb из docker compose); failover-evidence SHIFT=1 — сдвиг round robin на вторую реплику"
check:
$(PYTHON) scripts/check_contract.py
validate: PYTHON := $(if $(wildcard $(VENV)/bin/python),$(VENV)/bin/python,$(PYTHON))
validate:
$(PYTHON) scripts/validate_openapi.py
@@ -63,32 +90,34 @@ install:
lint:
$(VENV)/bin/ruff check src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ruff format --check src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ty check src student-tests
$(VENV)/bin/ty check src student-tests alembic
format:
$(VENV)/bin/ruff check --fix src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ruff format src student-tests $(SCRIPTS_PY)
typecheck:
$(VENV)/bin/ty check src student-tests
$(VENV)/bin/ty check src student-tests alembic
student-test:
$(VENV)/bin/pytest
$(VENV)/bin/pytest $(PYTEST_ARGS)
secrets:
sh scripts/secrets.sh
sh scripts/broker-secrets.sh
sh scripts/secrets-lab6.sh
up: secrets
$(COMPOSE) build
$(COMPOSE) up -d --wait
scripts/lb-wait.sh
down:
$(COMPOSE) down
$(COMPOSE) --profile bench --profile tools --profile reconcile down
reset-data:
@echo "Удаляются volumes с учебными данными стенда"
$(COMPOSE) down -v --remove-orphans
$(COMPOSE) --profile bench --profile tools --profile reconcile down -v --remove-orphans
migrate:
$(COMPOSE) run --rm migrate
@@ -105,19 +134,23 @@ restore:
lb-check:
$(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json
retry-check:
$(PYTHON) scripts/probe.py retry --out evidence/retry.json
limits:
$(PYTHON) scripts/limits.py --out evidence/limits.txt
detection:
sh scripts/detection.sh evidence/detection
rm -rf evidence/detection
for run in 1 2 3; do sh scripts/detection.sh evidence/detection/run$$run || exit 1; done
failover-each:
RATE=$(RATE) sh scripts/failover-each.sh evidence/failover-each
rolling-update:
ifneq ($(FROM_REV),)
git archive "$(FROM_REV)" | docker build -q -t "$(API_IMAGE)" -
$(COMPOSE) up -d --no-deps --force-recreate --wait api
git archive "$(FROM_REV)" | docker build -q -t "mtusi-messenger:$(FROM_REV)" -
API_IMAGE="mtusi-messenger:$(FROM_REV)" $(COMPOSE) up -d --no-deps --force-recreate --wait api
scripts/lb-wait.sh
endif
$(COMPOSE) build -q api
@@ -144,10 +177,28 @@ upload-faults:
status=$$?; cat evidence/upload-faults.txt; exit $$status
upload-memory:
$(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)"
$(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)" $(ARGS)
load-evidence:
sh scripts/load-evidence.sh
overload-leftovers:
sh scripts/overload-leftovers.sh
clean-clone-evidence:
sh scripts/clean-clone.sh
reconcile:
$(COMPOSE) run --rm --no-deps reconciler $(ARGS)
reconcile-demo:
$(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)" $(ARGS)
reconcile-cost:
$(PYTHON) scripts/reconcile_track.py cost $(ARGS)
failover-evidence:
sh scripts/failover-evidence.sh "$(BASE_URL)"
sh scripts/failover-evidence.sh "$(BASE_URL)" "$(SHIFT)"
stack-evidence:
sh scripts/evidence-stack.sh
@@ -166,6 +217,12 @@ cache-evidence:
test-faults:
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack
faults-evidence:
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; echo "\$$ make test-faults"; \
$(MAKE) --no-print-directory test-faults 2>&1; } | sed 's#$(CURDIR)#<repo>#g' > evidence/faults.txt
cat evidence/faults.txt
grep -qE '^=+ [0-9]+ passed in' evidence/faults.txt
seed:
$(PSQL) -f - < scripts/sql/seed-history.sql
@@ -226,3 +283,26 @@ dlq:
pipeline-evidence:
$(PYTHON) scripts/pipeline_evidence.py --base-url "$(BASE_URL)" > evidence/pipeline-stand.txt; \
status=$$?; cat evidence/pipeline-stand.txt; exit $$status
obs-validate:
sh scripts/obs-validate.sh
telemetry-check:
$(OBS_CHECK) grafana > evidence/grafana.json
$(OBS_CHECK) dashboards --strict > evidence/dashboards.json
$(OBS_CHECK) request --base-url "$(BASE_URL)" > evidence/trace-request.json
$(OBS_CHECK) job --base-url "$(BASE_URL)" > evidence/trace-job.json
$(OBS_CHECK) leaks --base-url "$(BASE_URL)" > evidence/leaks.json
$(OBS_CHECK) query 'count by (__name__, job) ($(APP_METRICS))' \
'count by (http_route, http_request_method) (http_server_request_duration_seconds_count)' \
'count by (job_type, outcome) (messenger_jobs_processed_total)' > evidence/cardinality.json
@echo "telemetry-check: evidence/grafana.json dashboards.json trace-request.json trace-job.json leaks.json cardinality.json"
.PHONY: track2 reprocess
track2:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 5 --only "$(TRACK2_SCENARIOS)" --output "$(TRACK2_OUTPUT)"
reprocess:
@test -n "$(ID)" || (echo "Set ID=<attachment_id>"; exit 2)
$(COMPOSE) run --rm --no-deps -T worker python -m messenger.admin reprocess --attachment-id "$(ID)" --pipeline-version "$(PIPELINE_VERSION)"
+4
View File
@@ -0,0 +1,4 @@
[alembic]
script_location = %(here)s/alembic
prepend_sys_path = %(here)s/src %(here)s/alembic
path_separator = space
+102
View File
@@ -0,0 +1,102 @@
import asyncio
from alembic import context
from alembic.script import ScriptDirectory
from legacy import adopt_legacy
from sqlalchemy import Connection, func, make_url, pool, select
from sqlalchemy.ext.asyncio import create_async_engine
from messenger.attachments.infrastructure.postgres import (
AttachmentRow,
JobEventRow,
JobRow,
OutboxRow,
VariantRow,
)
from messenger.identity.infrastructure.postgres import SessionRow, UserRow
from messenger.messaging.infrastructure.postgres import (
ChatMemberRow,
ChatRow,
IdempotencyKeyRow,
MessageAttachmentRow,
MessageRow,
)
from messenger.shared.config import PlatformSettings
LOCK_KEY = 20_260_002
MODELS = (
UserRow,
SessionRow,
ChatRow,
ChatMemberRow,
MessageRow,
IdempotencyKeyRow,
AttachmentRow,
JobRow,
OutboxRow,
JobEventRow,
VariantRow,
MessageAttachmentRow,
)
target_metadata = UserRow.metadata
config = context.config
def include_object(_obj, name, _type, _reflected, _compare_to):
return name != "schema_migrations"
def run_migrations(connection: Connection) -> None:
with connection.engine.connect() as lock:
lock.execute(select(func.pg_advisory_lock(LOCK_KEY)))
lock.commit()
try:
with connection.begin():
adopt_legacy(connection, ScriptDirectory.from_config(config))
context.configure(
connection=connection,
target_metadata=target_metadata,
transaction_per_migration=True,
compare_type=True,
include_object=include_object,
)
with context.begin_transaction():
context.run_migrations()
finally:
lock.execute(select(func.pg_advisory_unlock(LOCK_KEY)))
lock.commit()
async def run_async_migrations() -> None:
dsn = config.attributes.get("database_url")
if dsn is None:
dsn = PlatformSettings().database_url.get_secret_value()
engine = create_async_engine(
make_url(dsn).set(drivername="postgresql+asyncpg"),
poolclass=pool.NullPool,
hide_parameters=True,
connect_args={
"timeout": 10,
"server_settings": {"lock_timeout": "10000", "statement_timeout": "60000"},
},
)
try:
async with engine.connect() as connection:
await connection.run_sync(run_migrations)
finally:
await engine.dispose()
if context.is_offline_mode():
context.configure(
dialect_name="postgresql",
target_metadata=target_metadata,
literal_binds=True,
transaction_per_migration=True,
)
with context.begin_transaction():
context.run_migrations()
elif config.attributes.get("connection") is not None:
run_migrations(config.attributes["connection"])
else:
asyncio.run(run_async_migrations())
+94
View File
@@ -0,0 +1,94 @@
from uuid import uuid4
from alembic.migration import MigrationContext
from alembic.operations import Operations
from alembic.script import ScriptDirectory
from sqlalchemy import Connection, MetaData, Table, func, inspect, select, text
from sqlalchemy.schema import CreateSchema
LEGACY = {
"0001_init": "9a13618f078ae9056431391e2e5197a31b981ae1fff047a9b3421ac750d3fd0d",
"0002_idempotency_keys": "d9159a35553249a1e51af5f2e4e7c39b014e97593c49748e13f3799839063dd0",
"0003_attachments": "c4417f560538a083efba1c67f660147dc94532fcbb2fa91f3f3e8258a0f01f2e",
"0004_jobs": "c3887fa7a886ae8153a216aa43fd4c3c6190a2074e8ce20c3e77f29bfb01550e",
"0005_failed_registry": "02b80061a08a6929feb092148c453ad970e3d001901c651b5a73b90b1e40f145",
"0006_attachment_quarantine": (
"541f5399d87fac3584c4102b4e0bf92c23cd372cc20a7d56dad03e771773202c"
),
}
SNAPSHOT = text("""
WITH relations AS (
SELECT c.* FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = :schema AND c.relkind IN ('r', 'p')
AND c.relname NOT IN ('schema_migrations', 'alembic_version')
)
SELECT 'table' AS kind, c.relname AS name,
concat_ws('|', c.relkind, c.relpersistence,
c.relrowsecurity, c.relforcerowsecurity) AS definition
FROM relations c
UNION ALL
SELECT 'column', c.relname || '.' || a.attname,
concat_ws('|', a.attnum, format_type(a.atttypid, a.atttypmod), a.attnotnull,
a.attidentity, a.attgenerated, a.attcollation::regcollation,
pg_get_expr(d.adbin, d.adrelid))
FROM relations c JOIN pg_attribute a ON a.attrelid = c.oid
LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum
WHERE a.attnum > 0 AND NOT a.attisdropped
UNION ALL
SELECT 'constraint', c.relname || '.' || k.conname,
concat_ws('|', pg_get_constraintdef(k.oid), k.convalidated)
FROM relations c JOIN pg_constraint k ON k.conrelid = c.oid
UNION ALL
SELECT 'index', c.relname || '.' || x.relname,
concat_ws('|', pg_get_indexdef(i.indexrelid), i.indisvalid, i.indisready)
FROM relations c JOIN pg_index i ON i.indrelid = c.oid
JOIN pg_class x ON x.oid = i.indexrelid
UNION ALL
SELECT 'trigger', c.relname || '.' || t.tgname,
concat_ws('|', pg_get_triggerdef(t.oid), t.tgenabled)
FROM relations c JOIN pg_trigger t ON t.tgrelid = c.oid WHERE NOT t.tgisinternal
ORDER BY kind, name
""")
def snapshot(connection: Connection, schema: str) -> list[tuple[str, str, str]]:
return [
(kind, name, definition.replace(f"{schema}.", ""))
for kind, name, definition in connection.execute(SNAPSHOT, {"schema": schema})
]
def adopt_legacy(connection: Connection, scripts: ScriptDirectory) -> None:
migration = MigrationContext.configure(connection)
if migration.get_current_heads():
return
tables = set(inspect(connection).get_table_names(schema="public"))
if tables <= {"alembic_version"}:
return
if "schema_migrations" not in tables:
raise RuntimeError("Existing schema has no legacy migration journal")
journal = Table("schema_migrations", MetaData(), schema="public", autoload_with=connection)
applied = dict(connection.execute(select(journal.c.version, journal.c.checksum)).tuples().all())
expected = dict(list(LEGACY.items())[: len(applied)])
if applied != expected:
raise RuntimeError("Unknown or changed legacy migrations")
if not applied:
if tables - {"schema_migrations", "alembic_version"}:
raise RuntimeError("Existing schema has an empty migration journal")
return
revision = list(expected)[-1]
actual = snapshot(connection, "public")
reference = f"alembic_verify_{uuid4().hex}"
savepoint = connection.begin_nested()
try:
connection.execute(CreateSchema(reference))
connection.execute(select(func.set_config("search_path", reference, True)))
with Operations.context(MigrationContext.configure(connection)):
for script in reversed(list(scripts.walk_revisions(base="base", head=revision))):
script.module.upgrade()
wanted = snapshot(connection, reference)
finally:
savepoint.rollback()
if actual != wanted:
raise RuntimeError("Legacy schema differs from the recorded migrations")
migration.stamp(scripts, revision)
+16
View File
@@ -0,0 +1,16 @@
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
+87
View File
@@ -0,0 +1,87 @@
import sqlalchemy as sa
from alembic import op
revision = "0001_init"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"users",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("username", sa.Text(), nullable=False),
sa.Column("display_name", sa.Text(), nullable=False),
sa.Column("password_hash", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("password_hash LIKE '$argon2id$%'", name="users_password_hash_check"),
sa.CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="users_username_format"),
sa.CheckConstraint(
"char_length(display_name) BETWEEN 1 AND 100", name="users_display_name_check"
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("username", name="users_username_key"),
)
op.create_table(
"sessions",
sa.Column("token_hash", sa.LargeBinary(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("expires_at > created_at", name="sessions_check"),
sa.CheckConstraint("octet_length(token_hash) = 32", name="sessions_token_hash_check"),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("token_hash"),
)
op.create_index("sessions_user_idx", "sessions", ["user_id"], unique=False)
op.create_table(
"chats",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("title", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("last_message_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="chats_title_check"),
sa.PrimaryKeyConstraint("id"),
)
op.create_table(
"chat_members",
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("position", sa.SmallInteger(), nullable=False),
sa.Column("chat_created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("position BETWEEN 1 AND 100", name="chat_members_position_check"),
sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(
["user_id"],
["users.id"],
),
sa.PrimaryKeyConstraint("chat_id", "user_id"),
)
op.create_index(
"chat_members_user_idx",
"chat_members",
["user_id", "chat_created_at", "chat_id"],
unique=False,
)
op.create_table(
"messages",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("sender_id", sa.Uuid(), nullable=False),
sa.Column("text", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("char_length(text) BETWEEN 1 AND 4000", name="messages_text_check"),
sa.ForeignKeyConstraint(
["chat_id", "sender_id"],
["chat_members.chat_id", "chat_members.user_id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"messages_history_idx", "messages", ["chat_id", "created_at", "id"], unique=False
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete application data")
+33
View File
@@ -0,0 +1,33 @@
import sqlalchemy as sa
from alembic import op
revision = "0002_idempotency_keys"
down_revision = "0001_init"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"idempotency_keys",
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("key", sa.Text(), nullable=False),
sa.Column("fingerprint", sa.LargeBinary(), nullable=False),
sa.Column("message_id", sa.Uuid(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="idempotency_keys_key_check"),
sa.CheckConstraint(
"octet_length(fingerprint) = 32", name="idempotency_keys_fingerprint_check"
),
sa.ForeignKeyConstraint(
["chat_id", "user_id"],
["chat_members.chat_id", "chat_members.user_id"],
),
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("chat_id", "user_id", "key"),
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete application data")
+77
View File
@@ -0,0 +1,77 @@
import sqlalchemy as sa
from alembic import op
revision = "0003_attachments"
down_revision = "0002_idempotency_keys"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.drop_constraint("messages_text_check", "messages", type_="check")
op.create_check_constraint("messages_text_check", "messages", "char_length(text) <= 4000")
op.create_table(
"attachments",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("owner_id", sa.Uuid(), nullable=False),
sa.Column("filename", sa.Text(), nullable=False),
sa.Column("content_type", sa.Text(), nullable=False),
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
sa.Column("sha256", sa.Text(), nullable=False),
sa.Column("width", sa.Integer(), nullable=False),
sa.Column("height", sa.Integer(), nullable=False),
sa.Column("object_key", sa.Text(collation="C"), nullable=False),
sa.Column("status", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("object_key", name="attachments_object_key_key"),
sa.UniqueConstraint("id", "chat_id", name="attachments_chat_key"),
sa.ForeignKeyConstraint(
["chat_id", "owner_id"],
["chat_members.chat_id", "chat_members.user_id"],
name="attachments_owner_fkey",
),
sa.CheckConstraint(
"char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_check"
),
sa.CheckConstraint(
"content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_check"
),
sa.CheckConstraint(
"size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_check"
),
sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachments_sha256_check"),
sa.CheckConstraint("width > 0", name="attachments_width_check"),
sa.CheckConstraint("height > 0", name="attachments_height_check"),
sa.CheckConstraint("status IN ('pending', 'ready')", name="attachments_status_check"),
)
op.create_index(
"attachments_pending_idx",
"attachments",
["created_at"],
postgresql_where=sa.text("status = 'pending'"),
)
op.create_table(
"message_attachments",
sa.Column("message_id", sa.Uuid(), nullable=False),
sa.Column("position", sa.SmallInteger(), nullable=False),
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("attachment_id", sa.Uuid(), nullable=False),
sa.PrimaryKeyConstraint("message_id", "position"),
sa.UniqueConstraint("message_id", "attachment_id"),
sa.CheckConstraint("position BETWEEN 0 AND 9", name="message_attachments_position_check"),
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(
["attachment_id", "chat_id"],
["attachments.id", "attachments.chat_id"],
name="message_attachments_attachment_fkey",
ondelete="RESTRICT",
),
)
op.create_index("message_attachments_attachment_idx", "message_attachments", ["attachment_id"])
def downgrade() -> None:
raise RuntimeError("Downgrade would delete application data")
+146
View File
@@ -0,0 +1,146 @@
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision = "0004_jobs"
down_revision = "0003_attachments"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.drop_constraint("attachments_status_check", "attachments", type_="check")
op.create_check_constraint(
"attachments_status_check",
"attachments",
"status IN ('pending', 'queued', 'processing', 'ready', 'failed')",
)
for name in ("staging_key", "original_key"):
op.add_column("attachments", sa.Column(name, sa.Text(collation="C"), nullable=True))
op.create_unique_constraint(f"attachments_{name}_key", "attachments", [name])
op.add_column(
"attachments",
sa.Column("published_version", sa.Integer(), nullable=False, server_default="0"),
)
op.create_check_constraint(
"attachments_published_version_check", "attachments", "published_version >= 0"
)
op.add_column("attachments", sa.Column("error_code", sa.Text(), nullable=True))
op.create_check_constraint(
"attachments_error_code_check",
"attachments",
"error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')",
)
op.alter_column("attachments", "object_key", nullable=True)
attachments = sa.table(
"attachments",
sa.column("status"),
sa.column("object_key"),
sa.column("original_key"),
sa.column("staging_key"),
sa.column("published_version"),
)
op.execute(
attachments.update()
.where(attachments.c.status == "ready")
.values(original_key=attachments.c.object_key, published_version=1)
)
op.execute(
attachments.update()
.where(attachments.c.status == "pending")
.values(staging_key=attachments.c.object_key)
)
op.create_index(
"attachments_staged_idx",
"attachments",
["updated_at"],
postgresql_where=sa.column("staging_key").is_not(None),
)
op.create_table(
"attachment_variants",
sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False),
sa.Column("pipeline_version", sa.Integer(), nullable=False),
sa.Column("name", sa.Text(), nullable=False),
sa.Column("object_key", sa.Text(collation="C"), nullable=False),
sa.Column("content_type", sa.Text(), nullable=False),
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
sa.Column("width", sa.Integer(), nullable=False),
sa.Column("height", sa.Integer(), nullable=False),
sa.Column("sha256", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("attachment_id", "pipeline_version", "name"),
sa.UniqueConstraint("object_key", name="attachment_variants_object_key_key"),
sa.CheckConstraint(
"pipeline_version >= 1", name="attachment_variants_pipeline_version_check"
),
sa.CheckConstraint(
"name IN ('thumbnail', 'crop', 'optimized')", name="attachment_variants_name_check"
),
sa.CheckConstraint(
"content_type IN ('image/jpeg', 'image/png', 'image/webp')",
name="attachment_variants_content_type_check",
),
sa.CheckConstraint("size_bytes >= 1", name="attachment_variants_size_bytes_check"),
sa.CheckConstraint("width >= 1", name="attachment_variants_width_check"),
sa.CheckConstraint("height >= 1", name="attachment_variants_height_check"),
sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachment_variants_sha256_check"),
)
op.create_table(
"jobs",
sa.Column("job_id", sa.Uuid(), primary_key=True),
sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False),
sa.Column("pipeline_version", sa.Integer(), nullable=False),
sa.Column("status", sa.Text(), nullable=False),
sa.Column("attempt", sa.Integer(), nullable=False),
sa.Column("claims", sa.Integer(), nullable=False, server_default="0"),
sa.Column("lease_token", sa.Uuid(), nullable=True),
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("attachment_id", "pipeline_version", name="jobs_effect_key"),
sa.CheckConstraint("pipeline_version >= 1", name="jobs_pipeline_version_check"),
sa.CheckConstraint(
"status IN ('queued', 'processing', 'done', 'failed', 'superseded')",
name="jobs_status_check",
),
sa.CheckConstraint("attempt >= 1", name="jobs_attempt_check"),
sa.CheckConstraint("claims >= 0", name="jobs_claims_check"),
)
op.create_index(
"jobs_lease_idx",
"jobs",
["lease_until"],
postgresql_where=sa.column("status") == "processing",
)
op.create_table(
"outbox",
sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True),
sa.Column("job_id", sa.Uuid(), sa.ForeignKey("jobs.job_id"), nullable=False),
sa.Column("payload", postgresql.JSONB(), nullable=False),
sa.Column("available_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("published_at", sa.DateTime(timezone=True), nullable=True),
)
op.create_index(
"outbox_pending_idx",
"outbox",
["available_at", "id"],
postgresql_where=sa.column("published_at").is_(None),
)
op.create_index("outbox_job_idx", "outbox", ["job_id"])
op.create_table(
"job_events",
sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True),
sa.Column("job_id", sa.Uuid(), nullable=False),
sa.Column("attachment_id", sa.Uuid(), nullable=False),
sa.Column("kind", sa.Text(), nullable=False),
sa.Column("attempt", sa.Integer(), nullable=True),
sa.Column("detail", postgresql.JSONB(), nullable=False, server_default="{}"),
sa.Column("at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
op.create_index("job_events_attachment_idx", "job_events", ["attachment_id", "id"])
def downgrade() -> None:
raise RuntimeError("Downgrade would delete pipeline data")
+20
View File
@@ -0,0 +1,20 @@
import sqlalchemy as sa
from alembic import op
revision = "0005_failed_registry"
down_revision = "0004_jobs"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_index(
"jobs_failed_idx",
"jobs",
[sa.column("updated_at").desc()],
postgresql_where=sa.column("status") == "failed",
)
def downgrade() -> None:
op.drop_index("jobs_failed_idx", table_name="jobs")
@@ -0,0 +1,28 @@
import sqlalchemy as sa
from alembic import op
revision = "0006_attachment_quarantine"
down_revision = "0005_failed_registry"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("attachments", sa.Column("quarantine_reason", sa.Text(), nullable=True))
op.create_check_constraint(
"attachments_quarantine_reason_check",
"attachments",
"quarantine_reason IN ('missing', 'size_mismatch', 'sha_mismatch')",
)
op.add_column(
"attachments", sa.Column("quarantined_at", sa.DateTime(timezone=True), nullable=True)
)
op.create_check_constraint(
"attachments_quarantine_check",
"attachments",
"(quarantine_reason IS NULL) = (quarantined_at IS NULL)",
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete quarantine data")
+250 -14
View File
@@ -1,5 +1,5 @@
x-app: &app
image: ${API_IMAGE:-mtusi-messenger:lab6}
image: ${API_IMAGE:-mtusi-messenger:${COMPOSE_PROJECT_NAME}}
networks: [backend]
x-s3-env: &s3-env
@@ -19,7 +19,7 @@ x-s3-env: &s3-env
UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100}
x-pipeline: &pipeline
image: ${API_IMAGE:-mtusi-messenger:lab6}
image: ${API_IMAGE:-mtusi-messenger:${COMPOSE_PROJECT_NAME}}
pull_policy: never
restart: unless-stopped
init: true
@@ -44,8 +44,33 @@ x-pipeline: &pipeline
broker:
condition: service_healthy
x-otel-env: &otel-env
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-2}
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
OTEL_SEMCONV_STABILITY_OPT_IN: http
OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-10000}
OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048}
OTEL_BLRP_MAX_QUEUE_SIZE: ${OTEL_BLRP_MAX_QUEUE_SIZE:-2048}
TELEMETRY_SHUTDOWN_SECONDS: ${TELEMETRY_SHUTDOWN_SECONDS:-2}
x-obs-logging: &obs-logging
driver: json-file
options:
max-size: 10m
max-file: "3"
x-obs-hardening: &obs-hardening
restart: unless-stopped
stop_grace_period: 10s
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
logging: *obs-logging
x-pipeline-env: &pipeline-env
<<: *s3-env
<<: [*s3-env, *otel-env]
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
DB_POOL_MAX_SIZE: ${PIPELINE_DB_POOL_MAX_SIZE:-4}
@@ -64,10 +89,17 @@ x-s3-client: &s3-client
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
RC_CONFIG_DIR: /tmp/rc
user: "65534:65534"
volumes: ["./infra/s3:/s3:ro"]
secrets:
[s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key,
s3_worker_access_key_id, s3_worker_secret_access_key]
- s3_root_user
- s3_root_password
- s3_access_key_id
- s3_secret_access_key
- s3_reconciler_access_key_id
- s3_reconciler_secret_access_key
- s3_worker_access_key_id
- s3_worker_secret_access_key
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
restart: "no"
@@ -83,15 +115,16 @@ services:
API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64}
volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
ports:
- "127.0.0.1:${LB_HTTP_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
- "127.0.0.1:${LB_HTTP_PORT:-18086}:8080"
- "127.0.0.1:${LB_STATS_PORT:-26085}:8404"
networks: [edge, backend]
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
interval: 2s
timeout: 2s
retries: 3
start_period: 10s
start_period: 60s
start_interval: 1s
sysctls:
net.ipv4.ip_local_port_range: "10240 65535"
net.ipv4.tcp_tw_reuse: "1"
@@ -108,7 +141,8 @@ services:
<<: *app
build: .
environment:
<<: *s3-env
<<: [*s3-env, *otel-env]
OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-messenger-api}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
@@ -117,10 +151,11 @@ services:
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
MIN_TRANSFER_RATE_BYTES: ${MIN_TRANSFER_RATE_BYTES:-131072}
DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1}
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2}
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-6}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
MALLOC_MMAP_THRESHOLD_: ${MALLOC_MMAP_THRESHOLD_:-131072}
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
@@ -130,7 +165,7 @@ services:
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
start_interval: 200ms
stop_grace_period: 8s
restart: unless-stopped
read_only: true
@@ -153,7 +188,8 @@ services:
migrate:
<<: *app
pull_policy: never
command: ["python", "-m", "messenger.migrate"]
init: true
command: ["alembic", "upgrade", "head"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
LOG_LEVEL: ${LOG_LEVEL:-INFO}
@@ -192,12 +228,14 @@ services:
secrets: [postgres_password]
volumes: [pgdata:/var/lib/postgresql/data]
shm_size: 256m
cpuset: ${PG_CPUSET:-}
healthcheck:
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
start_interval: 200ms
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
@@ -245,6 +283,7 @@ services:
RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn}
secrets: [s3_root_user, s3_root_password]
volumes: [s3data:/data]
cpuset: ${S3_CPUSET:-}
read_only: true
tmpfs: ["/tmp:size=64m"]
cap_drop: [ALL]
@@ -260,7 +299,36 @@ services:
networks: [backend]
deploy:
resources:
limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"}
limits: {cpus: "${S3_CPUS:-1}", memory: "${S3_MEMORY:-512m}"}
reconciler:
<<: *app
pull_policy: never
profiles: [reconcile]
init: true
entrypoint: ["python", "-m", "messenger.attachments.entrypoints.reconcile"]
command: ["--dry-run"]
environment:
<<: *s3-env
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
S3_READ_TIMEOUT_SECONDS: ${RECONCILE_S3_READ_TIMEOUT_SECONDS:-10}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key]
healthcheck:
disable: true
restart: "no"
read_only: true
cpuset: ${RECONCILE_CPUSET:-}
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
deploy:
resources:
limits: {cpus: "${RECONCILE_CPUS:-1}", memory: "${RECONCILE_MEMORY:-256m}"}
s3-init:
<<: *s3-client
@@ -284,7 +352,7 @@ services:
security_opt: ["no-new-privileges:true"]
cap_drop: [ALL]
ports:
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672"
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-26086}:15672"
networks: [backend, edge]
volumes:
- broker-data:/var/lib/rabbitmq
@@ -312,6 +380,7 @@ services:
command: ["python", "-m", "messenger.relay"]
environment:
<<: *pipeline-env
OTEL_SERVICE_NAME: messenger-relay
OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS:-1}
SWEEP_INTERVAL_SECONDS: ${SWEEP_INTERVAL_SECONDS:-5}
secrets:
@@ -329,6 +398,7 @@ services:
command: ["python", "-m", "messenger.worker"]
environment:
<<: *pipeline-env
OTEL_SERVICE_NAME: messenger-worker
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
RENDER_TIMEOUT_SECONDS: ${RENDER_TIMEOUT_SECONDS:-30}
RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-448}
@@ -356,6 +426,161 @@ services:
resources:
limits: {cpus: "${K6_CPUS:-2}", memory: "${K6_MEMORY:-512m}"}
healthtools:
image: busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092
command: ["cp", "/bin/busybox", "/hc/busybox"]
restart: "no"
network_mode: none
volumes:
- healthtools:/hc
logging: *obs-logging
otel-collector:
<<: *obs-hardening
image: otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849
command: ["--config=/etc/otelcol/collector.yaml"]
networks: [backend, edge]
environment:
GOMEMLIMIT: 200MiB
volumes:
- ./infra/otel:/etc/otelcol:ro
- healthtools:/hc:ro
depends_on:
healthtools:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "/hc/busybox", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:13133/"]
interval: 10s
timeout: 3s
retries: 6
start_period: 10s
start_interval: 2s
deploy:
resources:
limits: {cpus: "0.50", memory: 256M}
prometheus:
<<: *obs-hardening
image: prom/prometheus:v3.15.1@sha256:fa3302279710be2f21a8301ec17a8b4f2301fd829bd113839b3b6b02b6f191a2
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d}
- --storage.tsdb.retention.size=1GB
- --web.enable-otlp-receiver
- --enable-feature=exemplar-storage
networks: [backend, edge]
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
- prometheus-data:/prometheus
ports:
- "127.0.0.1:${PROMETHEUS_PORT:-26090}:9090"
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
interval: 10s
timeout: 3s
retries: 6
start_period: 15s
start_interval: 2s
deploy:
resources:
limits: {cpus: "0.50", memory: 512M}
loki:
<<: *obs-hardening
image: grafana/loki:3.7.9@sha256:0db216b630ae1f348a1584c67f3cead8b4c1fc839169a39501db664744158e28
command: ["-config.file=/etc/loki/loki.yaml"]
networks: [edge]
environment:
GOMEMLIMIT: 300MiB
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki-data:/loki
- healthtools:/hc:ro
depends_on:
healthtools:
condition: service_completed_successfully
ports:
- "127.0.0.1:${LOKI_PORT:-26310}:3100"
healthcheck:
test: ["CMD", "/hc/busybox", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3100/ready"]
interval: 10s
timeout: 3s
retries: 10
start_period: 30s
start_interval: 2s
deploy:
resources:
limits: {cpus: "0.50", memory: 384M}
tempo:
<<: *obs-hardening
image: grafana/tempo:2.10.10@sha256:12b0820f13ffa10ce956b6e9acc3b03f3cffded99109be6cc09d106fe56e5d38
command: ["-config.file=/etc/tempo/tempo.yaml"]
networks: [edge]
environment:
GOMEMLIMIT: 300MiB
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo-data:/var/tempo
- healthtools:/hc:ro
depends_on:
healthtools:
condition: service_completed_successfully
ports:
- "127.0.0.1:${TEMPO_PORT:-26320}:3200"
healthcheck:
test: ["CMD", "/hc/busybox", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3200/ready"]
interval: 10s
timeout: 3s
retries: 10
start_period: 30s
start_interval: 2s
deploy:
resources:
limits: {cpus: "0.50", memory: 384M}
grafana:
<<: *obs-hardening
image: grafana/grafana:12.4.14@sha256:ae509d9ae8c1021ca32ec370a9ca975d40165eda9fccd1e1102d317ff8787cd7
networks: [edge]
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true"
GF_LOG_LEVEL: warn
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/etc/grafana/dashboards:ro
- grafana-data:/var/lib/grafana
depends_on:
prometheus:
condition: service_started
loki:
condition: service_started
tempo:
condition: service_started
ports:
- "127.0.0.1:${GRAFANA_PORT:-26300}:3000"
healthcheck:
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:3000/api/health"]
interval: 10s
timeout: 3s
retries: 10
start_period: 30s
start_interval: 2s
deploy:
resources:
limits: {cpus: "0.50", memory: 384M}
secrets:
postgres_password:
file: ./secrets/postgres_password
@@ -387,12 +612,23 @@ secrets:
file: ./secrets/broker_url_api
broker_url_worker:
file: ./secrets/broker_url_worker
grafana_admin_password:
file: ./secrets/grafana_admin_password
s3_reconciler_access_key_id:
file: ./secrets/s3_reconciler_access_key_id
s3_reconciler_secret_access_key:
file: ./secrets/s3_reconciler_secret_access_key
volumes:
pgdata:
valkeydata:
s3data:
broker-data:
healthtools:
prometheus-data:
loki-data:
tempo-data:
grafana-data:
networks:
edge:
+863
View File
@@ -0,0 +1,863 @@
{
"uid": "messenger-api",
"title": "Messenger · API RED",
"description": "RED по endpoint, SLI/SLO, логи и трассы API.",
"tags": [
"messenger",
"lab6",
"red"
],
"timezone": "browser",
"editable": false,
"graphTooltip": 1,
"refresh": "10s",
"schemaVersion": 41,
"version": 1,
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"fiscalYearStartMonth": 0,
"liveNow": false,
"links": [
{
"title": "Messenger",
"type": "dashboards",
"tags": [
"messenger"
],
"asDropdown": true,
"includeVars": false,
"keepTime": true
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"enable": true,
"iconColor": "red",
"name": "Firing alerts",
"expr": "ALERTS{alertstate=\"firing\"}",
"step": "15s",
"titleFormat": "{{alertname}}",
"textFormat": "{{alertname}} firing",
"useValueForTime": false
}
]
},
"templating": {
"list": [
{
"name": "route",
"label": "Маршрут",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"query": {
"query": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, http_route)",
"refId": "PrometheusVariableQueryEditor-VariableQuery",
"qryType": 1
},
"definition": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, http_route)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"sort": 1
},
{
"name": "instance",
"label": "Реплика",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"query": {
"query": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, instance)",
"refId": "PrometheusVariableQueryEditor-VariableQuery",
"qryType": 1
},
"definition": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, instance)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"sort": 1
}
]
},
"panels": [
{
"type": "stat",
"title": "RPS бизнес-запросов",
"description": "Все маршруты, кроме /health/*; сумма по репликам.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval]))",
"legendFormat": "__auto",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"id": 1,
"gridPos": {
"x": 0,
"y": 0,
"w": 6,
"h": 4
}
},
{
"type": "stat",
"title": "Доля 5xx (2m)",
"description": "SLI доступности: 5xx / все бизнес-запросы без /health/*. Неверный пароль (401) не считается отказом.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "messenger:http_errors:ratio_rate2m",
"legendFormat": "__auto",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 0.01
},
{
"color": "red",
"value": 0.05
}
]
},
"decimals": 2
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"id": 2,
"gridPos": {
"x": 6,
"y": 0,
"w": 6,
"h": 4
}
},
{
"type": "stat",
"title": "p95 latency (5m)",
"description": "histogram_quantile по сумме buckets всех реплик, не среднее p95.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "messenger:http_request_duration_seconds:p95_5m",
"legendFormat": "__auto",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 0.3
},
{
"color": "red",
"value": 0.5
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"id": 3,
"gridPos": {
"x": 12,
"y": 0,
"w": 6,
"h": 4
}
},
{
"type": "stat",
"title": "Доля не-5xx (окно 1h)",
"description": "Доля не-5xx бизнес-запросов за 1h; цель 99%.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "1 - messenger:http_errors:ratio_rate1h",
"legendFormat": "__auto",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"decimals": 3
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"id": 4,
"gridPos": {
"x": 18,
"y": 0,
"w": 6,
"h": 4
}
},
{
"type": "row",
"title": "Rate / Errors / Duration по endpoint",
"collapsed": false,
"panels": [],
"id": 5,
"gridPos": {
"x": 0,
"y": 4,
"w": 24,
"h": 1
}
},
{
"type": "timeseries",
"title": "RPS по маршруту и классу статуса",
"description": "Recording rule messenger:http_requests:rate2m; status class выводится из http.response.status_code.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum by (http_route, http_request_method, status_class) (messenger:http_requests:rate2m{http_route=~\"$route\"})",
"legendFormat": "{{http_request_method}} {{http_route}} {{status_class}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 6,
"gridPos": {
"x": 0,
"y": 5,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "Доля ошибок",
"description": "Числитель SLI — только 5xx; 4xx показаны отдельно и в SLO не входят.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\", http_response_status_code=~\"5..\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "5xx / все (SLI)",
"range": true,
"exemplar": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "B",
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\", http_response_status_code=~\"4..\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "4xx / все (клиент)",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 7,
"gridPos": {
"x": 12,
"y": 5,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "Latency p50 / p95 / p99",
"description": "Из histogram buckets (секунды) всех выбранных реплик; exemplars ведут в Tempo.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "p50",
"range": true,
"exemplar": true
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "p95",
"range": true,
"exemplar": true
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "p99",
"range": true,
"exemplar": true
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 8,
"gridPos": {
"x": 0,
"y": 13,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "p95 по маршруту",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "{{http_route}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 9,
"gridPos": {
"x": 12,
"y": 13,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "RPS по реплике",
"description": "instance = service.instance.id (реплика + pid процесса gunicorn).",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum by (instance) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "{{instance}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 10,
"gridPos": {
"x": 0,
"y": 21,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "Активные запросы",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum by (instance) (http_server_active_requests{job=\"messenger-api\"})",
"legendFormat": "{{instance}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 11,
"gridPos": {
"x": 12,
"y": 21,
"w": 12,
"h": 8
}
},
{
"type": "row",
"title": "Логи и трассы API",
"collapsed": false,
"panels": [],
"id": 12,
"gridPos": {
"x": 0,
"y": 29,
"w": 24,
"h": 1
}
},
{
"type": "logs",
"title": "Логи API (ошибки и предупреждения)",
"datasource": {
"type": "loki",
"uid": "loki"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"refId": "A",
"expr": "{service_name=\"messenger-api\"} | severity_text=~\"(?i)error|warn.*\"",
"queryType": "range"
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"sortOrder": "Descending",
"enableLogDetails": true,
"dedupStrategy": "none",
"prettifyLogMessage": false
},
"id": 13,
"gridPos": {
"x": 0,
"y": 30,
"w": 12,
"h": 10
}
},
{
"type": "table",
"title": "Трассы API с ошибкой",
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"targets": [
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"refId": "A",
"queryType": "traceql",
"query": "{ resource.service.name = \"messenger-api\" && status = error }",
"limit": 20,
"tableType": "traces"
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {},
"id": 14,
"gridPos": {
"x": 12,
"y": 30,
"w": 12,
"h": 10
}
}
]
}
@@ -0,0 +1,307 @@
{
"uid": "messenger-pipeline",
"title": "Messenger · Worker",
"description": "Задания worker: исходы, длительность обработки, логи и трассы.",
"tags": [
"messenger",
"lab6",
"worker"
],
"timezone": "browser",
"editable": false,
"graphTooltip": 1,
"refresh": "10s",
"schemaVersion": 41,
"version": 1,
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"fiscalYearStartMonth": 0,
"liveNow": false,
"links": [
{
"title": "Messenger",
"type": "dashboards",
"tags": [
"messenger"
],
"asDropdown": true,
"includeVars": false,
"keepTime": true
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"enable": true,
"iconColor": "red",
"name": "Firing alerts",
"expr": "ALERTS{alertstate=\"firing\"}",
"step": "15s",
"titleFormat": "{{alertname}}",
"textFormat": "{{alertname}} firing",
"useValueForTime": false
}
]
},
"templating": {
"list": []
},
"panels": [
{
"type": "row",
"title": "Worker",
"collapsed": false,
"panels": [],
"id": 8,
"gridPos": {
"x": 0,
"y": 0,
"w": 24,
"h": 1
}
},
{
"type": "timeseries",
"title": "Jobs по исходу",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum by (job_type, outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))",
"legendFormat": "{{job_type}} {{outcome}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "normal",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 9,
"gridPos": {
"x": 0,
"y": 1,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "Длительность обработки job",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket{}[$__rate_interval])))",
"legendFormat": "p50 {{job_type}}",
"range": true,
"exemplar": true
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket{}[$__rate_interval])))",
"legendFormat": "p95 {{job_type}}",
"range": true,
"exemplar": true
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket{}[$__rate_interval])))",
"legendFormat": "p99 {{job_type}}",
"range": true,
"exemplar": true
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 10,
"gridPos": {
"x": 12,
"y": 1,
"w": 12,
"h": 8
}
},
{
"type": "row",
"title": "Логи и трассы worker",
"collapsed": false,
"panels": [],
"id": 13,
"gridPos": {
"x": 0,
"y": 9,
"w": 24,
"h": 1
}
},
{
"type": "logs",
"title": "Логи worker",
"datasource": {
"type": "loki",
"uid": "loki"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"refId": "A",
"expr": "{service_name=\"messenger-worker\"}",
"queryType": "range"
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"sortOrder": "Descending",
"enableLogDetails": true,
"dedupStrategy": "none",
"prettifyLogMessage": false
},
"id": 14,
"gridPos": {
"x": 0,
"y": 10,
"w": 12,
"h": 10
}
},
{
"type": "table",
"title": "Трассы worker",
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"targets": [
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"refId": "A",
"queryType": "traceql",
"query": "{ resource.service.name = \"messenger-worker\" }",
"limit": 20,
"tableType": "traces"
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {},
"id": 15,
"gridPos": {
"x": 12,
"y": 10,
"w": 12,
"h": 10
}
}
]
}
@@ -0,0 +1 @@
apiVersion: 1
@@ -0,0 +1,14 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
folderUid: messenger
type: file
disableDeletion: true
allowUiUpdates: false
updateIntervalSeconds: 30
options:
path: /etc/grafana/dashboards
foldersFromFilesStructure: false
@@ -0,0 +1,79 @@
apiVersion: 1
prune: true
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
httpMethod: POST
prometheusType: Prometheus
prometheusVersion: 3.15.1
timeInterval: 15s
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: tempo
urlDisplayLabel: Trace
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
maxLines: 1000
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: "$${__value.raw}"
urlDisplayLabel: Trace
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
nodeGraph:
enabled: true
search:
hide: false
traceQuery:
timeShiftEnabled: true
spanStartTimeShift: -30m
spanEndTimeShift: 30m
streamingEnabled:
search: true
metrics: false
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -5m
spanEndTimeShift: 5m
filterByTraceID: true
filterBySpanID: false
tags:
- key: service.name
value: service_name
tracesToMetrics:
datasourceUid: prometheus
spanStartTimeShift: -5m
spanEndTimeShift: 5m
tags:
- key: service.name
value: job
queries:
- name: RPS маршрута
query: sum(rate(http_server_request_duration_seconds_count{$$__tags}[1m]))
- name: p95 маршрута
query: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{$$__tags}[5m])))
lokiSearch:
datasourceUid: loki
@@ -0,0 +1,3 @@
apiVersion: 1
apps: []
+77
View File
@@ -0,0 +1,77 @@
auth_enabled: false
server:
http_listen_address: 0.0.0.0
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
replication_factor: 1
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: "2026-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
ingester:
chunk_idle_period: 5m
max_chunk_age: 1h
limits_config:
allow_structured_metadata: true
volume_enabled: true
retention_period: 168h
reject_old_samples: true
reject_old_samples_max_age: 24h
ingestion_rate_mb: 8
ingestion_burst_size_mb: 16
max_query_lookback: 168h
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes:
- service.name
- deployment.environment.name
- action: structured_metadata
attributes:
- service.instance.id
- service.version
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
compaction_interval: 10m
query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 64
pattern_ingester:
enabled: false
ruler:
enable_api: false
analytics:
reporting_enabled: false
+105
View File
@@ -0,0 +1,105 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 8388608
processors:
memory_limiter:
check_interval: 1s
limit_percentage: 75
spike_limit_percentage: 20
redaction:
allow_all_keys: true
summary: silent
blocked_key_patterns:
- "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$"
- "^http\\.(request|response)\\.header\\..+"
- "^url\\.(query|full)$"
- "^http\\.(url|target)$"
- "^(messenger\\.)?(message\\.)?(text|body)$"
- "^(file\\.)?filename$"
blocked_values:
- "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}"
- "(?i)basic\\s+[A-Za-z0-9+/=]{8,}"
- "(?i)x-amz-(signature|credential)=[^&\\s]+"
- "-----BEGIN [A-Z ]*PRIVATE KEY-----"
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
exporters:
otlp_http/tempo:
traces_endpoint: http://tempo:4318/v1/traces
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/prometheus:
metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 50000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/loki:
logs_endpoint: http://loki:3100/otlp/v1/logs
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
service:
extensions: [health_check]
telemetry:
logs:
level: info
metrics:
level: normal
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
without_type_suffix: false
without_units: false
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/loki]
+41
View File
@@ -0,0 +1,41 @@
global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/rules/*.yml
otlp:
translation_strategy: UnderscoreEscapingWithSuffixes
promote_resource_attributes:
- service.name
- service.version
- deployment.environment.name
- container.name
- compose.project
- compose.service
keep_identifying_resource_attributes: true
storage:
tsdb:
out_of_order_time_window: 30m
exemplars:
max_exemplars: 100000
scrape_configs:
- job_name: prometheus
static_configs:
- targets: ["localhost:9090"]
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8888"]
- job_name: loki
static_configs:
- targets: ["loki:3100"]
- job_name: tempo
static_configs:
- targets: ["tempo:3200"]
- job_name: grafana
static_configs:
- targets: ["grafana:3000"]
@@ -0,0 +1,32 @@
groups:
- name: messenger-http
interval: 15s
rules:
- record: messenger:http_requests:rate2m
expr: |
sum by (http_route, http_request_method, status_class) (
label_replace(
rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[2m]),
"status_class", "${1}xx", "http_response_status_code", "([1-5])[0-9][0-9]"
)
)
- record: messenger:http_errors:ratio_rate2m
expr: |
(sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_code=~"5.."}[2m])) or vector(0))
/
sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[2m]))
- record: messenger:http_errors:ratio_rate1h
expr: |
(sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_code=~"5.."}[1h])) or vector(0))
/
sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1h]))
- record: messenger:http_request_duration_seconds:p50_5m
expr: histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m])))
- record: messenger:http_request_duration_seconds:p95_5m
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m])))
- record: messenger:http_request_duration_seconds:p99_5m
expr: histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m])))
- record: messenger:jobs:rate5m
expr: sum by (job_type, outcome) (rate(messenger_jobs_processed_total[5m]))
- record: messenger:job_duration_seconds:p95_5m
expr: histogram_quantile(0.95, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket[5m])))
+1 -13
View File
@@ -4,20 +4,8 @@
{
"Sid": "MessengerApiObjects",
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},
{
"Sid": "MessengerApiBucket",
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+2 -1
View File
@@ -4,7 +4,8 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
endpoint="${S3_ENDPOINT:-http://s3:9000}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}"
export RC_HOST_root RC_HOST_api
RC_HOST_reconciler="${endpoint%%://*}://$(secret s3_reconciler_access_key_id):$(secret s3_reconciler_secret_access_key)@${endpoint#*://}"
export RC_HOST_root RC_HOST_api RC_HOST_reconciler
rc alias set anon "$endpoint" --anonymous >/dev/null
case "${1:-}" in
sh|wget) exec "$@" ;;
+12 -13
View File
@@ -4,23 +4,22 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
bucket="${S3_BUCKET:-messenger-attachments}"
endpoint="${S3_ENDPOINT:-http://s3:9000}"
policy="${S3_API_POLICY:-messenger-api}"
reconciler="${S3_RECONCILER_POLICY:-messenger-reconciler}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
export RC_HOST_root
api_key=$(secret s3_access_key_id)
grant() {
sed "s/messenger-attachments/$bucket/g" "/s3/$2" > "/tmp/$2"
rc admin policy create root "$1" "/tmp/$2" >/dev/null
rc admin user add root "$(secret "$3")" "$(secret "$4")" >/dev/null
rc admin policy attach root "$1" --user "$(secret "$3")" >/dev/null
rc admin user info root "$(secret "$3")" --json | grep -q "\"$1\""
}
rc ready root --timeout 10 >/dev/null
rc bucket create --ignore-existing "root/$bucket" >/dev/null
rc bucket anonymous set private "root/$bucket" >/dev/null
sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json
rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null
rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null
rc admin policy attach root "$policy" --user "$api_key" >/dev/null
worker_key=$(secret s3_worker_access_key_id)
sed "s/messenger-attachments/$bucket/g" /s3/worker-policy.json > /tmp/worker-policy.json
rc admin policy create root messenger-worker /tmp/worker-policy.json >/dev/null
rc admin user add root "$worker_key" "$(secret s3_worker_secret_access_key)" >/dev/null
rc admin policy attach root messenger-worker --user "$worker_key" >/dev/null
grant "$policy" api-policy.json s3_access_key_id s3_secret_access_key
grant "$reconciler" reconciler-policy.json s3_reconciler_access_key_id s3_reconciler_secret_access_key
grant messenger-worker worker-policy.json s3_worker_access_key_id s3_worker_secret_access_key
rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null
rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"'
rc admin user info root "$api_key" --json | grep -q "\"$policy\""
rc admin user info root "$worker_key" --json | grep -q '"messenger-worker"'
echo "s3-init: bucket=$bucket anonymous=private policy=$policy worker=messenger-worker lifecycle=abort-incomplete-uploads"
echo "s3-init: bucket=$bucket anonymous=private policies=$policy,$reconciler lifecycle=abort-incomplete-uploads"
+17
View File
@@ -0,0 +1,17 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "MessengerReconcilerObjects",
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},
{
"Sid": "MessengerReconcilerListing",
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+48
View File
@@ -0,0 +1,48 @@
stream_over_http_enabled: true
server:
http_listen_address: 0.0.0.0
http_listen_port: 3200
grpc_listen_port: 9095
log_level: warn
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
trace_idle_period: 10s
compactor:
compaction:
block_retention: 168h
query_frontend:
search:
duration_slo: 5s
trace_by_id:
duration_slo: 5s
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
overrides:
defaults:
ingestion:
rate_limit_bytes: 15000000
burst_size_bytes: 20000000
max_traces_per_user: 50000
global:
max_bytes_per_trace: 5000000
usage_report:
reporting_enabled: false
@@ -0,0 +1,7 @@
[
{
"severity": "minor",
"file": "scripts/limits.py:33",
"problem": "Унаследованный POLICY и проверка опубликованных портов в .github/workflows/application.yml:85 ещё не учитывают существующий OTel WIP. Вызов declared() с текущей Compose-конфигурацией помечает все шесть OTel/helper-сервисов как service is not covered by the policy; CI разрешает порты только lb/broker. Это ограничение вспомогательного аудита; smoke и проверки переноса проходят. Полный CI в этой фазе не запускался."
}
]
@@ -1,16 +1,16 @@
## 1. Основа до грейдов ЛР6
- [ ] 1.1 Зафиксировать источник, WIP и контрольные суммы защищённых файлов.
- [ ] 1.2 Перенести зависимости, платформу SQLAlchemy и Alembic с legacy adoption.
- [ ] 1.3 Перенести identity и собственные проверки зоны.
- [ ] 1.4 Перенести messaging и собственные проверки зоны.
- [ ] 1.5 Перенести attachments, reconciler, pipeline и собственные проверки зоны.
- [ ] 1.6 Объединить точки сборки и OTel-порты, сохранив ack после COMMIT.
- [ ] 1.7 Объединить Dockerfile, Compose, Makefile, свои скрипты/CI; сохранить OTel WIP и порты ЛР6.
- [ ] 1.8 Выполнить make check, make lint, make student-test и проверить отсутствие raw asyncpg адаптеров.
- [x] 1.1 Зафиксировать источник, WIP и контрольные суммы защищённых файлов.
- [x] 1.2 Перенести зависимости, платформу SQLAlchemy и Alembic с legacy adoption.
- [x] 1.3 Перенести identity и собственные проверки зоны.
- [x] 1.4 Перенести messaging и собственные проверки зоны.
- [x] 1.5 Перенести attachments, reconciler, pipeline и собственные проверки зоны.
- [x] 1.6 Объединить точки сборки и OTel-порты, сохранив ack после COMMIT.
- [x] 1.7 Объединить Dockerfile, Compose, Makefile, свои скрипты/CI; сохранить OTel WIP и порты ЛР6.
- [x] 1.8 Выполнить make check, make lint, make student-test и проверить отсутствие raw asyncpg адаптеров.
## 2. Проверка и завершение переноса
- [ ] 2.1 Поднять свой стенд, выполнить публичный smoke и повторный migrate; записать Docker-фон.
- [ ] 2.2 Остановить свои стенды без удаления постоянных volumes, сверить защищённые файлы.
- [x] 2.1 Поднять свой стенд, выполнить публичный smoke и повторный migrate; записать Docker-фон.
- [x] 2.2 Остановить свои стенды без удаления постоянных volumes, сверить защищённые файлы.
- [ ] 2.3 Записать реальные результаты и ограничения в verification.md; синхронизировать specs и архивировать change.
@@ -0,0 +1,36 @@
# Проверка переноса основы ЛР5 → ЛР6
Источник: immutable snapshot `lab5-f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071`, SHA `f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071`. Перенесён разрешённый diff от `64d5c67`. Исходный HEAD ЛР6: `a8836ce`; proposal: `dc9256b`. Дата: 2026-11-06.
Перенесены SQLAlchemy 2.0.45/Alembic 1.16.5, поздние исправления identity/messaging, versioned pipeline и reconciler, собственные тесты, Dockerfile/Compose, зависимости, скрипты и CI. Существующие OTel-порты, SDK и инфраструктура сохранены. Версии всех общих зависимостей совпадают с snapshot, исключая намеренную версию проекта 1.6.0 вместо 1.5.0. Raw asyncpg адаптеров больше нет; asyncpg остаётся драйвером SQLAlchemy и точкой инструментации. Legacy SQL только в тестовых fixtures.
## Команды и результаты
- `make check` — PASS, контракт 1.6.0 и синтаксис Python 3.9 скриптов.
- `make validate` — PASS, OpenAPI 3.1 и JSON Schema.
- `make lint` — ruff check, format check, ty: PASS.
- Первый `make student-test` — 513 passed, 1 skipped, 19 deselected, 2 errors за 213.00 с. Обе ошибки — ready-probe RustFS через системный HTTP-прокси macOS. Они не объявляются успехом.
- `NO_PROXY=localhost,127.0.0.1,::1 make student-test PYTEST_ARGS=student-tests/adapters/test_s3_storage.py` — 3 passed за 6.91 с: обе ошибки устранены настройкой окружения, тесты и assertions не менялись.
- Итоговый `NO_PROXY=localhost,127.0.0.1,::1 make student-test` — **515 passed, 1 skipped, 19 deselected за 154.75 с**, exit 0. Включает 36 проверок миграций, наследуемые сценарии pipeline и сохранённые OTel-тесты.
- `docker compose config --quiet`, `make secrets`, `docker compose build`, `docker compose up -d --wait` — exit 0; все долгоживущие сервисы healthy, migrate/s3-init/healthtools завершены. Использованы переменные окружения ниже.
- Первый `make test BASE_URL=http://127.0.0.1:18086` — FAIL RemoteDisconnected из-за системного proxy. Прямой curl без proxy — 200.
- `NO_PROXY=localhost,127.0.0.1,::1 make test BASE_URL=http://127.0.0.1:18086` — `PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.`
- `make migrate` — повторный Alembic upgrade head, exit 0.
- `make down` — exit 0; project mtusi2026lab6 остановлен, девять постоянных volumes сохранены.
- SHA-256 сравнение 22 защищённых файлов и 14 исходных OTel WIP файлов — без изменений; исходный snapshot остался чистым. `git diff --check` — PASS.
## Условия Docker
Все Compose-команды выполнялись из основного каталога с `COMPOSE_PROJECT_NAME=mtusi2026lab6`, `LB_HTTP_PORT=18086`, `LB_STATS_PORT=26085`, `BROKER_MANAGEMENT_PORT=26086`, `PROMETHEUS_PORT=26090`, `LOKI_PORT=26310`, `TEMPO_PORT=26320`, `GRAFANA_PORT=26300`. При up задано `OTEL_SDK_DISABLED=false`; это подтверждено для обеих реплик API, relay и worker. Порты тестовых контейнеров — 26000–26999.
При первой проверке docker ps был пуст. К 09:20:06 UTC появился чужой project `hlpipe-lab4-forward-reconcile-20261106`: API ×2, PostgreSQL, Valkey, S3 и LB. Он не останавливался и не изменялся. Одновременно работали собственные одноразовые контейнеры тестов. Фоновая нагрузка неизвестна; нагрузочных замеров в этой фазе нет. Эти результаты подтверждают функциональность, не производительность. Повтор затронутого suite выполнен с NO_PROXY; позднее полный suite повторён с тем же окружением для единого результата.
Локальные исходные логи сохранены в gitignored `.verification/forward/`: student-test*.txt, smoke*.txt, build.txt, up.txt, migrate-rerun.txt, compose-ps.txt, telemetry-env.json, docker-background.json, down.txt, limits-wip.json. Они не содержат дампов и не читают значения файлов secrets.
## Ограничения для отдельного отчётного этапа
Один тест RLIMIT_AS пропускается на macOS и должен выполняться на Linux; он не объявлен пройденным. Девятнадцать stack-тестов исключены штатным маркером; вместо полного отказного suite в этой фазе выполнен заданный публичный smoke. Грейды 3/4/5 ЛР6, нагрузка, telemetry-check, публикация и Telegram не выполнялись. Активный change `add-lab6-telemetry-pipeline` не закрывается этим переносом.
Minor из findings.json: вспомогательный POLICY в scripts/limits.py и CI-allowlist портов ещё не учитывают OTel WIP. Это сохранено для раздела ограничений будущего REPORT; текущие REPORT.md, AGENTS.md и evidence не изменены по прямой инструкции пользователя. Замечание не является дефектом переноса бизнес-API и не требует fix в этой фазе.
Полный повтор завершён после остановки стенда ЛР6; одноразовые контейнеры тестов удалены fixture-finalizers. Конфигурации наблюдаемости сохранены как исходный WIP, а не как завершённый грейд ЛР6.
+2 -2
View File
@@ -41,7 +41,7 @@ context: |
Python 3.14 (аннотации вычисляются лениво по PEP 649, `TYPE_CHECKING` и
`from __future__ import annotations` не используются), FastAPI, gunicorn с несколькими
`UvicornWorker` (пакет `uvicorn-worker`, `preload_app = False`), dishka (DI, интеграции FastAPI
и FastStream), pydantic v2 + pydantic-settings, structlog, asyncpg, redis-py для Valkey,
и FastStream), pydantic v2 + pydantic-settings, structlog, SQLAlchemy 2.x async (asyncpg), Alembic, redis-py для Valkey,
argon2-cffi, aiobotocore, Pillow, FastStream 0.7 с `faststream[rabbit,otel]` поверх aio-pika,
OpenTelemetry Python (API/SDK, OTLP/HTTP protobuf exporter без gRPC, инструментации FastAPI и
asyncpg, версии в `uv.lock`), pytest + pytest-asyncio + httpx. Зависимости — uv и `uv.lock`;
@@ -68,7 +68,7 @@ context: |
main.py сборка HTTP-приложения: вклады зон, связи зон портами, телеметрия процесса
relay.py сборка процесса relay: outbox → брокер, возврат просроченных аренд
worker.py сборка процесса worker: FastStream-подписчик очереди заданий
migrate.py одноразовая точка входа: применяет SQL-миграции под advisory lock
alembic/ async миграции с legacy adoption и advisory lock
identity/ пользователи, пароли и сессии
messaging/ чаты, история и идемпотентная отправка, ссылки сообщения на вложения
attachments/ вложения: приём и staging, задания и outbox, обработка и варианты, порты
+14 -9
View File
@@ -19,27 +19,32 @@ PostgreSQL SHALL хранить пользователей, сессии, чат
### Requirement: Версионируемые миграции
Схема SHALL создаваться только миграциями: упорядоченными SQL-файлами, каждый из которых применяется в своей транзакции и записывается в журнал применённых версий. Миграции SHALL применяться одним одноразовым процессом до старта реплик API, а не каждой репликой; одновременный запуск двух процессов SHALL сериализоваться. Повторный запуск SHALL NOT менять схему и данные. Изменение текста уже применённой миграции SHALL останавливать запуск с ошибкой. Новая миграция SHALL быть аддитивной: работающая предыдущая версия API SHALL продолжать работать на схеме после неё.
Схема SHALL создаваться версионируемыми миграциями, каждая в своей транзакции. Одноразовый процесс `migrate` SHALL завершаться до старта API; одновременные запуски SHALL сериализоваться. Повторный запуск SHALL NOT менять данные. При переходе со старого SQL-журнала процесс SHALL проверить контрольные суммы и фактическую схему до принятия версии. Неизвестная или изменённая история SHALL останавливать запуск без изменения пользовательских данных. Аддитивные изменения SHALL сохранять совместимость с предыдущей версией приложения. Основание: обязательное постоянное хранение и сохранение lab1–lab4 в README.
#### Scenario: Миграции с пустого volume
- **WHEN** стенд поднимается на пустом volume PostgreSQL
- **THEN** процесс миграций применяет все версии по порядку и завершается кодом 0, после чего стартуют реплики API
- **WHEN** стенд запускается с пустой PostgreSQL
- **THEN** все миграции применяются по порядку, migrate завершается кодом 0 и запускается API
#### Scenario: Повторный запуск
- **WHEN** процесс миграций запускается повторно на той же БД
- **THEN** он завершается кодом 0, журнал версий и число строк во всех таблицах не меняются
- **WHEN** migrate повторно запускается на той же БД
- **THEN** он завершается кодом 0 без изменения пользовательских строк
#### Scenario: Переход старого стенда ЛР5
- **WHEN** БД содержит проверенный SQL-журнал ЛР5, вложения, задания, outbox и варианты
- **THEN** переход сохраняет строки, ограничения, состояния и ссылки на объекты
#### Scenario: Изменённая миграция
- **WHEN** текст применённой миграции изменён и процесс миграций запущен
- **THEN** процесс завершается ненулевым кодом и схему не трогает
- **WHEN** контрольная сумма SQL-журнала или фактическая схема не совпадает с ожидаемой
- **THEN** принятие истории завершается ошибкой без изменения данных и отметки версии
#### Scenario: Реплики не мигрируют схему
- **WHEN** стартует или перезапускается реплика API
- **THEN** она не применяет миграции; журнал версий меняет только процесс `migrate`
- **WHEN** стартует реплика API, relay или worker
- **THEN** только отдельный процесс migrate изменяет журнал миграций
### Requirement: Атомарное создание чата
@@ -0,0 +1,39 @@
# lab4-storage-reconciler Specification
## Purpose
Сохраняет совместимость перенесённой из ЛР4 сверки PostgreSQL и S3 с ранее записанными объектами и ограничивает её область, чтобы не вмешиваться в асинхронную обработку ЛР5. Основание — сохранение предыдущего решения при переносе; не является заявкой на грейд ЛР5.
## Requirements
### Requirement: Ручная сверка legacy-объектов
Одноразовый reconciler SHALL по умолчанию работать в dry-run над прежним префиксом `attachments/`, выдавая JSON со счётчиками расхождений. Apply SHALL сохранять свежие объекты и повторно проверять ссылку в БД перед удалением старого orphan. Одновременный второй проход SHALL завершаться с признаком skipped.
#### Scenario: Dry-run без изменения данных
- **WHEN** оператор запускает reconciler без apply для legacy-объектов
- **THEN** он возвращает JSON и не меняет строки или байты
#### Scenario: Повторная проверка защищает объект
- **WHEN** после листинга старого orphan появляется строка с его ключом
- **THEN** apply сохраняет объект
### Requirement: Карантин сохраняет данные
При подтверждённом расхождении готового legacy-вложения apply SHALL поместить метаданные в карантин без удаления строки и объекта. GET `/api/v1/attachments/{id}/content` SHALL возвращать 404 с ошибкой not_found для карантина, метаданные остаются доступны участнику. Снятие карантина SHALL требовать совпадения SHA-256 даже без deep.
#### Scenario: Возвращены другие байты той же длины
- **WHEN** объект в карантине заменён байтами прежнего размера, но другого SHA-256
- **THEN** карантин остаётся и скачивание не разрешается
### Requirement: Разделение со staging и pipeline
Перенесённая сверка SHALL NOT обходить или удалять объекты из `staging/`, `originals/` и `variants/`. Этими префиксами управляет pipeline ЛР5 и его существующая уборка staging.
#### Scenario: Готовое вложение ЛР5 вне legacy-префикса
- **WHEN** у нового вложения оригинал и варианты опубликованы pipeline
- **THEN** legacy-сверка не считает их orphan и не изменяет их
@@ -0,0 +1,51 @@
# lab5-versioned-pipeline Specification
## Purpose
Определяет наблюдаемые гарантии versioned pipeline и воспроизводимое доказательство идемпотентности для трека 2 оценки «5» README лабораторной 5.
## Requirements
### Requirement: Явная постановка версии
Оператор SHALL иметь команду reprocess с attachment-id и pipeline-version 1 или 2. Повтор той же версии SHALL возвращать noop без новой job или outbox. Для неизвестного вложения возвращается not_found, для pending/failed — conflict без изменения состояния; failed восстанавливается существующим replay. Успех queued SHALL означать durable job и outbox. Требование трассируется на README, трек 2.
#### Scenario: Одновременные операторские запросы
- **WHEN** несколько команд запрашивают версию 2 одного вложения
- **THEN** ровно одна получает queued, остальные noop; создаётся одна job версии 2 и одно намерение доставки
#### Scenario: Ошибочный запрос
- **WHEN** указана неподдерживаемая версия, неизвестный id или failed-вложение
- **THEN** новая работа не создаётся, оператор получает явную ошибку/результат отказа
### Requirement: Монотонная публикация различимых версий
Версии 1 и 2 SHALL давать детерминированные варианты с различимыми recipe при сохранении форматов/размеров контракта. API SHALL отдавать только опубликованную версию, published_version SHALL возрастать, а ready SHALL NOT откатываться. Работа или ошибка старой версии SHALL NOT перезаписывать новую. Ошибка одной версии SHALL NOT завершать всё вложение в failed, пока остаётся активная версия. Требование трассируется на README, трек 2 и contracts/attachment-states.md.
#### Scenario: Новая версия завершилась первой
- **WHEN** версия 2 достигла ready до доставки версии 1
- **THEN** последующая версия 1 не меняет опубликованные варианты; оригинал и байты версии 2 сохраняются
#### Scenario: Старый держатель завершился позже
- **WHEN** worker версии 1 задержан после записи объектов, а версия 2 опубликована
- **THEN** завершение версии 1 не перезаписывает опубликованный результат версии 2
#### Scenario: Уборка при активной версии
- **WHEN** новая версия уже готова, а старый worker задержан до или после side effect и запускается очистка staging
- **THEN** исходник активного worker сохраняется до его завершения; последующая обработка ready-вложения использует устойчивый опубликованный оригинал и не зависит от незавершённой уборки
#### Scenario: Ошибка при нескольких версиях
- **WHEN** одна версия завершилась ошибкой, другая ещё обрабатывается
- **THEN** вложение не становится failed до завершения оставшейся работы, а уже ready продолжает скачиваться
### Requirement: Воспроизводимые инварианты при отказах
Одна команда make track2 SHALL запускать контроль без отказа и автоматические duplicate, reorder, crash до/после эффекта, SIGKILL и stale job после версии 2. Результат SHALL включать UTC/SHA, фактические контейнеры, отсутствие потери подтверждённых 202, лишних эффектов и отката ready/версии, соответствие байтов метаданным и объектов собственных вложений ожидаемым ключам. Исследование SHALL содержать три последовательных повтора, сравнение, ограничения и объяснение at-least-once доставки. Требование трассируется на README, трек 2; унаследованные негативные сценарии остаются в спецификациях уровней 3/4.
#### Scenario: Повтор после эффекта
- **WHEN** worker падает после PUT либо после COMMIT до ack и получает ту же job снова
- **THEN** итоговое вложение доступно, лишнего набора вариантов нет, принятая job не потеряна
#### Scenario: Проверяемая серия
- **WHEN** три раза последовательно выполняется make track2 с разными output paths
- **THEN** каждый отчёт проверяет собственные принятые IDs, содержит реальные результаты и не удаляет данные предыдущих прогонов
+10 -3
View File
@@ -13,6 +13,8 @@ dependencies = [
"gunicorn>=26.2.0",
"orjson>=3.12.0",
"asyncpg>=0.32.0",
"sqlalchemy[asyncio]==2.0.45",
"alembic==1.16.5",
"argon2-cffi>=25.1.0",
"uvicorn-worker>=0.4.0",
"redis>=8.1.0",
@@ -34,6 +36,7 @@ dev = [
"httpx>=0.28.0",
"ruff>=0.14.0",
"ty>=0.0.1a21",
"openapi-spec-validator==0.7.2",
]
[build-system]
@@ -87,6 +90,7 @@ ignore = [
[tool.ruff.lint.per-file-ignores]
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "ARG002", "T20", "TID251"]
"tools/*.py" = ["T20"]
"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"]
"scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"]
"scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"]
@@ -96,10 +100,10 @@ ignore = [
"src/messenger/*/entrypoints/**" = ["TID251", "N818", "PLR0917", "S105"]
"src/messenger/shared/config.py" = ["ARG002", "ARG003"]
"src/messenger/main.py" = ["TID251"]
"src/messenger/migrate.py" = ["TID251"]
"src/messenger/relay.py" = ["TID251"]
"src/messenger/worker.py" = ["TID251"]
"src/messenger/admin.py" = ["TID251"]
"alembic/**" = ["TID251"]
"src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"]
[tool.ruff.lint.flake8-tidy-imports.banned-api]
@@ -112,7 +116,10 @@ ignore = [
known-first-party = ["messenger"]
[tool.ty.src]
include = ["src", "student-tests"]
include = ["src", "student-tests", "alembic"]
[tool.ty.environment]
extra-paths = ["alembic"]
[tool.ty.rules]
unresolved-import = "error"
@@ -123,7 +130,7 @@ possibly-missing-attribute = "warn"
testpaths = ["student-tests"]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
addopts = "-q --strict-markers -m 'not stack'"
addopts = "--strict-markers -m 'not stack'"
markers = [
"stack: black-box проверки против поднятого compose-стенда (make test-faults)",
]
+73
View File
@@ -0,0 +1,73 @@
import argparse
import json
import os
import shlex
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SETTINGS = (
"COMPOSE_PROJECT_NAME",
"COMPOSE_FILE",
"LB_HTTP_PORT",
"LB_STATS_PORT",
"BASE_URL",
"EVIDENCE_DIR",
"RECONCILE_EVIDENCE_DIR",
"NO_PROXY",
"S3_CPUS",
"S3_MEMORY",
"UPLOAD_GRACE_SECONDS",
"API_IMAGE",
"API_REPLICAS",
"WEB_CONCURRENCY",
"MALLOC_MMAP_THRESHOLD_",
)
def main():
parser = argparse.ArgumentParser(description="Capture a command without overwriting evidence")
parser.add_argument("output", type=Path)
parser.add_argument("command", nargs=argparse.REMAINDER)
args = parser.parse_args()
command = args.command[1:] if args.command[:1] == ["--"] else args.command
if not command:
parser.error("command is required")
args.output.parent.mkdir(parents=True, exist_ok=True)
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
started = time.monotonic()
with args.output.open("x") as output:
def write(line):
line = line.replace(str(ROOT), "<repo>")
output.write(line)
output.flush()
sys.stdout.write(line)
sys.stdout.flush()
write(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit: {commit}\n")
write(
"environment: "
+ json.dumps({k: os.environ[k] for k in SETTINGS if k in os.environ})
+ "\n"
)
write("$ " + shlex.join(command) + "\n")
with subprocess.Popen(
command,
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
errors="replace",
) as process:
for line in process.stdout:
write(line)
code = process.wait()
write(f"exit_code: {code} elapsed_seconds: {time.monotonic() - started:.3f}\n")
return code
if __name__ == "__main__":
sys.exit(main())
+54
View File
@@ -0,0 +1,54 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
repo=$PWD
project="hlpipe-lab6-clean-$(date -u +%Y%m%d%H%M%S)"
dir="$repo/.verification/$project"
out="$repo/${EVIDENCE_DIR:-evidence}/clean-clone.txt"
sha=$(git rev-parse HEAD)
[ ! -e "$out" ] || { echo "Choose a new EVIDENCE_DIR" >&2; exit 2; }
mkdir -p "$(dirname "$dir")" "$(dirname "$out")"
exec 3>&1
exec > "$out" 2>&1
cleanup() {
status=$?
trap - EXIT HUP INT TERM
if [ -d "$dir/.git" ]; then
cd "$dir"
docker compose --profile bench --profile tools --profile reconcile down || status=1
docker volume ls --filter "label=com.docker.compose.project=$project"
fi
echo "exit_code: $status; volumes retained"
sed "s#$repo#<repo>#g" "$out" >&3
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' HUP INT TERM
step() { echo "\$ $*"; "$@"; }
export COMPOSE_PROJECT_NAME="$project" COMPOSE_FILE=compose.yaml
export LB_HTTP_PORT=18086 LB_STATS_PORT=26085 BROKER_MANAGEMENT_PORT=26086 BASE_URL=http://127.0.0.1:18086
export PROMETHEUS_PORT=26090 LOKI_PORT=26310 TEMPO_PORT=26320 GRAFANA_PORT=26300
export NO_PROXY=localhost,127.0.0.1,::1
unset API_IMAGE
echo "date: $(date -u +%FT%TZ) commit: $sha project: $project"
echo "HTTP=$LB_HTTP_PORT stats=$LB_STATS_PORT"
step git clone --quiet --no-hardlinks "$repo" "$dir"
cd "$dir"
step git checkout --quiet --detach "$sha"
step make secrets
step docker compose --env-file .env config --quiet
step make up
step docker compose ps -a --format '{{.Service}} {{.State}} {{.Status}} {{.Ports}}'
step make test
step make test-persistence ACTION_SCRIPT=scripts/restart.sh
step make test-failover ACTION_SCRIPT=scripts/stop-one.sh
step docker compose ps -a --format '{{.Service}} {{.Name}} {{.State}}' api
step make restore
step make scenario SCENARIO=failover-upload
step make restore
step make scenario SCENARIO=s3-outage
step sh scripts/s3-check.sh
step make reconcile
step make migrate
step make down
step docker volume ls --filter "label=com.docker.compose.project=$project"
+1 -1
View File
@@ -8,7 +8,7 @@ replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
detect() { python3 scripts/probe.py detect "$@"; }
failed=0
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
-- docker stop -t "${STOP_TIMEOUT:-7}" "$(replica)" || failed=1
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
-- docker kill "$(replica)" || failed=1
hung=$(replica)
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
out=${1:?usage: docker-stats.sh <csv>}
vm=$(dirname "$out")/vm_stat.csv
cd "$(dirname "$0")/.."
project=$(docker compose config | sed -n 's/^name: //p')
echo 'ts,name,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids' > "$out"
echo 'ts,busy,total,cpus' > "$vm"
while :; do
ts=$(date +%s)
docker compose exec -T valkey cat /proc/stat 2>/dev/null | awk -v ts="$ts" '/^cpu /{b=$2+$3+$4+$7+$8+$9; t=b+$5+$6} /^cpu[0-9]/{n++} END{if (t) print ts "," b "," t "," n}' >> "$vm" || true
ids=$(docker ps -q --filter "label=com.docker.compose.project=$project")
[ -z "$ids" ] || docker stats --no-stream --format "$ts,{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true
sleep "${INTERVAL:-1}"
done
+5 -3
View File
@@ -1,12 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; }
value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; }
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
bucket=${S3_BUCKET:-messenger-attachments}
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'"
docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}'
for table in attachments message_attachments; do
@@ -36,5 +38,5 @@ bucket=${S3_BUCKET:-messenger-attachments}
docker compose config --format json | python3 -c 'import json,sys
for name, service in sorted(json.load(sys.stdin)["services"].items()):
print(name, sorted(s["source"] for s in service.get("secrets", [])))'
} > evidence/storage-layout.txt
cat evidence/storage-layout.txt
} > "$out"/storage-layout.txt
cat "$out"/storage-layout.txt
+19 -8
View File
@@ -11,14 +11,25 @@ cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
cli dbsize
cli "info stats" | grep -E '^keyspace_(hits|misses)'
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
hash=${key#session:}
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
echo "\$ количество отметок отзыва и закэшированных сессий"
revoked=0; cached=0
echo "\$ valkey-cli --scan --pattern 'session:*' и вид значения (хеш сокращён, значения не печатаются)"
revoked=0; cached=0; key=""; stored=""
for k in $(cli "--scan --pattern session:* --count 1000"); do
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
if [ "$(cli "get $k")" = "-" ]; then
revoked=$((revoked + 1)); printf '%.16s… отметка отзыва "-"\n' "$k"; continue
fi
cached=$((cached + 1)); printf '%.16s… сессия\n' "$k"
if [ -z "$key" ]; then
stored=$(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('${k#session:}', 'hex')")
if [ -n "$stored" ]; then key=$k; fi
fi
done
echo "tombstones: $revoked, cached sessions: $cached"
if [ "$revoked" -eq 0 ] || [ -z "$key" ]; then
echo "нужны закэшированная сессия из БД и отметка отзыва: сначала make test" >&2
exit 1
fi
expires=$(cli "pexpiretime $key")
echo "\$ срок ключа кэша против срока сессии для $(printf '%.16s' "$key")…"
echo "valkey pexpiretime, ms: $expires"
echo "postgres expires_at, ms: $stored"
if [ "$expires" -le "$stored" ]; then echo "ключ не переживает сессию: да"; else echo "ключ не переживает сессию: НЕТ"; exit 1; fi
+7 -5
View File
@@ -1,9 +1,11 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
base=${1:-http://$(docker compose port lb 8080)}
stats=http://$(docker compose port lb 8404)/stats\;csv
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
{
header
echo "\$ docker compose --env-file .env config --quiet && echo valid"
@@ -25,7 +27,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
} > evidence/stack.txt
} > "$out"/stack.txt
{
header
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
@@ -38,7 +40,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)"
echo "packages (uv.lock):"
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
for name in fastapi uvicorn uvicorn-worker gunicorn sqlalchemy alembic asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
faststream aio-pika aiormq dishka-faststream; do
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
done
@@ -47,5 +49,5 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
done | sort -u
} > evidence/environment.txt
cat evidence/stack.txt evidence/environment.txt
} > "$out"/environment.txt
cat "$out"/stack.txt "$out"/environment.txt
+16 -13
View File
@@ -1,20 +1,22 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM alembic_version) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, username = 'seed_author' AS seed_stub, count(*) AS users, min(length(password_hash)) AS min_length, max(length(password_hash)) AS max_length FROM users GROUP BY 1, 2 ORDER BY 2"
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
{
header
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments alembic_version; do
echo "\$ psql -c '\\d+ $table'"
psql "\\d+ $table"
done
echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'"
psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations"
} > evidence/schema.txt
echo "\$ psql -c 'SELECT version_num FROM alembic_version'"
psql "SELECT version_num FROM alembic_version"
} > "$out"/schema.txt
{
header
echo "\$ psql -c \"$counts\""
@@ -23,14 +25,14 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
docker compose run --rm migrate 2>/dev/null
echo "\$ psql -c \"$counts\""
psql "$counts"
} > evidence/migrations.txt
} > "$out"/migrations.txt
{
header
echo "\$ psql -c \"$kdf\""
psql "$kdf"
echo "\$ psql -c \"$tokens\""
psql "$tokens"
} > evidence/kdf.txt
} > "$out"/kdf.txt
{
header
echo "\$ git ls-files secrets .env backups volumes"
@@ -38,7 +40,8 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
echo "\$ docker volume ls --filter name=mtusi2026lab6"
docker volume ls --filter name=mtusi2026lab6
} > evidence/secrets.txt
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
project=$(docker compose config --format json | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])')
echo "\$ docker volume ls --filter label=com.docker.compose.project=$project"
docker volume ls --filter "label=com.docker.compose.project=$project"
} > "$out"/secrets.txt
echo ""$out"/{schema,migrations,kdf,secrets}.txt обновлены"
+1 -1
View File
@@ -10,7 +10,7 @@ failed=0
set --
for host in $hosts; do
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
-- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1
scripts/restore.sh
set -- "$@" "$out/$host/result.json"
done
+11 -6
View File
@@ -1,28 +1,33 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
replicas() {
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
}
tmp=evidence/failover.tmp
url=${1:-http://localhost:8080}
shift=0
while [ "$shift" -lt "${2:-0}" ]; do curl -fsS -o /dev/null "$url/health/live"; shift=$((shift + 1)); done
tmp="$out"/failover.tmp
since=$(date -u +%FT%TZ)
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
replicas
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$url"
echo "== after: api replicas"
replicas
} > "$tmp" 2>&1
stopped=$(replicas | awk '$2 == "exited" {print $1}')
{
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|service.draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
echo "\$ scripts/restore.sh"
scripts/restore.sh
echo "== after restore"
replicas
} >> "$tmp" 2>&1
mv "$tmp" "evidence/failover-$stopped.txt"
cat "evidence/failover-$stopped.txt"
mv "$tmp" "$out/failover-$stopped.txt"
cat "$out/failover-$stopped.txt"
+34 -2
View File
@@ -23,6 +23,19 @@ def compose(*args, stdin=None):
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
def containers(service):
ids = compose("ps", "-aq", service).stdout.split()
fields = ("hostname", "status", "exit_code", "started_at", "finished_at")
template = (
"{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}"
" {{.State.StartedAt}} {{.State.FinishedAt}}"
)
out = subprocess.run(
["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False
)
return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()]
def upload(c, actor, chat, expected=202):
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
schema = "Attachment" if expected == 202 else None
@@ -90,6 +103,10 @@ def failover_upload(c, args):
recovered = time.monotonic() - started
served = got["headers"].get("X-Instance-Id")
check(served != target, "download served by the stopped replica")
during = containers(args.service)
states = {item["hostname"]: item["status"] for item in during}
check(states.get(target) == "exited", f"target {target} is not exited: {states}")
check(states.get(served) == "running", f"neighbour {served} is not running: {states}")
digest = hashlib.sha256(got["bytes"]).hexdigest()
check(
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
@@ -104,7 +121,9 @@ def failover_upload(c, args):
"sha256_equal": True,
"hook_seconds": round(hook_seconds, 3),
"download_ok_after_seconds": round(recovered, 3),
"replicas_at_download": during,
"restored": args.restore,
"replicas_after_restore": containers(args.service) if args.restore else None,
}
@@ -124,6 +143,7 @@ def s3_outage(c, args):
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
}
stopped = containers(args.s3_service)
finally:
started = time.monotonic()
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
@@ -131,8 +151,11 @@ def s3_outage(c, args):
got = wait(lambda: content(c, bob, att), 30)
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
expected = {"upload": 503, "download": 503, "text_message": 201}
check(all(during[k]["status"] == v for k, v in expected.items()), f"S3 outage: {during}")
check(all(during[k]["seconds"] < 6 for k in ("upload", "download")), "S3 timeout exceeded")
return {
"scenario": "s3-outage",
"s3_container_during_outage": stopped,
"during_outage": during,
"expected": expected,
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
@@ -180,6 +203,8 @@ def abort_upload(c, args):
)
rows = out.stdout.split()
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
check(out.returncode == 0, "cannot inspect aborted upload")
check(counts.get("ready", 0) == 0, "aborted upload became ready")
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
return {
"scenario": "abort-upload",
@@ -216,10 +241,17 @@ def main():
"s3-outage": s3_outage,
"abort-upload": abort_upload,
}[args.scenario]
commit = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, check=False
)
stamp = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"commit": commit.stdout.decode().strip(),
}
try:
report = {"ok": True, **run(c, args)}
report = {"ok": True, **stamp, **run(c, args)}
except Failure as exc:
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
out=${1:?usage: lb-stats.sh <csv>}
cd "$(dirname "$0")/.."
url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv}
curl -fsS "$url" | sed -n '1s/^# /ts,/p' > "$out"
while :; do
ts=$(date +%s)
curl -fsS "$url" | grep -E '^(public|api),' | sed "s/^/$ts,/" >> "$out" || true
sleep "${INTERVAL:-1}"
done
+29 -5
View File
@@ -9,12 +9,20 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CLIENT_SECRETS = [
"s3_access_key_id",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
"s3_worker_access_key_id",
"s3_worker_secret_access_key",
"s3_root_password",
"s3_root_user",
"s3_secret_access_key",
"s3_worker_access_key_id",
"s3_worker_secret_access_key",
]
RECONCILER_SECRETS = [
"database_url",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
]
OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli")
POLICY = {
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
"api": {
@@ -29,6 +37,11 @@ POLICY = {
"networks": ["backend"],
},
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
"reconciler": {
"secrets": RECONCILER_SECRETS,
"volumes": [],
"networks": ["backend"],
},
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
@@ -94,6 +107,9 @@ def declared(name, service, networks, problems):
problems.append(f"{name}: publishes {ports} beyond loopback")
if name == "api" and not service.get("read_only"):
problems.append("api: root filesystem is writable")
user = str(service.get("user", ""))
if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"):
problems.append(f"{name}: runs as root")
internal = [n for n in nets if networks.get(n, {}).get("internal")]
return [
name,
@@ -103,6 +119,7 @@ def declared(name, service, networks, problems):
",".join(volumes) or "-",
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
str(service.get("read_only", False)).lower(),
user or "image",
]
@@ -149,6 +166,8 @@ def main():
"bench",
"--profile",
"tools",
"--profile",
"reconcile",
"config",
"--format",
"json",
@@ -162,9 +181,14 @@ def main():
if not any(row[0] == "api" for row in containers):
problems.append("no running api replicas to inspect")
lines = [
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
"$ docker compose --profile bench --profile tools config --format json (declared)",
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} "
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
" (declared)",
*table(
["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"],
services,
),
"",
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
*table(["service", "hostname", "cpus", "memory", "published"], containers),
+62
View File
@@ -0,0 +1,62 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
runs="${RUNS:-3}"
pause="${PAUSE:-15}"
project=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$(docker compose ps -q lb)")
network="${project}_edge"
image=$(sed -n 's/^FROM \(python:[^ ]*\) AS runtime$/\1/p' Dockerfile)
name="$project-load-client"
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out/load"
[ -z "$(ls -A "$out/load")" ] || { echo "Choose an empty EVIDENCE_DIR/load" >&2; exit 2; }
client() {
docker run --rm --name "$name" --network "$network" --cpus 2 --memory 512m \
--sysctl net.ipv4.tcp_tw_reuse=1 -v "$PWD/tests:/w/tests:ro" -v "$PWD/contracts:/w/contracts:ro" \
-v "$PWD/lab.json:/w/lab.json:ro" -v "$PWD/$out/load:/out" "$image" python /w/tests/load.py "$@"
}
handled() {
docker compose logs --no-log-prefix --since "$1" api | python3 -c 'import collections, json, sys
seen = collections.Counter()
for line in sys.stdin:
if line.startswith("{"):
event = json.loads(line)
if event.get("event") == "request.handled":
seen[(event["status"], event["method"], event["route"])] += 1
for (status, method, route), count in sorted(seen.items()):
print(status, method, route, count)'
}
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) network: $network"
echo "# make load-evidence (scripts/load-evidence.sh): закрытый контур tests/load.py, 64 потока, 8 с, все записи"
echo "# прогона в один новый чат, $runs повтора с паузой $pause с; клиент — одноразовый контейнер в сети <проект>_edge"
echo "# стенда (2 CPU, 512 MiB, базовый образ API по digest, tests/, contracts/ и lab.json только для чтения,"
echo "# tcp_tw_reuse=1), адрес http://lb:8080. На 4-й секунде — docker stats; после прогона — request.handled реплик."
echo "\$ docker ps --format '{{.Names}}' | grep -v '^$project-' # чужие контейнеры на VM во время серии"
docker ps --format '{{.Names}}' | grep -v "^$project-" || true
echo "\$ client() { docker run --rm --name $name --network $network --cpus 2 --memory 512m --sysctl net.ipv4.tcp_tw_reuse=1 -v \"\$PWD/tests:/w/tests:ro\" -v \"\$PWD/contracts:/w/contracts:ro\" -v \"\$PWD/lab.json:/w/lab.json:ro\" -v \"\$PWD/$out/load:/out\" $image python /w/tests/load.py \"\$@\"; }"
run=1
while [ "$run" -le "$runs" ]; do
since=$(date -u +%FT%TZ)
echo "\$ client --base-url http://lb:8080 --duration 8 --concurrency 64 --output /out/run$run.json"
client --base-url http://lb:8080 --duration 8 --concurrency 64 --output "/out/run$run.json" > "$work/out" 2>&1 &
pid=$!
sleep 4
stats=$(docker stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}}' $(docker compose ps -q) "$name" | sed "s/^$project-//" | sort)
if wait "$pid"; then code=0; else code=$?; fi
echo "exit $code"
cat "$work/out"
echo "\$ docker stats --no-stream # 4-я секунда прогона: контейнер CPU% память"
echo "$stats"
echo "\$ docker compose logs --since $since api | request.handled: статус метод маршрут число"
handled "$since"
[ "$code" -eq 0 ] || { exit "$code"; }
[ "$run" -lt "$runs" ] && sleep "$pause"
run=$((run + 1))
done
echo "\$ docker compose exec -T postgres psql -c 'SELECT count(*) FROM messages'"
docker compose exec -T postgres psql -U messenger -d messenger -qAt -c 'SELECT count(*) FROM messages'
} > "$out/load.txt"
cat "$out/load.txt"
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
otel=otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849
prom=prom/prometheus:v3.15.1@sha256:fa3302279710be2f21a8301ec17a8b4f2301fd829bd113839b3b6b02b6f191a2
loki=grafana/loki:3.7.9@sha256:0db216b630ae1f348a1584c67f3cead8b4c1fc839169a39501db664744158e28
tempo=grafana/tempo:2.10.10@sha256:12b0820f13ffa10ce956b6e9acc3b03f3cffded99109be6cc09d106fe56e5d38
docker run --rm --network none -v "$PWD/infra/otel:/etc/otelcol:ro" "$otel" validate --config=/etc/otelcol/collector.yaml
echo "otelcol-contrib validate collector.yaml: OK"
docker run --rm --network none --entrypoint promtool -v "$PWD/infra/prometheus:/etc/prometheus:ro" "$prom" check config /etc/prometheus/prometheus.yml
docker run --rm --network none -v "$PWD/infra/loki/loki.yaml:/etc/loki/loki.yaml:ro" "$loki" -config.file=/etc/loki/loki.yaml -verify-config
echo "loki -verify-config: OK"
docker run --rm --network none -v "$PWD/infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro" "$tempo" -config.file=/etc/tempo/tempo.yaml -config.verify=true
echo "tempo -config.verify: OK"
for board in infra/grafana/dashboards/*.json; do
python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d["uid"] and d["panels"]' "$board"
done
echo "grafana dashboards: valid JSON with uid and panels"
+489
View File
@@ -0,0 +1,489 @@
#!/usr/bin/env python3
import argparse
import base64
import json
import os
import re
import secrets
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PROMQL_WORDS = {
"sum", "min", "max", "avg", "count", "by", "without", "on", "ignoring", "group_left", "group_right", "and", "or",
"unless", "offset", "bool", "rate", "irate", "increase", "delta", "deriv", "histogram_quantile", "label_replace",
"label_join", "absent", "absent_over_time", "vector", "scalar", "clamp_min", "clamp_max", "topk", "bottomk",
"max_over_time", "min_over_time", "avg_over_time", "sum_over_time", "count_over_time", "quantile_over_time",
"time", "timestamp", "sort", "sort_desc", "round", "abs", "le", "inf", "nan", "predict_linear", "changes", "resets",
}
class Failure(Exception):
pass
def now():
return datetime.now(timezone.utc).isoformat(timespec="seconds")
def http(method, url, body=None, headers=None, timeout=10, expect=None):
data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode()
head = {"Accept": "application/json"}
if body is not None and not isinstance(body, bytes):
head["Content-Type"] = "application/json"
head.update(headers or {})
request = urllib.request.Request(url, data=data, headers=head, method=method)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
status, raw, got = response.status, response.read(), dict(response.headers)
except urllib.error.HTTPError as error:
status, raw, got = error.code, error.read(), dict(error.headers)
if expect is not None and status not in expect:
raise Failure(f"{method} {url.split('?')[0]} -> {status}: {raw[:200]!r}")
try:
parsed = json.loads(raw) if raw else None
except ValueError:
parsed = None
return status, parsed, got, raw
def get_json(url, headers=None, params=None):
if params:
url += ("&" if "?" in url else "?") + urllib.parse.urlencode(params)
return http("GET", url, headers=headers, expect=(200,))[1]
def wait_for(what, check, timeout, interval=2.0):
deadline = time.monotonic() + timeout
while True:
try:
result = check()
if result:
return result
except Failure:
pass
if time.monotonic() > deadline:
raise Failure(f"timeout waiting for {what}")
time.sleep(interval)
class Backends:
def __init__(self, args):
self.prometheus = args.prometheus.rstrip("/")
self.loki = args.loki.rstrip("/")
self.tempo = args.tempo.rstrip("/")
self.grafana = args.grafana.rstrip("/")
self.grafana_password_file = args.grafana_password_file
def grafana_headers(self):
password = Path(self.grafana_password_file).read_text().strip()
token = base64.b64encode(f"admin:{password}".encode()).decode()
return {"Authorization": "Basic " + token}
def promql(self, query, at=None, base=None):
params = {"query": query}
if at is not None:
params["time"] = f"{at:.3f}"
result = get_json((base or self.prometheus) + "/api/v1/query", params=params)
if result.get("status") != "success":
raise Failure(f"PromQL failed: {query}")
return result["data"]["result"]
def logs(self, query, start, end, limit=100, base=None):
params = {"query": query, "start": str(int(start * 1e9)), "end": str(int(end * 1e9)), "limit": str(limit),
"direction": "backward"}
result = get_json((base or self.loki) + "/loki/api/v1/query_range", params=params)
return [(stream["stream"], line) for stream in result["data"]["result"] for _, line in stream["values"]]
def traceql(self, query, start, end, limit=20, base=None):
params = {"q": query, "start": str(int(start)), "end": str(int(end) + 1), "limit": str(limit)}
return get_json((base or self.tempo) + "/api/search", params=params).get("traces") or []
def trace(self, trace_id, base=None):
status, body, _, raw = http("GET", (base or self.tempo) + f"/api/v2/traces/{trace_id}", expect=(200, 404))
return (body, raw.decode("utf-8", "replace")) if status == 200 else (None, "")
def otlp_attrs(values):
return [{"key": k, "value": {"intValue": str(v)} if isinstance(v, int) else {"stringValue": str(v)}}
for k, v in values.items()]
def otlp_resource(service, instance):
return {"attributes": otlp_attrs({"service.name": service, "service.version": "probe",
"service.instance.id": instance, "deployment.environment.name": "lab"})}
def tempo_spans(body):
spans = []
for batch in (body or {}).get("trace", {}).get("resourceSpans", []):
service = next((a["value"].get("stringValue") for a in batch["resource"].get("attributes", [])
if a["key"] == "service.name"), "?")
for scope in batch.get("scopeSpans", []):
for span in scope.get("spans", []):
values = {a["key"]: next(iter(a["value"].values()), None) for a in span.get("attributes", [])}
duration = (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6
spans.append({"service": service, "name": span["name"], "kind": span.get("kind"),
"status": span.get("status", {}).get("code", "STATUS_CODE_UNSET"),
"duration_ms": round(duration, 3), "links": len(span.get("links", [])),
"attributes": values})
return spans
def probe(args, b):
leak = "probe-secret-" + secrets.token_hex(8)
rid = uuid.uuid4().hex
trace_id, server, producer, consumer = secrets.token_hex(16), secrets.token_hex(8), secrets.token_hex(8), secrets.token_hex(8)
t0 = time.time()
ns = time.time_ns()
api, worker = args.service_name, args.service_name + "-worker"
scope = {"name": "obs-check", "version": "1"}
def span(span_id, parent, name, kind, start_ms, end_ms, values, links=()):
return {"traceId": trace_id, "spanId": span_id, "parentSpanId": parent, "name": name, "kind": kind,
"startTimeUnixNano": str(ns + start_ms * 1_000_000), "endTimeUnixNano": str(ns + end_ms * 1_000_000),
"attributes": otlp_attrs(values), "status": {"code": 1},
"links": [{"traceId": trace_id, "spanId": s} for s in links]}
traces = {"resourceSpans": [
{"resource": otlp_resource(api, api + "-1"), "scopeSpans": [{"scope": scope, "spans": [
span(server, "", "POST /api/v1/probe", 2, 0, 40, {
"http.request.method": "POST", "http.route": "/api/v1/probe", "http.response.status_code": 202,
"messenger.request_id": rid, "http.request.header.authorization": "Bearer " + leak,
"url.query": "filename=" + leak, "user.password": leak, "messenger.session.token": leak}),
span(producer, server, "probe.jobs publish", 4, 30, 35, {"messaging.operation.type": "send"})]}]},
{"resource": otlp_resource(worker, worker + "-1"), "scopeSpans": [{"scope": scope, "spans": [
span(consumer, producer, "probe.jobs process", 5, 50, 90, {"messaging.operation.type": "process"}, [producer])]}]},
]}
logs = {"resourceLogs": [{"resource": otlp_resource(api, api + "-1"), "scopeLogs": [{"scope": scope, "logRecords": [{
"timeUnixNano": str(ns), "observedTimeUnixNano": str(ns), "severityNumber": 9, "severityText": "INFO",
"body": {"stringValue": "probe.request.completed"}, "traceId": trace_id, "spanId": server,
"attributes": otlp_attrs({"event": "probe.request.completed", "request_id": rid, "authorization": "Bearer " + leak})}]}]}]}
point = {"attributes": otlp_attrs({"probe.id": rid}), "startTimeUnixNano": str(ns - 10_000_000_000), "timeUnixNano": str(ns)}
metrics = {"resourceMetrics": [{"resource": otlp_resource(api, api + "-1"), "scopeMetrics": [{"scope": scope, "metrics": [
{"name": "obs.probe.heartbeat", "unit": "{probe}", "sum": {"aggregationTemporality": 2, "isMonotonic": True,
"dataPoints": [dict(point, asInt="1")]}},
{"name": "obs.probe.latency", "unit": "s", "histogram": {"aggregationTemporality": 2, "dataPoints": [dict(
point, count="1", sum=0.04, bucketCounts=["0", "1", "0"], explicitBounds=[0.01, 0.1])]}},
]}]}]}
otlp = args.otlp.rstrip("/")
for path, payload in (("/v1/traces", traces), ("/v1/logs", logs), ("/v1/metrics", metrics)):
http("POST", otlp + path, payload, expect=(200,))
report = {"probe_id": rid, "trace_id": trace_id, "service": api, "sent_at": now()}
def tempo_ok(base=None):
body, raw = b.trace(trace_id, base)
spans = tempo_spans(body)
return {"spans": len(spans), "services": sorted({s["service"] for s in spans}), "raw": raw} if len(spans) == 3 else None
tempo = wait_for("trace in Tempo", tempo_ok, args.timeout)
found = wait_for("TraceQL search", lambda: b.traceql(f'{{ span.messenger.request_id = "{rid}" }}', t0 - 60, time.time() + 60), args.timeout)
loki_query = f'{{service_name="{api}"}} | request_id="{rid}"'
lines = wait_for("log in Loki", lambda: b.logs(loki_query, t0 - 60, time.time() + 60), args.timeout)
series = wait_for("metrics in Prometheus", lambda: (lambda r: r if len(r) == 2 else None)(b.promql(
f'{{__name__=~"obs_probe_heartbeat_total|obs_probe_latency_seconds_count", probe_id="{rid}"}}')), args.timeout)
labels = lines[0][0]
raw_logs = json.dumps(lines)
report.update({
"tempo": {"spans": tempo["spans"], "services": tempo["services"], "traceql_hits": len(found)},
"loki": {"lines": len(lines), "trace_id_matches": labels.get("trace_id") == trace_id,
"indexed_labels": sorted(k for k in labels if k in {"service_name", "deployment_environment_name"})},
"prometheus": {"series": sorted(s["metric"].get("__name__", "") for s in series),
"job": series[0]["metric"].get("job")},
"redaction": {"secret_in_tempo": leak in tempo["raw"], "secret_in_loki": leak in raw_logs,
"masked_in_tempo": "****" in tempo["raw"]},
})
if not args.no_grafana:
g = b.grafana + "/api/datasources/proxy/uid/"
headers = b.grafana_headers()
report["grafana"] = {
"prometheus": len(get_json(g + "prometheus/api/v1/query", headers, {"query": f'obs_probe_heartbeat_total{{probe_id="{rid}"}}'})["data"]["result"]),
"loki": len(get_json(g + "loki/loki/api/v1/query_range", headers, {"query": loki_query, "start": str(int((t0 - 60) * 1e9)), "end": str(int((time.time() + 60) * 1e9))})["data"]["result"]),
"tempo": http("GET", g + f"tempo/api/v2/traces/{trace_id}", headers=headers)[0],
}
print(json.dumps(report, ensure_ascii=False, indent=2))
ok = (report["tempo"]["spans"] == 3 and report["loki"]["trace_id_matches"] and len(series) == 2
and not report["redaction"]["secret_in_tempo"] and not report["redaction"]["secret_in_loki"]
and ("grafana" not in report or (report["grafana"]["prometheus"] and report["grafana"]["loki"] and report["grafana"]["tempo"] == 200)))
if not ok:
raise Failure("probe verification failed")
class Api:
def __init__(self, base):
self.base = base.rstrip("/")
def call(self, method, path, body=None, headers=None, expect=(200,)):
status, parsed, got, _ = http(method, self.base + path, body, headers, expect=expect)
return parsed, got.get("X-Request-Id") or got.get("x-request-id")
def register(self):
username = "u_obs_" + secrets.token_hex(6)
password = "T3st-" + secrets.token_hex(16)
user, rid = self.call("POST", "/api/v1/users", {"username": username, "display_name": "obs check", "password": password}, expect=(201,))
return user, username, password, rid
def login(self, username, password):
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
session, _ = self.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, expect=(201,))
return {"Authorization": "Bearer " + session["token"]}, session["token"]
def chat(self, auth, user):
chat, _ = self.call("POST", "/api/v1/chats", {"title": "obs", "member_ids": [user["id"]]}, auth, expect=(201,))
return chat
def logout(self, auth):
self.call("DELETE", "/api/v1/auth/sessions/current", headers=auth, expect=(204,))
def request_check(args, b):
api = Api(args.base_url)
t0 = time.time()
_, _, _, rid = api.register()
query = f'{{service_name="{args.api_service}"}} | {args.log_request_id_key}="{rid}"'
lines = wait_for("API log by request_id", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout)
labels, line = lines[0]
trace_id = labels.get("trace_id")
if not trace_id:
raise Failure("log record has no trace_id")
spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout)
print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id,
"span_id": labels.get("span_id"),
"service_name": labels.get("service_name"),
"severity": labels.get("severity_text") or labels.get("detected_level")},
"trace": [{k: s[k] for k in ("service", "name", "kind", "status", "duration_ms")} for s in spans]},
ensure_ascii=False, indent=2))
def job_check(args, b):
api = Api(args.base_url)
t0 = time.time()
user, username, password, _ = api.register()
auth, _ = api.login(username, password)
chat = api.chat(auth, user)
image = Path(args.image).read_bytes()
headers = dict(auth, **{"Content-Type": "image/png"})
attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=obs.png", image, headers, expect=(202,))
ready = wait_for("attachment ready", lambda: (lambda a: a if a["status"] in ("ready", "failed") else None)(
api.call("GET", f"/api/v1/attachments/{attachment['id']}", headers=auth)[0]), 60, 0.5)
key = args.attachment_attr
traces = wait_for("traces for attachment", lambda: b.traceql(f'{{ span.{key} = "{attachment["id"]}" }}', t0 - 30, time.time() + 30), args.timeout)
chain = []
for item in traces:
for span in tempo_spans(b.trace(item["traceID"])[0]):
chain.append(dict({k: span[k] for k in ("service", "name", "kind", "status", "duration_ms", "links")},
trace_id=item["traceID"], attempt=span["attributes"].get(args.attempt_attr)))
query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"'
worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout)
api.logout(auth)
print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"],
"traces": sorted({c["trace_id"] for c in chain}), "spans": chain,
"worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")}
for labels, line in worker_logs[:10]]}, ensure_ascii=False, indent=2))
def leaks_check(args, b):
api = Api(args.base_url)
t0 = time.time()
user, username, password, _ = api.register()
auth, token = api.login(username, password)
chat = api.chat(auth, user)
text = "obs-leak-text-" + secrets.token_hex(8)
filename = "obs-leak-file-" + secrets.token_hex(6) + ".png"
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": text}, dict(auth, **{"Idempotency-Key": uuid.uuid4().hex}), expect=(201,))
if Path(args.image).is_file():
api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=" + urllib.parse.quote(filename),
Path(args.image).read_bytes(), dict(auth, **{"Content-Type": "image/png"}), expect=(202,))
api.logout(auth)
time.sleep(args.flush)
needles = {"password": password, "token": token, "message_text": text, "filename": filename}
end = time.time() + 5
hits = {}
for name, needle in needles.items():
lines = b.logs(f'{{service_name=~"{args.service_regex}"}} |= `{needle}`', t0 - 30, end, limit=50)
hits[name] = {"loki": len(lines)}
traces = b.traceql(f'{{ resource.service.name =~ "{args.service_regex}" }}', t0 - 30, end, limit=500)
raws = [b.trace(t["traceID"])[1] for t in traces]
for name, needle in needles.items():
hits[name]["tempo"] = sum(needle in raw for raw in raws)
report = {"checked_at": now(), "window_seconds": round(end - t0), "traces_scanned": len(raws), "hits": hits}
print(json.dumps(report, indent=2))
if any(v["loki"] or v["tempo"] for v in hits.values()):
raise Failure("secret or message body found in telemetry")
def uploads_burst(args, b):
api = Api(args.base_url)
user, username, password, _ = api.register()
auth, _ = api.login(username, password)
chat = api.chat(auth, user)
image = Path(args.image).read_bytes()
accepted = []
for index in range(args.count):
attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=burst-{index}.png", image,
dict(auth, **{"Content-Type": "image/png"}), expect=(202,))
accepted.append({"attachment_id": attachment["id"], "request_id": rid, "status": attachment["status"]})
time.sleep(args.interval)
api.logout(auth)
print(json.dumps({"ts": now(), "uploaded": accepted}, ensure_ascii=False))
def query(args, b):
results = {expr: [{"labels": {k: v for k, v in r["metric"].items() if k != "__name__"}, "value": r["value"][1]}
for r in b.promql(expr)] for expr in args.expr}
print(json.dumps({"ts": now(), "results": results}, ensure_ascii=False))
def expand(expr):
expr = re.sub(r"\$\{?__(rate_interval|interval|range)\}?", "1m", expr)
return re.sub(r"\$\{?(\w+)(?::\w+)?\}?", ".*", expr)
def metric_names(expr):
stripped = re.sub(r'"[^"]*"|`[^`]*`|\{[^}]*\}|\b(by|without|on|ignoring|group_left|group_right)\s*\([^)]*\)', " ", expr)
stripped = re.sub(r"\[[^\]]*\]", " ", stripped)
return {t for t in re.findall(r"[a-zA-Z_:][a-zA-Z0-9_:]*", stripped) if ("_" in t or ":" in t) and t not in PROMQL_WORDS}
def walk_panels(dashboard):
for panel in dashboard.get("panels", []):
yield panel
yield from panel.get("panels", [])
def dashboards_check(args, b):
known = set(get_json(b.prometheus + "/api/v1/label/__name__/values")["data"])
problems, report = [], []
for path in sorted(Path(args.dir).glob("*.json")):
dashboard = json.loads(path.read_text())
for panel in walk_panels(dashboard):
for target in panel.get("targets", []):
uid = (target.get("datasource") or panel.get("datasource") or {}).get("uid")
entry = {"dashboard": path.name, "panel": panel.get("title"), "datasource": uid}
try:
if uid == "prometheus":
expr = expand(target["expr"])
result = b.promql(expr)
missing = sorted(metric_names(expr) - known)
entry.update(series=len(result), missing_metrics=missing)
if missing and args.strict:
problems.append(f"{path.name}: {panel.get('title')}: missing {missing}")
elif uid == "loki":
entry["lines"] = len(b.logs(expand(target["expr"]), time.time() - 3600, time.time(), limit=5))
elif uid == "tempo":
entry["traces"] = len(b.traceql(expand(target["query"]), time.time() - 3600, time.time(), limit=5))
except Failure as error:
problems.append(f"{path.name}: {panel.get('title')}: {error}")
report.append(entry)
print(json.dumps({"checked_at": now(), "targets": report, "problems": problems}, ensure_ascii=False, indent=2))
if problems:
raise Failure(f"{len(problems)} dashboard queries failed")
def grafana_check(args, b):
headers = b.grafana_headers()
health = get_json(b.grafana + "/api/health")
sources = {}
for uid in ("prometheus", "loki", "tempo"):
status, body, _, _ = http("GET", b.grafana + f"/api/datasources/uid/{uid}/health", headers=headers)
sources[uid] = {"http": status, "status": (body or {}).get("status"), "message": (body or {}).get("message")}
found = get_json(b.grafana + "/api/search", headers, {"type": "dash-db"})
boards = {}
for item in found:
status, body, _, _ = http("GET", b.grafana + f"/api/dashboards/uid/{item['uid']}", headers=headers)
meta = (body or {}).get("meta", {})
boards[item["uid"]] = {"title": item["title"], "http": status, "provisioned": meta.get("provisioned"),
"folder": meta.get("folderTitle"), "panels": len(list(walk_panels((body or {}).get("dashboard", {}))))}
expected = {json.loads(p.read_text())["uid"] for p in Path(args.dir).glob("*.json")}
print(json.dumps({"checked_at": now(), "grafana": health.get("version"), "datasources": sources, "dashboards": boards,
"missing_dashboards": sorted(expected - set(boards))}, ensure_ascii=False, indent=2))
if expected - set(boards) or any(s["http"] != 200 or s["status"] != "OK" for s in sources.values()):
raise Failure("grafana provisioning incomplete")
def alerts_watch(args, b):
want_state, _, wanted = args.until.partition(":") if args.until else ("", "", "")
names = set(filter(None, wanted.split("|")))
only = {n for value in args.only for n in value.split("|")}
deadline = time.monotonic() + args.timeout
last = None
output = open(args.output, "a") if args.output else None
while True:
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
state = sorted((a["labels"]["alertname"], a["state"], a.get("activeAt", "")) for a in alerts
if not only or a["labels"]["alertname"] in only)
if state != last:
line = json.dumps({"ts": now(), "alerts": [{"alertname": n, "state": s, "active_at": at} for n, s, at in state]}, ensure_ascii=False)
print(line, flush=True)
if output:
output.write(line + "\n")
output.flush()
last = state
current = {n: s for n, s, _ in state}
firing = {n for n in names if current.get(n) == "firing"}
if names and ((want_state == "firing" and firing) or (want_state == "resolved" and not firing)):
return
if time.monotonic() > deadline:
raise Failure(f"timeout waiting for {args.until}")
time.sleep(args.interval)
def main():
parser = argparse.ArgumentParser(description="Telemetry checks through public HTTP APIs of the messenger and of the observability stack")
parser.add_argument("--prometheus", default=os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090"))
parser.add_argument("--loki", default=os.environ.get("LOKI_URL", "http://127.0.0.1:3100"))
parser.add_argument("--tempo", default=os.environ.get("TEMPO_URL", "http://127.0.0.1:3200"))
parser.add_argument("--grafana", default=os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000"))
parser.add_argument("--grafana-password-file", default=os.environ.get("GRAFANA_PASSWORD_FILE", str(ROOT / "secrets/grafana_admin_password")))
parser.add_argument("--timeout", type=float, default=60)
sub = parser.add_subparsers(dest="command", required=True)
p = sub.add_parser("probe", help="send a synthetic OTLP trace, log and metric and find them in every backend and in Grafana")
p.add_argument("--otlp", default=os.environ.get("OTLP_URL", "http://127.0.0.1:4318"))
p.add_argument("--service-name", default="obs-probe")
p.add_argument("--no-grafana", action="store_true")
for name, fn in (("request", request_check), ("job", job_check), ("leaks", leaks_check), ("uploads", uploads_burst)):
q = sub.add_parser(name)
q.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
q.add_argument("--api-service", default="messenger-api")
q.add_argument("--worker-service", default="messenger-worker")
q.add_argument("--service-regex", default="messenger-.*")
q.add_argument("--log-request-id-key", default="request_id")
q.add_argument("--log-attachment-key", default="attachment_id")
q.add_argument("--attachment-attr", default="messenger.attachment_id")
q.add_argument("--attempt-attr", default="messenger.job.attempt")
q.add_argument("--image", default=str(ROOT / "tests/fixtures/sample.png"))
q.add_argument("--flush", type=float, default=15)
q.add_argument("--count", type=int, default=10)
q.add_argument("--interval", type=float, default=1)
q.set_defaults(fn=fn)
q = sub.add_parser("query", help="print instant PromQL results with a timestamp")
q.add_argument("expr", nargs="+")
q.set_defaults(fn=query)
for name, fn in (("dashboards", dashboards_check), ("grafana", grafana_check)):
q = sub.add_parser(name)
q.add_argument("--dir", default=str(ROOT / "infra/grafana/dashboards"))
q.add_argument("--strict", action="store_true", help="fail when a referenced metric does not exist yet")
q.set_defaults(fn=fn)
w = sub.add_parser("alerts", help="record alert state changes until firing:<a|b> or resolved:<a|b>")
w.add_argument("--until", default="")
w.add_argument("--only", nargs="*", default=[])
w.add_argument("--interval", type=float, default=5)
w.add_argument("--output", default="")
p.set_defaults(fn=probe)
w.set_defaults(fn=alerts_watch)
args = parser.parse_args()
try:
args.fn(args, Backends(args))
except (Failure, OSError, KeyError, ValueError) as error:
print(f"FAIL: {error}", file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main()
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
limit="${WAIT_SECONDS:-1500}"
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
sql() { docker compose exec -T postgres psql -U messenger -d messenger -X "$@"; }
pending() { sql -tAc "SELECT count(*) FROM attachments WHERE status = 'pending'"; }
statuses="SELECT status, count(*), min(created_at), max(created_at) FROM attachments GROUP BY status ORDER BY status"
since=$(date -u +%FT%TZ)
{
echo "date: $since commit: $(git rev-parse HEAD)"
echo "# make overload-leftovers (scripts/overload-leftovers.sh) после серии памяти: невидимые намерения от 503 по"
echo "# таймауту видны сверке свежими, уборка API удаляет их после UPLOAD_GRACE_SECONDS; стенд тот же"
echo "\$ make reconcile # dry-run"
docker compose run --rm --no-deps -T reconciler 2>/dev/null
echo "\$ psql -c \"$statuses\""
sql -c "$statuses"
echo "\$ until [ \"\$(psql -tAc \"SELECT count(*) FROM attachments WHERE status = 'pending'\")\" = 0 ]; do sleep 10; done"
started=$(date +%s)
until [ "$(pending)" = 0 ] || [ $(( $(date +%s) - started )) -gt "$limit" ]; do sleep 10; done
echo "pending = $(pending) в $(date -u +%FT%TZ)"
[ "$(pending)" = 0 ] || { echo "FAIL: pending cleanup timed out"; exit 1; }
echo "\$ docker compose logs --since $since api | audit.attachment_discarded и attachments.swept"
docker compose logs --no-log-prefix --since "$since" api | python3 -c 'import collections, json, sys
reasons, swept, times = collections.Counter(), 0, []
for line in sys.stdin:
if not line.startswith("{"):
continue
event = json.loads(line)
if event.get("event") == "audit.attachment_discarded":
reasons[event["reason"]] += 1
times.append(event["ts"][:19] + "Z")
elif event.get("event") == "attachments.swept":
swept += event["count"]
print(dict(reasons), "swept", swept, "first", min(times, default="-"), "last", max(times, default="-"))'
echo "\$ psql -c \"$statuses\""
sql -c "$statuses"
echo "\$ make reconcile # dry-run после уборки"
docker compose run --rm --no-deps -T reconciler 2>/dev/null
} > "$out/overload-leftovers.txt"
cat "$out/overload-leftovers.txt"
+1 -1
View File
@@ -6,7 +6,7 @@ started() {
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
done | sort
}
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
echo "== before: service hostname StartedAt status"
started
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
mode=${1:?usage: pg-stats.sh activity <csv> | statements <csv> | size | reset}
cd "$(dirname "$0")/.."
psql() {
docker compose exec -T postgres sh -c 'exec psql -X -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "${POSTGRES_DB:-$POSTGRES_USER}" "$@"' psql "$@"
}
case $mode in
reset)
psql -c 'set client_min_messages = warning' -c 'create extension if not exists pg_stat_statements' -c 'select pg_stat_statements_reset()' >/dev/null
;;
size)
psql -At -F, -c "select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
;;
statements)
psql -c "copy (select calls, round(total_exec_time::numeric, 3) as total_ms, round(mean_exec_time::numeric, 3) as mean_ms, rows, shared_blks_hit, shared_blks_read, left(regexp_replace(query, '\s+', ' ', 'g'), 200) as query from pg_stat_statements where dbid = (select oid from pg_database where datname = current_database()) and query not like '%pg_stat%' order by total_exec_time desc limit ${TOP:-20}) to stdout with csv header" > "${2:?csv}"
;;
activity)
echo 'ts,state,wait_event_type,wait_event,backends,max_connections' > "${2:?csv}"
while :; do
ts=$(date +%s)
psql -At -F, -c "select coalesce(state, ''), coalesce(wait_event_type, ''), coalesce(wait_event, ''), count(*), current_setting('max_connections') from pg_stat_activity where backend_type = 'client backend' and pid <> pg_backend_pid() group by 1, 2, 3" | sed "s/^/$ts,/" >> "$2" || true
sleep "${INTERVAL:-1}"
done
;;
*)
echo "unknown mode $mode" >&2
exit 2
;;
esac
+5
View File
@@ -53,6 +53,11 @@ def stop(service):
check=True,
).stdout.strip()
say(f"$ docker compose stop -t 10 {service} -> {seconds:.1f} s, exit code {code}")
say(
compose(
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", service
).rstrip()
)
check(seconds <= 10 and code == "0", f"{service} did not stop cleanly on SIGTERM")
+130 -25
View File
@@ -107,9 +107,11 @@ staging_left:
WHERE id = ANY(:'ids'::uuid[]) AND status = 'ready' AND staging_key IS NOT NULL
object_keys:
SELECT object_key AS key FROM attachment_variants
UNION ALL SELECT original_key FROM attachments WHERE original_key IS NOT NULL
UNION ALL SELECT staging_key FROM attachments WHERE staging_key IS NOT NULL
SELECT object_key AS key FROM attachment_variants WHERE attachment_id = ANY(:'ids'::uuid[])
UNION ALL SELECT original_key FROM attachments
WHERE id = ANY(:'ids'::uuid[]) AND original_key IS NOT NULL
UNION ALL SELECT staging_key FROM attachments
WHERE id = ANY(:'ids'::uuid[]) AND staging_key IS NOT NULL
columns:
SELECT table_name, column_name FROM information_schema.columns
@@ -252,6 +254,35 @@ class Stack:
out = self.rabbitmqadmin(*command, json.dumps(properties), stdin=json.dumps(payload))
check("routed successfully" in out, "broker did not route the injected job")
def s3_json(self, *argv, missing=False):
done = subprocess.run(
[*self.base, "run", "--rm", "--no-deps", "-T", "s3-cli", *argv, "--json"],
capture_output=True,
text=True,
timeout=60,
check=False,
)
body = json.loads(done.stdout or done.stderr)
if missing and done.returncode == 5 and body.get("details", {}).get("type") == "not_found":
return None
check(done.returncode == 0, "S3 inventory command failed: exit " + str(done.returncode))
return body
def objects(self, ids):
bucket = self.dc("exec", "-T", self.args.relay, "printenv", "S3_BUCKET").strip()
keys = set()
for ident in ids:
for prefix in ("staging", "originals"):
body = self.s3_json(
"object", "stat", f"root/{bucket}/{prefix}/{ident}", missing=True
)
if body is not None:
keys.add(body["name"])
body = self.s3_json("object", "list", "--recursive", f"root/{bucket}/variants/{ident}/")
check(not body["truncated"], "S3 variant inventory is truncated")
keys.update(item["key"] for item in body["items"])
return keys
def admin(self, *argv):
command = shlex.split(self.args.admin) + list(argv)
out = self.dc("run", "--rm", "--no-deps", "-T", self.args.worker, *command)
@@ -396,6 +427,9 @@ class Suite:
code = self.stack.wait_exit(service)
check(code == 137, f"{service} exited with {code}, expected 137 at {point}")
seen = observe(ident)
seen["containers"] = self.stack.dc(
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", service
).splitlines()
self.stack.launch(service)
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox")
@@ -424,7 +458,12 @@ class Suite:
steps.append({"at": now(), "step": step, **extra})
self.stack.stop(self.args.worker)
mark("worker stopped")
mark(
"worker stopped",
containers=self.stack.dc(
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", self.args.worker
).splitlines(),
)
att = self.api.fixture()
mark("upload 202 queued", attachment_id=att["id"])
self.api.content(att["id"], expected=409)
@@ -594,15 +633,39 @@ class Suite:
self.api.final[ident] = {"failed"}
kept = self.record(ident)["staging_key"]
check(kept is not None, "failed staging was removed before the retention period")
try:
self.stack.launch(self.args.relay, FAULT_STAGING_RETENTION_SECONDS="1")
poll(lambda: self.record(ident)["staging_key"] is None, 30, "failed staging cleanup")
finally:
self.stack.launch(self.args.relay)
retention = int(
self.stack.dc("exec", "-T", self.args.relay, "printenv", "STAGING_RETENTION_SECONDS")
)
aged = self.stack.dc(
"exec",
"-T",
self.args.postgres,
"sh",
"-c",
PSQL,
"sh",
"-v",
"id=" + ident,
"-v",
"chat=" + self.api.chat["id"],
"-v",
"age=" + str(retention + 1),
stdin="UPDATE attachments SET created_at = created_at - :age * interval '1 second', "
"updated_at = updated_at - :age * interval '1 second' "
"WHERE id = :'id' AND chat_id = :'chat' AND status = 'failed' RETURNING id;\n",
)
check(aged.strip() == ident, "did not age exactly the suite's failed attachment")
poll(lambda: self.record(ident)["staging_key"] is None, 30, "failed staging cleanup")
replay = self.stack.admin("replay", "--attachment-id", ident)
check(replay["result"] == "expired", f"replay after retention gave {replay}")
check(self.api.get(ident)["status"] == "failed", "expired replay changed the status")
return {"attachment_id": ident, "staging_before": "kept", "replay": replay}
return {
"attachment_id": ident,
"staging_before": "kept",
"retention_seconds": retention,
"aged_only": ident,
"replay": replay,
}
def image_matrix(self):
folder = Path(self.args.images)
@@ -696,11 +759,19 @@ class Suite:
poll(lambda: self.record(ident)["status"] == "processing", 30, "processing")
self.stack.kill(self.args.worker)
code = self.stack.wait_exit(self.args.worker)
containers = self.stack.dc(
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", self.args.worker
).splitlines()
self.stack.launch(self.args.worker)
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
self.drained()
self.verify(ready)
return {"attachment_id": ident, "exit_code": code, "jobs": self.stack.sql("jobs", id=ident)}
return {
"attachment_id": ident,
"exit_code": code,
"containers": containers,
"jobs": self.stack.sql("jobs", id=ident),
}
def duplicate_in_flight(self):
self.stack.stop(self.args.worker)
@@ -743,36 +814,60 @@ class Suite:
def reorder_versions(self):
self.stack.stop(self.args.worker, self.args.relay)
ident = self.api.fixture()["id"]
self.reprocess(ident)
check(self.reprocess(ident)["result"] == "queued", "version 2 was not scheduled")
check(self.reprocess(ident)["result"] == "noop", "duplicate reprocess was not a noop")
self.stack.publish(self.payload(ident, 2))
self.queued(1)
self.stack.launch(self.args.relay)
self.queued(3)
self.stack.launch(self.args.worker)
self.published(ident, 2)
self.api.wait(ident)
before = self.snapshot(ident)
self.stack.launch(self.args.relay)
poll(
lambda: (
{j["pipeline_version"]: j["status"] for j in self.stack.sql("jobs", id=ident)}
== {1: "superseded", 2: "done"}
),
self.args.recovery_timeout,
"old version superseded",
)
poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox")
self.drained()
final = self.snapshot(ident)
check(final["published"] == 2, f"published_version rolled back to {final['published']}")
check(before["public"] == final["public"], "late v1 changed the published v2 variants")
served = {v["name"]: v["sha256"] for v in final["public"]}
stored = {r["name"]: r["sha256"] for r in final["rows"] if r["pipeline_version"] == 2}
check(served == stored, "API serves variants of a version that is not published")
jobs = self.stack.sql("jobs", id=ident)
check(
{j["pipeline_version"]: j["status"] for j in jobs} == {1: "superseded", 2: "done"},
"v1 did not finish after v2",
)
self.verify(self.api.get(ident))
return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)}
return {"attachment_id": ident, "jobs": jobs, "v2_before_v1": before, "after_v1": final}
def stale_after_new_version(self):
ident = self.api.fixture()["id"]
self.api.wait(ident)
self.reprocess(ident)
first = self.snapshot(ident)
check(self.reprocess(ident)["result"] == "queued", "version 2 was not scheduled")
check(self.api.get(ident)["status"] == "ready", "reprocess rolled back ready")
self.published(ident, 2)
self.drained()
before = self.snapshot(ident)
check(first["public"] != before["public"], "recipes did not produce distinct results")
self.stack.publish(self.payload(ident, 1))
self.drained()
time.sleep(1)
after = self.snapshot(ident)
check(before == after and after["published"] == 2, "stale v1 job changed the v2 result")
return {"attachment_id": ident, "published_version": after["published"]}
return {
"attachment_id": ident,
"published_version": after["published"],
"v1": first,
"v2_before_stale": before,
"v2_after_stale": after,
}
def invariants(self, dead_before):
ids = self.api.accepted
@@ -819,13 +914,21 @@ class Suite:
check(queues[QUEUE] == (0, 0), "messages left in " + QUEUE)
expected = dead_before + self.dead_letters
check(queues[DLQ][0] == expected, f"unexpected dead letters: {queues[DLQ]}")
if not self.args.s3_list:
return {"queues": queues, "s3": "SKIP: --s3-list is not set"}
keys = {r["key"] for r in self.stack.sql("object_keys")}
listing = run(shlex.split(self.args.s3_list)).splitlines()
stray = [k for k in (x.split()[-1] for x in listing if x.strip()) if k not in keys]
check(not stray, f"S3 objects without metadata: {stray[:5]}")
return {"queues": queues, "s3_objects": len(listing)}
keys = {r["key"] for r in self.stack.sql("object_keys", ids=array)}
if self.args.s3_list:
listing = {
x.split()[-1]
for x in run(shlex.split(self.args.s3_list)).splitlines()
if x.strip()
}
listing = {key for key in listing if any("/" + ident in key for ident in ids)}
else:
listing = self.stack.objects(ids)
check(
listing == keys,
f"S3 mismatch: extra={sorted(listing - keys)}, missing={sorted(keys - listing)}",
)
return {"queues": queues, "s3_objects": len(listing), "scoped_attachments": len(ids)}
def no_loss():
known = {r["id"] for r in self.stack.sql("known", ids=array)}
@@ -904,6 +1007,7 @@ class Suite:
report = {
"suite": "lab5-pipeline-idempotency",
"started_at": now(),
"source_sha": run(["git", "rev-parse", "HEAD"]).strip(),
"config": self.config(),
}
if self.args.preflight:
@@ -925,6 +1029,7 @@ class Suite:
finally:
self.stack.up(self.args.s3)
self.stack.restore()
report["accepted_ids"] = self.api.accepted if self.api else []
report["scenarios"] = results
return self.finish(report, results + report.get("invariants", []))
+163 -7
View File
@@ -10,12 +10,16 @@ import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from datetime import datetime
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
PSQL += ["-X", "-q", "-At"]
@functools.cache
@@ -207,16 +211,112 @@ def check(args):
return 0 if passed else 1
def output(*command):
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return done.stdout + done.stderr
def replicas():
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
def answered(container, request_id):
for line in output("docker", "logs", container).splitlines():
if request_id in line and "request.handled" in line:
return json.loads(line).get("status")
return None
def lost(alice, bob, chat, kill, args):
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
request_id = "lost-" + key[:12]
lock = subprocess.Popen(
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
)
lock.stdin.write(
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
)
lock.stdin.flush()
lock.stdout.readline()
sent = time.time()
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
path = f"/api/v1/chats/{chat}/messages"
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
waiting = set(output(*PSQL, "-c", sql).split())
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
if kill and holder[0]:
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
lock.stdin.close()
lock.wait()
retried = time.time()
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
while status != 201 and attempts < 20:
time.sleep(0.2)
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
copies = [m for m in page["items"] if m["text"] == text]
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
report = {
"variant": "kill" if kill else "timeout",
"lost_attempt": {
"status": first,
"client_timeout_s": args.timeout,
"chat_lock_held_s": args.hold,
"replica": holder[1],
"replica_killed": bool(kill and holder[0]),
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
},
"retry": {
"after_s": round(retried - sent, 3),
"status": status,
"attempts": attempts,
"replica": instance,
},
"copies_in_history": len(copies),
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
"written_by": None
if stamp is None
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
}
if kill and holder[0]:
subprocess.call(
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
report["pass"] = (
first == 0
and holder[0] is not None
and status == 201
and report["copies_in_history"] == 1
and report["same_message"]
and (kill or report["lost_attempt"]["replica_answered"] == 201)
)
return report
def retry(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
passed = all(v["pass"] for v in variants)
emit({"variants": variants, "pass": passed}, args.out)
return 0 if passed else 1
def record(stop, alice, chat):
probes, trace = [], []
probes, trace, confirmed = [], [], []
def prober():
key = uuid.uuid4().hex
while not stop.is_set():
started = time.time()
status, instance, _ = send(alice, chat, "probe " + key, key)
status, instance, body = send(alice, chat, "probe " + key, key)
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
if status == 201:
confirmed.append(body)
key = uuid.uuid4().hex
time.sleep(0.1)
@@ -228,7 +328,38 @@ def record(stop, alice, chat):
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
for thread in threads:
thread.start()
return probes, trace, threads
return probes, trace, threads, confirmed
def history_check(reader, chat, confirmed):
items, cursors = [], set()
cursor = None
for _ in range(len(confirmed) // 100 + 2):
path = f"/api/v1/chats/{chat}/messages?limit=100"
if cursor:
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
status, _, page = call("GET", path, headers=reader["auth"])
if status != 200 or not isinstance(page, dict):
break
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"])
return {
"confirmed": len(confirmed),
"stored": len(items),
"exact_match": bool(confirmed) and same,
"unique": len({m["id"] for m in items}) == len(items),
}
if cursor in cursors:
break
cursors.add(cursor)
return {
"confirmed": len(confirmed),
"stored": len(items),
"exact_match": False,
"unique": False,
}
def write_csv(path, header, rows):
@@ -241,6 +372,10 @@ def summarize(args, probes, trace, action, expected):
t0, end = action["start"], action["end"]
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0]
recovery = (
next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0
)
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
outage = [t for t in down if not back or t < back[0]] or [end]
@@ -256,13 +391,15 @@ def summarize(args, probes, trace, action, expected):
"action_finished_before_load_end": end < action["load_end"],
"lb_up_before": expected,
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
"lb_samples_missing": sum(up is None for _, up in trace),
"detection_seconds": round(down[0] - t0, 3) if down else None,
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
"probe": {
"attempts": len(probes),
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
"first_failure_s": round(failures[0], 3) if failures else None,
"recovery_s": round(failures[-1], 3) if failures else 0.0,
"first_success_s": round(successes[0][1], 3) if successes else None,
"recovery_s": round(recovery, 3) if recovery is not None else None,
"instances_before": dict(before),
"instances_during_outage": dict(during),
"instances_after": dict(after),
@@ -275,7 +412,13 @@ def verdict(args, result):
checks = {
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
"load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0,
"lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0,
"write_recovery_ok": all(
probe[k] is not None and probe[k] <= args.max_recovery
for k in ("first_success_s", "recovery_s")
),
"history_preserved": result["history"]["exact_match"] and result["history"]["unique"],
}
if args.expect == "dip":
checks["detected"] = result["detection_seconds"] is not None
@@ -296,7 +439,8 @@ def load(args):
expected = lb_up() or 0
alice, bob = actor("alice"), actor("bob")
stop, action = threading.Event(), {}
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
chat = chat_of(alice, bob)
probes, trace, threads, confirmed = record(stop, alice, chat)
def act():
time.sleep(args.warmup + args.lead)
@@ -327,6 +471,7 @@ def load(args):
)
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
result = summarize(args, probes, trace, action, expected)
result["history"] = history_check(bob, chat, confirmed)
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
result["k6"] = dict(
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
@@ -344,10 +489,13 @@ def detect(args):
started = time.time()
process = subprocess.Popen(args.command, cwd=str(ROOT))
detected = acted = None
missing = 0
while time.time() - started < args.timeout and (detected is None or acted is None):
if acted is None and process.poll() is not None:
acted = time.time() - started
if detected is None and (lb_up() or 0) <= until:
up = lb_up()
missing += up is None
if detected is None and up is not None and up <= until:
detected = time.time() - started
time.sleep(0.05)
code = process.wait()
@@ -362,6 +510,7 @@ def detect(args):
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
checks = {
"action_ok": code == 0,
"lb_observed": before >= 2 and missing == 0,
"detected_in_time": detected is not None and detected <= args.max + args.slack,
"lb_answer_ok": status == args.lb_status,
}
@@ -374,6 +523,7 @@ def detect(args):
"action_exit": code,
"action_seconds": round(acted, 3),
"lb_up_before": before,
"lb_samples_missing": missing,
"lb_up_counted_as_detected": until,
"detection_seconds": None if detected is None else round(detected, 3),
"declared_max_seconds": args.max,
@@ -413,6 +563,10 @@ def main():
under.add_argument("--max-failed-ratio", type=float, default=0.01)
under.add_argument("--max-recovery", type=float, default=10.0)
under.add_argument("command", nargs=argparse.REMAINDER)
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
again.add_argument("--out", help="JSON file, written only after a complete report")
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
@@ -426,6 +580,8 @@ def main():
args = parser.parse_args()
if args.mode == "check":
return check(args)
if args.mode == "retry":
return retry(args)
args.command = [c for c in args.command if c != "--"]
if not args.command:
parser.error(f"{args.mode} needs an action command after --")
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
kind="${1:?usage: reconcile-inject.sh orphan|missing|corrupt|bitflip|stale-pending|pending-object|multipart [count]}"
count="${2:-1}"
age="${INJECT_AGE_SECONDS:-7200}"
prefix="${S3_PREFIX:-attachments/}"
sql() { docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -qAt -F ' ' -U "${PGUSER:-messenger}" -d "${PGDATABASE:-messenger}" -c "$1"; }
s3() { docker compose run --rm --no-deps -T -e ROWS="$1" s3-cli sh -c "$2"; }
ready() { sql "SELECT id, object_key, size_bytes FROM attachments WHERE status = 'ready' ${INJECT_IDS:+AND id = ANY('{$INJECT_IDS}'::uuid[])} ORDER BY random() LIMIT $count"; }
pending() {
sql "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at)
SELECT g.id, s.chat_id, s.owner_id, 'injected.png', 'image/png', 2048, repeat('0', 64), 1, 1, '$prefix' || g.id, 'pending',
clock_timestamp() - make_interval(secs => $age), clock_timestamp() - make_interval(secs => $age)
FROM (SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1) s,
(SELECT gen_random_uuid() AS id FROM generate_series(1, $count)) g
RETURNING id, object_key"
}
fresh() { sql "SELECT g, '$prefix' || g FROM (SELECT gen_random_uuid() AS g FROM generate_series(1, $count)) s"; }
put_random='printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; head -c "${size:-2048}" /dev/urandom | rc pipe "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done'
case "$kind" in
orphan) rows=$(fresh); s3 "$rows" "$put_random" | sed "s/^/orphan /" ;;
missing) rows=$(ready); s3 "$rows" 'printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; rc object remove --force "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' | sed "s/^/missing /" ;;
corrupt) rows=$(ready | awk '{print $1, $2, int($3 / 2) + 1}'); s3 "$rows" "$put_random" | sed "s/^/corrupt /" ;;
bitflip) rows=$(ready); s3 "$rows" "$put_random" | sed "s/^/bitflip /" ;;
stale-pending) pending | sed "s/^/stale-pending /" ;;
pending-object) rows=$(pending); s3 "$rows" "$put_random" | sed "s/^/pending-object /" ;;
multipart)
fresh | while read -r _ key; do
docker compose exec -T -e KEY="$key" -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" s3 sh -c 'u="http://127.0.0.1:9000/$S3_BUCKET/$KEY"; a="$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"; s="aws:amz:${RUSTFS_REGION:-us-east-1}:s3"
upload=$(curl -fsS --aws-sigv4 "$s" --user "$a" -X POST "$u?uploads" | sed -n "s:.*<UploadId>\(.*\)</UploadId>.*:\1:p")
head -c 4096 /dev/urandom | curl -fsS --aws-sigv4 "$s" --user "$a" -X PUT --data-binary @- "$u?partNumber=1&uploadId=$upload" -o /dev/null' </dev/null
echo "multipart - $key"
done
;;
*) echo "unknown kind: $kind" >&2; exit 2 ;;
esac
+199
View File
@@ -0,0 +1,199 @@
#!/usr/bin/env python3
import argparse
import datetime as dt
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SQL = (
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256,"
" floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint,"
" EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id),"
" (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL"
" FROM attachments a ORDER BY a.object_key"
)
HASH = (
'while read -r k; do printf "%s %s\\n" "$k" '
'"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done'
)
CLASSES = (
"orphan_stale",
"orphan_fresh",
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"pending_stale_with_object",
"pending_stale_without_object",
"pending_fresh",
"quarantined",
"incomplete_stale",
"incomplete_fresh",
"lost_ready",
)
TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"}
BROKEN = (
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"quarantined",
)
def timed(cmd, stdin=None):
started = time.monotonic()
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
if out.returncode:
raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}")
return out.stdout, round(time.monotonic() - started, 3)
def s3(*args, stdin=None):
return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin)
def age(stamp, now):
return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp()
def load(bucket, prefix, deep):
listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}")
uploads, uploads_seconds = s3(
"object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}"
)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"]
rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL])
objects = {}
for item in json.loads(listing).get("items", []):
key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"]
objects[key] = item
rows = {}
for line in rows_raw.splitlines():
rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t")
rows[key] = {
"id": rid,
"status": status,
"size": int(size),
"sha256": sha,
"age": int(row_age),
"referenced": referenced == "t",
"quarantined": quarantined == "t",
}
hashes, deep_seconds = {}, 0.0
if deep:
present = "".join(
f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects
)
out, deep_seconds = s3("sh", "-c", HASH, stdin=present)
hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
seconds = {
"s3_list": list_seconds,
"s3_incomplete": uploads_seconds,
"pg_query": pg_seconds,
"deep_hash": deep_seconds,
}
return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds
def row_class(key, row, objects, hashes, grace):
if row["quarantined"]:
return "quarantined"
if row["status"] == "pending":
stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object"
return "pending_fresh" if row["age"] <= grace else stale
if key not in objects:
return "missing_referenced" if row["referenced"] else "missing_unreferenced"
if objects[key]["size_bytes"] != row["size"]:
return "size_mismatch"
return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None
def classify(objects, rows, hashes, uploads, grace):
now = time.time()
classes = {name: [] for name in CLASSES}
for key, item in objects.items():
if key not in rows:
classes[
"orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh"
].append(key)
for key, row in rows.items():
name = row_class(key, row, objects, hashes, grace)
if name:
classes[name].append(key if name.startswith("orphan") else row["id"])
for item in uploads:
classes[
"incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh"
].append(item["key"])
return classes
def main():
p = argparse.ArgumentParser(
description="Independent PG <-> S3 invariant check for lab4 attachments"
)
p.add_argument(
"--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600"))
)
p.add_argument(
"--deep", action="store_true", help="stream every present object and compare sha256"
)
p.add_argument(
"--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments"
)
p.add_argument(
"--expect-ready",
metavar="FILE",
help="fail if any attachment from FILE is no longer healthy",
)
p.add_argument("--limit", type=int, default=20, help="max ids per class in the report")
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
started = time.monotonic()
prefix = os.environ.get("S3_PREFIX", "attachments/")
objects, rows, hashes, uploads, seconds = load(
os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep
)
classes = classify(objects, rows, hashes, uploads, args.grace)
bad = {i for c in BROKEN for i in classes[c]}
healthy = {
r["id"]: r["sha256"]
for k, r in rows.items()
if r["status"] == "ready" and k in objects and r["id"] not in bad
}
if args.expect_ready:
expected = json.loads(Path(args.expect_ready).read_text())
classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha)
if args.save_ready:
Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True))
violations = [c for c in CLASSES if classes[c] and c not in TOLERATED]
seconds["total"] = round(time.monotonic() - started, 3)
report = {
"checked_at": dt.datetime.now(dt.timezone.utc)
.isoformat(timespec="seconds")
.replace("+00:00", "Z"),
"grace_seconds": args.grace,
"deep": args.deep,
"objects": len(objects),
"rows": len(rows),
"ready_healthy": len(healthy),
"counts": {c: len(v) for c, v in classes.items()},
"samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v},
"seconds": seconds,
"violations": violations,
"ok": not violations,
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()
+823
View File
@@ -0,0 +1,823 @@
#!/usr/bin/env python3
import argparse
import http.client
import json
import os
import re
import statistics
import struct
import subprocess
import sys
import threading
import time
import uuid
import zlib
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
GRACE = 60
FLAGS = ("--grace", str(GRACE), "--skew", "0")
INTENT_AGE = 300
LOCK_KEY = 20260004
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale")
USER_HZ = 100
BUSY = (0, 1, 2, 5, 6, 7)
def run(cmd, env=None, stdin=None):
started = time.monotonic()
done = subprocess.run(
cmd,
cwd=ROOT,
input=stdin,
capture_output=True,
check=False,
env=dict(os.environ, **(env or {})),
)
out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace")
return done.returncode, out, err, round(time.monotonic() - started, 3)
def must(cmd, env=None, stdin=None):
code, out, err, _ = run(cmd, env, stdin)
if code:
raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}")
return out
def compose(*args, env=None, stdin=None):
return must(["docker", "compose", *args], env, stdin)
def psql(sql):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-v",
"ON_ERROR_STOP=1",
"-qAt",
"-U",
"messenger",
"-d",
"messenger",
"-c",
sql,
).strip()
def reconcile(*flags, env=None):
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags]
code, out, err, wall = run(cmd, env)
if code not in (0, 1) or not out.strip():
raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}")
report = json.loads(out)
audit = [json.loads(line) for line in err.splitlines() if '"audit.' in line]
report.update(exit_code=code, wall_seconds=wall, audit=audit)
return report
def oracle(*flags):
code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags])
if not out.strip():
raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}")
return json.loads(out)
def inject(kind, count, ids=(), age=None):
env = {"INJECT_IDS": ",".join(ids)} if ids else {}
if age is not None:
env["INJECT_AGE_SECONDS"] = str(age)
out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env)
return [line.split()[1] for line in out.splitlines() if line.strip()]
def head():
return must(["git", "rev-parse", "HEAD"]).strip()
def evidence_root():
return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
def save(name, data, *, stamp=True):
path = evidence_root() / name
path.parent.mkdir(parents=True, exist_ok=True)
stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data}
path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n")
return path
class Checks:
def __init__(self):
self.items = []
def __call__(self, name, ok, **detail):
self.items.append(dict(name=name, ok=bool(ok), **detail))
sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n")
@property
def ok(self):
return all(item["ok"] for item in self.items)
def chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
def tiny_png():
raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8))
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw))
+ chunk(b"IEND", b"")
)
def pick(found, names):
return {name: found.get(name, 0) for name in names}
def faults(c, check, pre):
alice, bob, _, chat = c.bootstrap()
ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)]
linked = ids[:6]
for part in (linked[:3], linked[3:]):
c.message(alice, chat, "", attachments=part)
targets = {
"missing": [linked[0], linked[1], ids[6]],
"corrupt": [linked[2], ids[7], ids[8]],
"bitflip": [linked[3], ids[9], ids[10]],
}
damaged = {i for group in targets.values() for i in group}
ready = evidence_root() / "ready-before.json"
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
save("ready-untouched.json", untouched, stamp=False)
started = time.monotonic()
injected = {
"orphan_stale": inject("orphan", 3),
"missing": inject("missing", 3, targets["missing"]),
"size_mismatch": inject("corrupt", 3, targets["corrupt"]),
"sha_mismatch": inject("bitflip", 3, targets["bitflip"]),
"pending_stale": inject("stale-pending", 3, age=INTENT_AGE)
+ inject("pending-object", 3, age=INTENT_AGE),
}
time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started)))
injected["orphan_fresh"] = inject("orphan", 2)
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
expected = {name: len(keys) for name, keys in injected.items()}
expected.update(quarantined=pre["counts"]["quarantined"])
before = oracle(
"--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json")
)
dry = reconcile("--dry-run", "--deep", *FLAGS)
save("dry-run.json", dry)
after_dry = oracle("--grace", str(GRACE), "--deep")
named = {k: sorted(s.rsplit("/", 1)[-1] for s in dry["samples"].get(k, [])) for k in injected}
check(
"dry-run находит каждый внедрённый класс, и найдено ровно внедрённое",
pick(dry["found"], expected) == expected
and named == {name: sorted(ids) for name, ids in injected.items()},
expected=expected,
found=dry["found"],
)
check(
"dry-run ничего не меняет",
not any(dry["fixed"].values())
and after_dry["counts"] == before["counts"]
and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]),
objects=[before["objects"], after_dry["objects"]],
rows=[before["rows"], after_dry["rows"]],
)
first = reconcile("--apply", "--deep", *FLAGS)
save("apply-1.json", first)
discarded = sorted(e["attachment_id"] for e in first["audit"] if e["reason"] == "reconcile")
check(
"apply исправляет ровно найденное dry-run, удаление намерений — в журнале аудита",
pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE)
and not first["errors"]
and discarded == sorted(injected["pending_stale"]),
dry_run=pick(dry["found"], FIXABLE),
fixed=pick(first["fixed"], FIXABLE),
audit_discarded=len(discarded),
)
second = reconcile("--apply", "--deep", *FLAGS)
save("apply-2.json", second)
check(
"повторный apply — 0 действий",
not any(second["fixed"].values()),
fixed=second["fixed"],
found=second["found"],
)
after = oracle(
"--grace",
str(GRACE),
"--deep",
"--expect-ready",
str(evidence_root() / "ready-untouched.json"),
"--output",
str(evidence_root() / "check-after.json"),
)
tolerated = ("orphan_fresh", "pending_fresh")
check(
"независимая проверка после apply: нарушений нет, готовые не потеряны",
after["ok"]
and not after["counts"]["lost_ready"]
and pick(after["counts"], tolerated) == pick(before["counts"], tolerated)
and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9,
counts=after["counts"],
)
marks = dict(
line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split()
)
links = psql("SELECT attachment_id FROM message_attachments").split()
kept = {i: marks.get(i) for i in sorted(damaged)}
check(
"строки с расхождениями и ссылки сообщений сохранены",
all(kept.values()) and sum(i in linked for i in links) == len(linked),
quarantine=kept,
message_links=sum(i in linked for i in links),
)
meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment")
blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob)
intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob)
check(
"карантин: метаданные 200, /content 404; нетронутое скачивается",
meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE,
)
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
for attachment, data in ((linked[0], FIXTURE), (linked[1], os.urandom(len(FIXTURE)))):
compose(
"run",
"--rm",
"--no-deps",
"-T",
"s3-cli",
"pipe",
f"root/{bucket}/attachments/{attachment}",
stdin=data,
)
healed = reconcile("--apply", *FLAGS)
save("healed.json", healed)
restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob)
foreign = c.call("GET", f"/api/v1/attachments/{linked[1]}/content", 404, actor=bob)
check(
"вернули те же байты — карантин снят; другие байты той же длины — остался",
healed["fixed"]["healed"] == 1
and healed["samples"].get("healed") == [linked[0]]
and linked[1] in healed["samples"].get("quarantined", [])
and restored["bytes"] == FIXTURE
and foreign["status"] == 404,
fixed=healed["fixed"],
s3_get=healed["cost"]["requests"]["s3_get"],
)
return {"injected": injected, "targets": targets, "linked": linked}
def single(check):
holder = subprocess.Popen(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-c",
f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)",
],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
time.sleep(3)
busy = reconcile("--apply", *FLAGS)
finally:
holder.wait()
save("lock-busy.json", busy)
check(
"второй одновременный проход пропущен",
busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()),
)
def healthy(service, seconds=90):
container = compose("ps", "-q", service).strip()
probe = ["docker", "inspect", "-f", "{{.State.Health.Status}}", container]
deadline = time.monotonic() + seconds
while must(probe).strip() != "healthy":
if time.monotonic() > deadline:
raise SystemExit(f"{service} is not healthy after {seconds} s")
time.sleep(1)
def interrupt(check):
name = f"hlpipe-lab6-reconcile-sigterm-{os.getpid()}"
compose("pause", "s3")
try:
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "--name", name, "reconciler"]
runner = subprocess.Popen(
[*cmd, "--apply", *FLAGS],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
time.sleep(4)
state = must(["docker", "inspect", "-f", "{{.State.Status}} {{.HostConfig.Init}}", name])
started = time.monotonic()
must(["docker", "stop", name])
stopped = round(time.monotonic() - started, 2)
code = runner.wait(timeout=60)
finally:
compose("unpause", "s3")
healthy("s3")
after = reconcile("--dry-run", *FLAGS)
status, init = state.split()
report = {
"s3": "paused",
"state_before_stop": status,
"init": init,
"stop_seconds": stopped,
"exit_code": code,
"next_pass_skipped": after["skipped"],
}
save("sigterm.json", report)
check(
"SIGTERM: идущий проход останавливается за секунды, блокировка освобождена",
status == "running" and code == 143 and stopped < 5 and not after["skipped"],
**report,
)
def outage(check):
before = oracle("--grace", str(GRACE))
compose("stop", "s3")
try:
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", "--apply"]
code, out, err, wall = run([*cmd, *FLAGS])
finally:
compose("up", "-d", "--wait", "--no-deps", "s3")
after = oracle("--grace", str(GRACE))
log = [line for line in err.splitlines() if "reconcile." in line]
same = all(after[k] == before[k] for k in ("objects", "rows", "counts"))
save("s3-down.json", {"exit_code": code, "wall_seconds": wall, "stdout": out, "log": log})
check(
"S3 остановлен: код 1 за ограниченное время, состояние не изменилось",
code == 1 and wall < 30 and not out.strip() and same,
exit_code=code,
wall_seconds=wall,
)
def uploader(base, token, chat, *, deadline, results, lock):
host = urlsplit(base).netloc
conn = http.client.HTTPConnection(host, timeout=30)
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
sent = 0
while time.monotonic() < deadline:
path = f"/api/v1/chats/{chat}/attachments?filename=load.png"
try:
conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"})
response = conn.getresponse()
status, data = response.status, response.read()
if status == 201 and sent % 10 == 0:
body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]})
extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}
conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra})
linked = conn.getresponse()
linked.read()
status = 201 if linked.status == 201 else -linked.status
except (OSError, http.client.HTTPException):
conn.close()
conn = http.client.HTTPConnection(host, timeout=30)
status, data = 0, b""
with lock:
results.append((status, json.loads(data) if status == 201 else None))
sent += 1
time.sleep(0.05)
def under_load(c, check, base, seconds, threads):
alice, _, _, chat = c.bootstrap()
ready = evidence_root() / "ready-load.json"
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
leftover = {
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
"pending_stale": left["pending_fresh"]
+ left["pending_stale_with_object"]
+ left["pending_stale_without_object"],
}
orphans = inject("orphan", 3)
intents = inject("stale-pending", 3, age=INTENT_AGE)
results, lock = [], threading.Lock()
deadline = time.monotonic() + seconds
workers = [
threading.Thread(
target=uploader,
args=(base, alice["token"], chat["id"]),
kwargs={"deadline": deadline, "results": results, "lock": lock},
)
for _ in range(threads)
]
for worker in workers:
worker.start()
runs = []
while time.monotonic() < deadline:
runs.append(reconcile("--apply", *FLAGS))
for worker in workers:
worker.join()
runs.append(reconcile("--apply", *FLAGS))
created = {att["id"]: att["sha256"] for status, att in results if att}
expected = dict(json.loads(ready.read_text()), **created)
save("ready-load.json", expected, stamp=False)
after = oracle(
"--grace",
str(GRACE),
"--deep",
"--expect-ready",
str(ready),
"--output",
str(evidence_root() / "check-under-load.json"),
)
statuses = {}
for status, _ in results:
statuses[str(status)] = statuses.get(str(status), 0) + 1
fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE}
rechecked = {}
for r in runs:
for name, count in r["rechecked"].items():
rechecked[name] = rechecked.get(name, 0) + count
seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")}
summary = {
"seconds": seconds,
"upload_threads": threads,
"uploads": statuses,
"created": len(created),
"linked_every": 10,
"reconciler_runs": len(runs),
"run_seconds": [r["cost"]["seconds"] for r in runs],
"fixed_total": fixed,
"fresh_seen_total": seen,
"rechecked_total": rechecked,
"errors": sum(r["errors"] for r in runs),
"skipped": sum(1 for r in runs if r["skipped"]),
"injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)},
"left_from_faults": leftover,
"check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")},
}
save("under-load.json", summary)
check(
"под нагрузкой: исправлены только внедрённые, ошибок нет",
fixed
== {
**dict.fromkeys(FIXABLE, 0),
"orphan_stale": leftover["orphan_stale"] + len(orphans),
"pending_stale": leftover["pending_stale"] + len(intents),
}
and not summary["errors"]
and not summary["skipped"],
fixed=fixed,
runs=len(runs),
)
check(
"под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0",
after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"},
uploads=statuses,
lost_ready=after["counts"]["lost_ready"],
)
def demo(args):
evidence_root().mkdir(parents=True, exist_ok=False)
c = client.Client(args.base_url, timeout=30)
check = Checks()
pre = oracle("--grace", str(GRACE))
dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"}
if dirty:
raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply")
started = time.monotonic()
result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()}
result["faults"] = faults(c, check, pre)
single(check)
interrupt(check)
outage(check)
under_load(c, check, args.base_url, args.load_seconds, args.load_threads)
result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok)
save("demo.json", result)
print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False))
sys.exit(0 if check.ok else 1)
def pins(args):
cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip())
sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory}
if args.no_pin or cpus < 8:
return sized
return dict(
sized,
S3_CPUSET=f"{cpus - 5}-{cpus - 4}",
PG_CPUSET=f"{cpus - 3}-{cpus - 2}",
RECONCILE_CPUSET=str(cpus - 1),
)
def cpu_set(env):
cpus = set()
for name in ("S3_CPUSET", "PG_CPUSET", "RECONCILE_CPUSET"):
for part in filter(None, env.get(name, "").split(",")):
low, _, high = part.partition("-")
cpus.update(range(int(low), int(high or low) + 1))
return cpus
def limits():
ids = compose("ps", "-q", "s3", "postgres").split()
fmt = (
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}"
" cpuset={{.HostConfig.CpusetCpus}}"
)
return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines()
def host(title):
stats = must(
[
"docker",
"stats",
"--no-stream",
"--all",
"--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}",
]
)
running = must(["docker", "ps", "--format", "{{.Names}} {{.Image}} {{.Status}}"])
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
load = " ".join(f"{v:.2f}" for v in os.getloadavg())
return (
f"== {title} date: {stamp} commit: {head()} host loadavg: {load}\n"
f"$ docker stats --no-stream --all\n{stats}\n$ docker ps\n{running}\n"
)
def usage(text):
return int(re.search(r"^usage_usec (\d+)$", text, re.M).group(1)) / 1e6
def sample():
vm = compose(
"exec", "-T", "postgres", "cat", "/proc/uptime", "/proc/stat", "/sys/fs/cgroup/cpu.stat"
)
s3 = compose("exec", "-T", "s3", "cat", "/sys/fs/cgroup/cpu.stat")
busy = {}
for line in vm.splitlines()[1:]:
name, *values = line.split() or [""]
if name.startswith("cpu") and name[3:].isdigit():
busy[int(name[3:])] = sum(int(values[i]) for i in BUSY) / USER_HZ
return float(vm.split()[0]), busy, usage(vm) + usage(s3)
def screen(first, last, own, pinned, args, *, started=True):
window = max(last[0] - first[0], 0.01)
busy = {cpu: spent - first[1].get(cpu, 0.0) for cpu, spent in last[1].items()}
ours = last[2] - first[2] + own
other = sum(busy.values()) - ours - args.start_other * started
on_pinned = sum(busy.get(cpu, 0.0) for cpu in pinned) - ours - args.start_pinned * started
other, on_pinned = max(0.0, other / window), max(0.0, on_pinned / window) if pinned else 0.0
return {
"window_seconds": round(window, 2),
"ours_cores": round(ours / window, 2),
"other_cores": round(other, 2),
"other_on_pinned_cores": round(on_pinned, 2),
"host_loadavg_1m": round(os.getloadavg()[0], 2),
"disturbed": other > args.max_other or on_pinned > args.max_pinned,
}
def wait_quiet(args, pinned):
deadline, calm = time.monotonic() + args.quiet_timeout, 0
while calm < 3 and time.monotonic() < deadline:
first = sample()
time.sleep(5)
quiet = screen(first, sample(), 0.0, pinned, args, started=False)
calm = 0 if quiet["disturbed"] else calm + 1
sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n")
return calm == 3
def measure(flags, env, args, pinned):
rejected = []
for attempt in range(args.retries + 1):
first = sample()
report = reconcile(*flags, "--samples", "3", env=env)
report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args)
if not report["screen"]["disturbed"]:
return report, rejected
rejected.append(report)
if attempt < args.retries and not wait_quiet(args, pinned):
break
return None, rejected
def seed(count, env):
if count <= 0:
return {"added": 0}
env = dict(env, API_CPUS="4", API_MEMORY="512m")
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
out = compose(
"run",
"--rm",
"--no-deps",
"-T",
"-v",
tools,
"--entrypoint",
"python",
"api",
"/tools/seed_storage.py",
"--count",
str(count),
"--concurrency",
"4",
env=env,
)
records = [json.loads(line) for line in out.strip().splitlines()]
return {**records[-1], "progress": records[:-1]}
def median(runs, path):
values = []
for r in runs:
value = r
for key in path:
value = value[key]
values.append(value)
return statistics.median(values)
def summarize(results, rejected):
rows = []
for (n, mode), runs in sorted(results.items()):
seconds = median(runs, ("cost", "seconds"))
rows.append(
{
"objects": n,
"mode": mode,
"runs": len(runs),
"seconds_median": seconds,
"seconds_all": [r["cost"]["seconds"] for r in runs],
"wall_seconds_median": median(runs, ("wall_seconds",)),
"us_per_object": round(seconds / n * 1e6, 1),
"cpu_seconds_median": median(runs, ("cost", "cpu_seconds")),
"s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")),
"sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")),
"requests": runs[0]["cost"]["requests"],
"max_rss_mib_median": median(runs, ("cost", "max_rss_mib")),
"other_cores_max": max(r["screen"]["other_cores"] for r in runs),
"other_on_pinned_cores_max": max(
r["screen"]["other_on_pinned_cores"] for r in runs
),
"disturbed_kept": sum(r["screen"]["disturbed"] for r in runs),
"rejected": rejected.get((n, mode), 0),
"scanned": runs[0]["scanned"],
"found_nonzero": {k: v for k, v in runs[0]["found"].items() if v},
}
)
return rows
def table(rows):
head = (
"objects mode runs t_med,s wall,s us/obj cpu,s list,s sql,s lists sqls"
" gets rss,MiB other pinned rejected"
)
lines = [head]
for r in rows:
q = r["requests"]
lines.append(
f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} "
f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} "
f"{r['cpu_seconds_median']:>5.2f} {r['s3_list_wait_median']:>6.3f} "
f"{r['sql_wait_median']:>5.3f} {q['s3_list']:>5} {q['sql']:>4} {q['s3_get']:>5} "
f"{r['max_rss_mib_median']:>7.1f} {r['other_cores_max']:>5.2f} "
f"{r['other_on_pinned_cores_max']:>6.2f} {r['rejected']:>8}"
)
return "\n".join(lines) + "\n"
def cost(args):
out = evidence_root() / "cost"
out.mkdir(parents=True, exist_ok=False)
(out / "disturbed").mkdir()
rows = int(psql("SELECT count(*) FROM attachments"))
if rows > args.sizes[0]:
raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project")
snapshot = out / "host-docker-stats.txt"
snapshot.write_text(host("before the series"))
env = pins(args)
pinned = cpu_set(env)
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env)
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []}
plan["screen"] = {
"max_other_cores": args.max_other,
"max_other_on_pinned_cores": args.max_pinned,
"start_allowance_core_seconds": {"vm": args.start_other, "pinned": args.start_pinned},
"retries": args.retries,
}
results, rejected = {}, {}
try:
for n in args.sizes:
plan["seeds"].append(dict(seed(n - rows, env), target=n))
rows = n
time.sleep(args.settle)
save("cost/plan.json", plan)
if not wait_quiet(args, pinned):
raise SystemExit("no quiet measurement window before timeout")
modes = [("shallow", ["--dry-run"])]
if n <= args.deep_max:
modes.append(("deep", ["--dry-run", "--deep"]))
if n == args.page_compare:
modes.append(("page100", ["--dry-run", "--page-size", "100"]))
for mode, flags in modes:
for attempt in range(1, args.repeats + 1):
report, dropped = measure(flags, env, args, pinned)
name = f"n{n}-{mode}-{attempt}"
for number, bad in enumerate(dropped, 1):
save(f"cost/disturbed/{name}-try{number}.json", bad)
rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped)
if report is None:
raise SystemExit(f"{name}: no clean run after retries; rejected preserved")
if (
report["exit_code"]
or report["errors"]
or report["skipped"]
or report["scanned"] != {"objects": n, "rows": n}
or any(report["found"].values())
):
save(f"cost/invalid/{name}.json", report)
raise SystemExit(f"{name}: inconsistent dataset or failed pass")
save(f"cost/{name}.json", report)
results.setdefault((n, mode), []).append(report)
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n")
finally:
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
with snapshot.open("a") as handle:
handle.write("\n" + host("after the series"))
summary = summarize(results, rejected)
save("cost/summary.json", {**plan, "results": summary})
(out / "summary.txt").write_text(
f"date: {stamp} commit: {plan['commit']} pins: {env}\n" + table(summary)
)
sys.stdout.write(table(summary))
def main():
p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass")
sub = p.add_subparsers(dest="command", required=True)
d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load")
d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
d.add_argument("--load-seconds", type=int, default=75)
d.add_argument("--load-threads", type=int, default=3)
k = sub.add_parser("cost", help="cost of a pass for N objects, repeated")
k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 30000, 100000])
k.add_argument("--repeats", type=int, default=3)
k.add_argument("--deep-max", type=int, default=100000)
k.add_argument("--page-compare", type=int, default=10000)
k.add_argument("--settle", type=float, default=10)
k.add_argument("--s3-memory", default="3g")
k.add_argument("--no-pin", action="store_true")
k.add_argument("--max-other", type=float, default=2.0, help="foreign CPU in the VM, cores")
k.add_argument("--max-pinned", type=float, default=0.25, help="foreign CPU on pinned cores")
k.add_argument("--start-other", type=float, default=1.5, help="container start, core-s")
k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores")
k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run")
k.add_argument("--quiet-timeout", type=float, default=900.0)
for command in (d, k):
command.add_argument(
"--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
)
args = p.parse_args()
os.environ["RECONCILE_EVIDENCE_DIR"] = args.out
if args.command == "demo":
demo(args)
else:
cost(args)
if __name__ == "__main__":
main()
+4 -2
View File
@@ -2,7 +2,8 @@
set -eu
cd "$(dirname "$0")/.."
service=${API_SERVICE:-api}
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; }
scripts/stand-usage.sh &
echo "$(date -u +%T) migrate once before the first replacement"
docker compose run --rm --no-deps -T migrate 2>&1
old=$(docker compose ps -q "$service")
@@ -13,8 +14,9 @@ for id in $old; do
scripts/lb-wait.sh $((count + 1))
new=$(docker compose ps -q "$service" | grep -v -F "$running")
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
docker stop -t "${STOP_TIMEOUT:-7}" "$id" >/dev/null
docker rm "$id" >/dev/null
done
scripts/lb-wait.sh "$count"
wait
echo "$(date -u +%T) done: $count replicas replaced"
+30 -7
View File
@@ -2,17 +2,30 @@
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
undated() {
python3 -c 'import json, sys
def clean(value):
if isinstance(value, dict):
return {k: clean(v) for k, v in value.items() if not k.endswith("_date")}
return [clean(v) for v in value] if isinstance(value, list) else value
text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0
while text[at:].strip():
at += len(text[at:]) - len(text[at:].lstrip())
value, at = decoder.raw_decode(text, at)
print(json.dumps(clean(value), indent=2, ensure_ascii=False))'
}
snapshot() {
cli sh -s <<'EOF' | grep -v '_date'
cli sh -s <<'EOF' | undated
ak=$(cat /run/secrets/s3_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
rk=$(cat /run/secrets/s3_reconciler_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do
rc $c --json
done | sed "s/$ak/<api-key>/g"
done | sed "s/$ak/<api-key>/g; s/$rk/<reconciler-key>/g"
EOF
}
failed=0
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) == S3 privacy and bootstrap checks"
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
before=$(snapshot)
@@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same
cli sh -s <<'EOF' || failed=1
b=$S3_BUCKET
key="attachments/s3-check-$(date +%s)-$$"
spare="$key-reconciler"
other="s3-check-$(date +%s)-$$"
head -c 4096 /dev/urandom > /tmp/probe
result=0
expect() {
want=$1; name=$2; shift 2
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
}
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
expect allow "api: HeadObject" rc object stat "api/$b/$key"
expect allow "api: GetObject" rc object show "api/$b/$key"
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
expect deny "api: ListBucket" rc object list "api/$b/attachments/"
rc put /tmp/probe "api/$b/$spare" >/dev/null
expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/"
expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key"
expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key"
expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put"
expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare"
expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other"
expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b"
expect deny "reconciler: admin ListUsers" rc admin user list reconciler
expect deny "api: CreateBucket" rc bucket create "api/$other"
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: admin ListUsers" rc admin user list api
rc bucket create --ignore-existing "root/$other" >/dev/null
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
+524
View File
@@ -0,0 +1,524 @@
#!/usr/bin/env python3
import argparse
import collections
import contextlib
import csv
import json
import math
import os
import re
import signal
import statistics
import subprocess
import sys
import time
from pathlib import Path
from probe import ROOT, k6_metrics, lb_up, run_k6
SCRIPTS = ROOT / "scripts"
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
PINNED = {
"PG_CPUSET": "0-1",
"LB_CPUSET": "2",
"VALKEY_CPUSET": "3",
"K6_CPUSET": "4-6",
"K6_CPUS": "3",
"K6_MEMORY": "1g",
}
API_FIRST_CPU = 7
FIELDS = [
"raw_dir",
"variant",
"config",
"replicas",
"cpus",
"memory_mb",
"workers",
"pool",
"maxconn",
"api_cpuset",
"rate",
"repeat",
"started",
"db_mb",
"messages",
"k6_exit",
"saturated",
"requests",
"rps",
"ok_rps",
"failed_ratio",
"dropped_iterations",
"p50_ms",
"p95_ms",
"p99_ms",
"p95_post_ms",
"p95_get_ms",
"cpu_api",
"cpu_postgres",
"cpu_lb",
"cpu_valkey",
"cpu_k6",
"mem_api_mb",
"mem_postgres_mb",
"mem_k6_mb",
"pg_backends_max",
"pg_active_avg",
"pg_active_max",
"pg_idle_avg",
"pg_idle_tx_avg",
"pg_lock_avg",
"pg_lock_max",
"pg_lwlock_avg",
"pg_io_avg",
"lb_scur_max",
"lb_qcur_max",
"lb_qtime_max_ms",
"lb_rtime_avg_ms",
"cpu_vm",
"cpu_other",
"gap_s",
"wall_s",
"disturbed",
]
def compose(*args, env=None):
command = ["docker", "compose", *args]
return subprocess.run(
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
).stdout
def size(text):
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
def rows(path, start, end):
with path.open() as handle:
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
def mean(values):
return round(statistics.mean(values), 2) if values else 0.0
def docker_usage(path, project, start, end):
ticks = collections.defaultdict(collections.Counter)
memory = collections.Counter()
for r in rows(path, start, end):
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
cpu = {
name: mean([t[name] for t in ticks.values() if name in t])
for name in ("api", "postgres", "lb", "valkey", "k6")
}
usage = {f"cpu_{name}": value for name, value in cpu.items()}
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
return usage
def vm_usage(path, start, end):
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
return 0.0
first, last = samples[0], samples[-1]
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
def gap(path, start, end):
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
edges = [start, *[t for t in ticks if start <= t <= end], end]
return max(b - a for a, b in zip(edges, edges[1:]))
def vm_counters():
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
values = [int(v) for v in lines[0].split()[1:]]
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
return sum(values) - values[3] - values[4], sum(values), cpus
def wait_quiet(args):
deadline, calm = time.monotonic() + args.quiet_timeout, 0
while calm < 3 and time.monotonic() < deadline:
first = vm_counters()
time.sleep(5)
last = vm_counters()
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
calm = calm + 1 if busy < args.max_other else 0
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
def pg_usage(path, start, end):
ticks = collections.defaultdict(collections.Counter)
for r in rows(path, start, end):
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
tick = ticks[r["ts"]]
tick["backends"] += count
tick["active"] += count * (state == "active")
tick["idle"] += count * (state == "idle")
tick["idle_tx"] += count * state.startswith("idle in transaction")
tick["lock"] += count * (wait == "Lock")
tick["lwlock"] += count * (wait == "LWLock")
tick["io"] += count * (wait == "IO")
series = list(ticks.values())
return {
"pg_backends_max": max((t["backends"] for t in series), default=0),
"pg_active_avg": mean([t["active"] for t in series]),
"pg_active_max": max((t["active"] for t in series), default=0),
"pg_idle_avg": mean([t["idle"] for t in series]),
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
"pg_lock_avg": mean([t["lock"] for t in series]),
"pg_lock_max": max((t["lock"] for t in series), default=0),
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
"pg_io_avg": mean([t["io"] for t in series]),
}
def lb_usage(path, start, end):
backend = [
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
]
def numbers(field):
return [int(r[field] or 0) for r in backend]
return {
"lb_scur_max": max(numbers("scur"), default=0),
"lb_qcur_max": max(numbers("qcur"), default=0),
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
"lb_rtime_avg_ms": mean(numbers("rtime")),
}
def trend(path, name):
metrics = json.loads(Path(path).read_text())["metrics"]
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
return round(found.get("values", {}).get("p(95)", 0.0), 2)
def sampler(*command):
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
def stop(process):
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
process.wait()
def wait_lb(expected, timeout=120):
deadline, stable = time.monotonic() + timeout, 0
while stable < 4:
if time.monotonic() > deadline:
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
stable = stable + 1 if lb_up() == expected else 0
time.sleep(0.5)
def layout(config, args, variant):
name, _, extra = variant.partition(":")
replicas, cpus = (float(v) for v in config.split("x"))
replicas = int(replicas)
workers = args.workers or max(1, int(cpus))
width = max(1, math.ceil(replicas * cpus))
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
memory = args.api_memory or args.api_memory_total // replicas
settings = {
"API_REPLICAS": str(replicas),
"API_CPUS": f"{cpus:g}",
"API_MEMORY": f"{memory}m",
"WEB_CONCURRENCY": str(workers),
"DB_POOL_MAX_SIZE": str(args.pool),
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
"API_CPUSET": cpuset if args.pin else "",
}
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
base = {
"variant": name,
"config": config,
"replicas": replicas,
"cpus": cpus,
"memory_mb": memory,
"workers": int(settings["WEB_CONCURRENCY"]),
"pool": int(settings["DB_POOL_MAX_SIZE"]),
"maxconn": int(settings["API_SERVER_MAXCONN"]),
"api_cpuset": settings["API_CPUSET"],
}
return base, settings
def apply(base, settings, env, out):
env = dict(env, **settings)
for attempt in range(3):
try:
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
break
except subprocess.CalledProcessError as error:
log = Path(out) / "compose-up-failed.txt"
with log.open("a") as handle:
stamp = time.strftime("%FT%TZ", time.gmtime())
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"date: {stamp} commit: {commit}\n")
handle.write(f"== {stamp} {base} attempt {attempt + 1}\n")
handle.write(error.stdout + error.stderr + "\n")
print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)}))
if attempt == 2:
raise
time.sleep(30)
wait_lb(base["replicas"])
return env
def deployment(env):
ids = compose("ps", "-q", env=env).split()
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
result = subprocess.run(
["docker", "inspect", "--format", template, *ids],
check=True,
capture_output=True,
text=True,
)
containers = []
for line in result.stdout.splitlines():
name, image, config = line.split(" ", 2)
limits = json.loads(config)
containers.append(
dict(
name=json.loads(name),
image=json.loads(image),
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
)
)
return containers
def provenance(folder, args, env):
stamp = time.strftime("%FT%TZ", time.gmtime())
metadata = {
"date": stamp,
"commit": args.commit,
"project": args.project,
"command": sys.argv,
"api_image": env.get("API_IMAGE", "compose default"),
"warmup_seconds": args.warmup,
"duration_seconds": args.duration,
"containers": args.deployment,
}
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
for path in folder.rglob("*"):
if path.suffix in {".csv", ".txt"}:
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
elif path.name == "summary.json":
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
path.write_text(json.dumps(data) + "\n")
def attempt_folder(out, base, rate, repeat):
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
attempt = 1
while (Path(out) / f"{name}_attempt{attempt}").exists():
attempt += 1
folder = Path(out) / f"{name}_attempt{attempt}"
folder.mkdir(parents=True)
return folder
def measure(folder, args, rate, env):
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
database = subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
).stdout.split(",")
samplers = [
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
]
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
began = time.time()
try:
code, started = run_k6(folder / "k6", load)
finally:
wall = time.time() - began
for process in samplers:
stop(process)
subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
)
if started is None:
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
return code, started + args.warmup, database, wall
def step(base, rate, repeat, args, env):
folder = attempt_folder(args.out, base, rate, repeat)
code, start, database, wall = measure(folder, args, rate, env)
end = start + args.duration
summary = folder / "k6/summary.json"
metrics = k6_metrics(summary, args.duration)
saturated = (
metrics["failed_ratio"] > args.max_error_ratio
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
or metrics["p95_ms"] > args.slo_p95_ms
)
row = dict(
base,
raw_dir=folder.name,
rate=rate,
repeat=repeat,
k6_exit=code,
saturated=saturated,
**metrics,
)
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
row["db_mb"] = round(int(database[0]) / 2**20, 1)
row["messages"] = int(database[1])
row["p95_post_ms"] = trend(summary, "post_message")
row["p95_get_ms"] = trend(summary, "get_history")
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
row.update(pg_usage(folder / "pg_activity.csv", start, end))
row.update(lb_usage(folder / "lb_stats.csv", start, end))
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
row["wall_s"] = round(wall, 1)
row["disturbed"] = (
row["cpu_other"] > args.max_other
or row["gap_s"] > args.max_gap
or wall > args.warmup + args.duration + args.max_overhead
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6"))
or not rows(folder / "pg_activity.csv", start, end)
or not rows(folder / "lb_stats.csv", start, end)
)
provenance(folder, args, env)
if code or not metrics["requests"]:
raise SystemExit(f"invalid k6 run: {folder}")
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
return row
def rates(args, base):
if args.rates:
return [int(r) for r in args.rates.split(",")]
factor, top = (float(v) for v in args.search.split(":"))
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
while value <= top:
values.append(int(round(value, -1)))
value *= factor
return values
def arguments():
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
parser.add_argument("--out", required=True)
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
parser.add_argument(
"--variant",
dest="variants",
action="append",
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
)
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
parser.add_argument("--rates", help="comma separated offered rates, rps")
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
parser.add_argument("--start-per-cpu", type=float, default=100.0)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat")
parser.add_argument("--warmup", type=int, default=10)
parser.add_argument("--duration", type=int, default=30)
parser.add_argument("--pause", type=float, default=5.0)
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
parser.add_argument("--max-error-ratio", type=float, default=0.01)
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
args = parser.parse_args()
args.variants = args.variants or ["base"]
if len(args.variants) > 1 and not args.rates:
parser.error("several variants need explicit --rates")
return args
def block(args, output, config, variant, repeats):
base, settings = layout(config, args, variant)
current, streak = apply(base, settings, args.base_env, args.out), 0
args.deployment = deployment(current)
for rate in rates(args, base):
results = []
for repeat in repeats:
for _ in range(args.retries + 1):
row = step(base, rate, repeat, args, current)
time.sleep(args.pause)
if not row["disturbed"]:
break
args.rejected.writerow(row)
wait_quiet(args)
else:
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
results.append(row)
output.writerows(results)
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
if streak == 2:
break
def series(args, output):
every = range(args.first_repeat, args.first_repeat + args.repeats)
configs = args.configs.split(",")
if not args.rates:
for config in configs:
block(args, output, config, args.variants[0], every)
return
for repeat in every:
for config in configs:
for variant in args.variants:
block(args, output, config, variant, [repeat])
@contextlib.contextmanager
def table(path):
fresh = not path.exists()
with path.open("a", newline="", buffering=1) as handle:
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
if fresh:
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
writer.writeheader()
yield writer
def main():
args = arguments()
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
args.project = json.loads(compose("config", "--format", "json"))["name"]
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
try:
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
series(args, writer)
finally:
if not args.keep:
compose("up", "-d", "--wait", "--wait-timeout", "180")
return 0
if __name__ == "__main__":
sys.exit(main())
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
[ -s secrets/grafana_admin_password ] || od -An -N24 -tx1 /dev/urandom | tr -d ' \n' > secrets/grafana_admin_password
chmod 644 secrets/grafana_admin_password
echo "secrets/grafana_admin_password is ready (value not printed)"
+2
View File
@@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)"
put s3_root_password "$(random)"
put s3_access_key_id "api$(random | cut -c1-16)"
put s3_secret_access_key "$(random)"
put s3_reconciler_access_key_id "rec$(random | cut -c1-16)"
put s3_reconciler_secret_access_key "$(random)"
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
size="select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
echo "$(date -u +%T) database bytes,messages: $(docker compose exec -T postgres psql -X -At -F, -U messenger -d messenger -c "$size")"
docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q)
+1 -1
View File
@@ -6,4 +6,4 @@ ids=$(docker compose ps -q "${API_SERVICE:-api}")
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
docker stop -t "${STOP_TIMEOUT:-7}" "$target" >/dev/null
+9 -2
View File
@@ -77,7 +77,9 @@ def main():
args = p.parse_args()
out = ROOT / args.out
out.mkdir(parents=True, exist_ok=True)
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
if any(out.iterdir()):
p.error("output directory must be empty; choose a new --out")
commit = seed.run(["git", "rev-parse", "HEAD"]).strip()
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
lines = [
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
@@ -102,7 +104,12 @@ def main():
link=0,
ids_out=None,
)
report = {"upload_concurrency": slots, **seed.over_http(run)}
report = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"commit": commit,
"upload_concurrency": slots,
**seed.over_http(run),
}
report["lb"] = balancer()
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
+2 -2
View File
@@ -3,7 +3,7 @@ import sys
from dishka import make_async_container
from messenger.attachments.domain.jobs import ReplayResult
from messenger.attachments.domain.jobs import ReplayResult, ReprocessResult
from messenger.attachments.entrypoints.admin import rendered, run
from messenger.attachments.entrypoints.ioc import AdminProvider
from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal
@@ -26,7 +26,7 @@ async def serve(argv: list[str]) -> int:
await bus.stop()
await container.close()
sys.stdout.write(rendered(report) + "\n")
return int(report.get("result") == ReplayResult.NOT_FOUND)
return int(report.get("result") in (ReplayResult.NOT_FOUND, ReprocessResult.CONFLICT))
def main() -> None:
@@ -47,3 +47,21 @@ class CleanStaging(Command):
class ReplayAttachment(Command):
attachment_id: UUID
operator: str
@dataclass(frozen=True, slots=True)
class ReprocessAttachment(Command):
attachment_id: UUID
pipeline_version: int
operator: str
@dataclass(frozen=True, slots=True)
class ReconcileStorage(Command):
apply: bool
grace: timedelta
skew: timedelta
deep: bool = False
page_size: int = 1000
deep_concurrency: int = 8
samples: int = 20
@@ -8,6 +8,7 @@ from messenger.attachments.application.commands import (
PublishOutbox,
RecoverExpiredLeases,
ReplayAttachment,
ReprocessAttachment,
)
from messenger.attachments.application.ports import (
AttachmentsUnitOfWork,
@@ -18,20 +19,26 @@ from messenger.attachments.application.ports import (
VariantRenderer,
)
from messenger.attachments.domain.errors import LeaseLost, UnreadableImage
from messenger.attachments.domain.events import AttachmentReplayRequested, JobSettled
from messenger.attachments.domain.events import (
AttachmentReplayRequested,
AttachmentReprocessRequested,
JobSettled,
)
from messenger.attachments.domain.jobs import (
PIPELINE_VERSIONS,
Claim,
FailureCode,
Job,
Lease,
Outcome,
ReplayResult,
ReprocessResult,
RetryPolicy,
)
from messenger.attachments.domain.models import Attachment, AttachmentStatus, original_key
from messenger.attachments.domain.variants import Variant, variant_key
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import IdGenerator
from messenger.shared.application.ports import Clock, IdGenerator
TRANSIENT_ERROR = "transient_error"
STALL_AFTER_CLAIM = "stall_after_claim"
@@ -299,3 +306,39 @@ class ReplayAttachmentHandler:
return ReplayResult.EXPIRED
job = await self._uow.jobs.replay(attachment, operator)
return ReplayResult.NOOP if job is None else ReplayResult.REPLAYED
class ReprocessAttachmentHandler:
__slots__ = ("_clock", "_ids", "_uow")
def __init__(self, uow: AttachmentsUnitOfWork, ids: IdGenerator, clock: Clock) -> None:
self._uow = uow
self._ids = ids
self._clock = clock
async def __call__(self, command: ReprocessAttachment) -> ReprocessResult:
if command.pipeline_version not in PIPELINE_VERSIONS:
raise ValueError("неподдерживаемая версия pipeline")
async with self._uow.transaction():
attachment = await self._uow.attachments.lock(command.attachment_id)
result = await self._schedule(attachment, command)
self._uow.record(
AttachmentReprocessRequested(
command.attachment_id, command.pipeline_version, command.operator, result.value
)
)
await self._uow.commit()
return result
async def _schedule(
self, attachment: Attachment | None, command: ReprocessAttachment
) -> ReprocessResult:
if attachment is None:
return ReprocessResult.NOT_FOUND
if attachment.status in (AttachmentStatus.PENDING, AttachmentStatus.FAILED):
return ReprocessResult.CONFLICT
if command.pipeline_version <= attachment.published_version:
return ReprocessResult.NOOP
job = Job(self._ids.new_id(), attachment.id, command.pipeline_version, 1, self._clock.now())
added = await self._uow.jobs.reprocess(job, command.operator)
return ReprocessResult.QUEUED if added else ReprocessResult.NOOP
@@ -1,4 +1,5 @@
from collections.abc import AsyncIterator, Sequence
from contextlib import AbstractAsyncContextManager
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Protocol, runtime_checkable
@@ -40,6 +41,13 @@ class StagedObject:
key: str
@dataclass(frozen=True, slots=True)
class StoredObject:
key: str
size: int
modified: datetime
@runtime_checkable
class ObjectStorage(Protocol):
async def put(self, key: str, data: bytes, content_type: str) -> None: ...
@@ -52,6 +60,10 @@ class ObjectStorage(Protocol):
async def delete(self, key: str) -> None: ...
def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]: ...
async def stat(self, key: str) -> StoredObject | None: ...
@runtime_checkable
class ImageInspector(Protocol):
@@ -95,6 +107,14 @@ class AttachmentRepository(Protocol):
async def remove(self, attachment: Attachment) -> None: ...
async def page_after(self, key: str, limit: int) -> Sequence[Attachment]: ...
async def by_key(self, key: str) -> Attachment | None: ...
async def claim(self, attachment_id: UUID) -> Attachment | None: ...
async def mark(self, attachment: Attachment) -> None: ...
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: ...
async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ...
@@ -104,6 +124,8 @@ class AttachmentRepository(Protocol):
class JobRepository(Protocol):
async def add(self, job: Job) -> None: ...
async def reprocess(self, job: Job, operator: str) -> bool: ...
async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: ...
async def complete(
@@ -158,3 +180,8 @@ class AttachmentDirectory(Protocol):
@runtime_checkable
class JobObserver(Protocol):
def observe(self, job: Job, outcome: str, seconds: float) -> None: ...
@runtime_checkable
class SingleRun(Protocol):
def hold(self) -> AbstractAsyncContextManager[bool]: ...
@@ -49,6 +49,9 @@ class OpenAttachment:
if found is None:
raise AttachmentNotFound("у вложения нет такого варианта")
key, content_type = found.object_key, found.content_type
if attachment.quarantine is not None:
self._logger.warning("attachment.quarantined", attachment_id=str(attachment.id))
raise AttachmentNotFound("вложение не найдено")
try:
content = await self._storage.open(str(key))
except ObjectMissingError as error:
@@ -0,0 +1,318 @@
import asyncio
from collections import Counter
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
from dataclasses import dataclass, field
from datetime import datetime
from hashlib import sha256
from time import monotonic
from uuid import UUID
from messenger.attachments.application.commands import ReconcileStorage
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.application.handlers import drop
from messenger.attachments.application.ports import (
AttachmentsUnitOfWork,
ObjectStorage,
SingleRun,
StoredObject,
)
from messenger.attachments.domain.models import (
OBJECT_PREFIX,
Attachment,
AttachmentStatus,
Quarantine,
)
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import Clock, EventPublisher, Logger
type UnitOfWorkFactory = Callable[[], AttachmentsUnitOfWork]
@dataclass(slots=True)
class Reconciliation:
apply: bool
skipped: bool = False
objects: int = 0
rows: int = 0
found: Counter[str] = field(default_factory=Counter)
fixed: Counter[str] = field(default_factory=Counter)
rechecked: Counter[str] = field(default_factory=Counter)
requests: Counter[str] = field(default_factory=Counter)
waited: dict[str, float] = field(default_factory=dict)
samples: dict[str, list[str]] = field(default_factory=dict)
hashed_bytes: int = 0
errors: int = 0
seconds: float = 0.0
class StorageReconciler:
__slots__ = ("_clock", "_logger", "_publisher", "_single", "_storage", "_uow")
def __init__(
self,
uow: UnitOfWorkFactory,
storage: ObjectStorage,
single: SingleRun,
clock: Clock,
*,
logger: Logger,
publisher: EventPublisher,
) -> None:
self._uow = uow
self._storage = storage
self._single = single
self._clock = clock
self._logger = logger
self._publisher = publisher
async def __call__(self, command: ReconcileStorage) -> Reconciliation:
report = Reconciliation(apply=command.apply)
started = monotonic()
async with self._single.hold() as held:
report.skipped = not held
if held:
before = self._clock.now() - command.grace - command.skew
await Pass(
self._uow,
self._storage,
command,
before=before,
report=report,
logger=self._logger,
publisher=self._publisher,
).run()
report.seconds = round(monotonic() - started, 3)
return report
class Pass:
__slots__ = (
"_before",
"_command",
"_logger",
"_publisher",
"_report",
"_slots",
"_storage",
"_uow",
)
def __init__(
self,
uow: UnitOfWorkFactory,
storage: ObjectStorage,
command: ReconcileStorage,
*,
before: datetime,
report: Reconciliation,
logger: Logger,
publisher: EventPublisher,
) -> None:
self._uow = uow
self._storage = storage
self._command = command
self._before = before
self._report = report
self._logger = logger
self._publisher = publisher
self._slots = asyncio.Semaphore(command.deep_concurrency)
async def run(self) -> None:
objects = aiter(self._objects())
item = await anext(objects, None)
try:
async with asyncio.TaskGroup() as checks:
async for row in self._rows():
while item is not None and item.key < row.object_key:
await self._orphan(item)
item = await anext(objects, None)
matched = item if item is not None and item.key == row.object_key else None
await self._row(row, matched, checks)
if matched is not None:
item = await anext(objects, None)
while item is not None:
await self._orphan(item)
item = await anext(objects, None)
except* Unavailable as failed:
error = failed.exceptions[0]
raise error from error.__cause__
async def _timed[ResultT](self, name: str, step: Awaitable[ResultT]) -> ResultT:
started = monotonic()
try:
return await step
finally:
spent = self._report.waited.get(name, 0.0) + monotonic() - started
self._report.waited[name] = spent
async def _objects(self) -> AsyncIterator[StoredObject]:
pages = aiter(self._storage.listing(OBJECT_PREFIX, self._command.page_size))
while (page := await self._timed("s3_list", anext(pages, None))) is not None:
self._report.requests["s3_list"] += 1
self._report.objects += len(page)
for item in page:
yield item
async def _rows(self) -> AsyncIterator[Attachment]:
after, size = "", self._command.page_size
while True:
page = await self._timed("sql", self._page(after, size + 1))
self._report.requests["sql"] += 1
rows = page[:size]
self._report.rows += len(rows)
for row in rows:
yield row
if len(page) <= size:
return
after = rows[-1].object_key
async def _page(self, after: str, size: int) -> Sequence[Attachment]:
uow = self._uow()
async with uow.transaction():
return await uow.attachments.page_after(after, size)
def _found(self, name: str, subject: str) -> None:
self._report.found[name] += 1
sample = self._report.samples.setdefault(name, [])
if len(sample) < self._command.samples:
sample.append(subject)
async def _safely(self, name: str, action: Awaitable[bool], subject: UUID | None) -> None:
attachment_id = None if subject is None else str(subject)
try:
done = await action
except Exception as error:
self._report.errors += 1
self._logger.warning(
"reconcile.action_failed",
reason=name,
attachment_id=attachment_id,
error=type(error).__name__,
)
return
(self._report.fixed if done else self._report.rechecked)[name] += 1
event = "reconcile.fixed" if done else "reconcile.rechecked"
self._logger.info(event, reason=name, attachment_id=attachment_id)
async def _orphan(self, item: StoredObject) -> None:
if item.modified >= self._before:
self._found("orphan_fresh", item.key)
return
self._found("orphan_stale", item.key)
if self._command.apply:
await self._safely("orphan_stale", self._delete_orphan(item), None)
async def _delete_orphan(self, item: StoredObject) -> bool:
uow = self._uow()
async with uow.transaction():
owner = await uow.attachments.by_key(item.key)
self._report.requests["sql"] += 1
if owner is not None:
return False
self._report.requests["s3_delete"] += 1
await self._storage.delete(item.key)
return True
async def _row(
self, row: Attachment, item: StoredObject | None, checks: asyncio.TaskGroup
) -> None:
if row.status is AttachmentStatus.PENDING:
await self._pending(row)
elif item is None:
await self._broken(row, Quarantine.MISSING)
elif item.size != row.size_bytes:
await self._broken(row, Quarantine.SIZE_MISMATCH)
elif self._command.deep or row.quarantine is not None:
await self._slots.acquire()
checks.create_task(self._verify(row))
async def _verify(self, row: Attachment) -> None:
try:
digest = await self._digest(row.object_key)
except Exception as error:
self._report.errors += 1
self._logger.warning(
"reconcile.hash_failed", attachment_id=str(row.id), error=type(error).__name__
)
return
finally:
self._slots.release()
if digest is None:
await self._broken(row, Quarantine.MISSING)
elif digest != row.sha256:
await self._broken(row, Quarantine.SHA_MISMATCH)
else:
await self._healthy(row)
async def _digest(self, key: str) -> str | None:
self._report.requests["s3_get"] += 1
try:
content = await self._storage.open(key)
except ObjectMissingError:
return None
digest = sha256()
async for chunk in content.chunks:
digest.update(chunk)
self._report.hashed_bytes += len(chunk)
return digest.hexdigest()
async def _broken(self, row: Attachment, reason: Quarantine) -> None:
if row.quarantine is not None:
self._found("quarantined", str(row.id))
return
self._found(reason.value, str(row.id))
if self._command.apply:
await self._safely(reason.value, self._isolate(row, reason), row.id)
async def _isolate(self, row: Attachment, reason: Quarantine) -> bool:
if reason is not Quarantine.SHA_MISMATCH:
self._report.requests["s3_head"] += 1
found = await self._storage.stat(row.object_key)
if found is not None and found.size == row.size_bytes:
return False
row.isolate(reason)
await self._mark(row)
return True
async def _healthy(self, row: Attachment) -> None:
if row.quarantine is None:
return
self._found("healed", str(row.id))
if self._command.apply:
await self._safely("healed", self._release(row), row.id)
async def _release(self, row: Attachment) -> bool:
row.release()
await self._mark(row)
return True
async def _mark(self, row: Attachment) -> None:
uow = self._uow()
async with uow.transaction():
await uow.attachments.mark(row)
await uow.commit()
self._report.requests["sql"] += 1
async def _pending(self, row: Attachment) -> None:
if row.created_at >= self._before:
self._found("pending_fresh", str(row.id))
return
self._found("pending_stale", str(row.id))
if self._command.apply:
await self._safely("pending_stale", self._drop(row), row.id)
async def _drop(self, row: Attachment) -> bool:
uow = self._uow()
async with uow.transaction():
current = await uow.attachments.claim(row.id)
self._report.requests["sql"] += 1
if (
current is None
or current.status is not AttachmentStatus.PENDING
or current.created_at >= self._before
):
return False
self._report.requests["s3_delete"] += 1
await drop(uow, self._storage, current, "reconcile")
await self._publisher.publish(uow.collect_new_events())
return True
@@ -33,6 +33,15 @@ class JobSettled(DomainEvent):
name: str = field(init=False, default="job.settled")
@dataclass(frozen=True, slots=True)
class AttachmentReprocessRequested(DomainEvent):
attachment_id: UUID
pipeline_version: int
operator: str
result: str
name: str = field(init=False, default="attachment.reprocess_requested")
@dataclass(frozen=True, slots=True)
class AttachmentReplayRequested(DomainEvent):
attachment_id: UUID
+8
View File
@@ -6,6 +6,7 @@ from uuid import UUID
JOB_TYPE = "attachment.process.v1"
SCHEMA_VERSION = 1
PIPELINE_VERSIONS = (1, 2)
class FailureCode(StrEnum):
@@ -21,6 +22,13 @@ class ReplayResult(StrEnum):
NOT_FOUND = "not_found"
class ReprocessResult(StrEnum):
QUEUED = "queued"
NOOP = "noop"
CONFLICT = "conflict"
NOT_FOUND = "not_found"
class Outcome(StrEnum):
DONE = "done"
DUPLICATE = "duplicate_noop"
@@ -8,6 +8,7 @@ from messenger.attachments.domain.values import Filename, ImageInfo
from messenger.attachments.domain.variants import Variant
from messenger.shared.domain.events import DomainEvent
OBJECT_PREFIX = "attachments/"
STAGING_PREFIX = "staging/"
ORIGINALS_PREFIX = "originals/"
@@ -24,6 +25,12 @@ def original_key(attachment_id: UUID) -> str:
return f"{ORIGINALS_PREFIX}{attachment_id}"
class Quarantine(StrEnum):
MISSING = "missing"
SIZE_MISMATCH = "size_mismatch"
SHA_MISMATCH = "sha_mismatch"
@dataclass(slots=True)
class Attachment:
id: UUID
@@ -41,6 +48,7 @@ class Attachment:
published_version: int = 0
error_code: str | None = None
variants: tuple[Variant, ...] = ()
quarantine: Quarantine | None = None
_events: list[DomainEvent] = field(init=False, default_factory=list, repr=False, compare=False)
@classmethod
@@ -70,6 +78,10 @@ class Attachment:
staging_key=f"{STAGING_PREFIX}{attachment_id}",
)
@property
def object_key(self) -> str:
return self.original_key or self.staging_key or f"{OBJECT_PREFIX}{self.id}"
@property
def content_type(self) -> str:
return self.image.kind.value
@@ -98,6 +110,13 @@ class Attachment:
AttachmentDiscarded(attachment_id=self.id, chat_id=self.chat_id, reason=reason)
)
def isolate(self, reason: Quarantine) -> None:
if self.status is AttachmentStatus.READY:
self.quarantine = reason
def release(self) -> None:
self.quarantine = None
def collect_events(self) -> list[DomainEvent]:
events, self._events = self._events, []
return events
+19 -5
View File
@@ -9,9 +9,9 @@ from uuid import UUID
from dishka import AsyncContainer
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import ReplayAttachment
from messenger.attachments.application.commands import ReplayAttachment, ReprocessAttachment
from messenger.attachments.application.ports import AttachmentDirectory
from messenger.attachments.domain.jobs import ReplayResult
from messenger.attachments.domain.jobs import PIPELINE_VERSIONS, ReplayResult, ReprocessResult
REGISTRY_LIMIT = 20
@@ -26,6 +26,10 @@ def parser() -> argparse.ArgumentParser:
replay = commands.add_parser("replay", help="вернуть failed-вложение в очередь")
replay.add_argument("--attachment-id", type=UUID, required=True)
replay.add_argument("--operator", default=os.environ.get("OPERATOR", "cli"))
reprocess = commands.add_parser("reprocess", help="поставить новую версию pipeline")
reprocess.add_argument("--attachment-id", type=UUID, required=True)
reprocess.add_argument("--pipeline-version", type=int, choices=PIPELINE_VERSIONS, required=True)
reprocess.add_argument("--operator", default=os.environ.get("OPERATOR", "cli"))
return root
@@ -36,10 +40,20 @@ async def run(container: AsyncContainer, argv: list[str]) -> dict[str, Any]:
directory = await scope.get(AttachmentDirectory)
return {"failed": [asdict(job) for job in await directory.failed_jobs(args.limit)]}
bus = await scope.get(AttachmentsCommandBus)
result: ReplayResult = await bus.execute(
ReplayAttachment(args.attachment_id, args.operator)
command = (
ReprocessAttachment(args.attachment_id, args.pipeline_version, args.operator)
if args.command == "reprocess"
else ReplayAttachment(args.attachment_id, args.operator)
)
return {"result": result.value, "attachment_id": args.attachment_id, "operator": args.operator}
result: ReplayResult | ReprocessResult = await bus.execute(command)
report = {
"result": result.value,
"attachment_id": args.attachment_id,
"operator": args.operator,
}
if args.command == "reprocess":
report["pipeline_version"] = args.pipeline_version
return report
def plain(value: object) -> str:
+8 -3
View File
@@ -12,6 +12,7 @@ from messenger.attachments.application.commands import (
PublishOutbox,
RecoverExpiredLeases,
ReplayAttachment,
ReprocessAttachment,
SweepAbandonedUploads,
UploadAttachment,
)
@@ -26,6 +27,7 @@ from messenger.attachments.application.pipeline import (
PublishOutboxHandler,
RecoverExpiredLeasesHandler,
ReplayAttachmentHandler,
ReprocessAttachmentHandler,
)
from messenger.attachments.application.ports import (
AttachmentDirectory,
@@ -76,7 +78,7 @@ async def object_storage(settings: AttachmentsSettings) -> AsyncIterator[ObjectS
yield S3ObjectStorage(client, settings.s3_bucket)
class AttachmentsProvider(Provider):
class AttachmentStorageProvider(Provider):
def __init__(self, settings: AttachmentsSettings) -> None:
super().__init__()
self._settings = settings
@@ -90,6 +92,8 @@ class AttachmentsProvider(Provider):
async with object_storage(settings) as storage:
yield storage
class AttachmentsProvider(AttachmentStorageProvider):
@provide(scope=Scope.APP)
def images(self, settings: AttachmentsSettings) -> ImageInspector:
return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency)
@@ -266,9 +270,10 @@ class AdminProvider(Provider):
@provide(scope=Scope.REQUEST)
def command_bus(
self, uow: AttachmentsUnitOfWork, publisher: EventPublisher
self, uow: AttachmentsUnitOfWork, publisher: EventPublisher, ids: IdGenerator, clock: Clock
) -> AttachmentsCommandBus:
handlers: dict[type[Command], CommandHandler] = {
ReplayAttachment: ReplayAttachmentHandler(uow)
ReplayAttachment: ReplayAttachmentHandler(uow),
ReprocessAttachment: ReprocessAttachmentHandler(uow, ids, clock),
}
return AttachmentsCommandBus(handlers, uow, publisher)
@@ -2,6 +2,7 @@ from messenger.attachments.domain.events import (
AttachmentDiscarded,
AttachmentQueued,
AttachmentReplayRequested,
AttachmentReprocessRequested,
JobSettled,
)
from messenger.shared.application.messagebus import EventBus
@@ -69,6 +70,15 @@ class AttachmentsJournal:
result=event.result,
)
async def on_reprocess_requested(self, event: AttachmentReprocessRequested) -> None:
self._logger.info(
"audit.attachment_reprocess",
attachment_id=str(event.attachment_id),
pipeline_version=event.pipeline_version,
operator=event.operator,
result=event.result,
)
def subscribe_journal(bus: EventBus, logger: Logger) -> None:
journal = AttachmentsJournal(logger)
@@ -76,3 +86,4 @@ def subscribe_journal(bus: EventBus, logger: Logger) -> None:
bus.subscribe(AttachmentDiscarded, journal.on_attachment_discarded)
bus.subscribe(JobSettled, journal.on_job_settled)
bus.subscribe(AttachmentReplayRequested, journal.on_replay_requested)
bus.subscribe(AttachmentReprocessRequested, journal.on_reprocess_requested)
@@ -0,0 +1,174 @@
import argparse
import asyncio
import json
import resource
import sys
from collections.abc import Callable, Sequence
from datetime import timedelta
from typing import Any
from dishka import Scope, make_async_container, provide
from messenger.attachments.application.commands import ReconcileStorage
from messenger.attachments.application.ports import ObjectStorage
from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler
from messenger.attachments.config import AttachmentsSettings
from messenger.attachments.domain.events import AttachmentDiscarded
from messenger.attachments.entrypoints.ioc import AttachmentStorageProvider
from messenger.attachments.entrypoints.journal import LOG_FIELDS, AttachmentsJournal
from messenger.attachments.infrastructure.postgres import (
PostgresAttachmentsUnitOfWork,
PostgresSingleRun,
)
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.messagebus import EventBus
from messenger.shared.application.ports import Clock, EventPublisher, Logger
from messenger.shared.config import PlatformSettings
from messenger.shared.entrypoints.ioc import PlatformProvider
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
from messenger.shared.infrastructure.postgres import Database
LOCK_KEY = 20_260_004
CLASSES = (
"orphan_stale",
"orphan_fresh",
"missing",
"size_mismatch",
"sha_mismatch",
"pending_stale",
"pending_fresh",
"quarantined",
"healed",
)
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale", "healed")
REQUESTS = ("s3_list", "s3_head", "s3_get", "s3_delete", "sql")
def bounded(low: int, high: int | None = None) -> Callable[[str], int]:
def parse(text: str) -> int:
value = int(text)
if value < low or (high is not None and value > high):
raise argparse.ArgumentTypeError(f"ожидается {low}..{high or '∞'}")
return value
return parse
def command_of(argv: Sequence[str] | None = None) -> ReconcileStorage:
parser = argparse.ArgumentParser(
prog="python -m messenger.attachments.entrypoints.reconcile",
description="Сверка метаданных вложений в PostgreSQL с объектами bucket S3",
)
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--dry-run", dest="apply", action="store_false", help="только отчёт")
mode.add_argument("--apply", dest="apply", action="store_true", help="исправить найденное")
parser.add_argument("--grace", type=bounded(60), default=3600, help="секунды, ≥ 60")
parser.add_argument("--skew", type=bounded(0), default=60, help="запас на часы, секунды")
parser.add_argument("--deep", action="store_true", help="сверить SHA-256 каждого объекта")
parser.add_argument("--page-size", type=bounded(1, 1000), default=1000)
parser.add_argument("--deep-concurrency", type=bounded(1, 48), default=8)
parser.add_argument("--samples", type=bounded(0, 1000), default=20)
args = parser.parse_args(argv)
return ReconcileStorage(
apply=args.apply,
grace=timedelta(seconds=args.grace),
skew=timedelta(seconds=args.skew),
deep=args.deep,
page_size=args.page_size,
deep_concurrency=args.deep_concurrency,
samples=args.samples,
)
class ReconcileProvider(AttachmentStorageProvider):
@provide(scope=Scope.APP)
def reconciler(
self,
database: Database,
storage: ObjectStorage,
clock: Clock,
logger: Logger,
publisher: EventPublisher,
) -> StorageReconciler:
return StorageReconciler(
lambda: PostgresAttachmentsUnitOfWork(database),
storage,
PostgresSingleRun(database, LOCK_KEY),
clock,
logger=logger,
publisher=publisher,
)
async def reconcile(
command: ReconcileStorage,
platform: PlatformSettings,
settings: AttachmentsSettings,
logger: Logger,
) -> Reconciliation:
container = make_async_container(
PlatformProvider(platform, logger), ReconcileProvider(settings)
)
bus = await container.get(EventBus)
bus.subscribe(AttachmentDiscarded, AttachmentsJournal(logger).on_attachment_discarded)
bus.start()
try:
reconciler = await container.get(StorageReconciler)
return await reconciler(command)
finally:
await bus.stop()
await container.close()
def failure(error: BaseException) -> str:
if isinstance(error, BaseExceptionGroup):
return failure(error.exceptions[0])
if isinstance(error, Unavailable) and error.__cause__ is not None:
return type(error.__cause__).__name__
return type(error).__name__
def report_of(command: ReconcileStorage, result: Reconciliation) -> dict[str, Any]:
usage = resource.getrusage(resource.RUSAGE_SELF)
return {
"mode": "apply" if command.apply else "dry-run",
"skipped": result.skipped,
"grace_seconds": int(command.grace.total_seconds()),
"skew_seconds": int(command.skew.total_seconds()),
"deep": command.deep,
"page_size": command.page_size,
"scanned": {"objects": result.objects, "rows": result.rows},
"found": {name: result.found[name] for name in CLASSES},
"fixed": {name: result.fixed[name] for name in FIXABLE},
"rechecked": dict(result.rechecked),
"errors": result.errors,
"samples": result.samples,
"cost": {
"seconds": result.seconds,
"requests": {name: result.requests[name] for name in REQUESTS},
"waited_seconds": {name: round(spent, 3) for name, spent in result.waited.items()},
"hashed_bytes": result.hashed_bytes,
"cpu_seconds": round(usage.ru_utime + usage.ru_stime, 3),
"max_rss_mib": round(usage.ru_maxrss / 1024, 1),
},
}
def main(argv: Sequence[str] | None = None) -> None:
command = command_of(argv)
platform = PlatformSettings(db_pool_max_size=command.deep_concurrency + 2)
settings = AttachmentsSettings()
logger = StructlogLogger(
level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS, stream=sys.stderr
)
try:
result = asyncio.run(reconcile(command, platform, settings, logger))
except Exception as error:
logger.error("reconcile.failed", error=failure(error))
raise SystemExit(1) from None
sys.stdout.write(json.dumps(report_of(command, result), ensure_ascii=False, indent=2) + "\n")
raise SystemExit(1 if result.errors else 0)
if __name__ == "__main__":
main()
@@ -40,7 +40,12 @@ async def handle_job(
outcome = (await bus.execute(ProcessJob(job))).value
except Exception as error:
observed(job, "error", started, observer, logger)
logger.warning("job.requeued", job_id=str(job.id), error=type(error).__name__)
logger.warning(
"job.requeued",
job_id=str(job.id),
error=type(error).__name__,
error_code=getattr(getattr(error, "orig", error), "sqlstate", None),
)
await asyncio.sleep(REQUEUE_DELAY_SECONDS)
await message.nack(requeue=True)
return
@@ -34,8 +34,8 @@ BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning)
JPEG_SCALE = 8
WIDE_MODES = frozenset({"I", "I;16", "I;16B", "I;16L", "I;16N"})
ALPHA_MODES = frozenset({"LA", "La", "PA", "RGBa"})
JPEG = {"quality": 85, "optimize": True}
WEBP = {"quality": 80, "method": 4}
JPEG = {1: {"quality": 85, "optimize": True}, 2: {"quality": 90, "optimize": True}}
WEBP = {1: {"quality": 80, "method": 4}, 2: {"quality": 85, "method": 4}}
ENCODINGS = {"JPEG": (ImageKind.JPEG.value, "jpg"), "WEBP": ("image/webp", "webp")}
REDUCING_GAP = 3.0
@@ -128,15 +128,27 @@ def decoded(data: bytes, max_pixels: int) -> tuple[Image.Image, int, int]:
raise UnreadableImage("изображение не декодируется") from error
def render_variants(data: bytes, limits: RenderLimits) -> list[Rendition]:
def render_variants(
data: bytes, limits: RenderLimits, pipeline_version: int = 1
) -> list[Rendition]:
if len(data) > limits.max_bytes:
raise ImageTooDemanding("оригинал больше допустимого размера")
upright, width, height = decoded(data, limits.max_pixels)
square = ImageOps.fit(upright, (CROP_SIDE, CROP_SIDE), Image.Resampling.LANCZOS)
return [
encoded(THUMBNAIL, scaled(upright, fit(width, height, THUMBNAIL_BOX)), "JPEG", **JPEG),
encoded(CROP, square, "JPEG", **JPEG),
encoded(OPTIMIZED, scaled(upright, fit(width, height, OPTIMIZED_BOX)), "WEBP", **WEBP),
encoded(
THUMBNAIL,
scaled(upright, fit(width, height, THUMBNAIL_BOX)),
"JPEG",
**JPEG[pipeline_version],
),
encoded(CROP, square, "JPEG", **JPEG[pipeline_version]),
encoded(
OPTIMIZED,
scaled(upright, fit(width, height, OPTIMIZED_BOX)),
"WEBP",
**WEBP[pipeline_version],
),
]
@@ -145,7 +157,7 @@ def limit_memory(limit_bytes: int) -> None:
resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes))
type Render = Callable[[bytes, RenderLimits], list[Rendition]]
type Render = Callable[[bytes, RenderLimits, int], list[Rendition]]
class ProcessVariantRenderer:
@@ -173,7 +185,9 @@ class ProcessVariantRenderer:
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
pool = await self._idle.get()
try:
future = asyncio.wrap_future(pool.submit(self._render, data, self._limits))
future = asyncio.wrap_future(
pool.submit(self._render, data, self._limits, pipeline_version)
)
return await asyncio.wait_for(future, self._timeout)
except TimeoutError as error:
pool = self._replace(pool)
File diff suppressed because it is too large Load Diff
+24 -2
View File
@@ -1,4 +1,4 @@
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Sequence
from contextlib import asynccontextmanager
from typing import Any
@@ -8,7 +8,7 @@ from botocore.exceptions import BotoCoreError, ClientError
from botocore.handlers import add_expect_header
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.application.ports import ObjectContent
from messenger.attachments.application.ports import ObjectContent, StoredObject
from messenger.shared.application.errors import Unavailable
CHUNK_BYTES = 64 * 1024
@@ -98,6 +98,28 @@ class S3ObjectStorage:
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]:
pages = self._client.get_paginator("list_objects_v2").paginate(
Bucket=self._bucket, Prefix=prefix, PaginationConfig={"PageSize": page_size}
)
try:
async for page in pages:
yield [
StoredObject(item["Key"], item["Size"], item["LastModified"])
for item in page.get("Contents", ())
]
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def stat(self, key: str) -> StoredObject | None:
try:
head = await self._client.head_object(Bucket=self._bucket, Key=key)
except FAILURES as error:
if missing(error):
return None
raise Unavailable("S3 недоступно") from error
return StoredObject(key, head["ContentLength"], head["LastModified"])
async def _chunks(self, body: Any) -> AsyncIterator[bytes]:
async with body as stream:
async for chunk in stream.iter_chunks(CHUNK_BYTES):
+6 -6
View File
@@ -1,5 +1,4 @@
import asyncio
import logging
import os
import sys
from typing import Any
@@ -10,12 +9,13 @@ from uvicorn.server import Server
from uvicorn_worker import UvicornWorker
from messenger.main import LOG_FIELDS
from messenger.shared.entrypoints.health import DRAINING
from messenger.shared.infrastructure.logging import configure
from messenger.shared.infrastructure.logging import StructlogLogger, configure
from messenger.shared.infrastructure.readiness import DRAINING
LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5"))
SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "6"))
DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2"))
LOGGER = StructlogLogger(level=LOG_LEVEL, allowed_fields=LOG_FIELDS)
class StructlogGunicornLogger(GunicornBaseLogger):
@@ -31,7 +31,7 @@ class DrainingServer(Server):
await super().main_loop()
if not self.force_exit:
DRAINING.set()
logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS)
LOGGER.info("service.draining", limit=DRAIN_SECONDS)
await asyncio.sleep(DRAIN_SECONDS)
@@ -40,7 +40,7 @@ class MessengerWorker(UvicornWorker):
"access_log": False,
"server_header": False,
"date_header": False,
"timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)),
"timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS) - 1),
}
async def _serve(self) -> None:
+13 -9
View File
@@ -4,7 +4,7 @@ from messenger.identity.application.commands import CloseSession, OpenSession, R
from messenger.identity.application.ports import (
IdentityUnitOfWork,
PasswordHasher,
SessionDirectory,
SessionRevocations,
)
from messenger.identity.domain.events import SessionClosed, SessionOpened
from messenger.identity.domain.models import Session, User
@@ -62,15 +62,19 @@ class OpenSessionHandler:
class CloseSessionHandler:
__slots__ = ("_sessions", "_uow")
__slots__ = ("_revocations", "_uow")
def __init__(self, uow: IdentityUnitOfWork, sessions: SessionDirectory) -> None:
def __init__(self, uow: IdentityUnitOfWork, revocations: SessionRevocations) -> None:
self._uow = uow
self._sessions = sessions
self._revocations = revocations
async def __call__(self, command: CloseSession) -> None:
await self._sessions.invalidate(command.session)
async with self._uow.transaction():
await self._uow.sessions.remove(command.session.token_hash)
self._uow.record(SessionClosed(user_id=command.session.user_id))
await self._uow.commit()
await self._revocations.revoke(command.session)
try:
async with self._uow.transaction():
await self._uow.sessions.remove(command.session.token_hash)
self._uow.record(SessionClosed(user_id=command.session.user_id))
await self._uow.commit()
except BaseException:
await self._revocations.restore(command.session)
raise
+6 -1
View File
@@ -38,7 +38,12 @@ class UserDirectory(Protocol):
class SessionDirectory(Protocol):
async def find(self, token_hash: bytes, now: datetime) -> Session | None: ...
async def invalidate(self, session: Session) -> None: ...
@runtime_checkable
class SessionRevocations(Protocol):
async def revoke(self, session: Session) -> None: ...
async def restore(self, session: Session) -> None: ...
@runtime_checkable
+8 -3
View File
@@ -17,6 +17,7 @@ from messenger.identity.application.ports import (
IdentityUnitOfWork,
PasswordHasher,
SessionDirectory,
SessionRevocations,
UserDirectory,
)
from messenger.identity.application.queries import Authenticate, GetCurrentUser, VerifyCredentials
@@ -27,7 +28,7 @@ from messenger.identity.infrastructure.postgres import (
PostgresSessionDirectory,
PostgresUserDirectory,
)
from messenger.identity.infrastructure.valkey import CachedSessionDirectory
from messenger.identity.infrastructure.valkey import CachedSessionDirectory, NoRevocations
from messenger.shared.application.commands import Command
from messenger.shared.application.messagebus import CommandHandler
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
@@ -76,6 +77,10 @@ class IdentityProvider(Provider):
finally:
await client.aclose()
@provide(scope=Scope.APP)
def revocations(self, sessions: SessionDirectory) -> SessionRevocations:
return sessions if isinstance(sessions, CachedSessionDirectory) else NoRevocations()
@provide(scope=Scope.APP)
def verify(self, users: UserDirectory, hasher: PasswordHasher) -> VerifyCredentials:
return VerifyCredentials(users, hasher)
@@ -100,13 +105,13 @@ class IdentityProvider(Provider):
clock: Clock,
ids: IdGenerator,
hasher: PasswordHasher,
sessions: SessionDirectory,
revocations: SessionRevocations,
settings: IdentitySettings,
) -> IdentityCommandBus:
ttl = timedelta(seconds=settings.session_ttl_seconds)
handlers: dict[type[Command], CommandHandler] = {
RegisterUser: RegisterUserHandler(uow, clock, ids, hasher),
OpenSession: OpenSessionHandler(uow, clock, ttl),
CloseSession: CloseSessionHandler(uow, sessions),
CloseSession: CloseSessionHandler(uow, revocations),
}
return IdentityCommandBus(handlers, uow, publisher)
+121 -63
View File
@@ -1,85 +1,126 @@
from datetime import datetime
from uuid import UUID
from asyncpg import Record
from sqlalchemy import (
CheckConstraint,
DateTime,
ForeignKey,
Index,
LargeBinary,
UniqueConstraint,
delete,
literal,
select,
)
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.orm import Mapped, mapped_column
from messenger.identity.domain.errors import UsernameAlreadyTaken
from messenger.identity.domain.models import Session, User
from messenger.identity.domain.values import DisplayName, Username
from messenger.shared.infrastructure.postgres import Connection, Database, PostgresUnitOfWork
INSERT_USER = """
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (username) DO NOTHING
RETURNING true
"""
USER_BY_ID = """
SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1
"""
USER_BY_USERNAME = """
SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1
"""
INSERT_SESSION = """
WITH expired AS (DELETE FROM sessions WHERE user_id = $2 AND expires_at <= $3)
INSERT INTO sessions (token_hash, user_id, created_at, expires_at) VALUES ($1, $2, $3, $4)
"""
DELETE_SESSION = "DELETE FROM sessions WHERE token_hash = $1"
ACTIVE_SESSION = """
SELECT token_hash, user_id, created_at, expires_at
FROM sessions
WHERE token_hash = $1 AND expires_at > $2
"""
from messenger.shared.infrastructure.postgres import (
Base,
Database,
DatabaseSession,
PostgresUnitOfWork,
)
def user_of(row: Record | None) -> User | None:
class UserRow(Base):
__tablename__ = "users"
__table_args__ = (
UniqueConstraint("username", name="users_username_key"),
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="users_username_format"),
CheckConstraint(
"char_length(display_name) BETWEEN 1 AND 100", name="users_display_name_check"
),
CheckConstraint("password_hash LIKE '$argon2id$%'", name="users_password_hash_check"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
username: Mapped[str]
display_name: Mapped[str]
password_hash: Mapped[str]
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class SessionRow(Base):
__tablename__ = "sessions"
__table_args__ = (
CheckConstraint("octet_length(token_hash) = 32", name="sessions_token_hash_check"),
CheckConstraint("expires_at > created_at", name="sessions_check"),
Index("sessions_user_idx", "user_id"),
)
token_hash: Mapped[bytes] = mapped_column(LargeBinary, primary_key=True)
user_id: Mapped[UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
def user_of(row: UserRow | None) -> User | None:
if row is None:
return None
return User(
id=row["id"],
username=Username(row["username"]),
display_name=DisplayName(row["display_name"]),
created_at=row["created_at"],
password_hash=row["password_hash"],
id=row.id,
username=Username(row.username),
display_name=DisplayName(row.display_name),
created_at=row.created_at,
password_hash=row.password_hash,
)
class PostgresUserRepository:
__slots__ = ("_connection",)
__slots__ = ("_session",)
def __init__(self, connection: Connection) -> None:
self._connection = connection
def __init__(self, session: DatabaseSession) -> None:
self._session = session
async def add(self, user: User) -> None:
inserted = await self._connection.fetchval(
INSERT_USER,
user.id,
user.username.value,
user.display_name.value,
user.password_hash,
user.created_at,
statement = (
insert(UserRow)
.values(
id=user.id,
username=user.username.value,
display_name=user.display_name.value,
password_hash=user.password_hash,
created_at=user.created_at,
)
.on_conflict_do_nothing(index_elements=["username"])
.returning(literal(True))
)
inserted = (await self._session.execute(statement)).scalar_one_or_none()
if inserted is None:
raise UsernameAlreadyTaken("username уже занят")
class PostgresSessionRepository:
__slots__ = ("_connection",)
__slots__ = ("_session",)
def __init__(self, connection: Connection) -> None:
self._connection = connection
def __init__(self, session: DatabaseSession) -> None:
self._session = session
async def add(self, session: Session) -> None:
await self._connection.execute(
INSERT_SESSION,
session.token_hash,
session.user_id,
session.created_at,
session.expires_at,
expired = (
delete(SessionRow)
.where(
SessionRow.user_id == session.user_id, SessionRow.expires_at <= session.created_at
)
.cte("expired")
)
await self._session.execute(
insert(SessionRow)
.values(
token_hash=session.token_hash,
user_id=session.user_id,
created_at=session.created_at,
expires_at=session.expires_at,
)
.add_cte(expired)
)
async def remove(self, token_hash: bytes) -> None:
await self._connection.execute(DELETE_SESSION, token_hash)
await self._session.execute(delete(SessionRow).where(SessionRow.token_hash == token_hash))
class PostgresIdentityUnitOfWork(PostgresUnitOfWork):
@@ -87,11 +128,11 @@ class PostgresIdentityUnitOfWork(PostgresUnitOfWork):
@property
def users(self) -> PostgresUserRepository:
return PostgresUserRepository(self.connection)
return PostgresUserRepository(self.session)
@property
def sessions(self) -> PostgresSessionRepository:
return PostgresSessionRepository(self.connection)
return PostgresSessionRepository(self.session)
class PostgresUserDirectory:
@@ -101,12 +142,18 @@ class PostgresUserDirectory:
self._database = database
async def find(self, user_id: UUID) -> User | None:
async with self._database.connection() as connection:
return user_of(await connection.fetchrow(USER_BY_ID, user_id))
async with self._database.session() as session:
row = (
await session.execute(select(UserRow).where(UserRow.id == user_id))
).scalar_one_or_none()
return user_of(row)
async def find_by_username(self, username: str) -> User | None:
async with self._database.connection() as connection:
return user_of(await connection.fetchrow(USER_BY_USERNAME, username))
async with self._database.session() as session:
row = (
await session.execute(select(UserRow).where(UserRow.username == username))
).scalar_one_or_none()
return user_of(row)
class PostgresSessionDirectory:
@@ -116,9 +163,20 @@ class PostgresSessionDirectory:
self._database = database
async def find(self, token_hash: bytes, now: datetime) -> Session | None:
async with self._database.connection() as connection:
row = await connection.fetchrow(ACTIVE_SESSION, token_hash, now)
return None if row is None else Session(**dict(row))
async def invalidate(self, session: Session) -> None:
return None
async with self._database.session() as session:
row = (
await session.execute(
select(SessionRow).where(
SessionRow.token_hash == token_hash,
SessionRow.expires_at > now,
)
)
).scalar_one_or_none()
if row is None:
return None
return Session(
token_hash=row.token_hash,
user_id=row.user_id,
created_at=row.created_at,
expires_at=row.expires_at,
)
@@ -37,6 +37,14 @@ def decode(token_hash: bytes, value: str) -> Session:
)
class NoRevocations:
async def revoke(self, session: Session) -> None:
return None
async def restore(self, session: Session) -> None:
return None
class CachedSessionDirectory:
__slots__ = ("_client", "_cooldown", "_limit", "_logger", "_origin", "_resume_at")
@@ -70,7 +78,7 @@ class CachedSessionDirectory:
await self._quietly(self._client.set(key, encode(session), pxat=until, nx=True))
return session
async def invalidate(self, session: Session) -> None:
async def revoke(self, session: Session) -> None:
if not self._closed():
raise Unavailable("кэш сессий недоступен")
try:
@@ -81,6 +89,9 @@ class CachedSessionDirectory:
self._trip(error)
raise Unavailable("кэш сессий недоступен") from error
async def restore(self, session: Session) -> None:
await self._quietly(self._client.delete(key_of(session.token_hash)))
async def _read(self, key: str) -> str | None:
if not self._closed():
return None
+2 -1
View File
@@ -42,7 +42,7 @@ from messenger.shared.entrypoints.telemetry import instrument_http
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
from messenger.shared.infrastructure.telemetry import start_telemetry
TRANSPORT_FIELDS = frozenset({"dropped_total", "queue", "instance"})
TRANSPORT_FIELDS = frozenset({"dropped_total"})
LOG_FIELDS = (
PLATFORM_FIELDS | TRANSPORT_FIELDS | IDENTITY_FIELDS | MESSAGING_FIELDS | ATTACHMENT_FIELDS
@@ -153,6 +153,7 @@ def create_app(
guard=guard,
instance=platform.instance_id,
limits=((UPLOAD_PATH, attachments.max_upload_bytes),),
min_rate=transport.min_transfer_rate_bytes,
)
errors.install(app, logger, ERROR_CODES, challenge_for, platform.instance_id)
app.include_router(health.router)
+31 -29
View File
@@ -1,9 +1,14 @@
from datetime import timedelta
from messenger.messaging.application.commands import CreateChat, PostMessage
from messenger.messaging.application.ports import AttachmentStates, MessagingUnitOfWork
from messenger.messaging.domain.errors import AttachmentUnavailable, ChatNotFound
from messenger.messaging.domain.models import Chat, Message, Receipt
from messenger.messaging.application.ports import ChatDirectory, MessageLog, MessagingUnitOfWork
from messenger.messaging.domain.errors import (
ChatNotFound,
InvalidAttachmentList,
InvalidMessageText,
)
from messenger.messaging.domain.models import Chat, Message, attachment_list
from messenger.messaging.domain.values import MessageText
from messenger.shared.application.ports import Clock, IdGenerator
@@ -30,44 +35,41 @@ class CreateChatHandler:
class PostMessageHandler:
__slots__ = ("_attachments", "_clock", "_ids", "_ttl", "_uow")
__slots__ = ("_chats", "_clock", "_ids", "_log", "_ttl", "_uow")
def __init__(
self,
uow: MessagingUnitOfWork,
attachments: AttachmentStates,
log: MessageLog,
chats: ChatDirectory,
clock: Clock,
ids: IdGenerator,
*,
ttl: timedelta,
) -> None:
self._uow = uow
self._attachments = attachments
self._log = log
self._chats = chats
self._clock = clock
self._ids = ids
self._ttl = ttl
async def __call__(self, command: PostMessage) -> Message:
async with self._uow.transaction():
chat = await self._uow.chats.get(command.chat_id)
if chat is None:
raise ChatNotFound("чат не найден")
now = self._clock.now()
message = chat.post(
message_id=self._ids.new_id(),
sender_id=command.sender_id,
text=command.text,
now=now,
attachment_ids=command.attachment_ids,
)
known = await self._uow.chats.receipt(
chat.id, command.sender_id, command.key, now - self._ttl
)
if known is not None:
return known.replay(command.fingerprint)
if message.attachment_ids and await self._attachments.unusable(
chat.id, message.attachment_ids
):
raise AttachmentUnavailable("вложение не обработано и не может быть отправлено")
await self._uow.chats.append(Receipt(command.key, command.fingerprint, message))
await self._uow.commit()
try:
text = MessageText(command.text)
attachments = attachment_list(text, command.attachment_ids)
except InvalidMessageText, InvalidAttachmentList:
if await self._chats.chat_for_member(command.chat_id, command.sender_id) is None:
raise ChatNotFound("чат не найден") from None
raise
now = self._clock.now()
draft = Message(
self._ids.new_id(), command.chat_id, command.sender_id, text, now, attachments
)
receipt = await self._log.post(draft, command.key, command.fingerprint, now - self._ttl)
if receipt is None:
raise ChatNotFound("чат не найден")
message = receipt.replay(command.fingerprint)
if message.id == draft.id:
self._uow.record(message.posted())
return message
+4 -5
View File
@@ -19,14 +19,13 @@ class CursorCodec(Protocol):
class ChatRepository(Protocol):
async def add(self, chat: Chat) -> None: ...
async def get(self, chat_id: UUID) -> Chat | None: ...
async def receipt(
self, chat_id: UUID, sender_id: UUID, key: str, since: datetime
@runtime_checkable
class MessageLog(Protocol):
async def post(
self, message: Message, key: str, fingerprint: bytes, since: datetime
) -> Receipt | None: ...
async def append(self, receipt: Receipt) -> None: ...
@runtime_checkable
class MessagingUnitOfWork(UnitOfWork, Protocol):
@@ -43,7 +43,7 @@ class ListChats:
async def __call__(self, user_id: UUID, *, limit: int, cursor: str | None) -> Page[Chat]:
scope = chats_scope(user_id)
after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None
after: HistoryKey | None = None if cursor is None else self._cursors.decode(scope, cursor)
items, has_more = await self._chats.chats_of(user_id, after=after, limit=limit)
next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None
return Page(items=items, next_cursor=next_cursor)
@@ -67,7 +67,7 @@ class ReadHistory:
if await self._chats.chat_for_member(chat_id, user_id) is None:
raise ChatNotFound("чат не найден")
scope = messages_scope(chat_id)
after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None
after: HistoryKey | None = None if cursor is None else self._cursors.decode(scope, cursor)
items, has_more = await self._chats.history(chat_id, after=after, limit=limit)
next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None
return Page(items=items, next_cursor=next_cursor)

Some files were not shown because too many files have changed in this diff Show More