ЛР 5 — БВТ2301 — Курлова — на 5

RabbitMQ + transactional outbox, отдельный worker (thumbnail/crop/WebP), lease и heartbeat, retry с backoff+jitter, DLQ и replay, admission control; тесты отказов, трек 1 и отчёт.
This commit is contained in:
alenchha
2026-11-06 12:10:16 +03:00
parent 276dbdad8a
commit 633fb96920
57 changed files with 5194 additions and 22 deletions
+88
View File
@@ -0,0 +1,88 @@
#!/usr/bin/env python3
"""Operator CLI for attachment jobs (run inside an API container: docker compose exec api-1 python scripts/jobs.py ...).
failed registry of failed jobs (the "DLQ" of business failures)
show <attachment_id> job state + audit trail
replay <attachment_id> --actor NAME explicit failed -> queued; refused (no-op) for ready/queued/processing
stats counts by state and outbox backlog
Poison messages that are not valid jobs are in the RabbitMQ queue attachments.dlq (see REPORT.md).
"""
import argparse
import asyncio
import json
import sys
sys.path.insert(0, "/app")
from starlette.concurrency import run_in_threadpool # noqa: E402
from app import jobs # noqa: E402
from app.cursor import fmt # noqa: E402
from app.db import Database # noqa: E402
from app.s3 import S3Storage # noqa: E402
def out(value) -> None:
print(json.dumps(value, ensure_ascii=False, indent=2, default=str))
async def main() -> int:
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("failed")
sub.add_parser("stats")
show = sub.add_parser("show")
show.add_argument("attachment_id")
rp = sub.add_parser("replay")
rp.add_argument("attachment_id")
rp.add_argument("--actor", required=True, help="who requests the replay (written to job_audit)")
args = parser.parse_args()
db = Database()
await db.open()
try:
if args.cmd == "failed":
async with db.connection() as conn:
out(await jobs.failed_jobs(conn))
elif args.cmd == "stats":
async with db.connection() as conn:
cur = await conn.execute("SELECT state, count(*) AS n FROM attachment_jobs GROUP BY state ORDER BY state")
states = {r["state"]: r["n"] for r in await cur.fetchall()}
cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL")
backlog = (await cur.fetchone())["n"]
out({"jobs": states, "outbox_unpublished": backlog})
elif args.cmd == "show":
async with db.connection() as conn:
cur = await conn.execute(
"SELECT j.id, j.state, j.attempt, j.max_attempts, j.error_code, j.lease_owner, j.lease_until,"
" a.status AS attachment_status FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
" WHERE j.attachment_id = %s", (args.attachment_id,))
job = await cur.fetchone()
cur = await conn.execute(
"SELECT action, actor, attempt, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id",
(args.attachment_id,))
trail = [{**r, "created_at": fmt(r["created_at"])} for r in await cur.fetchall()]
cur = await conn.execute(
"SELECT o.id, o.payload->>'attempt' AS attempt, o.available_at, o.published_at, o.publish_attempts"
" FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = %s ORDER BY o.id",
(args.attachment_id,))
outbox = await cur.fetchall()
out({"job": job, "audit": trail, "outbox": outbox})
elif args.cmd == "replay":
s3 = S3Storage()
await run_in_threadpool(s3.open)
async def exists(key):
return await run_in_threadpool(s3.exists, key)
result = await db.in_transaction(
lambda conn: jobs.replay(conn, args.attachment_id, "operator:" + args.actor, exists))
out(result)
return 0 if result["result"] == "queued" else 3
finally:
await db.close()
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run."""
import hashlib
import os
import pathlib
import sys
import psycopg
ROOT = pathlib.Path(__file__).resolve().parents[1]
MIGRATIONS = ROOT / "migrations"
LOCK_ID = 2026_0004
def database_url() -> str:
path = os.environ.get("DATABASE_URL_FILE")
if path:
return pathlib.Path(path).read_text(encoding="utf-8").strip()
url = os.environ.get("DATABASE_URL", "").strip()
if not url:
sys.exit("DATABASE_URL or DATABASE_URL_FILE is required")
return url
def main() -> None:
files = sorted(MIGRATIONS.glob("*.sql"))
if not files:
sys.exit("no migration files found in %s" % MIGRATIONS)
with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version text PRIMARY KEY,"
" checksum text NOT NULL,"
" applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()
applied = {row[0]: row[1] for row in rows}
for path in files:
version = path.stem
body = path.read_text(encoding="utf-8")
checksum = hashlib.sha256(body.encode("utf-8")).hexdigest()
if version in applied:
if applied[version] != checksum:
sys.exit("migration %s changed after it was applied" % version)
print("skip %s" % version)
continue
with conn.transaction():
conn.execute(body)
conn.execute(
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
(version, checksum),
)
print("apply %s" % version)
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
print("migrations complete: %d file(s)" % len(files))
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything."""
import argparse
import asyncio
import json
import sys
sys.path.insert(0, "/app")
from starlette.concurrency import run_in_threadpool # noqa: E402
from app import config, reconcile # noqa: E402
from app.db import Database # noqa: E402
from app.s3 import S3Storage # noqa: E402
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports")
parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS)
args = parser.parse_args()
db, s3 = Database(), S3Storage()
await db.open()
await run_in_threadpool(s3.open)
try:
print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True))
finally:
await db.close()
if __name__ == "__main__":
asyncio.run(main())
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
# Restart API, LB, worker and every store (postgres, valkey, minio, rabbitmq) keeping all volumes.
set -eu
cd "$(dirname "$0")/.."
docker compose restart -t 10 postgres valkey minio rabbitmq
docker compose restart -t 25 worker api-1 api-2 nginx
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
docker compose stop -t 3 "$TARGET_INSTANCE_ID"