forked from mtusi2026/mtusi2026lab5
ЛР 5 — БВТ2301 — Курлова — на 5
RabbitMQ + transactional outbox, отдельный worker (thumbnail/crop/WebP), lease и heartbeat, retry с backoff+jitter, DLQ и replay, admission control; тесты отказов, трек 1 и отчёт.
This commit is contained in:
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator CLI for attachment jobs (run inside an API container: docker compose exec api-1 python scripts/jobs.py ...).
|
||||
|
||||
failed registry of failed jobs (the "DLQ" of business failures)
|
||||
show <attachment_id> job state + audit trail
|
||||
replay <attachment_id> --actor NAME explicit failed -> queued; refused (no-op) for ready/queued/processing
|
||||
stats counts by state and outbox backlog
|
||||
Poison messages that are not valid jobs are in the RabbitMQ queue attachments.dlq (see REPORT.md).
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from starlette.concurrency import run_in_threadpool # noqa: E402
|
||||
|
||||
from app import jobs # noqa: E402
|
||||
from app.cursor import fmt # noqa: E402
|
||||
from app.db import Database # noqa: E402
|
||||
from app.s3 import S3Storage # noqa: E402
|
||||
|
||||
|
||||
def out(value) -> None:
|
||||
print(json.dumps(value, ensure_ascii=False, indent=2, default=str))
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
sub.add_parser("failed")
|
||||
sub.add_parser("stats")
|
||||
show = sub.add_parser("show")
|
||||
show.add_argument("attachment_id")
|
||||
rp = sub.add_parser("replay")
|
||||
rp.add_argument("attachment_id")
|
||||
rp.add_argument("--actor", required=True, help="who requests the replay (written to job_audit)")
|
||||
args = parser.parse_args()
|
||||
|
||||
db = Database()
|
||||
await db.open()
|
||||
try:
|
||||
if args.cmd == "failed":
|
||||
async with db.connection() as conn:
|
||||
out(await jobs.failed_jobs(conn))
|
||||
elif args.cmd == "stats":
|
||||
async with db.connection() as conn:
|
||||
cur = await conn.execute("SELECT state, count(*) AS n FROM attachment_jobs GROUP BY state ORDER BY state")
|
||||
states = {r["state"]: r["n"] for r in await cur.fetchall()}
|
||||
cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL")
|
||||
backlog = (await cur.fetchone())["n"]
|
||||
out({"jobs": states, "outbox_unpublished": backlog})
|
||||
elif args.cmd == "show":
|
||||
async with db.connection() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT j.id, j.state, j.attempt, j.max_attempts, j.error_code, j.lease_owner, j.lease_until,"
|
||||
" a.status AS attachment_status FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
|
||||
" WHERE j.attachment_id = %s", (args.attachment_id,))
|
||||
job = await cur.fetchone()
|
||||
cur = await conn.execute(
|
||||
"SELECT action, actor, attempt, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id",
|
||||
(args.attachment_id,))
|
||||
trail = [{**r, "created_at": fmt(r["created_at"])} for r in await cur.fetchall()]
|
||||
cur = await conn.execute(
|
||||
"SELECT o.id, o.payload->>'attempt' AS attempt, o.available_at, o.published_at, o.publish_attempts"
|
||||
" FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = %s ORDER BY o.id",
|
||||
(args.attachment_id,))
|
||||
outbox = await cur.fetchall()
|
||||
out({"job": job, "audit": trail, "outbox": outbox})
|
||||
elif args.cmd == "replay":
|
||||
s3 = S3Storage()
|
||||
await run_in_threadpool(s3.open)
|
||||
|
||||
async def exists(key):
|
||||
return await run_in_threadpool(s3.exists, key)
|
||||
|
||||
result = await db.in_transaction(
|
||||
lambda conn: jobs.replay(conn, args.attachment_id, "operator:" + args.actor, exists))
|
||||
out(result)
|
||||
return 0 if result["result"] == "queued" else 3
|
||||
finally:
|
||||
await db.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run."""
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
MIGRATIONS = ROOT / "migrations"
|
||||
LOCK_ID = 2026_0004
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
path = os.environ.get("DATABASE_URL_FILE")
|
||||
if path:
|
||||
return pathlib.Path(path).read_text(encoding="utf-8").strip()
|
||||
url = os.environ.get("DATABASE_URL", "").strip()
|
||||
if not url:
|
||||
sys.exit("DATABASE_URL or DATABASE_URL_FILE is required")
|
||||
return url
|
||||
|
||||
|
||||
def main() -> None:
|
||||
files = sorted(MIGRATIONS.glob("*.sql"))
|
||||
if not files:
|
||||
sys.exit("no migration files found in %s" % MIGRATIONS)
|
||||
with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version text PRIMARY KEY,"
|
||||
" checksum text NOT NULL,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
|
||||
rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()
|
||||
applied = {row[0]: row[1] for row in rows}
|
||||
for path in files:
|
||||
version = path.stem
|
||||
body = path.read_text(encoding="utf-8")
|
||||
checksum = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
if version in applied:
|
||||
if applied[version] != checksum:
|
||||
sys.exit("migration %s changed after it was applied" % version)
|
||||
print("skip %s" % version)
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(body)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
|
||||
(version, checksum),
|
||||
)
|
||||
print("apply %s" % version)
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
print("migrations complete: %d file(s)" % len(files))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything."""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from starlette.concurrency import run_in_threadpool # noqa: E402
|
||||
|
||||
from app import config, reconcile # noqa: E402
|
||||
from app.db import Database # noqa: E402
|
||||
from app.s3 import S3Storage # noqa: E402
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports")
|
||||
parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS)
|
||||
args = parser.parse_args()
|
||||
db, s3 = Database(), S3Storage()
|
||||
await db.open()
|
||||
await run_in_threadpool(s3.open)
|
||||
try:
|
||||
print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True))
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Restart API, LB, worker and every store (postgres, valkey, minio, rabbitmq) keeping all volumes.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart -t 10 postgres valkey minio rabbitmq
|
||||
docker compose restart -t 25 worker api-1 api-2 nginx
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop -t 3 "$TARGET_INSTANCE_ID"
|
||||
Reference in New Issue
Block a user