"""Операторские команды pipeline вложений (запускаются в контейнере dispatcher). docker compose exec dispatcher python ops.py status docker compose exec dispatcher python ops.py failed [--limit N] docker compose exec dispatcher python ops.py events docker compose exec dispatcher python ops.py replay [--reason TEXT] docker compose exec dispatcher python ops.py dlq [--limit N] docker compose exec dispatcher python ops.py dlq-replay [--limit N] Реестр failed — таблица attachment_jobs (status='failed') + журнал job_events: там error_code, число попыток, время. DLQ (attachments.process.v1.dlq) получает только сообщения, которые нельзя обработать как сообщение: не по схеме, неизвестный job_id, превышен x-delivery-limit. replay безопасен: переводит в queued только job со статусом failed и исходником в staging; ready, queued и processing не трогает («повторить» не значит исполнить дубль успешной job). Новая попытка идёт через outbox тем же job_id, эффект защищён ключом (attachment_id, pipeline_version, variant). dlq-replay разбирает DLQ: для валидного сообщения смотрит строку job в PostgreSQL (источник истины): failed -> replay; ready -> сообщение удаляется как no-op; queued/processing -> удаляется, работу ведёт outbox/аренда (при необходимости dispatcher опубликует заново). Невалидные сообщения остаются в DLQ для разбора. Вывод — JSON без секретов и ключей S3. """ import argparse import json import sys from uuid import UUID import psycopg from psycopg.rows import dict_row import broker import pipeline import settings ACTOR = "ops@" + settings.INSTANCE_ID def connect_db(): return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, options="-c TimeZone=UTC") def text(value): if isinstance(value, (UUID,)): return str(value) if hasattr(value, "isoformat"): return pipeline.utc_text(value) return value def rows_json(rows): return [{key: text(value) for key, value in row.items()} for row in rows] def cmd_status(conn, _args): jobs = {r["status"]: r["n"] for r in conn.execute( "SELECT status, count(*) AS n FROM attachment_jobs GROUP BY status").fetchall()} outbox = conn.execute( """SELECT count(*) FILTER (WHERE published_at IS NULL) AS pending, count(*) FILTER (WHERE published_at IS NULL AND available_at <= now()) AS due, coalesce(max(publish_failures) FILTER (WHERE published_at IS NULL), 0) AS max_publish_failures, extract(epoch FROM now() - min(available_at) FILTER (WHERE published_at IS NULL AND available_at <= now())) AS oldest_due_s FROM outbox""").fetchone() leases = conn.execute( """SELECT count(*) FILTER (WHERE lease_until >= now()) AS active, count(*) FILTER (WHERE lease_until < now()) AS expired FROM attachment_jobs WHERE status = 'processing'""").fetchone() staging = conn.execute("SELECT count(*) AS n FROM attachments WHERE source_key IS NOT NULL").fetchone()["n"] report = {"jobs": jobs, "outbox": {k: (round(float(v), 1) if k == "oldest_due_s" and v is not None else v) for k, v in outbox.items()}, "leases": leases, "attachments_with_staging_source": staging} try: connection = broker.connect() try: report["broker"] = broker.check_topology(connection.channel()) finally: connection.close() except broker.BrokerError as exc: report["broker"] = {"error": type(exc).__name__} return report def cmd_failed(conn, args): return rows_json(conn.execute( """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code, j.last_error, j.replays, j.finished_at, (a.source_key IS NOT NULL) AS replayable FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.status = 'failed' ORDER BY j.finished_at DESC LIMIT %s""", (args.limit,)).fetchall()) def cmd_events(conn, args): return rows_json(conn.execute( "SELECT id, attempt, event, detail, actor, created_at FROM job_events WHERE job_id = %s ORDER BY id", (args.job_id,)).fetchall()) def cmd_replay(conn, args): try: return {"replayed": pipeline.replay(conn, args.job_id, ACTOR, args.reason)} except pipeline.ReplayRefused as exc: return {"refused": str(exc), "job_id": str(args.job_id)} def dlq_messages(limit, handler): """Читает до limit сообщений DLQ (basic_get, ручной ack). handler(message|None) -> 'ack' | 'keep'.""" connection = broker.connect() results = [] try: channel = connection.channel() keep = [] for _ in range(limit): method, properties, body = channel.basic_get(pipeline.DLQ, auto_ack=False) if method is None: break death = (properties.headers or {}).get("x-death") or [{}] reason = death[0].get("reason") reason = reason.decode() if isinstance(reason, bytes) else reason try: message = pipeline.parse_message(body) except pipeline.InvalidMessage as exc: message, problem = None, str(exc) else: problem = None verdict, note = handler(message) results.append({"job_id": message["job_id"] if message else None, "invalid": problem, "dead_letter_reason": reason, "action": verdict, "note": note}) if verdict == "ack": channel.basic_ack(method.delivery_tag) else: keep.append(method.delivery_tag) for tag in keep: # вернуть непрочитанными в DLQ channel.basic_nack(tag, requeue=True) finally: connection.close() return results def cmd_dlq(_conn, args): return dlq_messages(args.limit, lambda message: ("keep", "peek")) def cmd_dlq_replay(conn, args): def handle(message): if message is None: return "keep", "invalid message is kept for analysis" job = conn.execute("SELECT status FROM attachment_jobs WHERE job_id = %s", (UUID(message["job_id"]),)).fetchone() if job is None: return "keep", "unknown job" if job["status"] == "failed": try: pipeline.replay(conn, UUID(message["job_id"]), ACTOR, "dlq replay") return "ack", "replayed" except pipeline.ReplayRefused as exc: return "keep", str(exc) if job["status"] == "ready": return "ack", "already ready, no-op" # queued/processing: работу ведёт PostgreSQL; если сообщения в полёте нет — опубликовать заново conn.execute("UPDATE attachment_jobs SET updated_at = now() - interval '1 day' WHERE job_id = %s " "AND status = 'queued'", (UUID(message["job_id"]),)) return "ack", f"job is {job['status']}, handled by outbox/lease recovery" return dlq_messages(args.limit, handle) def main(): parser = argparse.ArgumentParser(description="Attachment pipeline operator commands") sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("status") p = sub.add_parser("failed") p.add_argument("--limit", type=int, default=50) p = sub.add_parser("events") p.add_argument("job_id", type=UUID) p = sub.add_parser("replay") p.add_argument("job_id", type=UUID) p.add_argument("--reason", default="operator replay") p = sub.add_parser("dlq") p.add_argument("--limit", type=int, default=20) p = sub.add_parser("dlq-replay") p.add_argument("--limit", type=int, default=100) args = parser.parse_args() if args.command == "replay" and (len(args.reason) > 100 or not args.reason.isprintable()): parser.error("--reason: up to 100 printable characters") commands = {"status": cmd_status, "failed": cmd_failed, "events": cmd_events, "replay": cmd_replay, "dlq": cmd_dlq, "dlq-replay": cmd_dlq_replay} with connect_db() as conn: result = commands[args.command](conn, args) print(json.dumps(result, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": sys.exit(main())