Files
mtusi2026lab6/ops.py
T

202 lines
9.3 KiB
Python

"""Операторские команды pipeline вложений (запускаются в контейнере dispatcher).
docker compose exec dispatcher python ops.py status
docker compose exec dispatcher python ops.py failed [--limit N]
docker compose exec dispatcher python ops.py events <job_id>
docker compose exec dispatcher python ops.py replay <job_id> [--reason TEXT]
docker compose exec dispatcher python ops.py dlq [--limit N]
docker compose exec dispatcher python ops.py dlq-replay [--limit N]
Реестр failed — таблица attachment_jobs (status='failed') + журнал job_events: там error_code,
число попыток, время. DLQ (attachments.process.v1.dlq) получает только сообщения, которые нельзя
обработать как сообщение: не по схеме, неизвестный job_id, превышен x-delivery-limit.
replay безопасен: переводит в queued только job со статусом failed и исходником в staging; ready,
queued и processing не трогает («повторить» не значит исполнить дубль успешной job). Новая попытка
идёт через outbox тем же job_id, эффект защищён ключом (attachment_id, pipeline_version, variant).
dlq-replay разбирает DLQ: для валидного сообщения смотрит строку job в PostgreSQL (источник истины):
failed -> replay; ready -> сообщение удаляется как no-op; queued/processing -> удаляется, работу
ведёт outbox/аренда (при необходимости dispatcher опубликует заново). Невалидные сообщения
остаются в DLQ для разбора. Вывод — JSON без секретов и ключей S3.
"""
import argparse
import json
import sys
from uuid import UUID
import psycopg
from psycopg.rows import dict_row
import settings
import telemetry
telemetry.setup("messenger-ops")
import broker # noqa: E402
import pipeline # noqa: E402
ACTOR = "ops@" + settings.INSTANCE_ID
def connect_db():
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
options="-c TimeZone=UTC")
def text(value):
if isinstance(value, (UUID,)):
return str(value)
if hasattr(value, "isoformat"):
return pipeline.utc_text(value)
return value
def rows_json(rows):
return [{key: text(value) for key, value in row.items()} for row in rows]
def cmd_status(conn, _args):
jobs = {r["status"]: r["n"] for r in conn.execute(
"SELECT status, count(*) AS n FROM attachment_jobs GROUP BY status").fetchall()}
outbox = conn.execute(
"""SELECT count(*) FILTER (WHERE published_at IS NULL) AS pending,
count(*) FILTER (WHERE published_at IS NULL AND available_at <= now()) AS due,
coalesce(max(publish_failures) FILTER (WHERE published_at IS NULL), 0) AS max_publish_failures,
extract(epoch FROM now() - min(available_at) FILTER (WHERE published_at IS NULL
AND available_at <= now())) AS oldest_due_s
FROM outbox""").fetchone()
leases = conn.execute(
"""SELECT count(*) FILTER (WHERE lease_until >= now()) AS active,
count(*) FILTER (WHERE lease_until < now()) AS expired
FROM attachment_jobs WHERE status = 'processing'""").fetchone()
staging = conn.execute("SELECT count(*) AS n FROM attachments WHERE source_key IS NOT NULL").fetchone()["n"]
report = {"jobs": jobs, "outbox": {k: (round(float(v), 1) if k == "oldest_due_s" and v is not None else v)
for k, v in outbox.items()},
"leases": leases, "attachments_with_staging_source": staging}
try:
connection = broker.connect()
try:
report["broker"] = broker.check_topology(connection.channel())
finally:
connection.close()
except broker.BrokerError as exc:
report["broker"] = {"error": type(exc).__name__}
return report
def cmd_failed(conn, args):
return rows_json(conn.execute(
"""SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code,
j.last_error, j.replays, j.finished_at, (a.source_key IS NOT NULL) AS replayable
FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.status = 'failed' ORDER BY j.finished_at DESC LIMIT %s""", (args.limit,)).fetchall())
def cmd_events(conn, args):
return rows_json(conn.execute(
"SELECT id, attempt, event, detail, actor, created_at FROM job_events WHERE job_id = %s ORDER BY id",
(args.job_id,)).fetchall())
def cmd_replay(conn, args):
try:
return {"replayed": pipeline.replay(conn, args.job_id, ACTOR, args.reason)}
except pipeline.ReplayRefused as exc:
return {"refused": str(exc), "job_id": str(args.job_id)}
def dlq_messages(limit, handler):
"""Читает до limit сообщений DLQ (basic_get, ручной ack). handler(message|None) -> 'ack' | 'keep'."""
connection = broker.connect()
results = []
try:
channel = connection.channel()
keep = []
for _ in range(limit):
method, properties, body = channel.basic_get(pipeline.DLQ, auto_ack=False)
if method is None:
break
death = (properties.headers or {}).get("x-death") or [{}]
reason = death[0].get("reason")
reason = reason.decode() if isinstance(reason, bytes) else reason
try:
message = pipeline.parse_message(body)
except pipeline.InvalidMessage as exc:
message, problem = None, str(exc)
else:
problem = None
verdict, note = handler(message)
results.append({"job_id": message["job_id"] if message else None, "invalid": problem,
"dead_letter_reason": reason, "action": verdict, "note": note})
if verdict == "ack":
channel.basic_ack(method.delivery_tag)
else:
keep.append(method.delivery_tag)
for tag in keep: # вернуть непрочитанными в DLQ
channel.basic_nack(tag, requeue=True)
finally:
connection.close()
return results
def cmd_dlq(_conn, args):
return dlq_messages(args.limit, lambda message: ("keep", "peek"))
def cmd_dlq_replay(conn, args):
def handle(message):
if message is None:
return "keep", "invalid message is kept for analysis"
job = conn.execute("SELECT status FROM attachment_jobs WHERE job_id = %s",
(UUID(message["job_id"]),)).fetchone()
if job is None:
return "keep", "unknown job"
if job["status"] == "failed":
try:
pipeline.replay(conn, UUID(message["job_id"]), ACTOR, "dlq replay")
return "ack", "replayed"
except pipeline.ReplayRefused as exc:
return "keep", str(exc)
if job["status"] == "ready":
return "ack", "already ready, no-op"
# queued/processing: работу ведёт PostgreSQL; если сообщения в полёте нет — опубликовать заново
conn.execute("UPDATE attachment_jobs SET updated_at = now() - interval '1 day' WHERE job_id = %s "
"AND status = 'queued'", (UUID(message["job_id"]),))
return "ack", f"job is {job['status']}, handled by outbox/lease recovery"
return dlq_messages(args.limit, handle)
def main():
parser = argparse.ArgumentParser(description="Attachment pipeline operator commands")
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("status")
p = sub.add_parser("failed")
p.add_argument("--limit", type=int, default=50)
p = sub.add_parser("events")
p.add_argument("job_id", type=UUID)
p = sub.add_parser("replay")
p.add_argument("job_id", type=UUID)
p.add_argument("--reason", default="operator replay")
p = sub.add_parser("dlq")
p.add_argument("--limit", type=int, default=20)
p = sub.add_parser("dlq-replay")
p.add_argument("--limit", type=int, default=100)
args = parser.parse_args()
if args.command == "replay" and (len(args.reason) > 100 or not args.reason.isprintable()):
parser.error("--reason: up to 100 printable characters")
commands = {"status": cmd_status, "failed": cmd_failed, "events": cmd_events, "replay": cmd_replay,
"dlq": cmd_dlq, "dlq-replay": cmd_dlq_replay}
# Действие оператора — отдельная трасса; replay ссылается (link) на трассу исходной загрузки.
with telemetry.internal_span(f"ops {args.command}", {"messenger.ops.command": args.command}):
with connect_db() as conn:
result = commands[args.command](conn, args)
telemetry.shutdown()
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
sys.exit(main())