forked from mtusi2026/mtusi2026lab7
204 lines
9.4 KiB
Python
204 lines
9.4 KiB
Python
"""Операторские команды pipeline вложений (запускаются в контейнере dispatcher).
|
||
|
||
docker compose exec dispatcher python ops.py status
|
||
docker compose exec dispatcher python ops.py failed [--limit N]
|
||
docker compose exec dispatcher python ops.py events <job_id>
|
||
docker compose exec dispatcher python ops.py replay <job_id> [--reason TEXT]
|
||
docker compose exec dispatcher python ops.py dlq [--limit N]
|
||
docker compose exec dispatcher python ops.py dlq-replay [--limit N]
|
||
|
||
Реестр failed — таблица attachment_jobs (status='failed') + журнал job_events: там error_code,
|
||
число попыток, время. DLQ (attachments.process.v1.dlq) получает только сообщения, которые нельзя
|
||
обработать как сообщение: не по схеме, неизвестный job_id, превышен x-delivery-limit.
|
||
|
||
replay безопасен: переводит в queued только job со статусом failed и исходником в staging; ready,
|
||
queued и processing не трогает («повторить» не значит исполнить дубль успешной job). Новая попытка
|
||
идёт через outbox тем же job_id, эффект защищён ключом (attachment_id, pipeline_version, variant).
|
||
|
||
dlq-replay разбирает DLQ: для валидного сообщения смотрит строку job в PostgreSQL (источник истины):
|
||
failed -> replay; ready -> сообщение удаляется как no-op; queued/processing -> удаляется, работу
|
||
ведёт outbox/аренда (при необходимости dispatcher опубликует заново). Невалидные сообщения
|
||
остаются в DLQ для разбора. Вывод — JSON без секретов и ключей S3.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import sys
|
||
from uuid import UUID
|
||
|
||
import psycopg
|
||
from psycopg.rows import dict_row
|
||
|
||
import dbconn
|
||
import secretstore
|
||
import settings
|
||
import telemetry
|
||
|
||
telemetry.setup("messenger-ops")
|
||
|
||
import broker # noqa: E402
|
||
import pipeline # noqa: E402
|
||
|
||
ACTOR = "ops@" + settings.INSTANCE_ID
|
||
|
||
|
||
def connect_db():
|
||
# lab7: та же identity, что у dispatcher (ops.py запускается в его контейнере)
|
||
secretstore.store.start()
|
||
return dbconn.connect(autocommit=True, row_factory=dict_row, options="-c TimeZone=UTC")
|
||
|
||
|
||
def text(value):
|
||
if isinstance(value, (UUID,)):
|
||
return str(value)
|
||
if hasattr(value, "isoformat"):
|
||
return pipeline.utc_text(value)
|
||
return value
|
||
|
||
|
||
def rows_json(rows):
|
||
return [{key: text(value) for key, value in row.items()} for row in rows]
|
||
|
||
|
||
def cmd_status(conn, _args):
|
||
jobs = {r["status"]: r["n"] for r in conn.execute(
|
||
"SELECT status, count(*) AS n FROM attachment_jobs GROUP BY status").fetchall()}
|
||
outbox = conn.execute(
|
||
"""SELECT count(*) FILTER (WHERE published_at IS NULL) AS pending,
|
||
count(*) FILTER (WHERE published_at IS NULL AND available_at <= now()) AS due,
|
||
coalesce(max(publish_failures) FILTER (WHERE published_at IS NULL), 0) AS max_publish_failures,
|
||
extract(epoch FROM now() - min(available_at) FILTER (WHERE published_at IS NULL
|
||
AND available_at <= now())) AS oldest_due_s
|
||
FROM outbox""").fetchone()
|
||
leases = conn.execute(
|
||
"""SELECT count(*) FILTER (WHERE lease_until >= now()) AS active,
|
||
count(*) FILTER (WHERE lease_until < now()) AS expired
|
||
FROM attachment_jobs WHERE status = 'processing'""").fetchone()
|
||
staging = conn.execute("SELECT count(*) AS n FROM attachments WHERE source_key IS NOT NULL").fetchone()["n"]
|
||
report = {"jobs": jobs, "outbox": {k: (round(float(v), 1) if k == "oldest_due_s" and v is not None else v)
|
||
for k, v in outbox.items()},
|
||
"leases": leases, "attachments_with_staging_source": staging}
|
||
try:
|
||
connection = broker.connect()
|
||
try:
|
||
report["broker"] = broker.check_topology(connection.channel())
|
||
finally:
|
||
connection.close()
|
||
except broker.BrokerError as exc:
|
||
report["broker"] = {"error": type(exc).__name__}
|
||
return report
|
||
|
||
|
||
def cmd_failed(conn, args):
|
||
return rows_json(conn.execute(
|
||
"""SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code,
|
||
j.last_error, j.replays, j.finished_at, (a.source_key IS NOT NULL) AS replayable
|
||
FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
|
||
WHERE j.status = 'failed' ORDER BY j.finished_at DESC LIMIT %s""", (args.limit,)).fetchall())
|
||
|
||
|
||
def cmd_events(conn, args):
|
||
return rows_json(conn.execute(
|
||
"SELECT id, attempt, event, detail, actor, created_at FROM job_events WHERE job_id = %s ORDER BY id",
|
||
(args.job_id,)).fetchall())
|
||
|
||
|
||
def cmd_replay(conn, args):
|
||
try:
|
||
return {"replayed": pipeline.replay(conn, args.job_id, ACTOR, args.reason)}
|
||
except pipeline.ReplayRefused as exc:
|
||
return {"refused": str(exc), "job_id": str(args.job_id)}
|
||
|
||
|
||
def dlq_messages(limit, handler):
|
||
"""Читает до limit сообщений DLQ (basic_get, ручной ack). handler(message|None) -> 'ack' | 'keep'."""
|
||
connection = broker.connect()
|
||
results = []
|
||
try:
|
||
channel = connection.channel()
|
||
keep = []
|
||
for _ in range(limit):
|
||
method, properties, body = channel.basic_get(pipeline.DLQ, auto_ack=False)
|
||
if method is None:
|
||
break
|
||
death = (properties.headers or {}).get("x-death") or [{}]
|
||
reason = death[0].get("reason")
|
||
reason = reason.decode() if isinstance(reason, bytes) else reason
|
||
try:
|
||
message = pipeline.parse_message(body)
|
||
except pipeline.InvalidMessage as exc:
|
||
message, problem = None, str(exc)
|
||
else:
|
||
problem = None
|
||
verdict, note = handler(message)
|
||
results.append({"job_id": message["job_id"] if message else None, "invalid": problem,
|
||
"dead_letter_reason": reason, "action": verdict, "note": note})
|
||
if verdict == "ack":
|
||
channel.basic_ack(method.delivery_tag)
|
||
else:
|
||
keep.append(method.delivery_tag)
|
||
for tag in keep: # вернуть непрочитанными в DLQ
|
||
channel.basic_nack(tag, requeue=True)
|
||
finally:
|
||
connection.close()
|
||
return results
|
||
|
||
|
||
def cmd_dlq(_conn, args):
|
||
return dlq_messages(args.limit, lambda message: ("keep", "peek"))
|
||
|
||
|
||
def cmd_dlq_replay(conn, args):
|
||
def handle(message):
|
||
if message is None:
|
||
return "keep", "invalid message is kept for analysis"
|
||
job = conn.execute("SELECT status FROM attachment_jobs WHERE job_id = %s",
|
||
(UUID(message["job_id"]),)).fetchone()
|
||
if job is None:
|
||
return "keep", "unknown job"
|
||
if job["status"] == "failed":
|
||
try:
|
||
pipeline.replay(conn, UUID(message["job_id"]), ACTOR, "dlq replay")
|
||
return "ack", "replayed"
|
||
except pipeline.ReplayRefused as exc:
|
||
return "keep", str(exc)
|
||
if job["status"] == "ready":
|
||
return "ack", "already ready, no-op"
|
||
# queued/processing: работу ведёт PostgreSQL; если сообщения в полёте нет — опубликовать заново
|
||
conn.execute("UPDATE attachment_jobs SET updated_at = now() - interval '1 day' WHERE job_id = %s "
|
||
"AND status = 'queued'", (UUID(message["job_id"]),))
|
||
return "ack", f"job is {job['status']}, handled by outbox/lease recovery"
|
||
return dlq_messages(args.limit, handle)
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Attachment pipeline operator commands")
|
||
sub = parser.add_subparsers(dest="command", required=True)
|
||
sub.add_parser("status")
|
||
p = sub.add_parser("failed")
|
||
p.add_argument("--limit", type=int, default=50)
|
||
p = sub.add_parser("events")
|
||
p.add_argument("job_id", type=UUID)
|
||
p = sub.add_parser("replay")
|
||
p.add_argument("job_id", type=UUID)
|
||
p.add_argument("--reason", default="operator replay")
|
||
p = sub.add_parser("dlq")
|
||
p.add_argument("--limit", type=int, default=20)
|
||
p = sub.add_parser("dlq-replay")
|
||
p.add_argument("--limit", type=int, default=100)
|
||
args = parser.parse_args()
|
||
if args.command == "replay" and (len(args.reason) > 100 or not args.reason.isprintable()):
|
||
parser.error("--reason: up to 100 printable characters")
|
||
commands = {"status": cmd_status, "failed": cmd_failed, "events": cmd_events, "replay": cmd_replay,
|
||
"dlq": cmd_dlq, "dlq-replay": cmd_dlq_replay}
|
||
# Действие оператора — отдельная трасса; replay ссылается (link) на трассу исходной загрузки.
|
||
with telemetry.internal_span(f"ops {args.command}", {"messenger.ops.command": args.command}):
|
||
with connect_db() as conn:
|
||
result = commands[args.command](conn, args)
|
||
telemetry.shutdown()
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|