Files
mtusi2026lab7/ops.py
T
kalibr456 cb38fd5231
Skeleton contracts / contracts (pull_request) Canceled after 0s
лр7: Docker-прогон, evidence, исправления
2026-11-07 20:17:11 +03:00

204 lines
9.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Операторские команды pipeline вложений (запускаются в контейнере dispatcher).
docker compose exec dispatcher python ops.py status
docker compose exec dispatcher python ops.py failed [--limit N]
docker compose exec dispatcher python ops.py events <job_id>
docker compose exec dispatcher python ops.py replay <job_id> [--reason TEXT]
docker compose exec dispatcher python ops.py dlq [--limit N]
docker compose exec dispatcher python ops.py dlq-replay [--limit N]
Реестр failed — таблица attachment_jobs (status='failed') + журнал job_events: там error_code,
число попыток, время. DLQ (attachments.process.v1.dlq) получает только сообщения, которые нельзя
обработать как сообщение: не по схеме, неизвестный job_id, превышен x-delivery-limit.
replay безопасен: переводит в queued только job со статусом failed и исходником в staging; ready,
queued и processing не трогает («повторить» не значит исполнить дубль успешной job). Новая попытка
идёт через outbox тем же job_id, эффект защищён ключом (attachment_id, pipeline_version, variant).
dlq-replay разбирает DLQ: для валидного сообщения смотрит строку job в PostgreSQL (источник истины):
failed -> replay; ready -> сообщение удаляется как no-op; queued/processing -> удаляется, работу
ведёт outbox/аренда (при необходимости dispatcher опубликует заново). Невалидные сообщения
остаются в DLQ для разбора. Вывод — JSON без секретов и ключей S3.
"""
import argparse
import json
import sys
from uuid import UUID
import psycopg
from psycopg.rows import dict_row
import dbconn
import secretstore
import settings
import telemetry
telemetry.setup("messenger-ops")
import broker # noqa: E402
import pipeline # noqa: E402
ACTOR = "ops@" + settings.INSTANCE_ID
def connect_db():
# lab7: та же identity, что у dispatcher (ops.py запускается в его контейнере)
secretstore.store.start()
return dbconn.connect(autocommit=True, row_factory=dict_row, options="-c TimeZone=UTC")
def text(value):
if isinstance(value, (UUID,)):
return str(value)
if hasattr(value, "isoformat"):
return pipeline.utc_text(value)
return value
def rows_json(rows):
return [{key: text(value) for key, value in row.items()} for row in rows]
def cmd_status(conn, _args):
jobs = {r["status"]: r["n"] for r in conn.execute(
"SELECT status, count(*) AS n FROM attachment_jobs GROUP BY status").fetchall()}
outbox = conn.execute(
"""SELECT count(*) FILTER (WHERE published_at IS NULL) AS pending,
count(*) FILTER (WHERE published_at IS NULL AND available_at <= now()) AS due,
coalesce(max(publish_failures) FILTER (WHERE published_at IS NULL), 0) AS max_publish_failures,
extract(epoch FROM now() - min(available_at) FILTER (WHERE published_at IS NULL
AND available_at <= now())) AS oldest_due_s
FROM outbox""").fetchone()
leases = conn.execute(
"""SELECT count(*) FILTER (WHERE lease_until >= now()) AS active,
count(*) FILTER (WHERE lease_until < now()) AS expired
FROM attachment_jobs WHERE status = 'processing'""").fetchone()
staging = conn.execute("SELECT count(*) AS n FROM attachments WHERE source_key IS NOT NULL").fetchone()["n"]
report = {"jobs": jobs, "outbox": {k: (round(float(v), 1) if k == "oldest_due_s" and v is not None else v)
for k, v in outbox.items()},
"leases": leases, "attachments_with_staging_source": staging}
try:
connection = broker.connect()
try:
report["broker"] = broker.check_topology(connection.channel())
finally:
connection.close()
except broker.BrokerError as exc:
report["broker"] = {"error": type(exc).__name__}
return report
def cmd_failed(conn, args):
return rows_json(conn.execute(
"""SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code,
j.last_error, j.replays, j.finished_at, (a.source_key IS NOT NULL) AS replayable
FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.status = 'failed' ORDER BY j.finished_at DESC LIMIT %s""", (args.limit,)).fetchall())
def cmd_events(conn, args):
return rows_json(conn.execute(
"SELECT id, attempt, event, detail, actor, created_at FROM job_events WHERE job_id = %s ORDER BY id",
(args.job_id,)).fetchall())
def cmd_replay(conn, args):
try:
return {"replayed": pipeline.replay(conn, args.job_id, ACTOR, args.reason)}
except pipeline.ReplayRefused as exc:
return {"refused": str(exc), "job_id": str(args.job_id)}
def dlq_messages(limit, handler):
"""Читает до limit сообщений DLQ (basic_get, ручной ack). handler(message|None) -> 'ack' | 'keep'."""
connection = broker.connect()
results = []
try:
channel = connection.channel()
keep = []
for _ in range(limit):
method, properties, body = channel.basic_get(pipeline.DLQ, auto_ack=False)
if method is None:
break
death = (properties.headers or {}).get("x-death") or [{}]
reason = death[0].get("reason")
reason = reason.decode() if isinstance(reason, bytes) else reason
try:
message = pipeline.parse_message(body)
except pipeline.InvalidMessage as exc:
message, problem = None, str(exc)
else:
problem = None
verdict, note = handler(message)
results.append({"job_id": message["job_id"] if message else None, "invalid": problem,
"dead_letter_reason": reason, "action": verdict, "note": note})
if verdict == "ack":
channel.basic_ack(method.delivery_tag)
else:
keep.append(method.delivery_tag)
for tag in keep: # вернуть непрочитанными в DLQ
channel.basic_nack(tag, requeue=True)
finally:
connection.close()
return results
def cmd_dlq(_conn, args):
return dlq_messages(args.limit, lambda message: ("keep", "peek"))
def cmd_dlq_replay(conn, args):
def handle(message):
if message is None:
return "keep", "invalid message is kept for analysis"
job = conn.execute("SELECT status FROM attachment_jobs WHERE job_id = %s",
(UUID(message["job_id"]),)).fetchone()
if job is None:
return "keep", "unknown job"
if job["status"] == "failed":
try:
pipeline.replay(conn, UUID(message["job_id"]), ACTOR, "dlq replay")
return "ack", "replayed"
except pipeline.ReplayRefused as exc:
return "keep", str(exc)
if job["status"] == "ready":
return "ack", "already ready, no-op"
# queued/processing: работу ведёт PostgreSQL; если сообщения в полёте нет — опубликовать заново
conn.execute("UPDATE attachment_jobs SET updated_at = now() - interval '1 day' WHERE job_id = %s "
"AND status = 'queued'", (UUID(message["job_id"]),))
return "ack", f"job is {job['status']}, handled by outbox/lease recovery"
return dlq_messages(args.limit, handle)
def main():
parser = argparse.ArgumentParser(description="Attachment pipeline operator commands")
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("status")
p = sub.add_parser("failed")
p.add_argument("--limit", type=int, default=50)
p = sub.add_parser("events")
p.add_argument("job_id", type=UUID)
p = sub.add_parser("replay")
p.add_argument("job_id", type=UUID)
p.add_argument("--reason", default="operator replay")
p = sub.add_parser("dlq")
p.add_argument("--limit", type=int, default=20)
p = sub.add_parser("dlq-replay")
p.add_argument("--limit", type=int, default=100)
args = parser.parse_args()
if args.command == "replay" and (len(args.reason) > 100 or not args.reason.isprintable()):
parser.error("--reason: up to 100 printable characters")
commands = {"status": cmd_status, "failed": cmd_failed, "events": cmd_events, "replay": cmd_replay,
"dlq": cmd_dlq, "dlq-replay": cmd_dlq_replay}
# Действие оператора — отдельная трасса; replay ссылается (link) на трассу исходной загрузки.
with telemetry.internal_span(f"ops {args.command}", {"messenger.ops.command": args.command}):
with connect_db() as conn:
result = commands[args.command](conn, args)
telemetry.shutdown()
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
sys.exit(main())