Files
mtusi2026lab5/app/replay.py
T

110 lines
5.7 KiB
Python

"""
Реестр failed-задач и безопасный явный replay (lab5, уровень 4). Запуск в контейнере worker:
python -m app.replay --list # реестр failed (из БД)
python -m app.replay --dlq # последние записи DLQ-stream брокера
python -m app.replay --attachment <id> [--dry-run] [--actor имя]
python -m app.replay --all-failed [--error-code retry_exhausted] [--dry-run]
Почему «повторить» безопасно:
- replay меняет только job в статусе failed (FOR UPDATE): ready/queued/processing не трогает —
уже успешную задачу второй раз исполнить нельзя;
- job_id тот же, attempt растёт (attempt+1), бюджет попыток — ещё JOB_MAX_ATTEMPTS;
- эффект защищён ключом (attachment_id, pipeline_version, variant) и детерминированными
ключами S3: даже дубль сообщения не создаст второй набор вариантов;
- если исходник в staging уже удалён по retention — replay отказывает (повторять нечего);
- каждое действие пишется в job_events (audit: кто, когда, какая попытка).
"""
from __future__ import annotations
import argparse
import getpass
import json
from psycopg_pool import ConnectionPool
from . import broker
from .config import settings
def replay_one(pool: ConnectionPool, attachment_id: str, actor: str, dry_run: bool) -> str:
with pool.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT j.job_id, j.status, j.attempt, j.error_code, a.staging_key, j.pipeline_version "
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
"WHERE j.attachment_id = %s AND j.pipeline_version = %s FOR UPDATE OF j",
(attachment_id, settings.pipeline_version)).fetchone()
if row is None:
return "not found"
job_id, status, attempt, error_code, staging_key, pv = row
if status != "failed":
return f"refused: status is {status} (replay only for failed)"
if not staging_key:
return "refused: source already removed by staging retention"
if dry_run:
return f"would replay job {job_id} ({error_code}), next attempt {attempt + 1}"
nxt = attempt + 1
conn.execute(
"UPDATE attachment_jobs SET status = 'queued', attempt = %s, max_attempts = %s, "
"error_code = NULL, replays = replays + 1, finished_at = NULL, next_attempt_at = now(), "
"updated_at = now() WHERE job_id = %s",
(nxt, attempt + settings.job_max_attempts, job_id))
conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() "
"WHERE id = %s", (attachment_id,))
created = conn.execute("SELECT now()").fetchone()[0]
payload = broker.make_payload(job_id, attachment_id, pv, nxt, created)
conn.execute("INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) "
"VALUES (%s, %s, %s, now(), now())", (job_id, nxt, json.dumps(payload)))
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) "
"VALUES (%s, 'replay', %s, %s, %s)",
(job_id, nxt, actor, f"was failed: {error_code}"))
return f"replayed job {job_id}: attempt {nxt} queued (outbox → broker)"
def main() -> None:
ap = argparse.ArgumentParser(description="failed jobs registry and explicit replay")
g = ap.add_mutually_exclusive_group(required=True)
g.add_argument("--list", action="store_true")
g.add_argument("--dlq", action="store_true")
g.add_argument("--attachment")
g.add_argument("--all-failed", action="store_true")
ap.add_argument("--error-code")
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--actor", default=None)
args = ap.parse_args()
actor = args.actor or f"operator:{getpass.getuser()}"
if args.dlq:
r = broker.sync_client()
for msg_id, fields in r.xrevrange(settings.dlq_stream, count=20):
print(msg_id, json.dumps(fields, ensure_ascii=False))
return
with ConnectionPool(settings.database_url, min_size=1, max_size=2, open=True) as pool:
if args.list:
with pool.connection() as conn:
rows = conn.execute(
"SELECT j.attachment_id, j.job_id, j.attempt, j.error_code, j.finished_at, "
"a.staging_key IS NOT NULL FROM attachment_jobs j JOIN attachments a "
"ON a.id = j.attachment_id WHERE j.status = 'failed' ORDER BY j.finished_at").fetchall()
for aid, jid, att, code, fin, has_src in rows:
print(f"{aid} job={jid} attempt={att} error={code} at={fin:%Y-%m-%d %H:%M:%S} "
f"replayable={'yes' if has_src else 'no'}")
print(f"failed jobs: {len(rows)}")
return
targets = [args.attachment] if args.attachment else []
if args.all_failed:
with pool.connection() as conn:
q = "SELECT attachment_id FROM attachment_jobs WHERE status = 'failed'"
params: tuple = ()
if args.error_code:
q += " AND error_code = %s"
params = (args.error_code,)
targets = [str(r[0]) for r in conn.execute(q, params).fetchall()]
for aid in targets:
print(aid, "→", replay_one(pool, aid, actor, args.dry_run))
if __name__ == "__main__":
main()