forked from mtusi2026/mtusi2026lab5
110 lines
5.7 KiB
Python
110 lines
5.7 KiB
Python
"""
|
|
Реестр failed-задач и безопасный явный replay (lab5, уровень 4). Запуск в контейнере worker:
|
|
|
|
python -m app.replay --list # реестр failed (из БД)
|
|
python -m app.replay --dlq # последние записи DLQ-stream брокера
|
|
python -m app.replay --attachment <id> [--dry-run] [--actor имя]
|
|
python -m app.replay --all-failed [--error-code retry_exhausted] [--dry-run]
|
|
|
|
Почему «повторить» безопасно:
|
|
- replay меняет только job в статусе failed (FOR UPDATE): ready/queued/processing не трогает —
|
|
уже успешную задачу второй раз исполнить нельзя;
|
|
- job_id тот же, attempt растёт (attempt+1), бюджет попыток — ещё JOB_MAX_ATTEMPTS;
|
|
- эффект защищён ключом (attachment_id, pipeline_version, variant) и детерминированными
|
|
ключами S3: даже дубль сообщения не создаст второй набор вариантов;
|
|
- если исходник в staging уже удалён по retention — replay отказывает (повторять нечего);
|
|
- каждое действие пишется в job_events (audit: кто, когда, какая попытка).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import getpass
|
|
import json
|
|
|
|
from psycopg_pool import ConnectionPool
|
|
|
|
from . import broker
|
|
from .config import settings
|
|
|
|
|
|
def replay_one(pool: ConnectionPool, attachment_id: str, actor: str, dry_run: bool) -> str:
|
|
with pool.connection() as conn, conn.transaction():
|
|
row = conn.execute(
|
|
"SELECT j.job_id, j.status, j.attempt, j.error_code, a.staging_key, j.pipeline_version "
|
|
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
|
|
"WHERE j.attachment_id = %s AND j.pipeline_version = %s FOR UPDATE OF j",
|
|
(attachment_id, settings.pipeline_version)).fetchone()
|
|
if row is None:
|
|
return "not found"
|
|
job_id, status, attempt, error_code, staging_key, pv = row
|
|
if status != "failed":
|
|
return f"refused: status is {status} (replay only for failed)"
|
|
if not staging_key:
|
|
return "refused: source already removed by staging retention"
|
|
if dry_run:
|
|
return f"would replay job {job_id} ({error_code}), next attempt {attempt + 1}"
|
|
nxt = attempt + 1
|
|
conn.execute(
|
|
"UPDATE attachment_jobs SET status = 'queued', attempt = %s, max_attempts = %s, "
|
|
"error_code = NULL, replays = replays + 1, finished_at = NULL, next_attempt_at = now(), "
|
|
"updated_at = now() WHERE job_id = %s",
|
|
(nxt, attempt + settings.job_max_attempts, job_id))
|
|
conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() "
|
|
"WHERE id = %s", (attachment_id,))
|
|
created = conn.execute("SELECT now()").fetchone()[0]
|
|
payload = broker.make_payload(job_id, attachment_id, pv, nxt, created)
|
|
conn.execute("INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) "
|
|
"VALUES (%s, %s, %s, now(), now())", (job_id, nxt, json.dumps(payload)))
|
|
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) "
|
|
"VALUES (%s, 'replay', %s, %s, %s)",
|
|
(job_id, nxt, actor, f"was failed: {error_code}"))
|
|
return f"replayed job {job_id}: attempt {nxt} queued (outbox → broker)"
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser(description="failed jobs registry and explicit replay")
|
|
g = ap.add_mutually_exclusive_group(required=True)
|
|
g.add_argument("--list", action="store_true")
|
|
g.add_argument("--dlq", action="store_true")
|
|
g.add_argument("--attachment")
|
|
g.add_argument("--all-failed", action="store_true")
|
|
ap.add_argument("--error-code")
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
ap.add_argument("--actor", default=None)
|
|
args = ap.parse_args()
|
|
actor = args.actor or f"operator:{getpass.getuser()}"
|
|
|
|
if args.dlq:
|
|
r = broker.sync_client()
|
|
for msg_id, fields in r.xrevrange(settings.dlq_stream, count=20):
|
|
print(msg_id, json.dumps(fields, ensure_ascii=False))
|
|
return
|
|
|
|
with ConnectionPool(settings.database_url, min_size=1, max_size=2, open=True) as pool:
|
|
if args.list:
|
|
with pool.connection() as conn:
|
|
rows = conn.execute(
|
|
"SELECT j.attachment_id, j.job_id, j.attempt, j.error_code, j.finished_at, "
|
|
"a.staging_key IS NOT NULL FROM attachment_jobs j JOIN attachments a "
|
|
"ON a.id = j.attachment_id WHERE j.status = 'failed' ORDER BY j.finished_at").fetchall()
|
|
for aid, jid, att, code, fin, has_src in rows:
|
|
print(f"{aid} job={jid} attempt={att} error={code} at={fin:%Y-%m-%d %H:%M:%S} "
|
|
f"replayable={'yes' if has_src else 'no'}")
|
|
print(f"failed jobs: {len(rows)}")
|
|
return
|
|
targets = [args.attachment] if args.attachment else []
|
|
if args.all_failed:
|
|
with pool.connection() as conn:
|
|
q = "SELECT attachment_id FROM attachment_jobs WHERE status = 'failed'"
|
|
params: tuple = ()
|
|
if args.error_code:
|
|
q += " AND error_code = %s"
|
|
params = (args.error_code,)
|
|
targets = [str(r[0]) for r in conn.execute(q, params).fetchall()]
|
|
for aid in targets:
|
|
print(aid, "→", replay_one(pool, aid, actor, args.dry_run))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|