From a58d6beb5cb23abcb072bf1feebb0ca5423d1030 Mon Sep 17 00:00:00 2001 From: vasyukova Date: Fri, 23 Oct 2026 13:17:23 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A4=D0=B8=D0=BA=D1=81:=20=D0=B4=D0=BE=D0=B1?= =?UTF-8?q?=D0=B0=D0=B2=D0=B8=D0=BB=D0=B0=20=D1=88=D1=82=D0=B0=D1=82=D0=BD?= =?UTF-8?q?=D1=8B=D0=B9=20=D0=BE=D0=BF=D0=B5=D1=80=D0=B0=D1=82=D0=BE=D1=80?= =?UTF-8?q?=D1=81=D0=BA=D0=B8=D0=B9=20replay=20failed=20=E2=86=92=20queued?= =?UTF-8?q?:=20=D1=81=D0=BA=D1=80=D0=B8=D0=BF=D1=82=20python=20-m=20app.st?= =?UTF-8?q?orage.replay=5Ffailed=20=D1=81=D0=B1=D1=80=D0=B0=D1=81=D1=8B?= =?UTF-8?q?=D0=B2=D0=B0=D0=B5=D1=82=20=D1=82=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE?= =?UTF-8?q?=20failed=20=D0=B8=20=D0=BF=D1=83=D0=B1=D0=BB=D0=B8=D0=BA=D1=83?= =?UTF-8?q?=D0=B5=D1=82=20=D0=BD=D0=BE=D0=B2=D1=8B=D0=B9=20job=20=D1=87?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D0=B7=20outbox,=20=D0=B3=D0=BE=D1=82=D0=BE?= =?UTF-8?q?=D0=B2=D1=8B=D0=B5=20=D0=B7=D0=B0=D0=B4=D0=B0=D1=87=D0=B8=20?= =?UTF-8?q?=D0=B7=D0=B0=D1=89=D0=B8=D1=89=D0=B5=D0=BD=D1=8B.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 12 +++++ src/app/storage/postgres.py | 50 ++++++++++++++++++ src/app/storage/replay_failed.py | 88 ++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 src/app/storage/replay_failed.py diff --git a/REPORT.md b/REPORT.md index 11f8f405..f56d8342 100644 --- a/REPORT.md +++ b/REPORT.md @@ -66,6 +66,18 @@ docker compose down # volumes сохраняются docker compose down -v # ТОЛЬКО если осознанно стираем данные ``` +### Операторский replay failed-задач + +Причина ошибки устранена — вернуть задачи в обработку (failed → queued + новая строка outbox, готовые не трогаются): + +``` +docker compose --env-file .env exec -T worker python -m app.storage.replay_failed --all-failed +# или конкретные ID: +docker compose --env-file .env exec -T worker python -m app.storage.replay_failed [...] +``` + +Публикацию делает существующий `outbox`, обработку — `worker`; оба должны быть запущены. Проверка: `status=queued`, затем `ready`. + ## Архитектура и решения Схема процессов и хранилищ: diff --git a/src/app/storage/postgres.py b/src/app/storage/postgres.py index 2bc9519f..a36b7682 100644 --- a/src/app/storage/postgres.py +++ b/src/app/storage/postgres.py @@ -410,6 +410,56 @@ class PostgresStorage: conn.commit() return count + def replay_failed_to_queued(self, attachment_id: UUID) -> str: + """Операторский replay: failed → queued с новой записью в outbox. + + Возвращает: + - "requeued" — статус был failed, переведён в queued и запланирован job; + - "already" — запись уже queued/processing/ready (защита готовых), ничего не меняем; + - "missing" — такого вложения нет. + + Ключ эффекта (attachment_id, pipeline_version, variant) и ON CONFLICT + в finalize_ready гарантируют, что повтор не создаст дублей вариантов. + Новый job_id и строка outbox публикуются отдельным процессом outbox, + поэтому разрыв между COMMIT и publish закрыт тем же механизмом, что и при upload. + """ + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT status FROM attachments WHERE id = %s FOR UPDATE", + (attachment_id,), + ) + row = cur.fetchone() + if row is None: + conn.rollback() + return "missing" + status = row[0] + if status != "failed": + conn.rollback() + return "already" + + now = now_utc() + cur.execute( + "UPDATE attachments SET status='queued', error_code=NULL," + " processing_until=NULL, updated_at=%s WHERE id=%s", + (now, attachment_id), + ) + + job_id = uuid4() + payload = { + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": 1, + "created_at": now.isoformat().replace("+00:00", "Z"), + } + cur.execute( + "INSERT INTO outbox (job_id, payload) VALUES (%s, %s)", + (job_id, _json.dumps(payload)), + ) + conn.commit() + return "requeued" + def finalize_ready( self, attachment_id: UUID, diff --git a/src/app/storage/replay_failed.py b/src/app/storage/replay_failed.py new file mode 100644 index 00000000..07527b77 --- /dev/null +++ b/src/app/storage/replay_failed.py @@ -0,0 +1,88 @@ +"""Операторский replay failed-задач: failed → queued + запись в outbox. + +Использование: + python -m app.storage.replay_failed [ ...] + python -m app.storage.replay_failed --all-failed + +Защита: + - переводит ТОЛЬКО failed; queued/processing/ready не трогает (возвращает "already"); + - новая строка outbox с новым job_id — публикация через существующий outbox-процесс; + - идемпотентность эффекта по (attachment_id, pipeline_version, variant) + не даёт создать дубли вариантов, если повторная обработка уже была выполнена. +Логи пишутся только с ID и статусом, без секретов и содержимого файлов. +""" +import logging +import sys +import uuid + +from .db import init_pool, close_pool +from .postgres import storage + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, +) +log = logging.getLogger("replay_failed") + + +def _parse_args(argv): + if not argv: + print( + "usage: python -m app.storage.replay_failed [...] | --all-failed", + file=sys.stderr, + ) + return None + if argv == ["--all-failed"]: + return "all" + ids = [] + for raw in argv: + try: + ids.append(uuid.UUID(raw)) + except ValueError: + print("invalid attachment_id: %r" % raw, file=sys.stderr) + return None + return ids + + +def _list_failed_ids() -> list: + from .db import connection + with connection() as conn, conn.cursor() as cur: + cur.execute("SELECT id FROM attachments WHERE status='failed' ORDER BY updated_at") + return [r[0] for r in cur.fetchall()] + + +def main(argv) -> int: + parsed = _parse_args(argv) + if parsed is None: + return 2 + init_pool() + try: + if parsed == "all": + ids = _list_failed_ids() + else: + ids = parsed + requeued = already = missing = 0 + for aid in ids: + result = storage.replay_failed_to_queued(aid) + if result == "requeued": + requeued += 1 + log.info("event=replay_requeued attachment_id=%s", aid) + elif result == "already": + already += 1 + log.info("event=replay_skipped_not_failed attachment_id=%s", aid) + else: + missing += 1 + log.info("event=replay_missing attachment_id=%s", aid) + log.info( + "event=replay_done requested=%d requeued=%d skipped=%d missing=%d", + len(ids), requeued, already, missing, + ) + return 0 if missing == 0 else 1 + finally: + close_pool() + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file