Лаба 5: Redis Streams очередь + асинхронная обработка изображений (оценка 4)

Переиспользован стек lab4 (LB/2 реплики/Postgres/SeaweedFS). upload теперь
202 queued; отдельный worker-процесс генерирует thumbnail/crop/optimized.
Transactional outbox закрывает разрыв COMMIT/publish; durable retry
(job_retry_schedule) переживает падение worker во время backoff; lease
recovery через XAUTOCLAIM; DLQ-реестр (job_failures) с явным operator
replay (scripts/replay_failed.py). make test/test-persistence/test-failover/
validate зелёные; в процессе экспериментов найдены и исправлены два
реальных бага (неполный try/except вокруг S3-записи вариантов, ошибка типов
asyncpg в построении interval) — см. REPORT.md и evidence/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-10-30 01:21:42 +03:00
co-authored by Claude Sonnet 5
parent 276dbdad8a
commit bdba93f022
31 changed files with 2255 additions and 17 deletions
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""Explicit operator replay for a failed attachment: resets status to
'queued' and republishes a fresh job (new job_id, attempt=1). Never replays
an attachment that is already 'ready' -- the same protection the worker
itself applies against stale duplicates.
Usage: python3 scripts/replay_failed.py <attachment-id>
"""
import asyncio
import datetime as dt
import os
import sys
import uuid
import asyncpg
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from app import broker # noqa: E402
def read_secret(file_env, inline_env):
path = os.environ.get(file_env)
if path and os.path.isfile(path):
return open(path, encoding="utf-8").read().strip()
return os.environ[inline_env]
async def main():
if len(sys.argv) != 2:
print("usage: replay_failed.py <attachment-id>", file=sys.stderr)
return 2
attachment_id = uuid.UUID(sys.argv[1])
conn = await asyncpg.connect(read_secret("DATABASE_URL_FILE", "DATABASE_URL"))
record = await conn.fetchrow("SELECT status FROM attachments WHERE id=$1", attachment_id)
if record is None:
print("no such attachment", file=sys.stderr)
return 1
if record["status"] == "ready":
print("attachment is already ready; refusing to replay a finished job")
return 0
if record["status"] != "failed":
print(f"attachment status is '{record['status']}', not 'failed'; nothing to replay")
return 0
job_id = uuid.uuid4()
await conn.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=$1", attachment_id)
await conn.close()
broker.init(read_secret("CACHE_URL_FILE", "CACHE_URL"))
job = {
"schema_version": 1,
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": 1,
"attempt": 1,
"created_at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
}
await broker.publish(job)
print(f"replayed attachment {attachment_id} as job {job_id}")
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))