Files
mtusi2026lab5/app/worker.py
T
Artu-RitoandClaude Sonnet 5 bdba93f022 Лаба 5: Redis Streams очередь + асинхронная обработка изображений (оценка 4)
Переиспользован стек lab4 (LB/2 реплики/Postgres/SeaweedFS). upload теперь
202 queued; отдельный worker-процесс генерирует thumbnail/crop/optimized.
Transactional outbox закрывает разрыв COMMIT/publish; durable retry
(job_retry_schedule) переживает падение worker во время backoff; lease
recovery через XAUTOCLAIM; DLQ-реестр (job_failures) с явным operator
replay (scripts/replay_failed.py). make test/test-persistence/test-failover/
validate зелёные; в процессе экспериментов найдены и исправлены два
реальных бага (неполный try/except вокруг S3-записи вариантов, ошибка типов
asyncpg в построении interval) — см. REPORT.md и evidence/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-30 01:21:42 +03:00

202 lines
8.9 KiB
Python

"""Standalone worker process: consumes attachment.process.v1 jobs, generates
variants, writes results idempotently. Runs as its own container/command,
separate from the API (contracts/attachment-states.md requires this split).
"""
import asyncio
import hashlib
import logging
import os
import random
import socket
import uuid
from . import broker, db, pipeline, storage
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logger = logging.getLogger("worker")
CONSUMER_NAME = os.environ.get("INSTANCE_ID") or f"{socket.gethostname()}-{os.getpid()}"
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
RETRY_SCAN_INTERVAL_SECONDS = 1
def read_secret(file_env: str, inline_env: str) -> str:
path = os.environ.get(file_env)
if path and os.path.isfile(path):
return open(path, encoding="utf-8").read().strip()
value = os.environ.get(inline_env)
if not value:
raise RuntimeError(f"{file_env} or {inline_env} must be set")
return value
async def process_job(job: dict) -> None:
attachment_id = uuid.UUID(job["attachment_id"])
job_id = uuid.UUID(job["job_id"])
attempt = job["attempt"]
async with db.pool().acquire() as conn:
record = await conn.fetchrow("SELECT * FROM attachments WHERE id=$1", attachment_id)
if record is None:
logger.warning("job %s references missing attachment %s; dropping", job_id, attachment_id)
return
if record["status"] in ("ready", "failed"):
# Redelivery of an already-finished job, or a stale duplicate racing
# a newer attempt: never roll a finished result backwards.
logger.info("attachment %s already %s; no-op", attachment_id, record["status"])
return
async with db.pool().acquire() as conn:
await conn.execute(
"UPDATE attachments SET status='processing', updated_at=now() WHERE id=$1 AND status != 'ready'",
attachment_id,
)
content_types = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"}
try:
original = await storage.get_object(record["s3_key"])
generated = {
"thumbnail": pipeline.thumbnail(original),
"crop": pipeline.crop(original),
"optimized": pipeline.optimized(original),
}
uploaded = []
for name, (data, width, height) in generated.items():
key = f"attachments/{attachment_id}/{name}"
await storage.put_object(key, data, content_types[name]) # network call: can fail transiently too
uploaded.append((name, key, data, width, height))
except Exception: # noqa: BLE001 - covers S3 reachability (get+put) and image decode, not just decode
logger.exception("processing failed for attachment %s", attachment_id)
await _handle_failure(job, "processing_failed")
return
async with db.pool().acquire() as conn:
async with conn.transaction():
for name, key, data, width, height in uploaded:
digest = hashlib.sha256(data).hexdigest()
await conn.execute(
"""
INSERT INTO attachment_variants (attachment_id, name, content_type, size_bytes, width, height, sha256, s3_key)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (attachment_id, name) DO UPDATE SET
content_type=EXCLUDED.content_type, size_bytes=EXCLUDED.size_bytes,
width=EXCLUDED.width, height=EXCLUDED.height, sha256=EXCLUDED.sha256, s3_key=EXCLUDED.s3_key
""",
attachment_id, name, content_types[name], len(data), width, height, digest, key,
)
await conn.execute(
"UPDATE attachments SET status='ready', error_code=NULL, updated_at=now() WHERE id=$1 AND status != 'ready'",
attachment_id,
)
logger.info("attachment %s ready (attempt %d)", attachment_id, attempt)
async def _handle_failure(job: dict, error_code: str) -> None:
attachment_id = uuid.UUID(job["attachment_id"])
job_id = uuid.UUID(job["job_id"])
attempt = job["attempt"]
async with db.pool().acquire() as conn:
if attempt >= MAX_ATTEMPTS:
async with conn.transaction():
await conn.execute(
"UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1",
attachment_id, error_code,
)
await conn.execute(
"INSERT INTO job_failures (attachment_id, job_id, attempt, error_code) VALUES ($1,$2,$3,$4) "
"ON CONFLICT (attachment_id, job_id) DO NOTHING",
attachment_id, job_id, attempt, error_code,
)
logger.warning("attachment %s permanently failed after %d attempts: %s", attachment_id, attempt, error_code)
return
# Transient: schedule a durable delayed retry instead of sleeping in-process.
delay = min(30, 2 ** (attempt - 1)) + random.uniform(0, 1)
await conn.execute(
"""
INSERT INTO job_retry_schedule (attachment_id, job_id, attempt, next_attempt_at)
VALUES ($1, $2, $3, now() + ($4 * interval '1 second'))
ON CONFLICT (attachment_id, job_id) DO UPDATE SET
attempt=EXCLUDED.attempt, next_attempt_at=EXCLUDED.next_attempt_at
""",
attachment_id, job_id, attempt + 1, delay,
)
await conn.execute(
"UPDATE attachments SET status='queued', updated_at=now() WHERE id=$1 AND status != 'ready'",
attachment_id,
)
logger.info("attachment %s scheduled for retry #%d in %.1fs", attachment_id, attempt + 1, delay)
async def retry_scheduler_loop() -> None:
while True:
try:
async with db.pool().acquire() as conn:
due = await conn.fetch(
"SELECT attachment_id, job_id, attempt FROM job_retry_schedule WHERE next_attempt_at <= now() LIMIT 50"
)
for row in due:
job = {
"schema_version": 1,
"job_id": str(row["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(row["attachment_id"]),
"pipeline_version": 1,
"attempt": row["attempt"],
"created_at": "1970-01-01T00:00:00Z", # informational only; not used for ordering
}
await broker.publish(job)
await conn.execute(
"DELETE FROM job_retry_schedule WHERE attachment_id=$1 AND job_id=$2",
row["attachment_id"], row["job_id"],
)
except Exception: # noqa: BLE001 - scheduler must survive a transient hiccup
logger.exception("retry scheduler tick failed")
await asyncio.sleep(RETRY_SCAN_INTERVAL_SECONDS)
async def reclaim_loop() -> None:
"""Hand this worker any job another (likely dead) worker left unacked
past JOB_TIMEOUT_SECONDS -- lease recovery for a crashed worker."""
while True:
try:
for message_id, job in await broker.reclaim_stale(CONSUMER_NAME, JOB_TIMEOUT_SECONDS * 1000):
logger.info("reclaimed stale job %s (message %s)", job["job_id"], message_id)
await process_job(job)
await broker.ack(message_id)
except Exception: # noqa: BLE001
logger.exception("reclaim tick failed")
await asyncio.sleep(JOB_TIMEOUT_SECONDS / 2)
async def consume_loop() -> None:
while True:
try:
for message_id, job in await broker.read(CONSUMER_NAME, count=1, block_ms=2000):
await process_job(job)
await broker.ack(message_id)
except Exception: # noqa: BLE001 - keep the loop alive across transient errors
logger.exception("consume tick failed")
await asyncio.sleep(1)
async def main() -> None:
database_url = read_secret("DATABASE_URL_FILE", "DATABASE_URL")
cache_url = read_secret("CACHE_URL_FILE", "CACHE_URL")
await db.init_pool(database_url)
broker.init(cache_url)
await broker.ensure_group()
storage.init(
endpoint=os.environ["S3_ENDPOINT"],
region=os.environ.get("S3_REGION", "us-east-1"),
access_key=read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"),
secret_key=read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"),
bucket=os.environ.get("S3_BUCKET", "messenger-attachments"),
)
logger.info("worker %s starting", CONSUMER_NAME)
await asyncio.gather(consume_loop(), reclaim_loop(), retry_scheduler_loop())
if __name__ == "__main__":
asyncio.run(main())