Files
mtusi2026lab6/src/worker.py
T
Самохвалов Андрей Ремович c60b3e652b feat: Carry runnable lab5 business stack into lab6 (U1)
Import committed lab5 baseline 03899d7 via git archive: API replicas,
PostgreSQL, private S3, RabbitMQ, publisher, worker, migrations, and hooks.
Preserve lab6 contracts and the separate telemetry stack. Static checks pass;
Docker runtime integration remains pending.
2026-10-02 14:43:04 +03:00

217 lines
8.6 KiB
Python

"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects."""
import hashlib
import json
import logging
import os
import random
import re
import signal
import time
from contextlib import contextmanager
from datetime import datetime
from io import BytesIO
from pathlib import Path
from uuid import UUID
import pika
import psycopg
from minio.error import S3Error
from psycopg_pool import PoolTimeout
from .attachments import BUCKET, storage
from .db import pool
from .image_variants import InvalidImage, MAX_BYTES, render
from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired
logger = logging.getLogger(__name__)
QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
def parse_message(body: bytes) -> dict:
if len(body) > 4096:
raise ValueError("oversized job message")
item = json.loads(body)
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
if (not isinstance(item, dict) or not required <= item.keys() or
item.keys() - required - {"traceparent", "tracestate"} or
type(item["schema_version"]) is not int or item["schema_version"] != 1 or
item["type"] != "attachment.process.v1" or
type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or
type(item["attempt"]) is not int or item["attempt"] < 1 or
not isinstance(item["job_id"], str) or
not isinstance(item["attachment_id"], str) or
not isinstance(item["created_at"], str) or
("traceparent" in item and
(not isinstance(item["traceparent"], str) or
re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or
("tracestate" in item and
(not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))):
raise ValueError("invalid job message")
UUID(item["job_id"])
UUID(item["attachment_id"])
created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00"))
if created_at.utcoffset() is None:
raise ValueError("job timestamp lacks timezone")
return item
def _source_bytes(job) -> bytes:
if not 0 < job["size_bytes"] <= MAX_BYTES:
raise InvalidImage("invalid accepted size")
# Reprocessing uses the byte-identical final original, so ready staging
# can be reclaimed without making a future pipeline version impossible.
key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"]
response = storage().get_object(BUCKET, key)
try:
chunks = []
size = 0
digest = hashlib.sha256()
for chunk in response.stream(64 * 1024):
size += len(chunk)
if size > job["size_bytes"]:
raise InvalidImage("source exceeds accepted size")
chunks.append(chunk)
digest.update(chunk)
if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]:
raise InvalidImage("source checksum mismatch")
return b"".join(chunks)
finally:
response.close()
response.release_conn()
def _write_variants(job, variants: dict) -> dict:
extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"}
stored = {}
for name, variant in variants.items():
key = (job["object_key"] if name == "original" else
f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}")
data = variant["data"]
storage().put_object(
BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"]
)
stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"}
stored[name]["object_key"] = key
return stored
def _delay(attempt: int) -> float:
base = min(30, 2 ** min(attempt - 1, 5))
return base + random.uniform(0, base / 2)
@contextmanager
def _deadline(seconds: int):
def expired(_signal, _frame):
raise TimeoutError("image job exceeded worker deadline")
previous = signal.signal(signal.SIGALRM, expired)
signal.setitimer(signal.ITIMER_REAL, seconds)
try:
yield
finally:
signal.setitimer(signal.ITIMER_REAL, 0)
signal.signal(signal.SIGALRM, previous)
def process_message(item: dict) -> str:
with pool().connection() as conn:
job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS)
if job is None:
return "duplicate_or_stale"
token = job["lease_token"]
try:
with _deadline(LEASE_SECONDS):
data = _source_bytes(job)
variants = render(data, job["content_type"])
stored = _write_variants(job, variants)
with pool().connection() as conn:
current = complete_job(conn, job["id"], token, stored)
if current and job["staging_key"]:
try:
storage().remove_object(BUCKET, job["staging_key"])
except Exception as exc:
# Periodic cleanup recovers this post-COMMIT gap.
logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__)
return "ready"
except psycopg.Error:
raise
except Exception as exc:
if isinstance(exc, InvalidImage):
code, permanent = "invalid_image", True
elif isinstance(exc, TimeoutError):
code, permanent = "worker_timeout", False
elif isinstance(exc, S3Error) or isinstance(exc, OSError):
code, permanent = "storage_unavailable", False
elif isinstance(exc, ValueError):
code, permanent = "processing_failed", True
else:
code, permanent = "processing_failed", False
logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code)
with pool().connection() as conn:
try:
return finish_attempt(
conn, job["id"], token, max_attempts=MAX_ATTEMPTS,
delay_seconds=_delay(job["attempt"]), error_code=code,
permanent=permanent,
)
except ValueError:
# A lease recovery already owns the next attempt.
return "stale_lease"
def main() -> None:
logging.basicConfig(level=logging.INFO)
if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1:
raise SystemExit("invalid worker retry settings")
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
params = pika.URLParameters(url)
params.heartbeat = max(120, LEASE_SECONDS * 2)
params.socket_timeout = 5
params.blocked_connection_timeout = 10
while True:
broker = None
try:
broker = pika.BlockingConnection(params)
channel = broker.channel()
channel.queue_declare(queue=QUEUE, passive=True)
channel.basic_qos(prefetch_count=1)
last_recovery = 0.0
while broker.is_open:
if time.monotonic() - last_recovery >= 5:
with pool().connection() as conn:
recover_expired(conn, max_attempts=MAX_ATTEMPTS)
last_recovery = time.monotonic()
method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False)
if method is None:
broker.process_data_events(time_limit=1)
continue
try:
item = parse_message(body)
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
logger.warning("Rejecting invalid job message")
channel.basic_reject(method.delivery_tag, requeue=False)
continue
try:
result = process_message(item)
channel.basic_ack(method.delivery_tag)
logger.info("Job %s: %s", item["job_id"], result)
except (psycopg.Error, PoolTimeout, OSError) as exc:
logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__)
channel.basic_nack(method.delivery_tag, requeue=True)
time.sleep(2)
except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc:
logger.warning("Worker reconnecting after %s", type(exc).__name__)
time.sleep(2)
finally:
if broker is not None and broker.is_open:
broker.close()
if __name__ == "__main__":
main()