Files
mtusi2026lab6/src/messenger/worker.py
T

287 lines
11 KiB
Python

"""Separate serial worker (prefetch=1) and transactional-outbox dispatcher."""
import argparse
import hashlib
import json
import logging
import re
import signal
import subprocess
import sys
import tempfile
import time
from datetime import datetime
from pathlib import Path
from uuid import UUID, uuid4
import redis
from opentelemetry import propagate, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from . import telemetry
from .domain import Failure
from .jobs import GROUP, STREAM, Jobs, broker_client, fault
from .postgres import PostgresStore
from .storage import ObjectStorage
logger = logging.getLogger("messenger.worker")
logging.basicConfig(level=logging.INFO, format="%(message)s")
stopping = False
MAX_JOB_BYTES = 8192
def stop(signum, frame):
global stopping
stopping = True
def process(job, jobs, storage):
fault(jobs.store, "before_effect", job["attachment_id"])
with tempfile.TemporaryDirectory(prefix="image-job-") as directory:
root = Path(directory)
source = root / "source"
response = storage.get(job["source_key"])
try:
if response["ContentLength"] > 10_485_760:
raise ValueError("invalid_image")
with source.open("wb") as target:
while block := response["Body"].read(64 * 1024):
target.write(block)
if target.tell() > 10_485_760:
raise ValueError("invalid_image")
finally:
response["Body"].close()
with source.open("rb") as file:
digest = hashlib.file_digest(file, "sha256").hexdigest()
if digest != job["source_sha256"] or source.stat().st_size != job["source_size"]:
raise ValueError("invalid_image")
# Resize executes outside the API and is bounded in wall time, CPU and address space.
result = subprocess.run(
[
sys.executable,
"-m",
"messenger.image_pipeline",
str(source),
directory,
str(job["pipeline_version"]),
],
timeout=20,
capture_output=True,
check=False,
)
if result.returncode:
raise ValueError("invalid_image")
variants = json.loads((root / "variants.json").read_text())
prefix = f"attachments/{job['attachment_id']}/v{job['pipeline_version']}"
with source.open("rb") as file:
storage.put(
prefix + "/original", file, source.stat().st_size, job["source_content_type"]
)
for variant in variants:
path = root / variant["name"]
with path.open("rb") as file:
storage.put(
prefix + "/" + variant["name"],
file,
variant["size_bytes"],
variant["content_type"],
)
fault(jobs.store, "after_effect", job["attachment_id"])
published = jobs.complete(job, prefix + "/original", variants)
if published:
fault(jobs.store, "after_result_commit", job["attachment_id"])
cleanup_ready(jobs.store, storage, job["attachment_id"])
return published
def cleanup_ready(store, storage, attachment_id):
# Same row lock as reprocess/replay: cannot delete a source chosen by a new job.
with store.connection() as conn, conn.transaction():
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
).fetchone()
if (
a
and a["status"] == "ready"
and a["stage_key"]
and a["stage_key"].startswith("staging/")
):
storage.delete(a["stage_key"])
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (attachment_id,))
def validate_delivery(body):
if not isinstance(body, (str, bytes, bytearray)):
raise ValueError("invalid_job")
encoded = body.encode("utf-8") if isinstance(body, str) else body
if len(encoded) > MAX_JOB_BYTES:
raise ValueError("invalid_job")
try:
data = json.loads(encoded)
except RecursionError:
raise ValueError("invalid_job") from None
expected = {
"schema_version",
"job_id",
"type",
"attachment_id",
"pipeline_version",
"attempt",
"created_at",
}
optional = {"traceparent", "tracestate"}
if not isinstance(data, dict) or not expected <= set(data) or set(data) - expected - optional:
raise ValueError("invalid_job")
if (
type(data["schema_version"]) is not int
or data["schema_version"] != 1
or data["type"] != "attachment.process.v1"
):
raise ValueError("invalid_job")
if not isinstance(data["job_id"], str) or not isinstance(data["attachment_id"], str):
raise ValueError("invalid_job")
for field in ("job_id", "attachment_id"):
canonical = str(UUID(data[field]))
if data[field].lower() != canonical:
raise ValueError("invalid_job")
# RFC UUID hex is case-insensitive; PostgreSQL and state comparisons use
# the same canonical spelling, so a valid uppercase delivery is not lost.
data[field] = canonical
if type(data["pipeline_version"]) is not int or data["pipeline_version"] < 1:
raise ValueError("invalid_job")
if type(data["attempt"]) is not int or data["attempt"] < 1:
raise ValueError("invalid_job")
created = data["created_at"]
if (
not isinstance(created, str)
or len(created) > 64
or not re.fullmatch(
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z", created
)
):
raise ValueError("invalid_job")
datetime.fromisoformat(created[:-1] + "+00:00")
if "traceparent" in data and (
not isinstance(data["traceparent"], str)
or not re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"])
):
raise ValueError("invalid_job")
if "tracestate" in data and (
not isinstance(data["tracestate"], str) or len(data["tracestate"]) > 512
):
raise ValueError("invalid_job")
return data
def consume_one(store, jobs, storage, broker, entry):
entry_id, fields = entry
try:
body = validate_delivery(fields.get("body", fields.get(b"body")))
except (KeyError, ValueError, TypeError):
logger.error(json.dumps({"event": "invalid_delivery"}))
broker.xack(STREAM, GROUP, entry_id)
broker.xdel(STREAM, entry_id)
return
with telemetry.operation("attachments consume", context=propagate.extract(body), kind=SpanKind.CONSUMER):
consume_job(store, jobs, storage, broker, body)
# Durable state owns recovery even when notification acknowledgements repeat.
broker.xack(STREAM, GROUP, entry_id)
broker.xdel(STREAM, entry_id)
def consume_job(store, jobs, storage, broker, body):
job = jobs.claim(body["job_id"], body["attachment_id"], body["pipeline_version"])
t = telemetry.current()
started = time.perf_counter()
outcome = "duplicate"
if job:
try:
if t:
t.busy = 1
outcome = "success" if process(job, jobs, storage) else "superseded"
except ValueError:
jobs.fail(job, "invalid_image", permanent=True)
outcome = "failed"
except (Failure, subprocess.TimeoutExpired, OSError, RuntimeError) as exc:
logger.warning(
json.dumps(
{"event": "job_retry", "job_id": str(job["id"]), "kind": type(exc).__name__}
)
)
jobs.fail(job, "processing_failed")
outcome = "retry" if job["attempt"] < job["attempt_limit"] else "failed"
finally:
if t:
t.busy = 0
attrs = {"job_type": "attachment.process.v1", "outcome": outcome}
t.jobs.add(1, attrs)
t.job_duration.record(time.perf_counter() - started, attrs)
span = trace.get_current_span()
span.set_attributes({"job.type": "attachment.process.v1", "job.outcome": outcome})
if outcome in {"failed", "retry"}:
span.set_status(Status(StatusCode.ERROR, "processing_failed"))
logger.info(json.dumps({"event": "job_finished", "outcome": outcome}))
else:
trace.get_current_span().set_attributes({"job.type": "attachment.process.v1", "job.outcome": "duplicate"})
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--dispatcher", action="store_true")
args = parser.parse_args()
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
t = telemetry.setup("messenger-dispatcher" if args.dispatcher else "messenger-worker")
store = PostgresStore()
t.start_background(store)
jobs = Jobs(store)
broker = broker_client()
storage = None if args.dispatcher else ObjectStorage()
consumer = "worker-" + str(uuid4())
last_claim = 0
try:
while not stopping:
try:
if args.dispatcher:
jobs.recover_one()
if not jobs.publish_one(broker):
time.sleep(0.2)
continue
try:
broker.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
entries = []
if time.monotonic() - last_claim >= 5:
claimed = broker.xautoclaim(
STREAM,
GROUP,
consumer,
min_idle_time=jobs.lease_seconds * 1000,
start_id="0-0",
count=1,
)
entries = claimed[1]
last_claim = time.monotonic()
if not entries:
batches = broker.xreadgroup(GROUP, consumer, {STREAM: ">"}, count=1, block=500)
entries = batches[0][1] if batches else []
for entry in entries:
consume_one(store, jobs, storage, broker, entry)
except (redis.RedisError, Failure) as exc:
logger.warning(
json.dumps({"event": "dependency_retry", "kind": type(exc).__name__})
)
time.sleep(0.5)
finally:
broker.close()
if storage is not None:
storage.close()
t.close()
store.close()
if __name__ == "__main__":
main()