forked from mtusi2026/mtusi2026lab5
97 lines
3.1 KiB
Python
97 lines
3.1 KiB
Python
"""Dedicated durable Valkey Stream, one consumer group; never evict/trim pending work."""
|
|
|
|
import re
|
|
from datetime import datetime
|
|
from typing import Literal
|
|
from uuid import UUID
|
|
|
|
import redis
|
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
|
|
STREAM = "attachment-jobs"
|
|
GROUP = "image-workers-v1"
|
|
DLQ = "invalid-jobs"
|
|
|
|
|
|
class Job(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", strict=True)
|
|
schema_version: Literal[1]
|
|
job_id: str
|
|
type: Literal["attachment.process.v1"]
|
|
attachment_id: str
|
|
pipeline_version: int = Field(ge=1)
|
|
attempt: int = Field(ge=1)
|
|
created_at: str
|
|
traceparent: str | None = Field(
|
|
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
|
|
)
|
|
tracestate: str | None = Field(default=None, max_length=512)
|
|
|
|
@model_validator(mode="before")
|
|
@classmethod
|
|
def strict_carrier(cls, value):
|
|
if isinstance(value, dict):
|
|
if type(value.get("schema_version")) is not int:
|
|
raise ValueError("Integer schema version required")
|
|
if any(k in value and value[k] is None for k in ("traceparent", "tracestate")):
|
|
raise ValueError("Optional carrier fields must be strings when provided")
|
|
return value
|
|
|
|
@field_validator("job_id", "attachment_id")
|
|
@classmethod
|
|
def uuid(cls, value):
|
|
if str(UUID(value)) != value:
|
|
raise ValueError("Canonical UUID required")
|
|
return value
|
|
|
|
@field_validator("created_at")
|
|
@classmethod
|
|
def timestamp(cls, value):
|
|
if not re.fullmatch(r"\d{4}-\d\d-\d\dT.+(?:Z|[+-]\d\d:\d\d)", value):
|
|
raise ValueError("RFC3339 required")
|
|
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
return value
|
|
|
|
|
|
class Broker:
|
|
def __init__(self, settings):
|
|
self.client = redis.Redis.from_url(
|
|
settings.broker_url,
|
|
decode_responses=True,
|
|
socket_timeout=2,
|
|
socket_connect_timeout=1,
|
|
retry_on_timeout=False,
|
|
)
|
|
|
|
def group(self):
|
|
try:
|
|
self.client.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
|
except redis.ResponseError as exc:
|
|
if "BUSYGROUP" not in str(exc):
|
|
raise
|
|
|
|
def publish(self, job):
|
|
# XADD reply is the publisher boundary; appendfsync=always precedes that reply.
|
|
return self.client.xadd(STREAM, {"job": job.model_dump_json(exclude_none=True)})
|
|
|
|
def ack(self, id):
|
|
# A single consumer group only. Atomically remove completed entries, never MAXLEN.
|
|
self.client.eval(
|
|
"redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])",
|
|
1,
|
|
STREAM,
|
|
GROUP,
|
|
id,
|
|
)
|
|
|
|
def quarantine(self, id, reason):
|
|
# Never copy arbitrary attacker-supplied fields into DLQ/logs.
|
|
with self.client.pipeline(transaction=True) as tx:
|
|
tx.xadd(DLQ, {"stream_id": id, "reason": reason}, maxlen=1000)
|
|
tx.xack(STREAM, GROUP, id)
|
|
tx.xdel(STREAM, id)
|
|
tx.execute()
|
|
|
|
def close(self):
|
|
self.client.close()
|