Files
mtusi2026lab5/src/messenger/broker.py
T

96 lines
4.3 KiB
Python

"""Persistent Valkey Streams transport; schema fields contain IDs and metadata only."""
import datetime as dt
import json
import os
import re
import uuid
import valkey
from valkey.exceptions import ResponseError
from .config import read_secret
REQUIRED = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
OPTIONAL = {'traceparent', 'tracestate'}
def validate_message(payload):
"""Keep the wire contract strict even without a JSON Schema runtime dependency."""
if not isinstance(payload, dict) or not REQUIRED <= payload.keys() or payload.keys() - REQUIRED - OPTIONAL:
raise ValueError('invalid job fields')
if type(payload['schema_version']) is not int or payload['schema_version'] != 1:
raise ValueError('invalid schema version')
if payload['type'] != 'attachment.process.v1':
raise ValueError('invalid job type')
for name in ('job_id', 'attachment_id'):
if not isinstance(payload[name], str):
raise ValueError('invalid identifier')
uuid.UUID(payload[name])
for name in ('attempt', 'pipeline_version'):
if type(payload[name]) is not int or payload[name] < 1:
raise ValueError('invalid positive integer')
created = payload['created_at']
if not isinstance(created, str) or dt.datetime.fromisoformat(created.replace('Z', '+00:00')).tzinfo is None:
raise ValueError('invalid created_at')
if 'traceparent' in payload and (not isinstance(payload['traceparent'], str) or
re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', payload['traceparent']) is None):
raise ValueError('invalid traceparent')
if 'tracestate' in payload and (not isinstance(payload['tracestate'], str) or len(payload['tracestate']) > 512):
raise ValueError('invalid tracestate')
return payload
class InvalidDelivery(Exception):
def __init__(self, message_id):
self.message_id = message_id
super().__init__('invalid broker job')
class Broker:
def __init__(self):
self.stream = os.environ.get('JOB_STREAM', 'attachment-jobs')
self.group = os.environ.get('JOB_CONSUMER_GROUP', 'attachment-workers')
self.client = valkey.Valkey.from_url(read_secret('BROKER_URL'), decode_responses=True,
socket_connect_timeout=2, socket_timeout=4,
retry_on_timeout=False, health_check_interval=15)
def ensure_group(self):
try:
self.client.xgroup_create(self.stream, self.group, id='0', mkstream=True)
except ResponseError as error:
if not str(error).startswith('BUSYGROUP'):
raise
def publish(self, payload):
validate_message(payload)
# Broker runs AOF with appendfsync=always: successful XADD is its durable
# acceptance boundary. A lost response can cause a harmless duplicate.
return self.client.xadd(self.stream, {'payload': json.dumps(payload, separators=(',', ':'))})
def read(self, consumer, idle_ms):
# COUNT 1 bounds prefetch. Dead consumer pending entries are claimed only
# after the DB lease duration; DB fencing still decides who may commit.
pending = self.client.xautoclaim(self.stream, self.group, consumer, idle_ms,
start_id='0-0', count=1)
messages = pending[1]
if not messages:
entries = self.client.xreadgroup(self.group, consumer, {self.stream: '>'}, count=1, block=1000)
messages = entries[0][1] if entries else []
if not messages:
return None
message_id, fields = messages[0]
try:
if set(fields) != {'payload'} or len(fields['payload']) > 4096:
raise ValueError('invalid envelope')
return message_id, validate_message(json.loads(fields['payload']))
except (ValueError, TypeError, KeyError, AttributeError):
# Never acknowledge malformed work without a durable effect. Only
# our validated dispatcher can write to the private broker stream.
raise InvalidDelivery(message_id) from None
def ack(self, message_id):
self.client.xack(self.stream, self.group, message_id)
def close(self):
self.client.close()