96 lines
4.3 KiB
Python
96 lines
4.3 KiB
Python
"""Persistent Valkey Streams transport; schema fields contain IDs and metadata only."""
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import re
|
|
import uuid
|
|
|
|
import valkey
|
|
from valkey.exceptions import ResponseError
|
|
|
|
from .config import read_secret
|
|
|
|
REQUIRED = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
|
|
OPTIONAL = {'traceparent', 'tracestate'}
|
|
|
|
|
|
def validate_message(payload):
|
|
"""Keep the wire contract strict even without a JSON Schema runtime dependency."""
|
|
if not isinstance(payload, dict) or not REQUIRED <= payload.keys() or payload.keys() - REQUIRED - OPTIONAL:
|
|
raise ValueError('invalid job fields')
|
|
if type(payload['schema_version']) is not int or payload['schema_version'] != 1:
|
|
raise ValueError('invalid schema version')
|
|
if payload['type'] != 'attachment.process.v1':
|
|
raise ValueError('invalid job type')
|
|
for name in ('job_id', 'attachment_id'):
|
|
if not isinstance(payload[name], str):
|
|
raise ValueError('invalid identifier')
|
|
uuid.UUID(payload[name])
|
|
for name in ('attempt', 'pipeline_version'):
|
|
if type(payload[name]) is not int or payload[name] < 1:
|
|
raise ValueError('invalid positive integer')
|
|
created = payload['created_at']
|
|
if not isinstance(created, str) or dt.datetime.fromisoformat(created.replace('Z', '+00:00')).tzinfo is None:
|
|
raise ValueError('invalid created_at')
|
|
if 'traceparent' in payload and (not isinstance(payload['traceparent'], str) or
|
|
re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', payload['traceparent']) is None):
|
|
raise ValueError('invalid traceparent')
|
|
if 'tracestate' in payload and (not isinstance(payload['tracestate'], str) or len(payload['tracestate']) > 512):
|
|
raise ValueError('invalid tracestate')
|
|
return payload
|
|
|
|
|
|
class InvalidDelivery(Exception):
|
|
def __init__(self, message_id):
|
|
self.message_id = message_id
|
|
super().__init__('invalid broker job')
|
|
|
|
|
|
class Broker:
|
|
def __init__(self):
|
|
self.stream = os.environ.get('JOB_STREAM', 'attachment-jobs')
|
|
self.group = os.environ.get('JOB_CONSUMER_GROUP', 'attachment-workers')
|
|
self.client = valkey.Valkey.from_url(read_secret('BROKER_URL'), decode_responses=True,
|
|
socket_connect_timeout=2, socket_timeout=4,
|
|
retry_on_timeout=False, health_check_interval=15)
|
|
|
|
def ensure_group(self):
|
|
try:
|
|
self.client.xgroup_create(self.stream, self.group, id='0', mkstream=True)
|
|
except ResponseError as error:
|
|
if not str(error).startswith('BUSYGROUP'):
|
|
raise
|
|
|
|
def publish(self, payload):
|
|
validate_message(payload)
|
|
# Broker runs AOF with appendfsync=always: successful XADD is its durable
|
|
# acceptance boundary. A lost response can cause a harmless duplicate.
|
|
return self.client.xadd(self.stream, {'payload': json.dumps(payload, separators=(',', ':'))})
|
|
|
|
def read(self, consumer, idle_ms):
|
|
# COUNT 1 bounds prefetch. Dead consumer pending entries are claimed only
|
|
# after the DB lease duration; DB fencing still decides who may commit.
|
|
pending = self.client.xautoclaim(self.stream, self.group, consumer, idle_ms,
|
|
start_id='0-0', count=1)
|
|
messages = pending[1]
|
|
if not messages:
|
|
entries = self.client.xreadgroup(self.group, consumer, {self.stream: '>'}, count=1, block=1000)
|
|
messages = entries[0][1] if entries else []
|
|
if not messages:
|
|
return None
|
|
message_id, fields = messages[0]
|
|
try:
|
|
if set(fields) != {'payload'} or len(fields['payload']) > 4096:
|
|
raise ValueError('invalid envelope')
|
|
return message_id, validate_message(json.loads(fields['payload']))
|
|
except (ValueError, TypeError, KeyError, AttributeError):
|
|
# Never acknowledge malformed work without a durable effect. Only
|
|
# our validated dispatcher can write to the private broker stream.
|
|
raise InvalidDelivery(message_id) from None
|
|
|
|
def ack(self, message_id):
|
|
self.client.xack(self.stream, self.group, message_id)
|
|
|
|
def close(self):
|
|
self.client.close()
|