Add durable Redis Streams broker for attachment jobs

This commit is contained in:
2026-11-20 21:02:33 +03:00
parent d8332e17f6
commit 930963f1ea
4 changed files with 49 additions and 0 deletions
+18
View File
@@ -5,6 +5,7 @@ x-api: &api
environment: &app-env
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL: redis://redis:6379/0
BROKER_URL: redis://broker:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
S3_ENDPOINT: http://minio:9000
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
@@ -18,6 +19,8 @@ x-api: &api
condition: service_completed_successfully
s3init:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=4)"]
interval: 3s
@@ -103,6 +106,8 @@ services:
depends_on:
lb:
condition: service_started
broker:
condition: service_healthy
mem_limit: 512m
cpus: 1
db:
@@ -126,9 +131,22 @@ services:
command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
mem_limit: 128m
cpus: 0.5
broker:
image: redis:7.4.1-bookworm
command: [redis-server, --appendonly, yes, --appendfsync, everysec, --maxmemory, 128mb, --maxmemory-policy, noeviction]
volumes:
- brokerdata:/data
healthcheck:
test: [CMD, redis-cli, ping]
interval: 3s
timeout: 2s
retries: 20
mem_limit: 256m
cpus: 0.5
volumes:
pgdata:
s3data:
brokerdata:
secrets:
db_password:
file: ./secrets/db_password
+2
View File
@@ -7,6 +7,7 @@ from threading import BoundedSemaphore
from flask import Response, request, stream_with_context
from PIL import Image, UnidentifiedImageError
from src.storage import BUCKET, s3
from src.queue import enqueue
MAX_BYTES = 10 * 1024 * 1024
Image.MAX_IMAGE_PIXELS = 40_000_000
@@ -78,6 +79,7 @@ def register_routes(app, pool, actor, chat_access, uid, response, APIError, inva
(identifier,chat_id,user,filename,request.mimetype,length,digest.hexdigest(),key)).fetchone()
conn.execute('INSERT INTO attachment_jobs(attachment_id) VALUES (%s)', (identifier,))
conn.execute('DELETE FROM object_intents WHERE object_key=%s', (key,))
enqueue(identifier)
return response(public(row),202)
finally:
slots.release()
+17
View File
@@ -0,0 +1,17 @@
"""Durable Redis Streams queue with consumer-group acknowledgements."""
import os
import redis
STREAM = 'attachments.jobs'
GROUP = 'image-workers'
client = redis.Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'), decode_responses=True,
socket_connect_timeout=1, socket_timeout=2)
def ensure_group():
try: client.xgroup_create(STREAM, GROUP, id='0-0', mkstream=True)
except redis.ResponseError as error:
if 'BUSYGROUP' not in str(error): raise
def enqueue(attachment_id):
ensure_group()
return client.xadd(STREAM, {'attachment_id': str(attachment_id)}, maxlen=10000, approximate=True)
+12
View File
@@ -6,6 +6,7 @@ import time
from PIL import Image, ImageOps
from src.db import make_pool
from src.storage import BUCKET, s3
from src.queue import ensure_group, client as broker, STREAM, GROUP
MAX_ATTEMPTS = 5
@@ -41,7 +42,18 @@ def process(row, pool):
def loop():
pool=make_pool()
ensure_group()
while True:
# Repair the transactional-outbox -> stream gap after a crash between commit and XADD.
with pool.connection() as conn:
pending = conn.execute("SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until<clock_timestamp()) LIMIT 50").fetchall()
for pending_row in pending:
broker.xadd(STREAM, {'attachment_id': str(pending_row['attachment_id'])}, maxlen=10000, approximate=True)
messages = broker.xreadgroup(GROUP, 'worker-1', {STREAM:'>'}, count=1, block=1000)
if messages:
for _, entries in messages:
for message_id, fields in entries:
broker.xack(STREAM, GROUP, message_id)
with pool.connection() as conn:
job=conn.execute("UPDATE attachment_jobs SET attempts=attempts+1,locked_until=clock_timestamp()+interval '60 seconds' WHERE attachment_id=(SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until<clock_timestamp()) ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *").fetchone()
row=conn.execute('SELECT * FROM attachments WHERE id=%s',(job['attachment_id'],)).fetchone() if job else None