forked from mtusi2026/mtusi2026lab5
Add durable Redis Streams broker for attachment jobs
This commit is contained in:
@@ -5,6 +5,7 @@ x-api: &api
|
||||
environment: &app-env
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
BROKER_URL: redis://broker:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
@@ -18,6 +19,8 @@ x-api: &api
|
||||
condition: service_completed_successfully
|
||||
s3init:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=4)"]
|
||||
interval: 3s
|
||||
@@ -103,6 +106,8 @@ services:
|
||||
depends_on:
|
||||
lb:
|
||||
condition: service_started
|
||||
broker:
|
||||
condition: service_healthy
|
||||
mem_limit: 512m
|
||||
cpus: 1
|
||||
db:
|
||||
@@ -126,9 +131,22 @@ services:
|
||||
command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
broker:
|
||||
image: redis:7.4.1-bookworm
|
||||
command: [redis-server, --appendonly, yes, --appendfsync, everysec, --maxmemory, 128mb, --maxmemory-policy, noeviction]
|
||||
volumes:
|
||||
- brokerdata:/data
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
volumes:
|
||||
pgdata:
|
||||
s3data:
|
||||
brokerdata:
|
||||
secrets:
|
||||
db_password:
|
||||
file: ./secrets/db_password
|
||||
|
||||
@@ -7,6 +7,7 @@ from threading import BoundedSemaphore
|
||||
from flask import Response, request, stream_with_context
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from src.storage import BUCKET, s3
|
||||
from src.queue import enqueue
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
@@ -78,6 +79,7 @@ def register_routes(app, pool, actor, chat_access, uid, response, APIError, inva
|
||||
(identifier,chat_id,user,filename,request.mimetype,length,digest.hexdigest(),key)).fetchone()
|
||||
conn.execute('INSERT INTO attachment_jobs(attachment_id) VALUES (%s)', (identifier,))
|
||||
conn.execute('DELETE FROM object_intents WHERE object_key=%s', (key,))
|
||||
enqueue(identifier)
|
||||
return response(public(row),202)
|
||||
finally:
|
||||
slots.release()
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Durable Redis Streams queue with consumer-group acknowledgements."""
|
||||
import os
|
||||
import redis
|
||||
|
||||
STREAM = 'attachments.jobs'
|
||||
GROUP = 'image-workers'
|
||||
client = redis.Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'), decode_responses=True,
|
||||
socket_connect_timeout=1, socket_timeout=2)
|
||||
|
||||
def ensure_group():
|
||||
try: client.xgroup_create(STREAM, GROUP, id='0-0', mkstream=True)
|
||||
except redis.ResponseError as error:
|
||||
if 'BUSYGROUP' not in str(error): raise
|
||||
|
||||
def enqueue(attachment_id):
|
||||
ensure_group()
|
||||
return client.xadd(STREAM, {'attachment_id': str(attachment_id)}, maxlen=10000, approximate=True)
|
||||
@@ -6,6 +6,7 @@ import time
|
||||
from PIL import Image, ImageOps
|
||||
from src.db import make_pool
|
||||
from src.storage import BUCKET, s3
|
||||
from src.queue import ensure_group, client as broker, STREAM, GROUP
|
||||
|
||||
MAX_ATTEMPTS = 5
|
||||
|
||||
@@ -41,7 +42,18 @@ def process(row, pool):
|
||||
|
||||
def loop():
|
||||
pool=make_pool()
|
||||
ensure_group()
|
||||
while True:
|
||||
# Repair the transactional-outbox -> stream gap after a crash between commit and XADD.
|
||||
with pool.connection() as conn:
|
||||
pending = conn.execute("SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until<clock_timestamp()) LIMIT 50").fetchall()
|
||||
for pending_row in pending:
|
||||
broker.xadd(STREAM, {'attachment_id': str(pending_row['attachment_id'])}, maxlen=10000, approximate=True)
|
||||
messages = broker.xreadgroup(GROUP, 'worker-1', {STREAM:'>'}, count=1, block=1000)
|
||||
if messages:
|
||||
for _, entries in messages:
|
||||
for message_id, fields in entries:
|
||||
broker.xack(STREAM, GROUP, message_id)
|
||||
with pool.connection() as conn:
|
||||
job=conn.execute("UPDATE attachment_jobs SET attempts=attempts+1,locked_until=clock_timestamp()+interval '60 seconds' WHERE attachment_id=(SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until<clock_timestamp()) ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *").fetchone()
|
||||
row=conn.execute('SELECT * FROM attachments WHERE id=%s',(job['attachment_id'],)).fetchone() if job else None
|
||||
|
||||
Reference in New Issue
Block a user