diff --git a/compose.yaml b/compose.yaml index d1839a6d..8adae20b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,6 +5,7 @@ x-api: &api environment: &app-env DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL: redis://redis:6379/0 + BROKER_URL: redis://broker:6379/0 SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} S3_ENDPOINT: http://minio:9000 S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key @@ -18,6 +19,8 @@ x-api: &api condition: service_completed_successfully s3init: condition: service_completed_successfully + broker: + condition: service_healthy healthcheck: test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=4)"] interval: 3s @@ -103,6 +106,8 @@ services: depends_on: lb: condition: service_started + broker: + condition: service_healthy mem_limit: 512m cpus: 1 db: @@ -126,9 +131,22 @@ services: command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] mem_limit: 128m cpus: 0.5 + broker: + image: redis:7.4.1-bookworm + command: [redis-server, --appendonly, yes, --appendfsync, everysec, --maxmemory, 128mb, --maxmemory-policy, noeviction] + volumes: + - brokerdata:/data + healthcheck: + test: [CMD, redis-cli, ping] + interval: 3s + timeout: 2s + retries: 20 + mem_limit: 256m + cpus: 0.5 volumes: pgdata: s3data: + brokerdata: secrets: db_password: file: ./secrets/db_password diff --git a/src/attachments.py b/src/attachments.py index f071f717..722355cc 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -7,6 +7,7 @@ from threading import BoundedSemaphore from flask import Response, request, stream_with_context from PIL import Image, UnidentifiedImageError from src.storage import BUCKET, s3 +from src.queue import enqueue MAX_BYTES = 10 * 1024 * 1024 Image.MAX_IMAGE_PIXELS = 40_000_000 @@ -78,6 +79,7 @@ def register_routes(app, pool, actor, chat_access, uid, response, APIError, inva (identifier,chat_id,user,filename,request.mimetype,length,digest.hexdigest(),key)).fetchone() conn.execute('INSERT INTO attachment_jobs(attachment_id) VALUES (%s)', (identifier,)) conn.execute('DELETE FROM object_intents WHERE object_key=%s', (key,)) + enqueue(identifier) return response(public(row),202) finally: slots.release() diff --git a/src/queue.py b/src/queue.py new file mode 100644 index 00000000..63afa01b --- /dev/null +++ b/src/queue.py @@ -0,0 +1,17 @@ +"""Durable Redis Streams queue with consumer-group acknowledgements.""" +import os +import redis + +STREAM = 'attachments.jobs' +GROUP = 'image-workers' +client = redis.Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'), decode_responses=True, + socket_connect_timeout=1, socket_timeout=2) + +def ensure_group(): + try: client.xgroup_create(STREAM, GROUP, id='0-0', mkstream=True) + except redis.ResponseError as error: + if 'BUSYGROUP' not in str(error): raise + +def enqueue(attachment_id): + ensure_group() + return client.xadd(STREAM, {'attachment_id': str(attachment_id)}, maxlen=10000, approximate=True) diff --git a/src/worker.py b/src/worker.py index d691786d..a9316542 100644 --- a/src/worker.py +++ b/src/worker.py @@ -6,6 +6,7 @@ import time from PIL import Image, ImageOps from src.db import make_pool from src.storage import BUCKET, s3 +from src.queue import ensure_group, client as broker, STREAM, GROUP MAX_ATTEMPTS = 5 @@ -41,7 +42,18 @@ def process(row, pool): def loop(): pool=make_pool() + ensure_group() while True: + # Repair the transactional-outbox -> stream gap after a crash between commit and XADD. + with pool.connection() as conn: + pending = conn.execute("SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until'}, count=1, block=1000) + if messages: + for _, entries in messages: + for message_id, fields in entries: + broker.xack(STREAM, GROUP, message_id) with pool.connection() as conn: job=conn.execute("UPDATE attachment_jobs SET attempts=attempts+1,locked_until=clock_timestamp()+interval '60 seconds' WHERE attachment_id=(SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until