This commit is contained in:
Getlles
2026-11-16 20:34:32 +03:00
parent d475bf29ed
commit 749109aaf4
8 changed files with 363 additions and 178 deletions
+115 -44
View File
@@ -1,4 +1,5 @@
import os
import time
import base64
import secrets
import uuid
@@ -24,8 +25,38 @@ import redis
import boto3
from botocore.client import Config
import threading
app = FastAPI(title="MTUSI Messenger — lab5", version="1.5.0")
def outbox_publisher():
while True:
conn = None
try:
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute("""
SELECT id, stream, payload FROM outbox
WHERE published_at IS NULL
ORDER BY created_at
LIMIT 100
FOR UPDATE SKIP LOCKED
""")
for row in cur.fetchall():
redis_client.xadd(row["stream"], {"payload": json.dumps(row["payload"])})
cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],))
conn.commit()
except Exception as e:
print(f"OUTBOX PUBLISHER ERROR: {type(e).__name__} - {e}", flush=True)
finally:
if conn is not None and not conn.closed:
conn.close()
time.sleep(1)
@app.on_event("startup")
def _start_outbox_publisher():
threading.Thread(target=outbox_publisher, daemon=True).start()
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
INSTANCE_ID = os.getenv("INSTANCE_ID", "instance-1")
@@ -583,7 +614,10 @@ async def upload_attachment(
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id_str, user["id"]))
cur.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id_str, user["id"])
)
if not cur.fetchone():
conn.close()
raise HTTPException(status_code=404, detail="Chat not found or inaccessible")
@@ -633,6 +667,16 @@ async def upload_attachment(
except Exception:
raise HTTPException(status_code=503, detail="S3 storage unavailable")
job_payload = {
"schema_version": 1,
"job_id": job_id,
"type": "attachment.process.v1",
"attachment_id": attachment_id,
"pipeline_version": 1,
"attempt": 1,
"created_at": now
}
conn = get_db_connection()
try:
with conn.cursor() as cur:
@@ -643,30 +687,22 @@ async def upload_attachment(
pipeline_version, attempt, created_at, updated_at
)
VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', '[]', NULL, 1, 0, %s, %s)
""", (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now))
""", (attachment_id, chat_id_str, user["id"], filename,
content_type, size_bytes, sha256_hash, now, now))
cur.execute(
"INSERT INTO outbox (id, stream, payload, created_at) "
"VALUES (%s, %s, %s::jsonb, %s)",
(str(uuid.uuid4()), STREAM_NAME, json.dumps(job_payload), now)
)
conn.commit()
except Exception:
conn.rollback()
conn.close()
raise HTTPException(status_code=500, detail="Internal error saving attachment metadata")
raise HTTPException(status_code=500, detail="Internal error saving attachment")
finally:
if not conn.closed:
conn.close()
job_payload = {
"schema_version": 1,
"job_id": job_id,
"type": "attachment.process.v1",
"attachment_id": attachment_id,
"pipeline_version": 1,
"attempt": 1,
"created_at": now
}
try:
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(job_payload)})
except Exception:
raise HTTPException(status_code=503, detail="Broker unavailable")
return {
"id": attachment_id,
"chat_id": chat_id_str,
@@ -873,20 +909,18 @@ def create_message(
return msg_response
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, response: Response = None, authorization: Optional[str] = Header(None)):
def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
response: Response = None, authorization: Optional[str] = Header(None)):
req_id = generate_request_id()
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = INSTANCE_ID
user = get_current_user_from_token(authorization)
chat_id_str = str(chat_id)
conn = get_db_connection()
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id_str, user["id"])
)
cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",
(chat_id_str, user["id"]))
if not cur.fetchone():
conn.close()
raise HTTPException(status_code=404, detail="Chat not found")
@@ -895,17 +929,12 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
conn.close()
raise HTTPException(status_code=400, detail="Invalid limit")
query = """
SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages
WHERE chat_id = %s
"""
query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s"
params = [chat_id_str]
if cursor:
c_time, c_id = decode_cursor(cursor, chat_id_str)
query += " AND (created_at, id) > (%s, %s)"
params.extend([c_time, c_id])
query += " ORDER BY created_at ASC, id ASC LIMIT %s"
params.append(limit + 1)
@@ -917,23 +946,65 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
has_next = len(rows) > limit
page_items = rows[:limit]
formatted_msgs = []
formatted = []
for m in page_items:
raw_atts = m.get('attachment_ids')
atts = [str(aid) for aid in raw_atts if aid] if raw_atts else []
formatted_msgs.append({
"id": str(m['id']),
"chat_id": str(m['chat_id']),
"sender_id": str(m['sender_id']),
"text": m['text'],
"created_at": m['created_at'].isoformat().replace("+00:00", "Z") if isinstance(m['created_at'], datetime) else m['created_at'],
raw = m.get('attachment_ids')
atts = [str(a) for a in raw if a] if raw else []
formatted.append({
"id": str(m['id']), "chat_id": str(m['chat_id']),
"sender_id": str(m['sender_id']), "text": m['text'],
"created_at": m['created_at'].isoformat().replace("+00:00", "Z")
if isinstance(m['created_at'], datetime) else m['created_at'],
"attachment_ids": atts
})
next_cursor = None
if has_next and formatted_msgs:
last = formatted_msgs[-1]
if has_next and formatted:
last = formatted[-1]
next_cursor = encode_cursor(chat_id_str, last['created_at'], last['id'])
return {"items": formatted_msgs, "next_cursor": next_cursor}
return {"items": formatted, "next_cursor": next_cursor}
@app.post("/api/v1/attachments/{attachment_id}/replay", status_code=202)
def replay_attachment(attachment_id: UUID, response: Response,
authorization: Optional[str] = Header(None)):
req_id = generate_request_id()
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = INSTANCE_ID
user = get_current_user_from_token(authorization)
att_id = str(attachment_id)
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON a.chat_id = cm.chat_id
WHERE a.id=%s AND cm.user_id=%s
""", (att_id, user["id"]))
row = cur.fetchone()
if not row:
conn.close()
raise HTTPException(status_code=404, detail="Attachment not found")
if row["status"] != "failed":
conn.close()
raise HTTPException(status_code=409, detail="Only failed attachments can be replayed")
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
cur.execute("""
UPDATE attachments SET status='queued', attempt=0, error_code=NULL, updated_at=%s
WHERE id=%s
""", (now, att_id))
job = {"schema_version": 1, "job_id": str(uuid.uuid4()),
"type": "attachment.process.v1", "attachment_id": att_id,
"pipeline_version": 1, "attempt": 1, "created_at": now}
cur.execute("INSERT INTO outbox (id, stream, payload, created_at) "
"VALUES (%s, %s, %s::jsonb, %s)",
(str(uuid.uuid4()), STREAM_NAME, json.dumps(job), now))
conn.commit()
finally:
if not conn.closed:
conn.close()
return {"id": att_id, "status": "queued"}
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.6 KiB

Binary file not shown.
Binary file not shown.
+10 -1
View File
@@ -58,8 +58,17 @@ CREATE TABLE IF NOT EXISTS attachments (
updated_at TIMESTAMPTZ NOT NULL
);
CREATE TABLE IF NOT EXISTS outbox (
id UUID PRIMARY KEY,
stream VARCHAR(64) NOT NULL,
payload JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
published_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id);
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}';
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0;
CREATE INDEX IF NOT EXISTS idx_outbox_unpublished ON outbox(created_at) WHERE published_at IS NULL;
+1 -1
View File
@@ -45,7 +45,7 @@ def main():
for file in ROOT.rglob('*.json'):
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
continue
json.loads(file.read_text(encoding='utf-8'))
json.loads(file.read_text(encoding='utf-8-sig'))
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
assert (ROOT/required).is_file(), 'missing '+required
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
+237 -132
View File
@@ -3,21 +3,21 @@ import io
import json
import time
import signal
import random
import hashlib
import logging
from datetime import datetime, timezone
from concurrent.futures import ThreadPoolExecutor
import psycopg2
import psycopg2.extras
import redis
import boto3
import botocore.exceptions
from botocore.client import Config
from PIL import Image, ImageOps, UnidentifiedImageError
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s"
)
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger("attachment-worker")
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
@@ -28,28 +28,30 @@ S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq"
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4"))
CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
s3_client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
config=Config(signature_version="s3v4"),
region_name="us-east-1"
"s3", endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY,
config=Config(signature_version="s3v4"), region_name="us-east-1"
)
running = True
def handle_sigterm(signum, frame):
global running
logger.info("Received shutdown signal, finishing current job...")
logger.info("Received shutdown signal, finishing in-flight jobs...")
running = False
signal.signal(signal.SIGTERM, handle_sigterm)
@@ -60,73 +62,114 @@ def get_db_connection():
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
def ensure_consumer_group():
def ensure_consumer_groups():
while running:
try:
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
logger.info("Created consumer group '%s' on stream '%s'", GROUP_NAME, STREAM_NAME)
redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME)
return
except redis.exceptions.ResponseError as e:
if "BUSYGROUP" in str(e):
logger.info("Consumer group '%s' already exists", GROUP_NAME)
return
raise
except Exception as e:
logger.warning("Waiting for Redis broker: %s", type(e).__name__)
logger.warning("Waiting for Redis: %s", type(e).__name__)
time.sleep(1)
def build_thumbnail_jpeg(raw_bytes: bytes) -> tuple[bytes, int, int, str]:
with Image.open(io.BytesIO(raw_bytes)) as img:
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.split()[3])
img = background
elif img.mode != "RGB":
img = img.convert("RGB")
orig_w, orig_h = img.size
if orig_w <= 256 and orig_h <= 256:
target_w, target_h = orig_w, orig_h
else:
scale = min(256.0 / orig_w, 256.0 / orig_h)
target_w = max(1, int(orig_w * scale))
target_h = max(1, int(orig_h * scale))
if (target_w, target_h) != (orig_w, orig_h):
img = img.resize((target_w, target_h), Image.Resampling.LANCZOS)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85, optimize=True)
thumb_bytes = buf.getvalue()
thumb_sha256 = hashlib.sha256(thumb_bytes).hexdigest()
return thumb_bytes, target_w, target_h, thumb_sha256
def prepare_base_image(raw_bytes):
img = Image.open(io.BytesIO(raw_bytes))
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
bg = Image.new("RGB", rgba.size, (255, 255, 255))
bg.paste(rgba, mask=rgba.split()[3])
img = bg
elif img.mode != "RGB":
img = img.convert("RGB")
return img
def mark_failed(attachment_id: str, error_code: str):
def generate_grade4_variants(raw_bytes):
base_img = prepare_base_image(raw_bytes)
orig_w, orig_h = base_img.size
meta = []
blobs = {}
thumb = base_img.copy()
if orig_w <= 256 and orig_h <= 256:
t_w, t_h = orig_w, orig_h
else:
s = min(256.0 / orig_w, 256.0 / orig_h)
t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
if (t_w, t_h) != (orig_w, orig_h):
thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS)
buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85, optimize=True)
tb = buf.getvalue(); blobs["thumbnail"] = tb
meta.append({"name": "thumbnail", "content_type": "image/jpeg",
"size_bytes": len(tb), "width": t_w, "height": t_h,
"sha256": hashlib.sha256(tb).hexdigest()})
opt = base_img.copy()
if orig_w <= 1280 and orig_h <= 1280:
o_w, o_h = orig_w, orig_h
else:
s = min(1280.0 / orig_w, 1280.0 / orig_h)
o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
if (o_w, o_h) != (orig_w, orig_h):
opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS)
buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=85)
ob = buf.getvalue(); blobs["optimized"] = ob
meta.append({"name": "optimized", "content_type": "image/webp",
"size_bytes": len(ob), "width": o_w, "height": o_h,
"sha256": hashlib.sha256(ob).hexdigest()})
c = min(orig_w, orig_h)
left, top = (orig_w - c) // 2, (orig_h - c) // 2
crop = base_img.crop((left, top, left + c, top + c))
if c != 128:
crop = crop.resize((128, 128), Image.Resampling.LANCZOS)
buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85, optimize=True)
cb = buf.getvalue(); blobs["crop"] = cb
meta.append({"name": "crop", "content_type": "image/jpeg",
"size_bytes": len(cb), "width": 128, "height": 128,
"sha256": hashlib.sha256(cb).hexdigest()})
return meta, blobs
def mark_failed(attachment_id, error_code):
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments
SET status = 'failed', error_code = %s, updated_at = %s
WHERE id = %s AND status != 'ready'
""", (error_code, now, attachment_id))
cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=%s "
"WHERE id=%s AND status != 'ready'",
(error_code, now, attachment_id))
conn.commit()
finally:
conn.close()
def process_job(message_id: str, fields: dict):
PERMANENT_ERRORS = (
UnidentifiedImageError, Image.DecompressionBombError, ValueError,
psycopg2.errors.DataError,
)
TRANSIENT_ERRORS = (
TimeoutError,
botocore.exceptions.EndpointConnectionError,
botocore.exceptions.ReadTimeoutError,
psycopg2.OperationalError,
redis.exceptions.ConnectionError,
)
def process_job(message_id, fields):
try:
job = json.loads(fields.get("payload", "{}"))
except Exception:
logger.error("Malformed job JSON in message %s, acknowledging to drop", message_id)
logger.error("Malformed job %s, ack to drop", message_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
@@ -142,32 +185,38 @@ def process_job(message_id: str, fields: dict):
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,))
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
att = cur.fetchone()
if not att:
conn.commit()
logger.warning("Attachment %s not found in DB, acking job %s", attachment_id, job_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
conn.commit()
logger.info("Attachment %s already ready (or newer pipeline), no-op ack for job %s", attachment_id, job_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "failed":
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "processing":
updated = att["updated_at"]
if isinstance(updated, str):
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
conn.commit()
logger.info("Attachment %s already processing (age=%.1fs), ack duplicate",
attachment_id, age_seconds)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
cur.execute("""
UPDATE attachments
SET status = 'processing', attempt = %s, updated_at = %s
WHERE id = %s
""", (attempt, now, attachment_id))
cur.execute(
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
(attempt, now, attachment_id)
)
conn.commit()
finally:
conn.close()
@@ -175,7 +224,12 @@ def process_job(message_id: str, fields: dict):
chat_id = str(att["chat_id"])
staging_key = f"staging/{chat_id}/{attachment_id}/original"
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
final_thumb_key = f"attachments/{chat_id}/{attachment_id}/thumbnail"
start_time = time.monotonic()
def check_timeout():
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
raise TimeoutError("Job exceeded time limit")
try:
try:
@@ -183,117 +237,168 @@ def process_job(message_id: str, fields: dict):
except Exception:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
raw_bytes = obj["Body"].read()
check_timeout()
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
raise ValueError("Staging checksum mismatch")
thumb_bytes, thumb_w, thumb_h, thumb_sha256 = build_thumbnail_jpeg(raw_bytes)
variants_meta, variants_bytes = generate_grade4_variants(raw_bytes)
check_timeout()
s3_client.put_object(
Bucket=S3_BUCKET,
Key=final_original_key,
Body=raw_bytes,
ContentType=att["content_type"]
Bucket=S3_BUCKET, Key=final_original_key,
Body=raw_bytes, ContentType=att["content_type"]
)
s3_client.put_object(
Bucket=S3_BUCKET,
Key=final_thumb_key,
Body=thumb_bytes,
ContentType="image/jpeg"
)
variants = [
{
"name": "thumbnail",
"content_type": "image/jpeg",
"size_bytes": len(thumb_bytes),
"width": thumb_w,
"height": thumb_h,
"sha256": thumb_sha256
}
]
for var_name, var_bytes in variants_bytes.items():
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
s3_client.put_object(
Bucket=S3_BUCKET,
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
Body=var_bytes, ContentType=ct
)
check_timeout()
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments
SET status = 'ready',
variants = %s::jsonb,
error_code = NULL,
updated_at = %s
WHERE id = %s AND pipeline_version <= %s
""", (json.dumps(variants), ready_at, attachment_id, pipeline_version))
UPDATE attachments SET status='ready', variants=%s::jsonb,
error_code=NULL, updated_at=%s
WHERE id=%s AND pipeline_version <= %s
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
conn.commit()
finally:
conn.close()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
logger.info("Job %s completed: attachment %s is ready (thumbnail %dx%d)", job_id, attachment_id, thumb_w, thumb_h)
logger.info("Job %s completed: attachment %s ready", job_id, attachment_id)
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
pass
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError) as perm_err:
logger.error("Permanent image error for %s: %s", attachment_id, type(perm_err).__name__)
except (UnidentifiedImageError, Image.DecompressionBombError) as e:
logger.error("Permanent image error for %s: %s", attachment_id, type(e).__name__)
mark_failed(attachment_id, "invalid_image")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except Exception as transient_err:
except ValueError as e:
logger.error("Checksum error for %s: %s", attachment_id, e)
mark_failed(attachment_id, "invalid_checksum")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except psycopg2.errors.DataError as e:
logger.error("DB data error for %s: %s", attachment_id, type(e).__name__)
mark_failed(attachment_id, "data_error")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except TRANSIENT_ERRORS as e:
logger.warning(
"Transient error processing attachment %s (attempt %d/%d): %s",
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(transient_err).__name__
"Transient error for %s (attempt %d/%d): %s",
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(e).__name__
)
if attempt >= JOB_MAX_ATTEMPTS:
mark_failed(attachment_id, "retry_exhausted")
try:
dlq_job = dict(job)
dlq_job["attempt"] = 1
redis_client.xadd(
DLQ_STREAM_NAME,
{"payload": json.dumps(dlq_job), "error": str(e)}
)
except Exception:
pass
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
else:
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
time.sleep(delay)
next_job = dict(job)
next_job["attempt"] = attempt + 1
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except Exception as e:
logger.error("Unknown error for %s: %s, marking failed",
attachment_id, type(e).__name__)
mark_failed(attachment_id, "processing_failed")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
def run_staging_cleanup():
try:
cutoff = datetime.now(timezone.utc).timestamp() - 3600
paginator = s3_client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"):
for obj in page.get("Contents", []):
if obj["LastModified"].timestamp() < cutoff:
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"])
except Exception:
pass
except Exception as e:
logger.debug("Staging cleanup: %s", e)
def safe_process(msg_id, fields):
"""Wrapper - exceptions в threads иначе исчезают молча."""
try:
process_job(msg_id, fields)
except Exception as e:
logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__)
def main():
ensure_consumer_group()
logger.info("Worker '%s' started listening on stream '%s'", CONSUMER_NAME, STREAM_NAME)
ensure_consumer_groups()
logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY)
while running:
try:
pending = redis_client.xreadgroup(
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: "0"}, count=10
)
messages = pending[0][1] if pending else []
if not messages:
break
for msg_id, fields in messages:
if not running:
break
process_job(msg_id, fields)
except Exception as e:
logger.warning("Error reading pending messages: %s", type(e).__name__)
time.sleep(1)
executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY)
last_claim = 0
last_cleanup = 0
while running:
try:
resp = redis_client.xreadgroup(
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=1, block=2000
)
if not resp:
continue
for _, messages in resp:
for msg_id, fields in messages:
if not running:
break
process_job(msg_id, fields)
except Exception as e:
logger.warning("Broker read error, retrying: %s", type(e).__name__)
time.sleep(1)
try:
while running:
try:
now = time.time()
logger.info("Worker stopped cleanly")
if now - last_claim > 15:
last_claim = now
try:
claimed = redis_client.xautoclaim(
STREAM_NAME, GROUP_NAME, CONSUMER_NAME,
min_idle_time=CLAIM_MIN_IDLE_MS,
start_id="0-0", count=WORKER_CONCURRENCY
)
if claimed and len(claimed) > 1 and claimed[1]:
for msg_id, fields in claimed[1]:
if not running:
break
executor.submit(safe_process, msg_id, fields)
except Exception:
pass
if now - last_cleanup > 600:
last_cleanup = now
executor.submit(run_staging_cleanup)
resp = redis_client.xreadgroup(
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"},
count=WORKER_CONCURRENCY, block=2000
)
if not resp:
continue
for _, messages in resp:
for msg_id, fields in messages:
if not running:
break
executor.submit(safe_process, msg_id, fields)
except Exception as e:
logger.warning("Worker loop: %s", type(e).__name__)
time.sleep(1)
finally:
logger.info("Shutting down, waiting for in-flight jobs...")
executor.shutdown(wait=True)
logger.info("Worker stopped cleanly")
if __name__ == "__main__":