Files
mtusi2026lab5/worker.py
T
2026-11-16 17:38:53 +03:00

300 lines
10 KiB
Python

import os
import io
import json
import time
import signal
import hashlib
import logging
from datetime import datetime, timezone
import psycopg2
import psycopg2.extras
import redis
import boto3
from botocore.client import Config
from PIL import Image, ImageOps, UnidentifiedImageError
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s"
)
logger = logging.getLogger("attachment-worker")
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
s3_client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
config=Config(signature_version="s3v4"),
region_name="us-east-1"
)
running = True
def handle_sigterm(signum, frame):
global running
logger.info("Received shutdown signal, finishing current job...")
running = False
signal.signal(signal.SIGTERM, handle_sigterm)
signal.signal(signal.SIGINT, handle_sigterm)
def get_db_connection():
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
def ensure_consumer_group():
while running:
try:
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
logger.info("Created consumer group '%s' on stream '%s'", GROUP_NAME, STREAM_NAME)
return
except redis.exceptions.ResponseError as e:
if "BUSYGROUP" in str(e):
logger.info("Consumer group '%s' already exists", GROUP_NAME)
return
raise
except Exception as e:
logger.warning("Waiting for Redis broker: %s", type(e).__name__)
time.sleep(1)
def build_thumbnail_jpeg(raw_bytes: bytes) -> tuple[bytes, int, int, str]:
with Image.open(io.BytesIO(raw_bytes)) as img:
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.split()[3])
img = background
elif img.mode != "RGB":
img = img.convert("RGB")
orig_w, orig_h = img.size
if orig_w <= 256 and orig_h <= 256:
target_w, target_h = orig_w, orig_h
else:
scale = min(256.0 / orig_w, 256.0 / orig_h)
target_w = max(1, int(orig_w * scale))
target_h = max(1, int(orig_h * scale))
if (target_w, target_h) != (orig_w, orig_h):
img = img.resize((target_w, target_h), Image.Resampling.LANCZOS)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85, optimize=True)
thumb_bytes = buf.getvalue()
thumb_sha256 = hashlib.sha256(thumb_bytes).hexdigest()
return thumb_bytes, target_w, target_h, thumb_sha256
def mark_failed(attachment_id: str, error_code: str):
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments
SET status = 'failed', error_code = %s, updated_at = %s
WHERE id = %s AND status != 'ready'
""", (error_code, now, attachment_id))
conn.commit()
finally:
conn.close()
def process_job(message_id: str, fields: dict):
try:
job = json.loads(fields.get("payload", "{}"))
except Exception:
logger.error("Malformed job JSON in message %s, acknowledging to drop", message_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
attachment_id = job.get("attachment_id")
job_id = job.get("job_id")
pipeline_version = int(job.get("pipeline_version", 1))
attempt = int(job.get("attempt", 1))
if not attachment_id:
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,))
att = cur.fetchone()
if not att:
conn.commit()
logger.warning("Attachment %s not found in DB, acking job %s", attachment_id, job_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
conn.commit()
logger.info("Attachment %s already ready (or newer pipeline), no-op ack for job %s", attachment_id, job_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "failed":
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
cur.execute("""
UPDATE attachments
SET status = 'processing', attempt = %s, updated_at = %s
WHERE id = %s
""", (attempt, now, attachment_id))
conn.commit()
finally:
conn.close()
chat_id = str(att["chat_id"])
staging_key = f"staging/{chat_id}/{attachment_id}/original"
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
final_thumb_key = f"attachments/{chat_id}/{attachment_id}/thumbnail"
try:
try:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
raw_bytes = obj["Body"].read()
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
raise ValueError("Staging checksum mismatch")
thumb_bytes, thumb_w, thumb_h, thumb_sha256 = build_thumbnail_jpeg(raw_bytes)
s3_client.put_object(
Bucket=S3_BUCKET,
Key=final_original_key,
Body=raw_bytes,
ContentType=att["content_type"]
)
s3_client.put_object(
Bucket=S3_BUCKET,
Key=final_thumb_key,
Body=thumb_bytes,
ContentType="image/jpeg"
)
variants = [
{
"name": "thumbnail",
"content_type": "image/jpeg",
"size_bytes": len(thumb_bytes),
"width": thumb_w,
"height": thumb_h,
"sha256": thumb_sha256
}
]
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments
SET status = 'ready',
variants = %s::jsonb,
error_code = NULL,
updated_at = %s
WHERE id = %s AND pipeline_version <= %s
""", (json.dumps(variants), ready_at, attachment_id, pipeline_version))
conn.commit()
finally:
conn.close()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
logger.info("Job %s completed: attachment %s is ready (thumbnail %dx%d)", job_id, attachment_id, thumb_w, thumb_h)
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
pass
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError) as perm_err:
logger.error("Permanent image error for %s: %s", attachment_id, type(perm_err).__name__)
mark_failed(attachment_id, "invalid_image")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except Exception as transient_err:
logger.warning(
"Transient error processing attachment %s (attempt %d/%d): %s",
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(transient_err).__name__
)
if attempt >= JOB_MAX_ATTEMPTS:
mark_failed(attachment_id, "retry_exhausted")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
else:
next_job = dict(job)
next_job["attempt"] = attempt + 1
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
def main():
ensure_consumer_group()
logger.info("Worker '%s' started listening on stream '%s'", CONSUMER_NAME, STREAM_NAME)
while running:
try:
pending = redis_client.xreadgroup(
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: "0"}, count=10
)
messages = pending[0][1] if pending else []
if not messages:
break
for msg_id, fields in messages:
if not running:
break
process_job(msg_id, fields)
except Exception as e:
logger.warning("Error reading pending messages: %s", type(e).__name__)
time.sleep(1)
while running:
try:
resp = redis_client.xreadgroup(
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=1, block=2000
)
if not resp:
continue
for _, messages in resp:
for msg_id, fields in messages:
if not running:
break
process_job(msg_id, fields)
except Exception as e:
logger.warning("Broker read error, retrying: %s", type(e).__name__)
time.sleep(1)
logger.info("Worker stopped cleanly")
if __name__ == "__main__":
main()