forked from mtusi2026/mtusi2026lab5
300 lines
10 KiB
Python
300 lines
10 KiB
Python
import os
|
|
import io
|
|
import json
|
|
import time
|
|
import signal
|
|
import hashlib
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
import psycopg2
|
|
import psycopg2.extras
|
|
import redis
|
|
import boto3
|
|
from botocore.client import Config
|
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(message)s"
|
|
)
|
|
logger = logging.getLogger("attachment-worker")
|
|
|
|
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
|
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
|
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
|
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
|
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
|
|
|
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
|
|
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
|
|
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
|
|
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
|
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
|
|
|
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
|
|
|
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
|
|
s3_client = boto3.client(
|
|
"s3",
|
|
endpoint_url=S3_ENDPOINT,
|
|
aws_access_key_id=S3_ACCESS_KEY,
|
|
aws_secret_access_key=S3_SECRET_KEY,
|
|
config=Config(signature_version="s3v4"),
|
|
region_name="us-east-1"
|
|
)
|
|
|
|
running = True
|
|
|
|
def handle_sigterm(signum, frame):
|
|
global running
|
|
logger.info("Received shutdown signal, finishing current job...")
|
|
running = False
|
|
|
|
signal.signal(signal.SIGTERM, handle_sigterm)
|
|
signal.signal(signal.SIGINT, handle_sigterm)
|
|
|
|
|
|
def get_db_connection():
|
|
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
|
|
|
|
|
def ensure_consumer_group():
|
|
while running:
|
|
try:
|
|
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
|
|
logger.info("Created consumer group '%s' on stream '%s'", GROUP_NAME, STREAM_NAME)
|
|
return
|
|
except redis.exceptions.ResponseError as e:
|
|
if "BUSYGROUP" in str(e):
|
|
logger.info("Consumer group '%s' already exists", GROUP_NAME)
|
|
return
|
|
raise
|
|
except Exception as e:
|
|
logger.warning("Waiting for Redis broker: %s", type(e).__name__)
|
|
time.sleep(1)
|
|
|
|
|
|
def build_thumbnail_jpeg(raw_bytes: bytes) -> tuple[bytes, int, int, str]:
|
|
with Image.open(io.BytesIO(raw_bytes)) as img:
|
|
img = ImageOps.exif_transpose(img)
|
|
|
|
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
|
rgba = img.convert("RGBA")
|
|
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
|
background.paste(rgba, mask=rgba.split()[3])
|
|
img = background
|
|
elif img.mode != "RGB":
|
|
img = img.convert("RGB")
|
|
|
|
orig_w, orig_h = img.size
|
|
|
|
if orig_w <= 256 and orig_h <= 256:
|
|
target_w, target_h = orig_w, orig_h
|
|
else:
|
|
scale = min(256.0 / orig_w, 256.0 / orig_h)
|
|
target_w = max(1, int(orig_w * scale))
|
|
target_h = max(1, int(orig_h * scale))
|
|
|
|
if (target_w, target_h) != (orig_w, orig_h):
|
|
img = img.resize((target_w, target_h), Image.Resampling.LANCZOS)
|
|
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="JPEG", quality=85, optimize=True)
|
|
thumb_bytes = buf.getvalue()
|
|
thumb_sha256 = hashlib.sha256(thumb_bytes).hexdigest()
|
|
return thumb_bytes, target_w, target_h, thumb_sha256
|
|
|
|
|
|
def mark_failed(attachment_id: str, error_code: str):
|
|
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
conn = get_db_connection()
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
UPDATE attachments
|
|
SET status = 'failed', error_code = %s, updated_at = %s
|
|
WHERE id = %s AND status != 'ready'
|
|
""", (error_code, now, attachment_id))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def process_job(message_id: str, fields: dict):
|
|
try:
|
|
job = json.loads(fields.get("payload", "{}"))
|
|
except Exception:
|
|
logger.error("Malformed job JSON in message %s, acknowledging to drop", message_id)
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
return
|
|
|
|
attachment_id = job.get("attachment_id")
|
|
job_id = job.get("job_id")
|
|
pipeline_version = int(job.get("pipeline_version", 1))
|
|
attempt = int(job.get("attempt", 1))
|
|
|
|
if not attachment_id:
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
return
|
|
|
|
conn = get_db_connection()
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,))
|
|
att = cur.fetchone()
|
|
|
|
if not att:
|
|
conn.commit()
|
|
logger.warning("Attachment %s not found in DB, acking job %s", attachment_id, job_id)
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
return
|
|
|
|
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
|
|
conn.commit()
|
|
logger.info("Attachment %s already ready (or newer pipeline), no-op ack for job %s", attachment_id, job_id)
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
return
|
|
|
|
if att["status"] == "failed":
|
|
conn.commit()
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
return
|
|
|
|
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
cur.execute("""
|
|
UPDATE attachments
|
|
SET status = 'processing', attempt = %s, updated_at = %s
|
|
WHERE id = %s
|
|
""", (attempt, now, attachment_id))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
chat_id = str(att["chat_id"])
|
|
staging_key = f"staging/{chat_id}/{attachment_id}/original"
|
|
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
|
|
final_thumb_key = f"attachments/{chat_id}/{attachment_id}/thumbnail"
|
|
|
|
try:
|
|
try:
|
|
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
|
|
except Exception:
|
|
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
|
|
raw_bytes = obj["Body"].read()
|
|
|
|
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
|
|
raise ValueError("Staging checksum mismatch")
|
|
|
|
thumb_bytes, thumb_w, thumb_h, thumb_sha256 = build_thumbnail_jpeg(raw_bytes)
|
|
|
|
s3_client.put_object(
|
|
Bucket=S3_BUCKET,
|
|
Key=final_original_key,
|
|
Body=raw_bytes,
|
|
ContentType=att["content_type"]
|
|
)
|
|
s3_client.put_object(
|
|
Bucket=S3_BUCKET,
|
|
Key=final_thumb_key,
|
|
Body=thumb_bytes,
|
|
ContentType="image/jpeg"
|
|
)
|
|
|
|
variants = [
|
|
{
|
|
"name": "thumbnail",
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": len(thumb_bytes),
|
|
"width": thumb_w,
|
|
"height": thumb_h,
|
|
"sha256": thumb_sha256
|
|
}
|
|
]
|
|
|
|
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
conn = get_db_connection()
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
UPDATE attachments
|
|
SET status = 'ready',
|
|
variants = %s::jsonb,
|
|
error_code = NULL,
|
|
updated_at = %s
|
|
WHERE id = %s AND pipeline_version <= %s
|
|
""", (json.dumps(variants), ready_at, attachment_id, pipeline_version))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
logger.info("Job %s completed: attachment %s is ready (thumbnail %dx%d)", job_id, attachment_id, thumb_w, thumb_h)
|
|
|
|
try:
|
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
|
|
except Exception:
|
|
pass
|
|
|
|
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError) as perm_err:
|
|
logger.error("Permanent image error for %s: %s", attachment_id, type(perm_err).__name__)
|
|
mark_failed(attachment_id, "invalid_image")
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
|
|
except Exception as transient_err:
|
|
logger.warning(
|
|
"Transient error processing attachment %s (attempt %d/%d): %s",
|
|
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(transient_err).__name__
|
|
)
|
|
if attempt >= JOB_MAX_ATTEMPTS:
|
|
mark_failed(attachment_id, "retry_exhausted")
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
else:
|
|
next_job = dict(job)
|
|
next_job["attempt"] = attempt + 1
|
|
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
|
|
|
|
def main():
|
|
ensure_consumer_group()
|
|
logger.info("Worker '%s' started listening on stream '%s'", CONSUMER_NAME, STREAM_NAME)
|
|
|
|
while running:
|
|
try:
|
|
pending = redis_client.xreadgroup(
|
|
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: "0"}, count=10
|
|
)
|
|
messages = pending[0][1] if pending else []
|
|
if not messages:
|
|
break
|
|
for msg_id, fields in messages:
|
|
if not running:
|
|
break
|
|
process_job(msg_id, fields)
|
|
except Exception as e:
|
|
logger.warning("Error reading pending messages: %s", type(e).__name__)
|
|
time.sleep(1)
|
|
|
|
while running:
|
|
try:
|
|
resp = redis_client.xreadgroup(
|
|
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=1, block=2000
|
|
)
|
|
if not resp:
|
|
continue
|
|
for _, messages in resp:
|
|
for msg_id, fields in messages:
|
|
if not running:
|
|
break
|
|
process_job(msg_id, fields)
|
|
except Exception as e:
|
|
logger.warning("Broker read error, retrying: %s", type(e).__name__)
|
|
time.sleep(1)
|
|
|
|
logger.info("Worker stopped cleanly")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |