Files
mtusi2026lab5/worker.py
T
2026-11-16 20:34:32 +03:00

405 lines
15 KiB
Python

import os
import io
import json
import time
import signal
import random
import hashlib
import logging
from datetime import datetime, timezone
from concurrent.futures import ThreadPoolExecutor
import psycopg2
import psycopg2.extras
import redis
import boto3
import botocore.exceptions
from botocore.client import Config
from PIL import Image, ImageOps, UnidentifiedImageError
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger("attachment-worker")
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq"
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4"))
CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
s3_client = boto3.client(
"s3", endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY,
config=Config(signature_version="s3v4"), region_name="us-east-1"
)
running = True
def handle_sigterm(signum, frame):
global running
logger.info("Received shutdown signal, finishing in-flight jobs...")
running = False
signal.signal(signal.SIGTERM, handle_sigterm)
signal.signal(signal.SIGINT, handle_sigterm)
def get_db_connection():
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
def ensure_consumer_groups():
while running:
try:
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME)
return
except redis.exceptions.ResponseError as e:
if "BUSYGROUP" in str(e):
return
raise
except Exception as e:
logger.warning("Waiting for Redis: %s", type(e).__name__)
time.sleep(1)
def prepare_base_image(raw_bytes):
img = Image.open(io.BytesIO(raw_bytes))
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
bg = Image.new("RGB", rgba.size, (255, 255, 255))
bg.paste(rgba, mask=rgba.split()[3])
img = bg
elif img.mode != "RGB":
img = img.convert("RGB")
return img
def generate_grade4_variants(raw_bytes):
base_img = prepare_base_image(raw_bytes)
orig_w, orig_h = base_img.size
meta = []
blobs = {}
thumb = base_img.copy()
if orig_w <= 256 and orig_h <= 256:
t_w, t_h = orig_w, orig_h
else:
s = min(256.0 / orig_w, 256.0 / orig_h)
t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
if (t_w, t_h) != (orig_w, orig_h):
thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS)
buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85, optimize=True)
tb = buf.getvalue(); blobs["thumbnail"] = tb
meta.append({"name": "thumbnail", "content_type": "image/jpeg",
"size_bytes": len(tb), "width": t_w, "height": t_h,
"sha256": hashlib.sha256(tb).hexdigest()})
opt = base_img.copy()
if orig_w <= 1280 and orig_h <= 1280:
o_w, o_h = orig_w, orig_h
else:
s = min(1280.0 / orig_w, 1280.0 / orig_h)
o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
if (o_w, o_h) != (orig_w, orig_h):
opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS)
buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=85)
ob = buf.getvalue(); blobs["optimized"] = ob
meta.append({"name": "optimized", "content_type": "image/webp",
"size_bytes": len(ob), "width": o_w, "height": o_h,
"sha256": hashlib.sha256(ob).hexdigest()})
c = min(orig_w, orig_h)
left, top = (orig_w - c) // 2, (orig_h - c) // 2
crop = base_img.crop((left, top, left + c, top + c))
if c != 128:
crop = crop.resize((128, 128), Image.Resampling.LANCZOS)
buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85, optimize=True)
cb = buf.getvalue(); blobs["crop"] = cb
meta.append({"name": "crop", "content_type": "image/jpeg",
"size_bytes": len(cb), "width": 128, "height": 128,
"sha256": hashlib.sha256(cb).hexdigest()})
return meta, blobs
def mark_failed(attachment_id, error_code):
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=%s "
"WHERE id=%s AND status != 'ready'",
(error_code, now, attachment_id))
conn.commit()
finally:
conn.close()
PERMANENT_ERRORS = (
UnidentifiedImageError, Image.DecompressionBombError, ValueError,
psycopg2.errors.DataError,
)
TRANSIENT_ERRORS = (
TimeoutError,
botocore.exceptions.EndpointConnectionError,
botocore.exceptions.ReadTimeoutError,
psycopg2.OperationalError,
redis.exceptions.ConnectionError,
)
def process_job(message_id, fields):
try:
job = json.loads(fields.get("payload", "{}"))
except Exception:
logger.error("Malformed job %s, ack to drop", message_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
attachment_id = job.get("attachment_id")
job_id = job.get("job_id")
pipeline_version = int(job.get("pipeline_version", 1))
attempt = int(job.get("attempt", 1))
if not attachment_id:
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
att = cur.fetchone()
if not att:
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "failed":
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "processing":
updated = att["updated_at"]
if isinstance(updated, str):
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
conn.commit()
logger.info("Attachment %s already processing (age=%.1fs), ack duplicate",
attachment_id, age_seconds)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
cur.execute(
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
(attempt, now, attachment_id)
)
conn.commit()
finally:
conn.close()
chat_id = str(att["chat_id"])
staging_key = f"staging/{chat_id}/{attachment_id}/original"
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
start_time = time.monotonic()
def check_timeout():
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
raise TimeoutError("Job exceeded time limit")
try:
try:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
raw_bytes = obj["Body"].read()
check_timeout()
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
raise ValueError("Staging checksum mismatch")
variants_meta, variants_bytes = generate_grade4_variants(raw_bytes)
check_timeout()
s3_client.put_object(
Bucket=S3_BUCKET, Key=final_original_key,
Body=raw_bytes, ContentType=att["content_type"]
)
for var_name, var_bytes in variants_bytes.items():
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
s3_client.put_object(
Bucket=S3_BUCKET,
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
Body=var_bytes, ContentType=ct
)
check_timeout()
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments SET status='ready', variants=%s::jsonb,
error_code=NULL, updated_at=%s
WHERE id=%s AND pipeline_version <= %s
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
conn.commit()
finally:
conn.close()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
logger.info("Job %s completed: attachment %s ready", job_id, attachment_id)
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
pass
except (UnidentifiedImageError, Image.DecompressionBombError) as e:
logger.error("Permanent image error for %s: %s", attachment_id, type(e).__name__)
mark_failed(attachment_id, "invalid_image")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except ValueError as e:
logger.error("Checksum error for %s: %s", attachment_id, e)
mark_failed(attachment_id, "invalid_checksum")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except psycopg2.errors.DataError as e:
logger.error("DB data error for %s: %s", attachment_id, type(e).__name__)
mark_failed(attachment_id, "data_error")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except TRANSIENT_ERRORS as e:
logger.warning(
"Transient error for %s (attempt %d/%d): %s",
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(e).__name__
)
if attempt >= JOB_MAX_ATTEMPTS:
mark_failed(attachment_id, "retry_exhausted")
try:
dlq_job = dict(job)
dlq_job["attempt"] = 1
redis_client.xadd(
DLQ_STREAM_NAME,
{"payload": json.dumps(dlq_job), "error": str(e)}
)
except Exception:
pass
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
else:
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
time.sleep(delay)
next_job = dict(job)
next_job["attempt"] = attempt + 1
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except Exception as e:
logger.error("Unknown error for %s: %s, marking failed",
attachment_id, type(e).__name__)
mark_failed(attachment_id, "processing_failed")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
def run_staging_cleanup():
try:
cutoff = datetime.now(timezone.utc).timestamp() - 3600
paginator = s3_client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"):
for obj in page.get("Contents", []):
if obj["LastModified"].timestamp() < cutoff:
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"])
except Exception:
pass
except Exception as e:
logger.debug("Staging cleanup: %s", e)
def safe_process(msg_id, fields):
"""Wrapper - exceptions в threads иначе исчезают молча."""
try:
process_job(msg_id, fields)
except Exception as e:
logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__)
def main():
ensure_consumer_groups()
logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY)
executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY)
last_claim = 0
last_cleanup = 0
try:
while running:
try:
now = time.time()
if now - last_claim > 15:
last_claim = now
try:
claimed = redis_client.xautoclaim(
STREAM_NAME, GROUP_NAME, CONSUMER_NAME,
min_idle_time=CLAIM_MIN_IDLE_MS,
start_id="0-0", count=WORKER_CONCURRENCY
)
if claimed and len(claimed) > 1 and claimed[1]:
for msg_id, fields in claimed[1]:
if not running:
break
executor.submit(safe_process, msg_id, fields)
except Exception:
pass
if now - last_cleanup > 600:
last_cleanup = now
executor.submit(run_staging_cleanup)
resp = redis_client.xreadgroup(
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"},
count=WORKER_CONCURRENCY, block=2000
)
if not resp:
continue
for _, messages in resp:
for msg_id, fields in messages:
if not running:
break
executor.submit(safe_process, msg_id, fields)
except Exception as e:
logger.warning("Worker loop: %s", type(e).__name__)
time.sleep(1)
finally:
logger.info("Shutting down, waiting for in-flight jobs...")
executor.shutdown(wait=True)
logger.info("Worker stopped cleanly")
if __name__ == "__main__":
main()