forked from mtusi2026/mtusi2026lab5
grade4
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import time
|
||||
import base64
|
||||
import secrets
|
||||
import uuid
|
||||
@@ -24,8 +25,38 @@ import redis
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
|
||||
import threading
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger — lab5", version="1.5.0")
|
||||
|
||||
def outbox_publisher():
|
||||
while True:
|
||||
conn = None
|
||||
try:
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT id, stream, payload FROM outbox
|
||||
WHERE published_at IS NULL
|
||||
ORDER BY created_at
|
||||
LIMIT 100
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""")
|
||||
for row in cur.fetchall():
|
||||
redis_client.xadd(row["stream"], {"payload": json.dumps(row["payload"])})
|
||||
cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],))
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
print(f"OUTBOX PUBLISHER ERROR: {type(e).__name__} - {e}", flush=True)
|
||||
finally:
|
||||
if conn is not None and not conn.closed:
|
||||
conn.close()
|
||||
time.sleep(1)
|
||||
|
||||
@app.on_event("startup")
|
||||
def _start_outbox_publisher():
|
||||
threading.Thread(target=outbox_publisher, daemon=True).start()
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
INSTANCE_ID = os.getenv("INSTANCE_ID", "instance-1")
|
||||
@@ -583,7 +614,10 @@ async def upload_attachment(
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id_str, user["id"]))
|
||||
cur.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id_str, user["id"])
|
||||
)
|
||||
if not cur.fetchone():
|
||||
conn.close()
|
||||
raise HTTPException(status_code=404, detail="Chat not found or inaccessible")
|
||||
@@ -633,6 +667,16 @@ async def upload_attachment(
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="S3 storage unavailable")
|
||||
|
||||
job_payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": job_id,
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": attachment_id,
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": now
|
||||
}
|
||||
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
@@ -643,30 +687,22 @@ async def upload_attachment(
|
||||
pipeline_version, attempt, created_at, updated_at
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', '[]', NULL, 1, 0, %s, %s)
|
||||
""", (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now))
|
||||
""", (attachment_id, chat_id_str, user["id"], filename,
|
||||
content_type, size_bytes, sha256_hash, now, now))
|
||||
|
||||
cur.execute(
|
||||
"INSERT INTO outbox (id, stream, payload, created_at) "
|
||||
"VALUES (%s, %s, %s::jsonb, %s)",
|
||||
(str(uuid.uuid4()), STREAM_NAME, json.dumps(job_payload), now)
|
||||
)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
conn.close()
|
||||
raise HTTPException(status_code=500, detail="Internal error saving attachment metadata")
|
||||
raise HTTPException(status_code=500, detail="Internal error saving attachment")
|
||||
finally:
|
||||
if not conn.closed:
|
||||
conn.close()
|
||||
|
||||
job_payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": job_id,
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": attachment_id,
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": now
|
||||
}
|
||||
try:
|
||||
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(job_payload)})
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="Broker unavailable")
|
||||
|
||||
return {
|
||||
"id": attachment_id,
|
||||
"chat_id": chat_id_str,
|
||||
@@ -873,20 +909,18 @@ def create_message(
|
||||
return msg_response
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, response: Response = None, authorization: Optional[str] = Header(None)):
|
||||
def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
|
||||
response: Response = None, authorization: Optional[str] = Header(None)):
|
||||
req_id = generate_request_id()
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
user = get_current_user_from_token(authorization)
|
||||
|
||||
chat_id_str = str(chat_id)
|
||||
conn = get_db_connection()
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id_str, user["id"])
|
||||
)
|
||||
cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",
|
||||
(chat_id_str, user["id"]))
|
||||
if not cur.fetchone():
|
||||
conn.close()
|
||||
raise HTTPException(status_code=404, detail="Chat not found")
|
||||
@@ -895,17 +929,12 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
|
||||
conn.close()
|
||||
raise HTTPException(status_code=400, detail="Invalid limit")
|
||||
|
||||
query = """
|
||||
SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages
|
||||
WHERE chat_id = %s
|
||||
"""
|
||||
query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s"
|
||||
params = [chat_id_str]
|
||||
|
||||
if cursor:
|
||||
c_time, c_id = decode_cursor(cursor, chat_id_str)
|
||||
query += " AND (created_at, id) > (%s, %s)"
|
||||
params.extend([c_time, c_id])
|
||||
|
||||
query += " ORDER BY created_at ASC, id ASC LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
|
||||
@@ -917,23 +946,65 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
|
||||
has_next = len(rows) > limit
|
||||
page_items = rows[:limit]
|
||||
|
||||
formatted_msgs = []
|
||||
formatted = []
|
||||
for m in page_items:
|
||||
raw_atts = m.get('attachment_ids')
|
||||
atts = [str(aid) for aid in raw_atts if aid] if raw_atts else []
|
||||
|
||||
formatted_msgs.append({
|
||||
"id": str(m['id']),
|
||||
"chat_id": str(m['chat_id']),
|
||||
"sender_id": str(m['sender_id']),
|
||||
"text": m['text'],
|
||||
"created_at": m['created_at'].isoformat().replace("+00:00", "Z") if isinstance(m['created_at'], datetime) else m['created_at'],
|
||||
raw = m.get('attachment_ids')
|
||||
atts = [str(a) for a in raw if a] if raw else []
|
||||
formatted.append({
|
||||
"id": str(m['id']), "chat_id": str(m['chat_id']),
|
||||
"sender_id": str(m['sender_id']), "text": m['text'],
|
||||
"created_at": m['created_at'].isoformat().replace("+00:00", "Z")
|
||||
if isinstance(m['created_at'], datetime) else m['created_at'],
|
||||
"attachment_ids": atts
|
||||
})
|
||||
|
||||
next_cursor = None
|
||||
if has_next and formatted_msgs:
|
||||
last = formatted_msgs[-1]
|
||||
if has_next and formatted:
|
||||
last = formatted[-1]
|
||||
next_cursor = encode_cursor(chat_id_str, last['created_at'], last['id'])
|
||||
|
||||
return {"items": formatted_msgs, "next_cursor": next_cursor}
|
||||
return {"items": formatted, "next_cursor": next_cursor}
|
||||
|
||||
@app.post("/api/v1/attachments/{attachment_id}/replay", status_code=202)
|
||||
def replay_attachment(attachment_id: UUID, response: Response,
|
||||
authorization: Optional[str] = Header(None)):
|
||||
req_id = generate_request_id()
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
user = get_current_user_from_token(authorization)
|
||||
att_id = str(attachment_id)
|
||||
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT a.* FROM attachments a
|
||||
JOIN chat_members cm ON a.chat_id = cm.chat_id
|
||||
WHERE a.id=%s AND cm.user_id=%s
|
||||
""", (att_id, user["id"]))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
conn.close()
|
||||
raise HTTPException(status_code=404, detail="Attachment not found")
|
||||
if row["status"] != "failed":
|
||||
conn.close()
|
||||
raise HTTPException(status_code=409, detail="Only failed attachments can be replayed")
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
cur.execute("""
|
||||
UPDATE attachments SET status='queued', attempt=0, error_code=NULL, updated_at=%s
|
||||
WHERE id=%s
|
||||
""", (now, att_id))
|
||||
|
||||
job = {"schema_version": 1, "job_id": str(uuid.uuid4()),
|
||||
"type": "attachment.process.v1", "attachment_id": att_id,
|
||||
"pipeline_version": 1, "attempt": 1, "created_at": now}
|
||||
cur.execute("INSERT INTO outbox (id, stream, payload, created_at) "
|
||||
"VALUES (%s, %s, %s::jsonb, %s)",
|
||||
(str(uuid.uuid4()), STREAM_NAME, json.dumps(job), now))
|
||||
conn.commit()
|
||||
finally:
|
||||
if not conn.closed:
|
||||
conn.close()
|
||||
|
||||
return {"id": att_id, "status": "queued"}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.6 KiB |
Binary file not shown.
Binary file not shown.
+10
-1
@@ -58,8 +58,17 @@ CREATE TABLE IF NOT EXISTS attachments (
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS outbox (
|
||||
id UUID PRIMARY KEY,
|
||||
stream VARCHAR(64) NOT NULL,
|
||||
payload JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
published_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id);
|
||||
|
||||
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}';
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1;
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0;
|
||||
CREATE INDEX IF NOT EXISTS idx_outbox_unpublished ON outbox(created_at) WHERE published_at IS NULL;
|
||||
@@ -45,7 +45,7 @@ def main():
|
||||
for file in ROOT.rglob('*.json'):
|
||||
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
|
||||
continue
|
||||
json.loads(file.read_text(encoding='utf-8'))
|
||||
json.loads(file.read_text(encoding='utf-8-sig'))
|
||||
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
|
||||
assert (ROOT/required).is_file(), 'missing '+required
|
||||
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
|
||||
|
||||
@@ -3,21 +3,21 @@ import io
|
||||
import json
|
||||
import time
|
||||
import signal
|
||||
import random
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
import redis
|
||||
import boto3
|
||||
import botocore.exceptions
|
||||
from botocore.client import Config
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s"
|
||||
)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("attachment-worker")
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||
@@ -28,28 +28,30 @@ S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
||||
|
||||
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
|
||||
DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq"
|
||||
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
|
||||
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
|
||||
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
|
||||
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4"))
|
||||
|
||||
CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
||||
|
||||
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
|
||||
s3_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_SECRET_KEY,
|
||||
config=Config(signature_version="s3v4"),
|
||||
region_name="us-east-1"
|
||||
"s3", endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY,
|
||||
config=Config(signature_version="s3v4"), region_name="us-east-1"
|
||||
)
|
||||
|
||||
running = True
|
||||
|
||||
def handle_sigterm(signum, frame):
|
||||
global running
|
||||
logger.info("Received shutdown signal, finishing current job...")
|
||||
logger.info("Received shutdown signal, finishing in-flight jobs...")
|
||||
running = False
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_sigterm)
|
||||
@@ -60,73 +62,114 @@ def get_db_connection():
|
||||
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
|
||||
|
||||
def ensure_consumer_group():
|
||||
def ensure_consumer_groups():
|
||||
while running:
|
||||
try:
|
||||
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
|
||||
logger.info("Created consumer group '%s' on stream '%s'", GROUP_NAME, STREAM_NAME)
|
||||
redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
|
||||
logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME)
|
||||
return
|
||||
except redis.exceptions.ResponseError as e:
|
||||
if "BUSYGROUP" in str(e):
|
||||
logger.info("Consumer group '%s' already exists", GROUP_NAME)
|
||||
return
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.warning("Waiting for Redis broker: %s", type(e).__name__)
|
||||
logger.warning("Waiting for Redis: %s", type(e).__name__)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def build_thumbnail_jpeg(raw_bytes: bytes) -> tuple[bytes, int, int, str]:
|
||||
with Image.open(io.BytesIO(raw_bytes)) as img:
|
||||
img = ImageOps.exif_transpose(img)
|
||||
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.split()[3])
|
||||
img = background
|
||||
elif img.mode != "RGB":
|
||||
img = img.convert("RGB")
|
||||
|
||||
orig_w, orig_h = img.size
|
||||
|
||||
if orig_w <= 256 and orig_h <= 256:
|
||||
target_w, target_h = orig_w, orig_h
|
||||
else:
|
||||
scale = min(256.0 / orig_w, 256.0 / orig_h)
|
||||
target_w = max(1, int(orig_w * scale))
|
||||
target_h = max(1, int(orig_h * scale))
|
||||
|
||||
if (target_w, target_h) != (orig_w, orig_h):
|
||||
img = img.resize((target_w, target_h), Image.Resampling.LANCZOS)
|
||||
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
thumb_bytes = buf.getvalue()
|
||||
thumb_sha256 = hashlib.sha256(thumb_bytes).hexdigest()
|
||||
return thumb_bytes, target_w, target_h, thumb_sha256
|
||||
def prepare_base_image(raw_bytes):
|
||||
img = Image.open(io.BytesIO(raw_bytes))
|
||||
img = ImageOps.exif_transpose(img)
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
bg = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
bg.paste(rgba, mask=rgba.split()[3])
|
||||
img = bg
|
||||
elif img.mode != "RGB":
|
||||
img = img.convert("RGB")
|
||||
return img
|
||||
|
||||
|
||||
def mark_failed(attachment_id: str, error_code: str):
|
||||
def generate_grade4_variants(raw_bytes):
|
||||
base_img = prepare_base_image(raw_bytes)
|
||||
orig_w, orig_h = base_img.size
|
||||
meta = []
|
||||
blobs = {}
|
||||
|
||||
thumb = base_img.copy()
|
||||
if orig_w <= 256 and orig_h <= 256:
|
||||
t_w, t_h = orig_w, orig_h
|
||||
else:
|
||||
s = min(256.0 / orig_w, 256.0 / orig_h)
|
||||
t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
|
||||
if (t_w, t_h) != (orig_w, orig_h):
|
||||
thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS)
|
||||
buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
tb = buf.getvalue(); blobs["thumbnail"] = tb
|
||||
meta.append({"name": "thumbnail", "content_type": "image/jpeg",
|
||||
"size_bytes": len(tb), "width": t_w, "height": t_h,
|
||||
"sha256": hashlib.sha256(tb).hexdigest()})
|
||||
|
||||
opt = base_img.copy()
|
||||
if orig_w <= 1280 and orig_h <= 1280:
|
||||
o_w, o_h = orig_w, orig_h
|
||||
else:
|
||||
s = min(1280.0 / orig_w, 1280.0 / orig_h)
|
||||
o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
|
||||
if (o_w, o_h) != (orig_w, orig_h):
|
||||
opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS)
|
||||
buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=85)
|
||||
ob = buf.getvalue(); blobs["optimized"] = ob
|
||||
meta.append({"name": "optimized", "content_type": "image/webp",
|
||||
"size_bytes": len(ob), "width": o_w, "height": o_h,
|
||||
"sha256": hashlib.sha256(ob).hexdigest()})
|
||||
|
||||
c = min(orig_w, orig_h)
|
||||
left, top = (orig_w - c) // 2, (orig_h - c) // 2
|
||||
crop = base_img.crop((left, top, left + c, top + c))
|
||||
if c != 128:
|
||||
crop = crop.resize((128, 128), Image.Resampling.LANCZOS)
|
||||
buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
cb = buf.getvalue(); blobs["crop"] = cb
|
||||
meta.append({"name": "crop", "content_type": "image/jpeg",
|
||||
"size_bytes": len(cb), "width": 128, "height": 128,
|
||||
"sha256": hashlib.sha256(cb).hexdigest()})
|
||||
|
||||
return meta, blobs
|
||||
|
||||
|
||||
def mark_failed(attachment_id, error_code):
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = %s, updated_at = %s
|
||||
WHERE id = %s AND status != 'ready'
|
||||
""", (error_code, now, attachment_id))
|
||||
cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=%s "
|
||||
"WHERE id=%s AND status != 'ready'",
|
||||
(error_code, now, attachment_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def process_job(message_id: str, fields: dict):
|
||||
PERMANENT_ERRORS = (
|
||||
UnidentifiedImageError, Image.DecompressionBombError, ValueError,
|
||||
psycopg2.errors.DataError,
|
||||
)
|
||||
TRANSIENT_ERRORS = (
|
||||
TimeoutError,
|
||||
botocore.exceptions.EndpointConnectionError,
|
||||
botocore.exceptions.ReadTimeoutError,
|
||||
psycopg2.OperationalError,
|
||||
redis.exceptions.ConnectionError,
|
||||
)
|
||||
|
||||
|
||||
def process_job(message_id, fields):
|
||||
try:
|
||||
job = json.loads(fields.get("payload", "{}"))
|
||||
except Exception:
|
||||
logger.error("Malformed job JSON in message %s, acknowledging to drop", message_id)
|
||||
logger.error("Malformed job %s, ack to drop", message_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
@@ -142,32 +185,38 @@ def process_job(message_id: str, fields: dict):
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,))
|
||||
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
|
||||
att = cur.fetchone()
|
||||
|
||||
if not att:
|
||||
conn.commit()
|
||||
logger.warning("Attachment %s not found in DB, acking job %s", attachment_id, job_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
|
||||
conn.commit()
|
||||
logger.info("Attachment %s already ready (or newer pipeline), no-op ack for job %s", attachment_id, job_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
if att["status"] == "failed":
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
if att["status"] == "processing":
|
||||
updated = att["updated_at"]
|
||||
if isinstance(updated, str):
|
||||
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
|
||||
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
|
||||
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
|
||||
conn.commit()
|
||||
logger.info("Attachment %s already processing (age=%.1fs), ack duplicate",
|
||||
attachment_id, age_seconds)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
cur.execute("""
|
||||
UPDATE attachments
|
||||
SET status = 'processing', attempt = %s, updated_at = %s
|
||||
WHERE id = %s
|
||||
""", (attempt, now, attachment_id))
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
|
||||
(attempt, now, attachment_id)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -175,7 +224,12 @@ def process_job(message_id: str, fields: dict):
|
||||
chat_id = str(att["chat_id"])
|
||||
staging_key = f"staging/{chat_id}/{attachment_id}/original"
|
||||
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
|
||||
final_thumb_key = f"attachments/{chat_id}/{attachment_id}/thumbnail"
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
def check_timeout():
|
||||
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
|
||||
raise TimeoutError("Job exceeded time limit")
|
||||
|
||||
try:
|
||||
try:
|
||||
@@ -183,117 +237,168 @@ def process_job(message_id: str, fields: dict):
|
||||
except Exception:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
|
||||
raw_bytes = obj["Body"].read()
|
||||
check_timeout()
|
||||
|
||||
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
|
||||
raise ValueError("Staging checksum mismatch")
|
||||
|
||||
thumb_bytes, thumb_w, thumb_h, thumb_sha256 = build_thumbnail_jpeg(raw_bytes)
|
||||
variants_meta, variants_bytes = generate_grade4_variants(raw_bytes)
|
||||
check_timeout()
|
||||
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=final_original_key,
|
||||
Body=raw_bytes,
|
||||
ContentType=att["content_type"]
|
||||
Bucket=S3_BUCKET, Key=final_original_key,
|
||||
Body=raw_bytes, ContentType=att["content_type"]
|
||||
)
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=final_thumb_key,
|
||||
Body=thumb_bytes,
|
||||
ContentType="image/jpeg"
|
||||
)
|
||||
|
||||
variants = [
|
||||
{
|
||||
"name": "thumbnail",
|
||||
"content_type": "image/jpeg",
|
||||
"size_bytes": len(thumb_bytes),
|
||||
"width": thumb_w,
|
||||
"height": thumb_h,
|
||||
"sha256": thumb_sha256
|
||||
}
|
||||
]
|
||||
for var_name, var_bytes in variants_bytes.items():
|
||||
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
|
||||
Body=var_bytes, ContentType=ct
|
||||
)
|
||||
check_timeout()
|
||||
|
||||
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
UPDATE attachments
|
||||
SET status = 'ready',
|
||||
variants = %s::jsonb,
|
||||
error_code = NULL,
|
||||
updated_at = %s
|
||||
WHERE id = %s AND pipeline_version <= %s
|
||||
""", (json.dumps(variants), ready_at, attachment_id, pipeline_version))
|
||||
UPDATE attachments SET status='ready', variants=%s::jsonb,
|
||||
error_code=NULL, updated_at=%s
|
||||
WHERE id=%s AND pipeline_version <= %s
|
||||
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
logger.info("Job %s completed: attachment %s is ready (thumbnail %dx%d)", job_id, attachment_id, thumb_w, thumb_h)
|
||||
logger.info("Job %s completed: attachment %s ready", job_id, attachment_id)
|
||||
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError) as perm_err:
|
||||
logger.error("Permanent image error for %s: %s", attachment_id, type(perm_err).__name__)
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError) as e:
|
||||
logger.error("Permanent image error for %s: %s", attachment_id, type(e).__name__)
|
||||
mark_failed(attachment_id, "invalid_image")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except Exception as transient_err:
|
||||
except ValueError as e:
|
||||
logger.error("Checksum error for %s: %s", attachment_id, e)
|
||||
mark_failed(attachment_id, "invalid_checksum")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except psycopg2.errors.DataError as e:
|
||||
logger.error("DB data error for %s: %s", attachment_id, type(e).__name__)
|
||||
mark_failed(attachment_id, "data_error")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except TRANSIENT_ERRORS as e:
|
||||
logger.warning(
|
||||
"Transient error processing attachment %s (attempt %d/%d): %s",
|
||||
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(transient_err).__name__
|
||||
"Transient error for %s (attempt %d/%d): %s",
|
||||
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(e).__name__
|
||||
)
|
||||
if attempt >= JOB_MAX_ATTEMPTS:
|
||||
mark_failed(attachment_id, "retry_exhausted")
|
||||
try:
|
||||
dlq_job = dict(job)
|
||||
dlq_job["attempt"] = 1
|
||||
redis_client.xadd(
|
||||
DLQ_STREAM_NAME,
|
||||
{"payload": json.dumps(dlq_job), "error": str(e)}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
else:
|
||||
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
|
||||
time.sleep(delay)
|
||||
next_job = dict(job)
|
||||
next_job["attempt"] = attempt + 1
|
||||
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Unknown error for %s: %s, marking failed",
|
||||
attachment_id, type(e).__name__)
|
||||
mark_failed(attachment_id, "processing_failed")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
def run_staging_cleanup():
|
||||
try:
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - 3600
|
||||
paginator = s3_client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"):
|
||||
for obj in page.get("Contents", []):
|
||||
if obj["LastModified"].timestamp() < cutoff:
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"])
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.debug("Staging cleanup: %s", e)
|
||||
|
||||
|
||||
def safe_process(msg_id, fields):
|
||||
"""Wrapper - exceptions в threads иначе исчезают молча."""
|
||||
try:
|
||||
process_job(msg_id, fields)
|
||||
except Exception as e:
|
||||
logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__)
|
||||
|
||||
|
||||
def main():
|
||||
ensure_consumer_group()
|
||||
logger.info("Worker '%s' started listening on stream '%s'", CONSUMER_NAME, STREAM_NAME)
|
||||
ensure_consumer_groups()
|
||||
logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY)
|
||||
|
||||
while running:
|
||||
try:
|
||||
pending = redis_client.xreadgroup(
|
||||
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: "0"}, count=10
|
||||
)
|
||||
messages = pending[0][1] if pending else []
|
||||
if not messages:
|
||||
break
|
||||
for msg_id, fields in messages:
|
||||
if not running:
|
||||
break
|
||||
process_job(msg_id, fields)
|
||||
except Exception as e:
|
||||
logger.warning("Error reading pending messages: %s", type(e).__name__)
|
||||
time.sleep(1)
|
||||
executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY)
|
||||
last_claim = 0
|
||||
last_cleanup = 0
|
||||
|
||||
while running:
|
||||
try:
|
||||
resp = redis_client.xreadgroup(
|
||||
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=1, block=2000
|
||||
)
|
||||
if not resp:
|
||||
continue
|
||||
for _, messages in resp:
|
||||
for msg_id, fields in messages:
|
||||
if not running:
|
||||
break
|
||||
process_job(msg_id, fields)
|
||||
except Exception as e:
|
||||
logger.warning("Broker read error, retrying: %s", type(e).__name__)
|
||||
time.sleep(1)
|
||||
try:
|
||||
while running:
|
||||
try:
|
||||
now = time.time()
|
||||
|
||||
logger.info("Worker stopped cleanly")
|
||||
if now - last_claim > 15:
|
||||
last_claim = now
|
||||
try:
|
||||
claimed = redis_client.xautoclaim(
|
||||
STREAM_NAME, GROUP_NAME, CONSUMER_NAME,
|
||||
min_idle_time=CLAIM_MIN_IDLE_MS,
|
||||
start_id="0-0", count=WORKER_CONCURRENCY
|
||||
)
|
||||
if claimed and len(claimed) > 1 and claimed[1]:
|
||||
for msg_id, fields in claimed[1]:
|
||||
if not running:
|
||||
break
|
||||
executor.submit(safe_process, msg_id, fields)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if now - last_cleanup > 600:
|
||||
last_cleanup = now
|
||||
executor.submit(run_staging_cleanup)
|
||||
|
||||
resp = redis_client.xreadgroup(
|
||||
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"},
|
||||
count=WORKER_CONCURRENCY, block=2000
|
||||
)
|
||||
if not resp:
|
||||
continue
|
||||
for _, messages in resp:
|
||||
for msg_id, fields in messages:
|
||||
if not running:
|
||||
break
|
||||
executor.submit(safe_process, msg_id, fields)
|
||||
except Exception as e:
|
||||
logger.warning("Worker loop: %s", type(e).__name__)
|
||||
time.sleep(1)
|
||||
finally:
|
||||
logger.info("Shutting down, waiting for in-flight jobs...")
|
||||
executor.shutdown(wait=True)
|
||||
logger.info("Worker stopped cleanly")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user