From 749109aaf49bde9b93c97dafd52a1848e65c6ab3 Mon Sep 17 00:00:00 2001 From: Getlles Date: Mon, 16 Nov 2026 20:34:32 +0300 Subject: [PATCH] grade4 --- app.py | 159 +++++++++++----- evidence/g4-crop.jpg | Bin 0 -> 1820 bytes evidence/g4-thumbnail.jpg | Bin 0 -> 4679 bytes evidence/g4-variants.txt | Bin 0 -> 196 bytes evidence/smoke.txt | Bin 0 -> 294 bytes migrations/init.sql | 11 +- scripts/check_contract.py | 2 +- worker.py | 369 ++++++++++++++++++++++++-------------- 8 files changed, 363 insertions(+), 178 deletions(-) create mode 100644 evidence/g4-crop.jpg create mode 100644 evidence/g4-thumbnail.jpg create mode 100644 evidence/g4-variants.txt create mode 100644 evidence/smoke.txt diff --git a/app.py b/app.py index 36add4bf..fb41f90d 100644 --- a/app.py +++ b/app.py @@ -1,4 +1,5 @@ import os +import time import base64 import secrets import uuid @@ -24,8 +25,38 @@ import redis import boto3 from botocore.client import Config +import threading + app = FastAPI(title="MTUSI Messenger — lab5", version="1.5.0") +def outbox_publisher(): + while True: + conn = None + try: + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute(""" + SELECT id, stream, payload FROM outbox + WHERE published_at IS NULL + ORDER BY created_at + LIMIT 100 + FOR UPDATE SKIP LOCKED + """) + for row in cur.fetchall(): + redis_client.xadd(row["stream"], {"payload": json.dumps(row["payload"])}) + cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],)) + conn.commit() + except Exception as e: + print(f"OUTBOX PUBLISHER ERROR: {type(e).__name__} - {e}", flush=True) + finally: + if conn is not None and not conn.closed: + conn.close() + time.sleep(1) + +@app.on_event("startup") +def _start_outbox_publisher(): + threading.Thread(target=outbox_publisher, daemon=True).start() + DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) INSTANCE_ID = os.getenv("INSTANCE_ID", "instance-1") @@ -583,7 +614,10 @@ async def upload_attachment( conn = get_db_connection() with conn.cursor() as cur: - cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id_str, user["id"])) + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id_str, user["id"]) + ) if not cur.fetchone(): conn.close() raise HTTPException(status_code=404, detail="Chat not found or inaccessible") @@ -633,6 +667,16 @@ async def upload_attachment( except Exception: raise HTTPException(status_code=503, detail="S3 storage unavailable") + job_payload = { + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": attachment_id, + "pipeline_version": 1, + "attempt": 1, + "created_at": now + } + conn = get_db_connection() try: with conn.cursor() as cur: @@ -643,30 +687,22 @@ async def upload_attachment( pipeline_version, attempt, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', '[]', NULL, 1, 0, %s, %s) - """, (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now)) + """, (attachment_id, chat_id_str, user["id"], filename, + content_type, size_bytes, sha256_hash, now, now)) + + cur.execute( + "INSERT INTO outbox (id, stream, payload, created_at) " + "VALUES (%s, %s, %s::jsonb, %s)", + (str(uuid.uuid4()), STREAM_NAME, json.dumps(job_payload), now) + ) conn.commit() except Exception: conn.rollback() - conn.close() - raise HTTPException(status_code=500, detail="Internal error saving attachment metadata") + raise HTTPException(status_code=500, detail="Internal error saving attachment") finally: if not conn.closed: conn.close() - job_payload = { - "schema_version": 1, - "job_id": job_id, - "type": "attachment.process.v1", - "attachment_id": attachment_id, - "pipeline_version": 1, - "attempt": 1, - "created_at": now - } - try: - redis_client.xadd(STREAM_NAME, {"payload": json.dumps(job_payload)}) - except Exception: - raise HTTPException(status_code=503, detail="Broker unavailable") - return { "id": attachment_id, "chat_id": chat_id_str, @@ -873,20 +909,18 @@ def create_message( return msg_response @app.get("/api/v1/chats/{chat_id}/messages") -def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, response: Response = None, authorization: Optional[str] = Header(None)): +def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, + response: Response = None, authorization: Optional[str] = Header(None)): req_id = generate_request_id() response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = INSTANCE_ID user = get_current_user_from_token(authorization) - chat_id_str = str(chat_id) - conn = get_db_connection() + conn = get_db_connection() with conn.cursor() as cur: - cur.execute( - "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", - (chat_id_str, user["id"]) - ) + cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", + (chat_id_str, user["id"])) if not cur.fetchone(): conn.close() raise HTTPException(status_code=404, detail="Chat not found") @@ -895,17 +929,12 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, conn.close() raise HTTPException(status_code=400, detail="Invalid limit") - query = """ - SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages - WHERE chat_id = %s - """ + query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s" params = [chat_id_str] - if cursor: c_time, c_id = decode_cursor(cursor, chat_id_str) query += " AND (created_at, id) > (%s, %s)" params.extend([c_time, c_id]) - query += " ORDER BY created_at ASC, id ASC LIMIT %s" params.append(limit + 1) @@ -917,23 +946,65 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, has_next = len(rows) > limit page_items = rows[:limit] - formatted_msgs = [] + formatted = [] for m in page_items: - raw_atts = m.get('attachment_ids') - atts = [str(aid) for aid in raw_atts if aid] if raw_atts else [] - - formatted_msgs.append({ - "id": str(m['id']), - "chat_id": str(m['chat_id']), - "sender_id": str(m['sender_id']), - "text": m['text'], - "created_at": m['created_at'].isoformat().replace("+00:00", "Z") if isinstance(m['created_at'], datetime) else m['created_at'], + raw = m.get('attachment_ids') + atts = [str(a) for a in raw if a] if raw else [] + formatted.append({ + "id": str(m['id']), "chat_id": str(m['chat_id']), + "sender_id": str(m['sender_id']), "text": m['text'], + "created_at": m['created_at'].isoformat().replace("+00:00", "Z") + if isinstance(m['created_at'], datetime) else m['created_at'], "attachment_ids": atts }) next_cursor = None - if has_next and formatted_msgs: - last = formatted_msgs[-1] + if has_next and formatted: + last = formatted[-1] next_cursor = encode_cursor(chat_id_str, last['created_at'], last['id']) - return {"items": formatted_msgs, "next_cursor": next_cursor} \ No newline at end of file + return {"items": formatted, "next_cursor": next_cursor} + +@app.post("/api/v1/attachments/{attachment_id}/replay", status_code=202) +def replay_attachment(attachment_id: UUID, response: Response, + authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + att_id = str(attachment_id) + + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON a.chat_id = cm.chat_id + WHERE a.id=%s AND cm.user_id=%s + """, (att_id, user["id"])) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(status_code=404, detail="Attachment not found") + if row["status"] != "failed": + conn.close() + raise HTTPException(status_code=409, detail="Only failed attachments can be replayed") + + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + cur.execute(""" + UPDATE attachments SET status='queued', attempt=0, error_code=NULL, updated_at=%s + WHERE id=%s + """, (now, att_id)) + + job = {"schema_version": 1, "job_id": str(uuid.uuid4()), + "type": "attachment.process.v1", "attachment_id": att_id, + "pipeline_version": 1, "attempt": 1, "created_at": now} + cur.execute("INSERT INTO outbox (id, stream, payload, created_at) " + "VALUES (%s, %s, %s::jsonb, %s)", + (str(uuid.uuid4()), STREAM_NAME, json.dumps(job), now)) + conn.commit() + finally: + if not conn.closed: + conn.close() + + return {"id": att_id, "status": "queued"} \ No newline at end of file diff --git a/evidence/g4-crop.jpg b/evidence/g4-crop.jpg new file mode 100644 index 0000000000000000000000000000000000000000..754ee743b16930505bb987d16c52c3ec5aba9188 GIT binary patch literal 1820 zcmb7^2~ZPf6o5B`5D17w*bpd}kw6Kgf*@63R0xR>B$6Qslp`opX;2VxkV9z!+u(E? z1fh5V)g&Nc!qGSpgwtuQSb?cLg}H6^wIEIZD?pF6p$e zLrUXcZ@m(GjLFH{Lv3*BrkPB-8uJft7byE6d9cCrd<$VEXcQ7TuqEhR{gffdE9-6E zF0f2ydRL_8;~i=|dy`^zar?Yalz_Zywvfqz6ke_>-u2Q5k}6#4Clbk+N>ML+Q%_S^ z_;g`1C(pa$P`-tu+1T%#lC&@D3h;g20v$z!hq7){GGq$Lr_iYteA)t3+U$ zbG#PR%y@VZ#8Q6xLJ#T@xjv2%MQPkNHA|nKKMO%AoKCKXW+Mn<#v&71w#{_cKE%*x zPT?gy$zWm2=zNHZA$VHQaB&A_Jo?vg$swqzOHihWu#T<0s9D>ed$;T9@XyvaO4%YA zgHNT#ilf~F`!zU{Gl+rY9FnU0FWyKgLlIpuw}qtY0#i@nZ9&T+N+R*)a~b-e0m$o%)~Bt(?0{!@Myo*b+0H4lFGxm*C#Hwl;!4|6g5v?R9|&ri_+*5 zHm>ms|HcG}-bB^qoq$RL8B+zWEnk`<16F4bEH6SD>!S!2X$iz^AdKoup znMQwXvw`8H%B}I7Op38{yO3rgV_x8Av7hSHTRI_iz z2tR_(;VYM2HC6k0F_VU6(XUS44pD_4GUDgHr>T*~O5DtSt|?_uVKb82r8D#<tyx(5KBaYXy zW`__XWmcEN^1GFpEExUPA;bR2gl_-(V+V5{R2OTsqX(y#bQq9_TA$~W*jUbYRXuqSz>SkPCRl5Re4w_0~i<`BcTZ5L5KK6Y5Pzg3< zMe}=aH6Iaj=(CF~xb->o7qIRhYJPa(o#vZC9!I1;8Rz&*!KAAZjCwY=JX930vpl_L zRe*$h^j_n$$kz?o0n!z{`=B51I?J;Rq`~r%fkAWm%~7Q1N=w3HCHE&9Yec3UG$05< z$N9KIr?x}AP3?jr=*@Hid2gnRg?747X(I4%8#h0A#Q8hl^ojkUA2+-6=*tN?k{Yr3 zK%MQG^K+8X>n-2K9esgQyJN}1JLl8)5!3qw{VcYq{*CZ{O*r*>6Q38(4QOIUmw@HM z7RRmdm(tQYB5ymOJZcABSYC?gJ#CDUnq}h)9#Qs>zMP^Pk7QCKoIxJEos5&0(&OUP zX0L5p?ZgzH=~i9ZG$#v-PtH+sjz3`!MeTzBKy*N2`IW_!c9Xt@nia_t6cL8R!$E?6 o?~cd;2Y-3(9V-^8lY9Sx>D-V9j%ge7=sid$HTq=r+tCUbG>(-%EP^k52 zZT0o4nrbK%W&=hOi`CK5S+B0UQ5UyS8>fSVgCN1HD_5*kR!~sJsiIVI|L<$D4uMiY z+(05^k-7*O6jBz2TzrPmLV*9_3$c93%E`+hkSi6yms{2%WRNoQax&nNrLSdV4Bz-t(z5J(x2zrugRfTv~Tyk}G@s{w6`(Y%#Q z+qvzY@>j>BPT1sY$6EPCFAW>IY;nsU zw2lPkx+^P=*}mG(jxZ(SvWKc?`H-61hhNC?rQ3(}UguC=eev*ub2foX1<836uZ6G9 zwm%^VrSp$_wuv^AfA+G~j8e>%^FAv_{X{Y12z&UHt7%{rwBp62exEf*a0_ehs>I2w zfhwCj`d^98&b`ARXQc*Qzi;0scdRzm^$nAc581+wP-EHq2||N+JGlmJH|*coOY`Y1 z>W*v(@4y&mGS3{{zL@ z$FAAwh_BgsP*UJrOK$W0O2XcItA|80rx!3W75ufSP6JOKo)o7WJ~zP9&2%*{$T#MW zujn;fIBbS9(URYOGJ0A8q@^l>iG?KDlEPtXPtji^n+;cOue~u+Qc3Zyn4Yn^Uz_2f zz4P#!MJk67n9)?M`{d9|M$XPb$)VuR08gC3EPhB~J`y^us$|w(mRn^7JHO!;a zqOXbUDRzB1`(}b~7k3^mQ|YY^?@Dl1EiN5-S0gqbp|>&}GAs$j&m4!M;KlBI)A8PE z^8<5oD6Rgq>1ep%c&J!^I43cK`bfXSx6xLKiaK@fzWtmWSZ}oiZFtQSxx+=J6mDi^ ziD>`y15^HBE=KNfOibD`at)pRw#87m&;?X1cM(A=HXAk#<@%0K1cN)2e>@d?F-eke zu@%K<6X_|QD#8#LlTeQj)2D#vGJXJ0t>Cd=J!Si9l0fq3HOjzU^1MN*a}lve@TNw)EZU!%L&G2L z$3o@+&)>{Ams`C^To4~#UH<-bJlL38s-jvv{&=(EyY`Pe`d(IC zb-`8mavh*G_#=iRL}-Oy?QdU#=`#cEy#v&cs$HA?6B)Vj9Ud?6cg1$5j z2ceXvG6r9Muo+OxE?y9gJm*obsve1ry@dCY^It?f9yaYoM?4T$AFMJ>-lRd*%4d3% z-MrEOgLg`)9pu{t)o6bV3=GYoLFoMkpr%K`J`(J2Pm5;{R`JlJNpfoij{?KBn?Nns zTRFR6`K8w&>f5*AgOy(g8=V2lsT^M_pqRSBSSx@VpWT zh%IG4f}$_=LoAzX0L=;yP->1Q@2Vn^!C8xlpyG-tB(;bQljBW*cP&6utCF&|bBjmw zXnxo`ESU2`1%vN&!QHAU32p1QNq|lCg^%}U2|VrK(vg`Ru=y~3T*o2LLgbd|@;n!= zIW@lZ>nTZ58zhO8Seq+%T(yX5WDTjM@ePgNcJPPBTa@6%_L-TRwD%e|UEg@^qhRLr zZ($nMe>*lk=PLc${E_USXEiobjRbIw+8PoWVDx05w8FRIqYPP3 z1z36o|K_VimVt(H>-NF=`Yht6oW>grabN1-k4 z1khZoAdYjky{dr9Th)Ry(tdjU*UOH8`XH(37a8eUjb0<_^v7zD_{&`X=U&&MO<}t+ zYfl=pcES~TSIWDDH9z6)z<#=m_vYo2{GhOD*(8Z$4af|irCFhObq z9Dof-p4vI^Ua5TH0-%!>#S&=m$EL2EV7piav4=IGAte+$f&1`f_V z1bW}m(0HhWufmEv9?GX)vWVKss&_X>dFlZ+_?O~X%ghO37@~SFAgvW3jd)=5Qn7St z9kXbDhNYopqWM!ZP~IF&x++B6%^^-=T0<1=_S_wgS%=q!C?#&Gp|h^K0jvOOyC6|A zUuk^dI!73gWPgwwq|*46Z(f;b^oL91a2QUS5JEi35@({NvDt zHgjS12yhxdbwh1Im1xo!p}2|X%CN?tiZ`y>^|2G}-C!h13#0~&8<1m*uJ_iH*!3SX z4>fuLBtHabvR9o-2T~hCj`2{$usMNNJ&m(#AOK`)+s*?pBk z13A{5&#wJ^HpV8zH+tam!78-&r^~Q+!NXb;Z(*G7dS4W_&&A>hOv?;B=VVpBFGC}H z5ux|A(%^lZ>yN-B^vz`zqRj&_@{Ar}65yTN45&VQhJx^HFDnj7x-W9JTbF_ARtR<) zSlDSiPB4l4>Ft2pMEB>NNjhBeJV(+yR_@;qXt{* zi>{igRSm-??{48C*cMK@lNdxoVa=25xRDvgnfjc&fxy{$+-;&ZMslu^crZhl_qZ~B zZi`~ve3^vDir~FAk$we$i?fP&j{K&Q96MiGM2;SRcT2M83Ro1BT5DW(w%=_w2Gb9y z?+thYfail%g$HQ?JAN?%%3B)6rzXOnd?JCx%W?lX)9ImE*I-2%UjcO~PPm#r#VNBC_KaoubYkq>u~t@XCKc8*<>^@Ulj zw19Ni6apQnrWBMG@J2{+J|7o2f(6;X8I5;RqXFibe;5tlC8GgGwk^x;Y6COF-;GA_ zgYSLEWOxSwsv@?4IY`n}6iL2Cy}%8KVGZHKZ3%7{vPrS-&X{VP@~)$KN=%R{I{DXgcR&xRw(DD4Vu`R{JMV-iu!Oz9V51V2G3&UnHGbv%K%LbTm~Ju4A=si|DP^HCoS?g%xZ2wWhUhzX(!%n z9H1S`oA_{f*!&b1Zt~@PoiEW`&l{D7q5-%_ePegAy<53{%B@G#DkjxgY;B~JP94elVXKJ!3EsKS1D;+Kke zp^{SPzL;o+HqdN%C5%QgqjZy z4^_mAH#4Dnpxu?$>Z7EXyTnz;t%R~Dr*_fX@xcNbOPC|mjczztlpt zjyhu%c!1kzM;6g+z&-O0My(TZ0tN2#?Ca4!Q0u{0u-%e~Gn|w!3Uoc)me^%Ro7o*W c=uO^cHPdwT1Hbf{VVeRs*dC9X{o8K+0|#d`OaK4? literal 0 HcmV?d00001 diff --git a/migrations/init.sql b/migrations/init.sql index e6e466fd..444f86ca 100644 --- a/migrations/init.sql +++ b/migrations/init.sql @@ -58,8 +58,17 @@ CREATE TABLE IF NOT EXISTS attachments ( updated_at TIMESTAMPTZ NOT NULL ); +CREATE TABLE IF NOT EXISTS outbox ( + id UUID PRIMARY KEY, + stream VARCHAR(64) NOT NULL, + payload JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + published_at TIMESTAMPTZ +); + CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id); ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}'; ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1; -ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0; \ No newline at end of file +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0; +CREATE INDEX IF NOT EXISTS idx_outbox_unpublished ON outbox(created_at) WHERE published_at IS NULL; \ No newline at end of file diff --git a/scripts/check_contract.py b/scripts/check_contract.py index 0c75274b..614ddc10 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -45,7 +45,7 @@ def main(): for file in ROOT.rglob('*.json'): if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): continue - json.loads(file.read_text(encoding='utf-8')) + json.loads(file.read_text(encoding='utf-8-sig')) for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'): assert (ROOT/required).is_file(), 'missing '+required print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.') diff --git a/worker.py b/worker.py index 65242dd9..9f33be8f 100644 --- a/worker.py +++ b/worker.py @@ -3,21 +3,21 @@ import io import json import time import signal +import random import hashlib import logging from datetime import datetime, timezone +from concurrent.futures import ThreadPoolExecutor import psycopg2 import psycopg2.extras import redis import boto3 +import botocore.exceptions from botocore.client import Config from PIL import Image, ImageOps, UnidentifiedImageError -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s" -) +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("attachment-worker") DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") @@ -28,28 +28,30 @@ S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs") +DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq" GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers") CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1") JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30")) +WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4")) + +CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000 Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) s3_client = boto3.client( - "s3", - endpoint_url=S3_ENDPOINT, - aws_access_key_id=S3_ACCESS_KEY, - aws_secret_access_key=S3_SECRET_KEY, - config=Config(signature_version="s3v4"), - region_name="us-east-1" + "s3", endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), region_name="us-east-1" ) running = True def handle_sigterm(signum, frame): global running - logger.info("Received shutdown signal, finishing current job...") + logger.info("Received shutdown signal, finishing in-flight jobs...") running = False signal.signal(signal.SIGTERM, handle_sigterm) @@ -60,73 +62,114 @@ def get_db_connection(): return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) -def ensure_consumer_group(): +def ensure_consumer_groups(): while running: try: redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True) - logger.info("Created consumer group '%s' on stream '%s'", GROUP_NAME, STREAM_NAME) + redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True) + logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME) return except redis.exceptions.ResponseError as e: if "BUSYGROUP" in str(e): - logger.info("Consumer group '%s' already exists", GROUP_NAME) return raise except Exception as e: - logger.warning("Waiting for Redis broker: %s", type(e).__name__) + logger.warning("Waiting for Redis: %s", type(e).__name__) time.sleep(1) -def build_thumbnail_jpeg(raw_bytes: bytes) -> tuple[bytes, int, int, str]: - with Image.open(io.BytesIO(raw_bytes)) as img: - img = ImageOps.exif_transpose(img) - - if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info): - rgba = img.convert("RGBA") - background = Image.new("RGB", rgba.size, (255, 255, 255)) - background.paste(rgba, mask=rgba.split()[3]) - img = background - elif img.mode != "RGB": - img = img.convert("RGB") - - orig_w, orig_h = img.size - - if orig_w <= 256 and orig_h <= 256: - target_w, target_h = orig_w, orig_h - else: - scale = min(256.0 / orig_w, 256.0 / orig_h) - target_w = max(1, int(orig_w * scale)) - target_h = max(1, int(orig_h * scale)) - - if (target_w, target_h) != (orig_w, orig_h): - img = img.resize((target_w, target_h), Image.Resampling.LANCZOS) - - buf = io.BytesIO() - img.save(buf, format="JPEG", quality=85, optimize=True) - thumb_bytes = buf.getvalue() - thumb_sha256 = hashlib.sha256(thumb_bytes).hexdigest() - return thumb_bytes, target_w, target_h, thumb_sha256 +def prepare_base_image(raw_bytes): + img = Image.open(io.BytesIO(raw_bytes)) + img = ImageOps.exif_transpose(img) + if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info): + rgba = img.convert("RGBA") + bg = Image.new("RGB", rgba.size, (255, 255, 255)) + bg.paste(rgba, mask=rgba.split()[3]) + img = bg + elif img.mode != "RGB": + img = img.convert("RGB") + return img -def mark_failed(attachment_id: str, error_code: str): +def generate_grade4_variants(raw_bytes): + base_img = prepare_base_image(raw_bytes) + orig_w, orig_h = base_img.size + meta = [] + blobs = {} + + thumb = base_img.copy() + if orig_w <= 256 and orig_h <= 256: + t_w, t_h = orig_w, orig_h + else: + s = min(256.0 / orig_w, 256.0 / orig_h) + t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) + if (t_w, t_h) != (orig_w, orig_h): + thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS) + buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85, optimize=True) + tb = buf.getvalue(); blobs["thumbnail"] = tb + meta.append({"name": "thumbnail", "content_type": "image/jpeg", + "size_bytes": len(tb), "width": t_w, "height": t_h, + "sha256": hashlib.sha256(tb).hexdigest()}) + + opt = base_img.copy() + if orig_w <= 1280 and orig_h <= 1280: + o_w, o_h = orig_w, orig_h + else: + s = min(1280.0 / orig_w, 1280.0 / orig_h) + o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) + if (o_w, o_h) != (orig_w, orig_h): + opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS) + buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=85) + ob = buf.getvalue(); blobs["optimized"] = ob + meta.append({"name": "optimized", "content_type": "image/webp", + "size_bytes": len(ob), "width": o_w, "height": o_h, + "sha256": hashlib.sha256(ob).hexdigest()}) + + c = min(orig_w, orig_h) + left, top = (orig_w - c) // 2, (orig_h - c) // 2 + crop = base_img.crop((left, top, left + c, top + c)) + if c != 128: + crop = crop.resize((128, 128), Image.Resampling.LANCZOS) + buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85, optimize=True) + cb = buf.getvalue(); blobs["crop"] = cb + meta.append({"name": "crop", "content_type": "image/jpeg", + "size_bytes": len(cb), "width": 128, "height": 128, + "sha256": hashlib.sha256(cb).hexdigest()}) + + return meta, blobs + + +def mark_failed(attachment_id, error_code): now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") conn = get_db_connection() try: with conn.cursor() as cur: - cur.execute(""" - UPDATE attachments - SET status = 'failed', error_code = %s, updated_at = %s - WHERE id = %s AND status != 'ready' - """, (error_code, now, attachment_id)) + cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=%s " + "WHERE id=%s AND status != 'ready'", + (error_code, now, attachment_id)) conn.commit() finally: conn.close() -def process_job(message_id: str, fields: dict): +PERMANENT_ERRORS = ( + UnidentifiedImageError, Image.DecompressionBombError, ValueError, + psycopg2.errors.DataError, +) +TRANSIENT_ERRORS = ( + TimeoutError, + botocore.exceptions.EndpointConnectionError, + botocore.exceptions.ReadTimeoutError, + psycopg2.OperationalError, + redis.exceptions.ConnectionError, +) + + +def process_job(message_id, fields): try: job = json.loads(fields.get("payload", "{}")) except Exception: - logger.error("Malformed job JSON in message %s, acknowledging to drop", message_id) + logger.error("Malformed job %s, ack to drop", message_id) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return @@ -142,32 +185,38 @@ def process_job(message_id: str, fields: dict): conn = get_db_connection() try: with conn.cursor() as cur: - cur.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)) + cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)) att = cur.fetchone() - if not att: conn.commit() - logger.warning("Attachment %s not found in DB, acking job %s", attachment_id, job_id) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return - if att["status"] == "ready" or att["pipeline_version"] > pipeline_version: conn.commit() - logger.info("Attachment %s already ready (or newer pipeline), no-op ack for job %s", attachment_id, job_id) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return - if att["status"] == "failed": conn.commit() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return + if att["status"] == "processing": + updated = att["updated_at"] + if isinstance(updated, str): + updated = datetime.fromisoformat(updated.replace("Z", "+00:00")) + age_seconds = (datetime.now(timezone.utc) - updated).total_seconds() + if age_seconds < (JOB_TIMEOUT_SECONDS + 10): + conn.commit() + logger.info("Attachment %s already processing (age=%.1fs), ack duplicate", + attachment_id, age_seconds) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - cur.execute(""" - UPDATE attachments - SET status = 'processing', attempt = %s, updated_at = %s - WHERE id = %s - """, (attempt, now, attachment_id)) + cur.execute( + "UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s", + (attempt, now, attachment_id) + ) conn.commit() finally: conn.close() @@ -175,7 +224,12 @@ def process_job(message_id: str, fields: dict): chat_id = str(att["chat_id"]) staging_key = f"staging/{chat_id}/{attachment_id}/original" final_original_key = f"attachments/{chat_id}/{attachment_id}/original" - final_thumb_key = f"attachments/{chat_id}/{attachment_id}/thumbnail" + + start_time = time.monotonic() + + def check_timeout(): + if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS: + raise TimeoutError("Job exceeded time limit") try: try: @@ -183,117 +237,168 @@ def process_job(message_id: str, fields: dict): except Exception: obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key) raw_bytes = obj["Body"].read() + check_timeout() if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]: raise ValueError("Staging checksum mismatch") - thumb_bytes, thumb_w, thumb_h, thumb_sha256 = build_thumbnail_jpeg(raw_bytes) + variants_meta, variants_bytes = generate_grade4_variants(raw_bytes) + check_timeout() s3_client.put_object( - Bucket=S3_BUCKET, - Key=final_original_key, - Body=raw_bytes, - ContentType=att["content_type"] + Bucket=S3_BUCKET, Key=final_original_key, + Body=raw_bytes, ContentType=att["content_type"] ) - s3_client.put_object( - Bucket=S3_BUCKET, - Key=final_thumb_key, - Body=thumb_bytes, - ContentType="image/jpeg" - ) - - variants = [ - { - "name": "thumbnail", - "content_type": "image/jpeg", - "size_bytes": len(thumb_bytes), - "width": thumb_w, - "height": thumb_h, - "sha256": thumb_sha256 - } - ] + for var_name, var_bytes in variants_bytes.items(): + ct = "image/webp" if var_name == "optimized" else "image/jpeg" + s3_client.put_object( + Bucket=S3_BUCKET, + Key=f"attachments/{chat_id}/{attachment_id}/{var_name}", + Body=var_bytes, ContentType=ct + ) + check_timeout() ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") conn = get_db_connection() try: with conn.cursor() as cur: cur.execute(""" - UPDATE attachments - SET status = 'ready', - variants = %s::jsonb, - error_code = NULL, - updated_at = %s - WHERE id = %s AND pipeline_version <= %s - """, (json.dumps(variants), ready_at, attachment_id, pipeline_version)) + UPDATE attachments SET status='ready', variants=%s::jsonb, + error_code=NULL, updated_at=%s + WHERE id=%s AND pipeline_version <= %s + """, (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version)) conn.commit() finally: conn.close() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - logger.info("Job %s completed: attachment %s is ready (thumbnail %dx%d)", job_id, attachment_id, thumb_w, thumb_h) + logger.info("Job %s completed: attachment %s ready", job_id, attachment_id) try: s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) except Exception: pass - except (UnidentifiedImageError, Image.DecompressionBombError, ValueError) as perm_err: - logger.error("Permanent image error for %s: %s", attachment_id, type(perm_err).__name__) + except (UnidentifiedImageError, Image.DecompressionBombError) as e: + logger.error("Permanent image error for %s: %s", attachment_id, type(e).__name__) mark_failed(attachment_id, "invalid_image") redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - except Exception as transient_err: + except ValueError as e: + logger.error("Checksum error for %s: %s", attachment_id, e) + mark_failed(attachment_id, "invalid_checksum") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + + except psycopg2.errors.DataError as e: + logger.error("DB data error for %s: %s", attachment_id, type(e).__name__) + mark_failed(attachment_id, "data_error") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + + except TRANSIENT_ERRORS as e: logger.warning( - "Transient error processing attachment %s (attempt %d/%d): %s", - attachment_id, attempt, JOB_MAX_ATTEMPTS, type(transient_err).__name__ + "Transient error for %s (attempt %d/%d): %s", + attachment_id, attempt, JOB_MAX_ATTEMPTS, type(e).__name__ ) if attempt >= JOB_MAX_ATTEMPTS: mark_failed(attachment_id, "retry_exhausted") + try: + dlq_job = dict(job) + dlq_job["attempt"] = 1 + redis_client.xadd( + DLQ_STREAM_NAME, + {"payload": json.dumps(dlq_job), "error": str(e)} + ) + except Exception: + pass redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) else: + delay = min(2 ** attempt, 30) + random.uniform(0, 1) + time.sleep(delay) next_job = dict(job) next_job["attempt"] = attempt + 1 redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)}) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + except Exception as e: + logger.error("Unknown error for %s: %s, marking failed", + attachment_id, type(e).__name__) + mark_failed(attachment_id, "processing_failed") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + +def run_staging_cleanup(): + try: + cutoff = datetime.now(timezone.utc).timestamp() - 3600 + paginator = s3_client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"): + for obj in page.get("Contents", []): + if obj["LastModified"].timestamp() < cutoff: + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"]) + except Exception: + pass + except Exception as e: + logger.debug("Staging cleanup: %s", e) + + +def safe_process(msg_id, fields): + """Wrapper - exceptions в threads иначе исчезают молча.""" + try: + process_job(msg_id, fields) + except Exception as e: + logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__) + def main(): - ensure_consumer_group() - logger.info("Worker '%s' started listening on stream '%s'", CONSUMER_NAME, STREAM_NAME) + ensure_consumer_groups() + logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY) - while running: - try: - pending = redis_client.xreadgroup( - GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: "0"}, count=10 - ) - messages = pending[0][1] if pending else [] - if not messages: - break - for msg_id, fields in messages: - if not running: - break - process_job(msg_id, fields) - except Exception as e: - logger.warning("Error reading pending messages: %s", type(e).__name__) - time.sleep(1) + executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY) + last_claim = 0 + last_cleanup = 0 - while running: - try: - resp = redis_client.xreadgroup( - GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=1, block=2000 - ) - if not resp: - continue - for _, messages in resp: - for msg_id, fields in messages: - if not running: - break - process_job(msg_id, fields) - except Exception as e: - logger.warning("Broker read error, retrying: %s", type(e).__name__) - time.sleep(1) + try: + while running: + try: + now = time.time() - logger.info("Worker stopped cleanly") + if now - last_claim > 15: + last_claim = now + try: + claimed = redis_client.xautoclaim( + STREAM_NAME, GROUP_NAME, CONSUMER_NAME, + min_idle_time=CLAIM_MIN_IDLE_MS, + start_id="0-0", count=WORKER_CONCURRENCY + ) + if claimed and len(claimed) > 1 and claimed[1]: + for msg_id, fields in claimed[1]: + if not running: + break + executor.submit(safe_process, msg_id, fields) + except Exception: + pass + + if now - last_cleanup > 600: + last_cleanup = now + executor.submit(run_staging_cleanup) + + resp = redis_client.xreadgroup( + GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, + count=WORKER_CONCURRENCY, block=2000 + ) + if not resp: + continue + for _, messages in resp: + for msg_id, fields in messages: + if not running: + break + executor.submit(safe_process, msg_id, fields) + except Exception as e: + logger.warning("Worker loop: %s", type(e).__name__) + time.sleep(1) + finally: + logger.info("Shutting down, waiting for in-flight jobs...") + executor.shutdown(wait=True) + logger.info("Worker stopped cleanly") if __name__ == "__main__":