From d475bf29ed6c7936278125a29fca84343f958e97 Mon Sep 17 00:00:00 2001 From: Getlles Date: Mon, 16 Nov 2026 17:38:53 +0300 Subject: [PATCH] grade3 --- Dockerfile | 2 +- app.py | 214 ++++++++------- compose.yaml | 115 +++++--- evidence/bucket_private.txt | 1 - evidence/compose_ps.txt | 7 - evidence/demo-00-user.json | 6 + evidence/demo-01-session.json | 11 + evidence/demo-01b-chat.json | 8 + evidence/demo-01c-worker-stopped.txt | 10 + evidence/demo-02-upload-202.json | 1 + evidence/demo-03-message.json | 10 + evidence/demo-04-api-restarted.txt | 7 + evidence/demo-05-worker-started.txt | 10 + evidence/demo-06-ready.json | 23 ++ evidence/demo-07-thumbnail.jpg | Bin 0 -> 4679 bytes evidence/demo-08-after-broker-restart.json | 23 ++ evidence/failover.txt | 4 - evidence/level4.txt | 17 -- evidence/persistence.txt | 3 - evidence/reconcile_dry_run.txt | 4 - evidence/smoke.txt | 2 - evidence/upload_load.json | 7 - evidence/upload_load_stats.txt | 40 --- migrations/init.sql | 8 +- requirements.txt | 3 +- scripts/check_contract.py | 8 +- tests/client.py | 2 +- tests/resilience.py | 2 +- worker.py | 300 +++++++++++++++++++++ 29 files changed, 620 insertions(+), 228 deletions(-) delete mode 100644 evidence/bucket_private.txt delete mode 100644 evidence/compose_ps.txt create mode 100644 evidence/demo-00-user.json create mode 100644 evidence/demo-01-session.json create mode 100644 evidence/demo-01b-chat.json create mode 100644 evidence/demo-01c-worker-stopped.txt create mode 100644 evidence/demo-02-upload-202.json create mode 100644 evidence/demo-03-message.json create mode 100644 evidence/demo-04-api-restarted.txt create mode 100644 evidence/demo-05-worker-started.txt create mode 100644 evidence/demo-06-ready.json create mode 100644 evidence/demo-07-thumbnail.jpg create mode 100644 evidence/demo-08-after-broker-restart.json delete mode 100644 evidence/failover.txt delete mode 100644 evidence/level4.txt delete mode 100644 evidence/persistence.txt delete mode 100644 evidence/reconcile_dry_run.txt delete mode 100644 evidence/smoke.txt delete mode 100644 evidence/upload_load.json delete mode 100644 evidence/upload_load_stats.txt create mode 100644 worker.py diff --git a/Dockerfile b/Dockerfile index a33beebc..db3469e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN useradd -u 10001 -m appuser WORKDIR /home/appuser COPY --from=builder /install /usr/local -COPY app.py . +COPY app.py worker.py ./ USER appuser EXPOSE 8080 diff --git a/app.py b/app.py index a7d43e34..36add4bf 100644 --- a/app.py +++ b/app.py @@ -24,7 +24,7 @@ import redis import boto3 from botocore.client import Config -app = FastAPI(title="MTUSI Messenger — lab4", version="1.4.0") +app = FastAPI(title="MTUSI Messenger — lab5", version="1.5.0") DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) @@ -37,8 +37,10 @@ SECRET_HMAC_KEY = os.environ["SECRET_HMAC_KEY"] S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") -S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "minioadmin") -S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "minioadminpassword") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") +S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") + +STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs") s3_client = boto3.client( "s3", @@ -50,8 +52,7 @@ s3_client = boto3.client( ) def get_db_connection(): - conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) - return conn + return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) def generate_request_id() -> str: return uuid.uuid4().hex @@ -91,14 +92,18 @@ def decode_cursor(cursor: str, expected_chat_id: str) -> tuple: if len(parts) != 4: raise ValueError() chat_id, created_at, msg_id, signature = parts - + if chat_id != expected_chat_id: raise HTTPException(status_code=400, detail="Cursor belongs to another chat") - - expected_sig = hmac.new(SECRET_HMAC_KEY.encode(), f"{chat_id}|{created_at}|{msg_id}".encode(), hashlib.sha256).hexdigest()[:16] + + expected_sig = hmac.new( + SECRET_HMAC_KEY.encode(), + f"{chat_id}|{created_at}|{msg_id}".encode(), + hashlib.sha256 + ).hexdigest()[:16] if not hmac.compare_digest(signature, expected_sig): raise ValueError() - + return created_at, msg_id except HTTPException: raise @@ -175,7 +180,7 @@ class CreateUser(BaseModel): class CreateChat(BaseModel): model_config = ConfigDict(extra='forbid') title: str = Field(..., min_length=1, max_length=100) - member_ids: List[uuid.UUID] = Field(..., min_items=1, max_items=100) + member_ids: List[uuid.UUID] = Field(..., min_length=1, max_length=100) @field_validator("title") def not_blank(cls, v): @@ -186,7 +191,7 @@ class CreateChat(BaseModel): class CreateMessage(BaseModel): model_config = ConfigDict(extra='forbid') text: str = Field(default="", max_length=4000) - attachment_ids: List[uuid.UUID] = Field(default=[], max_items=10) + attachment_ids: List[uuid.UUID] = Field(default=[], max_length=10) @model_validator(mode="after") def check_content(self) -> 'CreateMessage': @@ -220,7 +225,7 @@ def register_user(body: CreateUser, response: Response): req_id = generate_request_id() response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = INSTANCE_ID - + password_hash = bcrypt.hashpw(body.password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8') user_id = str(uuid.uuid4()) created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") @@ -235,7 +240,10 @@ def register_user(body: CreateUser, response: Response): conn.commit() except psycopg2.errors.UniqueViolation: conn.rollback() - raise HTTPException(status_code=409, detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}}) + raise HTTPException( + status_code=409, + detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}} + ) finally: conn.close() @@ -254,18 +262,18 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon if not authorization or not authorization.startswith("Basic "): raise HTTPException( - status_code=401, + status_code=401, detail="Missing or invalid basic auth", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} ) - + try: encoded = authorization.split(" ")[1] decoded = base64.b64decode(encoded).decode("utf-8") username, password = decoded.split(":", 1) except Exception: raise HTTPException( - status_code=401, + status_code=401, detail="Invalid credentials format", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} ) @@ -274,11 +282,11 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon with conn.cursor() as cur: cur.execute("SELECT * FROM users WHERE username = %s", (username,)) user = cur.fetchone() - + conn.close() + if not user or not bcrypt.checkpw(password.encode('utf-8'), user['password_hash'].encode('utf-8')): - conn.close() raise HTTPException( - status_code=401, + status_code=401, detail="Invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} ) @@ -293,7 +301,7 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon "token_type": "Bearer", "expires_at": expires_at_str, "user": { - "id": user['id'], + "id": str(user['id']), "username": user['username'], "display_name": user['display_name'], "created_at": user['created_at'].isoformat().replace("+00:00", "Z") if isinstance(user['created_at'], datetime) else user['created_at'] @@ -303,7 +311,7 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> dict: if not authorization or not authorization.startswith("Bearer "): raise HTTPException( - status_code=401, + status_code=401, detail="Missing or invalid bearer token", headers={"WWW-Authenticate": 'Bearer realm="messenger"'} ) @@ -316,7 +324,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> if not user_id: raise HTTPException( - status_code=401, + status_code=401, detail="Invalid or expired session token", headers={"WWW-Authenticate": 'Bearer realm="messenger"'} ) @@ -329,7 +337,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> if not row: raise HTTPException( - status_code=401, + status_code=401, detail="User not found", headers={"WWW-Authenticate": 'Bearer realm="messenger"'} ) @@ -344,10 +352,10 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> @app.delete("/api/v1/auth/sessions/current", status_code=204) def revoke_session(authorization: Optional[str] = Header(None)): req_id = generate_request_id() - + if not authorization or not authorization.startswith("Bearer "): raise HTTPException( - status_code=401, + status_code=401, detail="Unauthorized", headers={"WWW-Authenticate": 'Bearer realm="messenger"'} ) @@ -357,7 +365,7 @@ def revoke_session(authorization: Optional[str] = Header(None)): if not deleted: raise HTTPException( - status_code=401, + status_code=401, detail="Invalid session token", headers={"WWW-Authenticate": 'Bearer realm="messenger"'} ) @@ -365,7 +373,7 @@ def revoke_session(authorization: Optional[str] = Header(None)): return Response( status_code=204, headers={ - "X-Request-Id": req_id, + "X-Request-Id": req_id, "X-Instance-Id": INSTANCE_ID }, ) @@ -383,7 +391,7 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = INSTANCE_ID user = get_current_user_from_token(authorization) - + member_ids_str = [str(mid) for mid in body.member_ids] if user["id"] not in member_ids_str: @@ -433,7 +441,7 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = INSTANCE_ID user = get_current_user_from_token(authorization) - + if limit < 1 or limit > 100: raise HTTPException(status_code=400, detail="Invalid limit") @@ -446,21 +454,18 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response ORDER BY c.created_at ASC, c.id ASC """, (user["id"],)) rows = cur.fetchall() - conn.close() - user_chats = [] - for r in rows: - conn = get_db_connection() - with conn.cursor() as cur: + user_chats = [] + for r in rows: cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],)) m_rows = cur.fetchall() - conn.close() - user_chats.append({ - "id": r['id'], - "title": r['title'], - "member_ids": [m['user_id'] for m in m_rows], - "created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at'] - }) + user_chats.append({ + "id": str(r['id']), + "title": r['title'], + "member_ids": [str(m['user_id']) for m in m_rows], + "created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at'] + }) + conn.close() start_idx = 0 if cursor: @@ -483,7 +488,7 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response if start_idx + limit < len(user_chats): last = page_items[-1] next_cursor = f"{last['created_at']}_{last['id']}" - + return {"items": page_items, "next_cursor": next_cursor} @app.get("/api/v1/chats/{chat_id}") @@ -492,7 +497,7 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = INSTANCE_ID user = get_current_user_from_token(authorization) - + chat_id_str = str(chat_id) conn = get_db_connection() with conn.cursor() as cur: @@ -511,16 +516,16 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H raise HTTPException(status_code=404, detail="Chat not found") return { - "id": chat['id'], + "id": str(chat['id']), "title": chat['title'], - "member_ids": [m['user_id'] for m in members], + "member_ids": [str(m['user_id']) for m in members], "created_at": chat['created_at'].isoformat().replace("+00:00", "Z") if isinstance(chat['created_at'], datetime) else chat['created_at'] } def validate_image_and_get_dimensions(data: bytes, content_type: str): if len(data) > 10 * 1024 * 1024: raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)") - + width, height = 0, 0 if content_type == "image/png": if not data.startswith(b'\x89PNG\r\n\x1a\n'): @@ -544,7 +549,13 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str): if 0xc0 <= marker_type <= 0xc3: height, width = struct.unpack('>HH', data[i+5:i+9]) break - i += 2 + struct.unpack('>H', data[i+2:i+4])[0] + if marker_type in {0xd8, 0xd9} or (0xd0 <= marker_type <= 0xd7): + i += 2 + continue + seg_len = struct.unpack('>H', data[i+2:i+4])[0] + if seg_len < 2: + raise HTTPException(status_code=415, detail="Invalid JPEG segment") + i += 2 + seg_len else: i += 1 else: @@ -556,7 +567,7 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str): if width * height > 40_000_000: raise HTTPException(status_code=413, detail="Image exceeds 40 million pixels limit") -@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) async def upload_attachment( chat_id: UUID, filename: str = Query(..., min_length=1, max_length=255), @@ -579,7 +590,7 @@ async def upload_attachment( conn.close() if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename) or '/' in filename or '\\' in filename: - raise HTTPException(400, "Invalid filename") + raise HTTPException(status_code=400, detail="Invalid filename") content_type = request.headers.get("content-type") if content_type not in ["image/png", "image/jpeg"]: @@ -608,30 +619,14 @@ async def upload_attachment( size_bytes = len(body_bytes) sha256_hash = hashlib.sha256(body_bytes).hexdigest() attachment_id = str(uuid.uuid4()) + job_id = str(uuid.uuid4()) now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - s3_key = f"attachments/{chat_id_str}/{attachment_id}/{filename}" - - conn = get_db_connection() - try: - with conn.cursor() as cur: - cur.execute(""" - INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, variants, error_code, created_at, updated_at) - VALUES (%s, %s, %s, %s, %s, %s, %s, 'pending', '[]', NULL, %s, %s) - """, (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now)) - conn.commit() - except Exception: - conn.rollback() - conn.close() - raise HTTPException(status_code=500, detail="Internal error initializing attachment") - finally: - if not conn.closed: - conn.close() - + staging_key = f"staging/{chat_id_str}/{attachment_id}/original" try: s3_client.put_object( Bucket=S3_BUCKET, - Key=s3_key, + Key=staging_key, Body=body_bytes, ContentType=content_type ) @@ -642,16 +637,36 @@ async def upload_attachment( try: with conn.cursor() as cur: cur.execute(""" - UPDATE attachments SET status = 'ready', updated_at = %s WHERE id = %s - """, (now, attachment_id)) + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, status, variants, error_code, + pipeline_version, attempt, created_at, updated_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', '[]', NULL, 1, 0, %s, %s) + """, (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now)) conn.commit() except Exception: conn.rollback() - raise HTTPException(status_code=500, detail="Internal error finalizing attachment status") + conn.close() + raise HTTPException(status_code=500, detail="Internal error saving attachment metadata") finally: if not conn.closed: conn.close() + job_payload = { + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": attachment_id, + "pipeline_version": 1, + "attempt": 1, + "created_at": now + } + try: + redis_client.xadd(STREAM_NAME, {"payload": json.dumps(job_payload)}) + except Exception: + raise HTTPException(status_code=503, detail="Broker unavailable") + return { "id": attachment_id, "chat_id": chat_id_str, @@ -660,7 +675,7 @@ async def upload_attachment( "content_type": content_type, "size_bytes": size_bytes, "sha256": sha256_hash, - "status": "ready", + "status": "queued", "variants": [], "error_code": None, "created_at": now, @@ -704,7 +719,12 @@ def get_attachment(attachment_id: UUID, response: Response, authorization: Optio } @app.get("/api/v1/attachments/{attachment_id}/content") -def download_attachment(attachment_id: UUID, response: Response, authorization: Optional[str] = Header(None)): +def download_attachment( + attachment_id: UUID, + variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"), + response: Response = None, + authorization: Optional[str] = Header(None) +): req_id = generate_request_id() response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = INSTANCE_ID @@ -725,9 +745,19 @@ def download_attachment(attachment_id: UUID, response: Response, authorization: raise HTTPException(status_code=404, detail="Attachment not found or inaccessible") if row['status'] != 'ready': - raise HTTPException(status_code=409, detail="Attachment not ready") + raise HTTPException(status_code=409, detail=f"Attachment is in status '{row['status']}'") + + if variant == "original": + s3_key = f"attachments/{row['chat_id']}/{row['id']}/original" + media_type = row['content_type'] + else: + variants_list = row['variants'] or [] + matched = [v for v in variants_list if v.get("name") == variant] + if not matched: + raise HTTPException(status_code=404, detail=f"Variant '{variant}' not found") + s3_key = f"attachments/{row['chat_id']}/{row['id']}/{variant}" + media_type = matched[0]["content_type"] - s3_key = f"attachments/{row['chat_id']}/{row['id']}/{row['filename']}" try: obj = s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key) data = obj['Body'].read() @@ -736,7 +766,7 @@ def download_attachment(attachment_id: UUID, response: Response, authorization: return Response( content=data, - media_type=row['content_type'], + media_type=media_type, headers={ "X-Request-Id": req_id, "X-Instance-Id": INSTANCE_ID, @@ -745,9 +775,9 @@ def download_attachment(attachment_id: UUID, response: Response, authorization: @app.post("/api/v1/chats/{chat_id}/messages", status_code=201) def create_message( - chat_id: UUID, - body: CreateMessage, - response: Response, + chat_id: UUID, + body: CreateMessage, + response: Response, authorization: Optional[str] = Header(None), idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key") ): @@ -755,14 +785,14 @@ def create_message( response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = INSTANCE_ID user = get_current_user_from_token(authorization) - + if not idempotency_key or not re.match(r"^[A-Za-z0-9._:-]{1,128}$", idempotency_key): raise HTTPException(status_code=400, detail="Missing or invalid Idempotency-Key") chat_id_str = str(chat_id) method = "POST" path = f"/api/v1/chats/{chat_id_str}/messages" - + body_json_str = json.dumps(body.model_dump(mode='json'), sort_keys=True) request_hash = hashlib.sha256(body_json_str.encode("utf-8")).hexdigest() @@ -794,7 +824,7 @@ def create_message( msg_id = str(uuid.uuid4()) created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - + msg_response = { "id": msg_id, "chat_id": chat_id_str, @@ -809,7 +839,7 @@ def create_message( VALUES (%s, %s, %s, %s, %s, %s, %s) ON CONFLICT (user_id, method, path, key) DO NOTHING """, (user["id"], method, path, idempotency_key, request_hash, 201, json.dumps(msg_response))) - + if cur.rowcount == 0: cur.execute(""" SELECT request_hash, response_status, response_body @@ -822,7 +852,7 @@ def create_message( if row["request_hash"] != request_hash: raise HTTPException(status_code=409, detail="Idempotency key conflict with different payload") - + return row["response_body"] cur.execute( @@ -832,7 +862,7 @@ def create_message( conn.commit() except HTTPException: raise - except Exception as exc: + except Exception: conn.rollback() conn.close() raise HTTPException(status_code=500, detail="Internal error processing message") @@ -890,16 +920,12 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, formatted_msgs = [] for m in page_items: raw_atts = m.get('attachment_ids') - atts = [] - if raw_atts: - for aid in raw_atts: - if aid: - atts.append(str(aid)) - + atts = [str(aid) for aid in raw_atts if aid] if raw_atts else [] + formatted_msgs.append({ - "id": m['id'], - "chat_id": m['chat_id'], - "sender_id": m['sender_id'], + "id": str(m['id']), + "chat_id": str(m['chat_id']), + "sender_id": str(m['sender_id']), "text": m['text'], "created_at": m['created_at'].isoformat().replace("+00:00", "Z") if isinstance(m['created_at'], datetime) else m['created_at'], "attachment_ids": atts diff --git a/compose.yaml b/compose.yaml index c6f67609..e4822cac 100644 --- a/compose.yaml +++ b/compose.yaml @@ -23,7 +23,7 @@ services: redis: image: redis:7.2.4-alpine - command: redis-server --save 60 1 --loglevel warning + command: redis-server --appendonly yes --appendfsync everysec --loglevel warning volumes: - redisdata:/data deploy: @@ -51,6 +51,44 @@ services: condition: service_healthy restart: "no" + s3: + image: chrislusf/seaweedfs:latest + command: ["server", "-s3", "-dir=/data", "-s3.port=8333"] + environment: + - AWS_ACCESS_KEY_ID=appuser + - AWS_SECRET_ACCESS_KEY=appsecretpassword + volumes: + - s3data:/data + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz || exit 1"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s + + s3-init: + image: python:3.11.7-slim-bookworm + volumes: + - ./scripts:/scripts + - ./requirements.txt:/requirements.txt + environment: + - S3_ENDPOINT=http://s3:8333 + - S3_BUCKET=messenger-attachments + - S3_ROOT_ACCESS_KEY=appuser + - S3_ROOT_SECRET_KEY=appsecretpassword + command: > + sh -c "pip install --no-cache-dir -r /requirements.txt && + python /scripts/bootstrap_s3.py" + depends_on: + s3: + condition: service_healthy + restart: "no" + api-1: build: . stop_grace_period: 15s @@ -64,6 +102,7 @@ services: - S3_BUCKET=messenger-attachments - S3_ACCESS_KEY_ID=appuser - S3_SECRET_ACCESS_KEY=appsecretpassword + - BROKER_STREAM=attachment_jobs deploy: resources: limits: @@ -97,6 +136,7 @@ services: - S3_BUCKET=messenger-attachments - S3_ACCESS_KEY_ID=appuser - S3_SECRET_ACCESS_KEY=appsecretpassword + - BROKER_STREAM=attachment_jobs deploy: resources: limits: @@ -117,6 +157,41 @@ services: timeout: 3s retries: 3 + worker: + build: . + command: ["python", "worker.py"] + stop_grace_period: 15s + environment: + - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger + - REDIS_URL=redis://redis:6379/0 + - S3_ENDPOINT=http://s3:8333 + - S3_BUCKET=messenger-attachments + - S3_ACCESS_KEY_ID=appuser + - S3_SECRET_ACCESS_KEY=appsecretpassword + - BROKER_STREAM=attachment_jobs + - BROKER_GROUP=attachment_workers + - JOB_MAX_ATTEMPTS=5 + - MAX_IMAGE_PIXELS=40000000 + deploy: + resources: + limits: + cpus: '0.50' + memory: 256M + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import os, redis; redis.Redis.from_url(os.environ['REDIS_URL']).ping()"] + interval: 5s + timeout: 3s + retries: 3 + nginx: image: nginx:1.25.4-alpine ports: @@ -137,44 +212,6 @@ services: timeout: 3s retries: 3 - s3: - image: chrislusf/seaweedfs:latest - command: "mini -dir=/data" - environment: - - AWS_ACCESS_KEY_ID=appuser - - AWS_SECRET_ACCESS_KEY=appsecretpassword - - S3_BUCKET=messenger-attachments - volumes: - - s3data:/data - deploy: - resources: - limits: - cpus: '0.50' - memory: 512M - healthcheck: - test: ["CMD", "nc", "-z", "localhost", "8333"] - interval: 5s - timeout: 3s - retries: 3 - - s3-init: - image: python:3.11-slim - volumes: - - ./scripts:/scripts - - ./requirements.txt:/requirements.txt - environment: - - S3_ENDPOINT=http://s3:8333 - - S3_BUCKET=messenger-attachments - - S3_ROOT_ACCESS_KEY=appuser - - S3_ROOT_SECRET_KEY=appsecretpassword - command: > - sh -c "pip install --no-cache-dir -r /requirements.txt && - python /scripts/bootstrap_s3.py" - depends_on: - s3: - condition: service_healthy - restart: "no" - volumes: pgdata: redisdata: diff --git a/evidence/bucket_private.txt b/evidence/bucket_private.txt deleted file mode 100644 index 01e3dccd..00000000 --- a/evidence/bucket_private.txt +++ /dev/null @@ -1 +0,0 @@ -PASS: anonymous GET -> 403 diff --git a/evidence/compose_ps.txt b/evidence/compose_ps.txt deleted file mode 100644 index 4d1a2278..00000000 --- a/evidence/compose_ps.txt +++ /dev/null @@ -1,7 +0,0 @@ -NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS -mtusi2026lab4-api-1-1 mtusi2026lab4-api-1 "uvicorn app:app --h…" api-1 15 minutes ago Up 12 minutes (healthy) 8080/tcp -mtusi2026lab4-api-2-1 mtusi2026lab4-api-2 "uvicorn app:app --h…" api-2 15 minutes ago Up 23 seconds (healthy) 8080/tcp -mtusi2026lab4-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.…" nginx 15 minutes ago Up 12 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp -mtusi2026lab4-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.s…" postgres 15 minutes ago Up 12 minutes (healthy) 5432/tcp -mtusi2026lab4-redis-1 redis:7.2.4-alpine "docker-entrypoint.s…" redis 15 minutes ago Up 12 minutes (healthy) 6379/tcp -mtusi2026lab4-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh mini…" s3 15 minutes ago Up 12 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp diff --git a/evidence/demo-00-user.json b/evidence/demo-00-user.json new file mode 100644 index 00000000..cbfec116 --- /dev/null +++ b/evidence/demo-00-user.json @@ -0,0 +1,6 @@ +{ + "id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242", + "username": "demo_75866", + "display_name": "Demo User", + "created_at": "2026-11-16T13:43:38.059472Z" +} diff --git a/evidence/demo-01-session.json b/evidence/demo-01-session.json new file mode 100644 index 00000000..272e0cfe --- /dev/null +++ b/evidence/demo-01-session.json @@ -0,0 +1,11 @@ +{ + "token": "grSsHp1gJpFLckF14UT44obEonSGQRvP878JhLyCaNzM6PGBH1LwrdgZIFF8Zu8U", + "token_type": "Bearer", + "expires_at": "2026-11-16T14:43:38.850596Z", + "user": { + "id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242", + "username": "demo_75866", + "display_name": "Demo User", + "created_at": "2026-11-16T13:43:38.059472Z" + } +} diff --git a/evidence/demo-01b-chat.json b/evidence/demo-01b-chat.json new file mode 100644 index 00000000..23e24ac2 --- /dev/null +++ b/evidence/demo-01b-chat.json @@ -0,0 +1,8 @@ +{ + "id": "5db90ff5-9604-4853-b002-db4144868bcb", + "title": "Demo chat", + "member_ids": [ + "fdeb71d4-ee4f-47a6-8968-ffc7740e5242" + ], + "created_at": "2026-11-16T13:43:39.240968Z" +} diff --git a/evidence/demo-01c-worker-stopped.txt b/evidence/demo-01c-worker-stopped.txt new file mode 100644 index 00000000..e1b8d4b0 --- /dev/null +++ b/evidence/demo-01c-worker-stopped.txt @@ -0,0 +1,10 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab5-api-1-1 mtusi2026lab5-api-1 "uvicorn app:app --hт�ж" api-1 30 minutes ago Up 6 minutes (healthy) 8080/tcp +mtusi2026lab5-api-2-1 mtusi2026lab5-api-2 "uvicorn app:app --hт�ж" api-2 30 minutes ago Up 6 minutes (healthy) 8080/tcp +mtusi2026lab5-migrate-1 postgres:15.5-alpine3.19 "docker-entrypoint.sт�ж" migrate 30 minutes ago Exited (0) 5 minutes ago +mtusi2026lab5-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.т�ж" nginx 30 minutes ago Up 25 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab5-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.sт�ж" postgres 30 minutes ago Up 30 minutes (healthy) 5432/tcp +mtusi2026lab5-redis-1 redis:7.2.4-alpine "docker-entrypoint.sт�ж" redis 30 minutes ago Up 4 minutes (healthy) 6379/tcp +mtusi2026lab5-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh servт�ж" s3 30 minutes ago Up 30 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi2026lab5-s3-init-1 python:3.11.7-slim-bookworm "sh -c 'pip install т�ж" s3-init 30 minutes ago Exited (0) 5 minutes ago +mtusi2026lab5-worker-1 mtusi2026lab5-worker "python worker.py" worker 30 minutes ago Exited (0) Less than a second ago diff --git a/evidence/demo-02-upload-202.json b/evidence/demo-02-upload-202.json new file mode 100644 index 00000000..4184ebb1 --- /dev/null +++ b/evidence/demo-02-upload-202.json @@ -0,0 +1 @@ +{"id":"9becbd32-e413-417a-8762-79bcc195e296","chat_id":"5db90ff5-9604-4853-b002-db4144868bcb","owner_id":"fdeb71d4-ee4f-47a6-8968-ffc7740e5242","filename":"sample.png","content_type":"image/png","size_bytes":111817,"sha256":"99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75","status":"queued","variants":[],"error_code":null,"created_at":"2026-11-16T13:43:41.734766Z","updated_at":"2026-11-16T13:43:41.734766Z"} diff --git a/evidence/demo-03-message.json b/evidence/demo-03-message.json new file mode 100644 index 00000000..7518fdd9 --- /dev/null +++ b/evidence/demo-03-message.json @@ -0,0 +1,10 @@ +{ + "id": "74dc2e91-a901-4067-8993-da9d2cc4ca55", + "chat_id": "5db90ff5-9604-4853-b002-db4144868bcb", + "sender_id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242", + "text": "", + "created_at": "2026-11-16T13:43:42.100558Z", + "attachment_ids": [ + "9becbd32-e413-417a-8762-79bcc195e296" + ] +} diff --git a/evidence/demo-04-api-restarted.txt b/evidence/demo-04-api-restarted.txt new file mode 100644 index 00000000..c8ec13cf --- /dev/null +++ b/evidence/demo-04-api-restarted.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab5-api-1-1 mtusi2026lab5-api-1 "uvicorn app:app --hт�ж" api-1 30 minutes ago Up Less than a second (health: starting) 8080/tcp +mtusi2026lab5-api-2-1 mtusi2026lab5-api-2 "uvicorn app:app --hт�ж" api-2 30 minutes ago Up Less than a second (health: starting) 8080/tcp +mtusi2026lab5-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.т�ж" nginx 30 minutes ago Up 25 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab5-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.sт�ж" postgres 30 minutes ago Up 30 minutes (healthy) 5432/tcp +mtusi2026lab5-redis-1 redis:7.2.4-alpine "docker-entrypoint.sт�ж" redis 30 minutes ago Up 4 minutes (healthy) 6379/tcp +mtusi2026lab5-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh servт�ж" s3 30 minutes ago Up 30 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp diff --git a/evidence/demo-05-worker-started.txt b/evidence/demo-05-worker-started.txt new file mode 100644 index 00000000..ce38db2d --- /dev/null +++ b/evidence/demo-05-worker-started.txt @@ -0,0 +1,10 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab5-api-1-1 mtusi2026lab5-api-1 "uvicorn app:app --hт�ж" api-1 30 minutes ago Up 10 seconds (healthy) 8080/tcp +mtusi2026lab5-api-2-1 mtusi2026lab5-api-2 "uvicorn app:app --hт�ж" api-2 30 minutes ago Up 9 seconds (healthy) 8080/tcp +mtusi2026lab5-migrate-1 postgres:15.5-alpine3.19 "docker-entrypoint.sт�ж" migrate 30 minutes ago Exited (0) 7 seconds ago +mtusi2026lab5-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.т�ж" nginx 30 minutes ago Up 26 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab5-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.sт�ж" postgres 30 minutes ago Up 30 minutes (healthy) 5432/tcp +mtusi2026lab5-redis-1 redis:7.2.4-alpine "docker-entrypoint.sт�ж" redis 30 minutes ago Up 4 minutes (healthy) 6379/tcp +mtusi2026lab5-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh servт�ж" s3 30 minutes ago Up 30 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi2026lab5-s3-init-1 python:3.11.7-slim-bookworm "sh -c 'pip install т�ж" s3-init 30 minutes ago Exited (0) Less than a second ago +mtusi2026lab5-worker-1 mtusi2026lab5-worker "python worker.py" worker 30 minutes ago Up Less than a second (health: starting) 8080/tcp diff --git a/evidence/demo-06-ready.json b/evidence/demo-06-ready.json new file mode 100644 index 00000000..03c68584 --- /dev/null +++ b/evidence/demo-06-ready.json @@ -0,0 +1,23 @@ +{ + "id": "9becbd32-e413-417a-8762-79bcc195e296", + "chat_id": "5db90ff5-9604-4853-b002-db4144868bcb", + "owner_id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242", + "filename": "sample.png", + "content_type": "image/png", + "size_bytes": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "status": "ready", + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "size_bytes": 4679, + "content_type": "image/jpeg" + } + ], + "error_code": null, + "created_at": "2026-11-16T13:43:41.734766Z", + "updated_at": "2026-11-16T13:43:55.375481Z" +} diff --git a/evidence/demo-07-thumbnail.jpg b/evidence/demo-07-thumbnail.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e33b369bb05652b5d5e440f515f3ef8d1f66d43e GIT binary patch literal 4679 zcmb7Hc|4T++n%wMBAQ~1WyTwo%GlRJ{m|&N$gzy3Mrr1WlWk6jWcewwG$zx@X)J|Q z)~3U#Fr7ka%+U~zrLmx>D-V9j%ge7=sid$HTq=r+tCUbG>(-%EP^k52 zZT0o4nrbK%W&=hOi`CK5S+B0UQ5UyS8>fSVgCN1HD_5*kR!~sJsiIVI|L<$D4uMiY z+(05^k-7*O6jBz2TzrPmLV*9_3$c93%E`+hkSi6yms{2%WRNoQax&nNrLSdV4Bz-t(z5J(x2zrugRfTv~Tyk}G@s{w6`(Y%#Q z+qvzY@>j>BPT1sY$6EPCFAW>IY;nsU zw2lPkx+^P=*}mG(jxZ(SvWKc?`H-61hhNC?rQ3(}UguC=eev*ub2foX1<836uZ6G9 zwm%^VrSp$_wuv^AfA+G~j8e>%^FAv_{X{Y12z&UHt7%{rwBp62exEf*a0_ehs>I2w zfhwCj`d^98&b`ARXQc*Qzi;0scdRzm^$nAc581+wP-EHq2||N+JGlmJH|*coOY`Y1 z>W*v(@4y&mGS3{{zL@ z$FAAwh_BgsP*UJrOK$W0O2XcItA|80rx!3W75ufSP6JOKo)o7WJ~zP9&2%*{$T#MW zujn;fIBbS9(URYOGJ0A8q@^l>iG?KDlEPtXPtji^n+;cOue~u+Qc3Zyn4Yn^Uz_2f zz4P#!MJk67n9)?M`{d9|M$XPb$)VuR08gC3EPhB~J`y^us$|w(mRn^7JHO!;a zqOXbUDRzB1`(}b~7k3^mQ|YY^?@Dl1EiN5-S0gqbp|>&}GAs$j&m4!M;KlBI)A8PE z^8<5oD6Rgq>1ep%c&J!^I43cK`bfXSx6xLKiaK@fzWtmWSZ}oiZFtQSxx+=J6mDi^ ziD>`y15^HBE=KNfOibD`at)pRw#87m&;?X1cM(A=HXAk#<@%0K1cN)2e>@d?F-eke zu@%K<6X_|QD#8#LlTeQj)2D#vGJXJ0t>Cd=J!Si9l0fq3HOjzU^1MN*a}lve@TNw)EZU!%L&G2L z$3o@+&)>{Ams`C^To4~#UH<-bJlL38s-jvv{&=(EyY`Pe`d(IC zb-`8mavh*G_#=iRL}-Oy?QdU#=`#cEy#v&cs$HA?6B)Vj9Ud?6cg1$5j z2ceXvG6r9Muo+OxE?y9gJm*obsve1ry@dCY^It?f9yaYoM?4T$AFMJ>-lRd*%4d3% z-MrEOgLg`)9pu{t)o6bV3=GYoLFoMkpr%K`J`(J2Pm5;{R`JlJNpfoij{?KBn?Nns zTRFR6`K8w&>f5*AgOy(g8=V2lsT^M_pqRSBSSx@VpWT zh%IG4f}$_=LoAzX0L=;yP->1Q@2Vn^!C8xlpyG-tB(;bQljBW*cP&6utCF&|bBjmw zXnxo`ESU2`1%vN&!QHAU32p1QNq|lCg^%}U2|VrK(vg`Ru=y~3T*o2LLgbd|@;n!= zIW@lZ>nTZ58zhO8Seq+%T(yX5WDTjM@ePgNcJPPBTa@6%_L-TRwD%e|UEg@^qhRLr zZ($nMe>*lk=PLc${E_USXEiobjRbIw+8PoWVDx05w8FRIqYPP3 z1z36o|K_VimVt(H>-NF=`Yht6oW>grabN1-k4 z1khZoAdYjky{dr9Th)Ry(tdjU*UOH8`XH(37a8eUjb0<_^v7zD_{&`X=U&&MO<}t+ zYfl=pcES~TSIWDDH9z6)z<#=m_vYo2{GhOD*(8Z$4af|irCFhObq z9Dof-p4vI^Ua5TH0-%!>#S&=m$EL2EV7piav4=IGAte+$f&1`f_V z1bW}m(0HhWufmEv9?GX)vWVKss&_X>dFlZ+_?O~X%ghO37@~SFAgvW3jd)=5Qn7St z9kXbDhNYopqWM!ZP~IF&x++B6%^^-=T0<1=_S_wgS%=q!C?#&Gp|h^K0jvOOyC6|A zUuk^dI!73gWPgwwq|*46Z(f;b^oL91a2QUS5JEi35@({NvDt zHgjS12yhxdbwh1Im1xo!p}2|X%CN?tiZ`y>^|2G}-C!h13#0~&8<1m*uJ_iH*!3SX z4>fuLBtHabvR9o-2T~hCj`2{$usMNNJ&m(#AOK`)+s*?pBk z13A{5&#wJ^HpV8zH+tam!78-&r^~Q+!NXb;Z(*G7dS4W_&&A>hOv?;B=VVpBFGC}H z5ux|A(%^lZ>yN-B^vz`zqRj&_@{Ar}65yTN45&VQhJx^HFDnj7x-W9JTbF_ARtR<) zSlDSiPB4l4>Ft2pMEB>NNjhBeJV(+yR_@;qXt{* zi>{igRSm-??{48C*cMK@lNdxoVa=25xRDvgnfjc&fxy{$+-;&ZMslu^crZhl_qZ~B zZi`~ve3^vDir~FAk$we$i?fP&j{K&Q96MiGM2;SRcT2M83Ro1BT5DW(w%=_w2Gb9y z?+thYfail%g$HQ?JAN?%%3B)6rzXOnd?JCx%W?lX)9ImE*I-2%UjcO~PPm#r#VNBC_KaoubYkq>u~t@XCKc8*<>^@Ulj zw19Ni6apQnrWBMG@J2{+J|7o2f(6;X8I5;RqXFibe;5tlC8GgGwk^x;Y6COF-;GA_ zgYSLEWOxSwsv@?4IY`n}6iL2Cy}%8KVGZHKZ3%7{vPrS-&X{VP@~)$KN=%R{I{DXgcR&xRw(DD4Vu`R{JMV-iu!Oz9V51V2G3&UnHGbv%K%LbTm~Ju4A=si|DP^HCoS?g%xZ2wWhUhzX(!%n z9H1S`oA_{f*!&b1Zt~@PoiEW`&l{D7q5-%_ePegAy<53{%B@G#DkjxgY;B~JP94elVXKJ!3EsKS1D;+Kke zp^{SPzL;o+HqdN%C5%Q=2) assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4) for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')): - ast.parse(file.read_text(),filename=str(file)) + ast.parse(file.read_text(encoding='utf-8'),filename=str(file)) for file in ROOT.rglob('*.json'): if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): continue - json.loads(file.read_text()) + json.loads(file.read_text(encoding='utf-8')) for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'): assert (ROOT/required).is_file(), 'missing '+required print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.') diff --git a/tests/client.py b/tests/client.py index f0499a29..38159a0c 100644 --- a/tests/client.py +++ b/tests/client.py @@ -14,7 +14,7 @@ import uuid ROOT = Path(__file__).resolve().parents[1] LAB = json.loads((ROOT / 'lab.json').read_text())['number'] -SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) +SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8')) class Failure(AssertionError): pass diff --git a/tests/resilience.py b/tests/resilience.py index f3fa700a..d5d67fe1 100644 --- a/tests/resilience.py +++ b/tests/resilience.py @@ -55,7 +55,7 @@ def main(): check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity') print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.') # No shell interpolation and no Docker commands in this test. - proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + proc = subprocess.Popen(['sh', str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) started = time.monotonic() successes,failures,after_instances = [],0,[] try: diff --git a/worker.py b/worker.py new file mode 100644 index 00000000..65242dd9 --- /dev/null +++ b/worker.py @@ -0,0 +1,300 @@ +import os +import io +import json +import time +import signal +import hashlib +import logging +from datetime import datetime, timezone + +import psycopg2 +import psycopg2.extras +import redis +import boto3 +from botocore.client import Config +from PIL import Image, ImageOps, UnidentifiedImageError + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s" +) +logger = logging.getLogger("attachment-worker") + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") +REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") +S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") + +STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs") +GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers") +CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1") +JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) + +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS + +redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) +s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + +running = True + +def handle_sigterm(signum, frame): + global running + logger.info("Received shutdown signal, finishing current job...") + running = False + +signal.signal(signal.SIGTERM, handle_sigterm) +signal.signal(signal.SIGINT, handle_sigterm) + + +def get_db_connection(): + return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) + + +def ensure_consumer_group(): + while running: + try: + redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True) + logger.info("Created consumer group '%s' on stream '%s'", GROUP_NAME, STREAM_NAME) + return + except redis.exceptions.ResponseError as e: + if "BUSYGROUP" in str(e): + logger.info("Consumer group '%s' already exists", GROUP_NAME) + return + raise + except Exception as e: + logger.warning("Waiting for Redis broker: %s", type(e).__name__) + time.sleep(1) + + +def build_thumbnail_jpeg(raw_bytes: bytes) -> tuple[bytes, int, int, str]: + with Image.open(io.BytesIO(raw_bytes)) as img: + img = ImageOps.exif_transpose(img) + + if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info): + rgba = img.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.split()[3]) + img = background + elif img.mode != "RGB": + img = img.convert("RGB") + + orig_w, orig_h = img.size + + if orig_w <= 256 and orig_h <= 256: + target_w, target_h = orig_w, orig_h + else: + scale = min(256.0 / orig_w, 256.0 / orig_h) + target_w = max(1, int(orig_w * scale)) + target_h = max(1, int(orig_h * scale)) + + if (target_w, target_h) != (orig_w, orig_h): + img = img.resize((target_w, target_h), Image.Resampling.LANCZOS) + + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85, optimize=True) + thumb_bytes = buf.getvalue() + thumb_sha256 = hashlib.sha256(thumb_bytes).hexdigest() + return thumb_bytes, target_w, target_h, thumb_sha256 + + +def mark_failed(attachment_id: str, error_code: str): + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + UPDATE attachments + SET status = 'failed', error_code = %s, updated_at = %s + WHERE id = %s AND status != 'ready' + """, (error_code, now, attachment_id)) + conn.commit() + finally: + conn.close() + + +def process_job(message_id: str, fields: dict): + try: + job = json.loads(fields.get("payload", "{}")) + except Exception: + logger.error("Malformed job JSON in message %s, acknowledging to drop", message_id) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + attachment_id = job.get("attachment_id") + job_id = job.get("job_id") + pipeline_version = int(job.get("pipeline_version", 1)) + attempt = int(job.get("attempt", 1)) + + if not attachment_id: + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)) + att = cur.fetchone() + + if not att: + conn.commit() + logger.warning("Attachment %s not found in DB, acking job %s", attachment_id, job_id) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + if att["status"] == "ready" or att["pipeline_version"] > pipeline_version: + conn.commit() + logger.info("Attachment %s already ready (or newer pipeline), no-op ack for job %s", attachment_id, job_id) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + if att["status"] == "failed": + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + cur.execute(""" + UPDATE attachments + SET status = 'processing', attempt = %s, updated_at = %s + WHERE id = %s + """, (attempt, now, attachment_id)) + conn.commit() + finally: + conn.close() + + chat_id = str(att["chat_id"]) + staging_key = f"staging/{chat_id}/{attachment_id}/original" + final_original_key = f"attachments/{chat_id}/{attachment_id}/original" + final_thumb_key = f"attachments/{chat_id}/{attachment_id}/thumbnail" + + try: + try: + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key) + except Exception: + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key) + raw_bytes = obj["Body"].read() + + if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]: + raise ValueError("Staging checksum mismatch") + + thumb_bytes, thumb_w, thumb_h, thumb_sha256 = build_thumbnail_jpeg(raw_bytes) + + s3_client.put_object( + Bucket=S3_BUCKET, + Key=final_original_key, + Body=raw_bytes, + ContentType=att["content_type"] + ) + s3_client.put_object( + Bucket=S3_BUCKET, + Key=final_thumb_key, + Body=thumb_bytes, + ContentType="image/jpeg" + ) + + variants = [ + { + "name": "thumbnail", + "content_type": "image/jpeg", + "size_bytes": len(thumb_bytes), + "width": thumb_w, + "height": thumb_h, + "sha256": thumb_sha256 + } + ] + + ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + UPDATE attachments + SET status = 'ready', + variants = %s::jsonb, + error_code = NULL, + updated_at = %s + WHERE id = %s AND pipeline_version <= %s + """, (json.dumps(variants), ready_at, attachment_id, pipeline_version)) + conn.commit() + finally: + conn.close() + + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + logger.info("Job %s completed: attachment %s is ready (thumbnail %dx%d)", job_id, attachment_id, thumb_w, thumb_h) + + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) + except Exception: + pass + + except (UnidentifiedImageError, Image.DecompressionBombError, ValueError) as perm_err: + logger.error("Permanent image error for %s: %s", attachment_id, type(perm_err).__name__) + mark_failed(attachment_id, "invalid_image") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + + except Exception as transient_err: + logger.warning( + "Transient error processing attachment %s (attempt %d/%d): %s", + attachment_id, attempt, JOB_MAX_ATTEMPTS, type(transient_err).__name__ + ) + if attempt >= JOB_MAX_ATTEMPTS: + mark_failed(attachment_id, "retry_exhausted") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + else: + next_job = dict(job) + next_job["attempt"] = attempt + 1 + redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)}) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + + +def main(): + ensure_consumer_group() + logger.info("Worker '%s' started listening on stream '%s'", CONSUMER_NAME, STREAM_NAME) + + while running: + try: + pending = redis_client.xreadgroup( + GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: "0"}, count=10 + ) + messages = pending[0][1] if pending else [] + if not messages: + break + for msg_id, fields in messages: + if not running: + break + process_job(msg_id, fields) + except Exception as e: + logger.warning("Error reading pending messages: %s", type(e).__name__) + time.sleep(1) + + while running: + try: + resp = redis_client.xreadgroup( + GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=1, block=2000 + ) + if not resp: + continue + for _, messages in resp: + for msg_id, fields in messages: + if not running: + break + process_job(msg_id, fields) + except Exception as e: + logger.warning("Broker read error, retrying: %s", type(e).__name__) + time.sleep(1) + + logger.info("Worker stopped cleanly") + + +if __name__ == "__main__": + main() \ No newline at end of file