This commit is contained in:
Getlles
2026-11-16 17:38:53 +03:00
parent 3571422836
commit d475bf29ed
29 changed files with 620 additions and 228 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ RUN useradd -u 10001 -m appuser
WORKDIR /home/appuser
COPY --from=builder /install /usr/local
COPY app.py .
COPY app.py worker.py ./
USER appuser
EXPOSE 8080
+120 -94
View File
@@ -24,7 +24,7 @@ import redis
import boto3
from botocore.client import Config
app = FastAPI(title="MTUSI Messenger — lab4", version="1.4.0")
app = FastAPI(title="MTUSI Messenger — lab5", version="1.5.0")
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
@@ -37,8 +37,10 @@ SECRET_HMAC_KEY = os.environ["SECRET_HMAC_KEY"]
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "minioadmin")
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "minioadminpassword")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
s3_client = boto3.client(
"s3",
@@ -50,8 +52,7 @@ s3_client = boto3.client(
)
def get_db_connection():
conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
return conn
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
def generate_request_id() -> str:
return uuid.uuid4().hex
@@ -91,14 +92,18 @@ def decode_cursor(cursor: str, expected_chat_id: str) -> tuple:
if len(parts) != 4:
raise ValueError()
chat_id, created_at, msg_id, signature = parts
if chat_id != expected_chat_id:
raise HTTPException(status_code=400, detail="Cursor belongs to another chat")
expected_sig = hmac.new(SECRET_HMAC_KEY.encode(), f"{chat_id}|{created_at}|{msg_id}".encode(), hashlib.sha256).hexdigest()[:16]
expected_sig = hmac.new(
SECRET_HMAC_KEY.encode(),
f"{chat_id}|{created_at}|{msg_id}".encode(),
hashlib.sha256
).hexdigest()[:16]
if not hmac.compare_digest(signature, expected_sig):
raise ValueError()
return created_at, msg_id
except HTTPException:
raise
@@ -175,7 +180,7 @@ class CreateUser(BaseModel):
class CreateChat(BaseModel):
model_config = ConfigDict(extra='forbid')
title: str = Field(..., min_length=1, max_length=100)
member_ids: List[uuid.UUID] = Field(..., min_items=1, max_items=100)
member_ids: List[uuid.UUID] = Field(..., min_length=1, max_length=100)
@field_validator("title")
def not_blank(cls, v):
@@ -186,7 +191,7 @@ class CreateChat(BaseModel):
class CreateMessage(BaseModel):
model_config = ConfigDict(extra='forbid')
text: str = Field(default="", max_length=4000)
attachment_ids: List[uuid.UUID] = Field(default=[], max_items=10)
attachment_ids: List[uuid.UUID] = Field(default=[], max_length=10)
@model_validator(mode="after")
def check_content(self) -> 'CreateMessage':
@@ -220,7 +225,7 @@ def register_user(body: CreateUser, response: Response):
req_id = generate_request_id()
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = INSTANCE_ID
password_hash = bcrypt.hashpw(body.password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')
user_id = str(uuid.uuid4())
created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
@@ -235,7 +240,10 @@ def register_user(body: CreateUser, response: Response):
conn.commit()
except psycopg2.errors.UniqueViolation:
conn.rollback()
raise HTTPException(status_code=409, detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}})
raise HTTPException(
status_code=409,
detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}}
)
finally:
conn.close()
@@ -254,18 +262,18 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
if not authorization or not authorization.startswith("Basic "):
raise HTTPException(
status_code=401,
status_code=401,
detail="Missing or invalid basic auth",
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
)
try:
encoded = authorization.split(" ")[1]
decoded = base64.b64decode(encoded).decode("utf-8")
username, password = decoded.split(":", 1)
except Exception:
raise HTTPException(
status_code=401,
status_code=401,
detail="Invalid credentials format",
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
)
@@ -274,11 +282,11 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
with conn.cursor() as cur:
cur.execute("SELECT * FROM users WHERE username = %s", (username,))
user = cur.fetchone()
conn.close()
if not user or not bcrypt.checkpw(password.encode('utf-8'), user['password_hash'].encode('utf-8')):
conn.close()
raise HTTPException(
status_code=401,
status_code=401,
detail="Invalid credentials",
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
)
@@ -293,7 +301,7 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
"token_type": "Bearer",
"expires_at": expires_at_str,
"user": {
"id": user['id'],
"id": str(user['id']),
"username": user['username'],
"display_name": user['display_name'],
"created_at": user['created_at'].isoformat().replace("+00:00", "Z") if isinstance(user['created_at'], datetime) else user['created_at']
@@ -303,7 +311,7 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> dict:
if not authorization or not authorization.startswith("Bearer "):
raise HTTPException(
status_code=401,
status_code=401,
detail="Missing or invalid bearer token",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
)
@@ -316,7 +324,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
if not user_id:
raise HTTPException(
status_code=401,
status_code=401,
detail="Invalid or expired session token",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
)
@@ -329,7 +337,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
if not row:
raise HTTPException(
status_code=401,
status_code=401,
detail="User not found",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
)
@@ -344,10 +352,10 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def revoke_session(authorization: Optional[str] = Header(None)):
req_id = generate_request_id()
if not authorization or not authorization.startswith("Bearer "):
raise HTTPException(
status_code=401,
status_code=401,
detail="Unauthorized",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
)
@@ -357,7 +365,7 @@ def revoke_session(authorization: Optional[str] = Header(None)):
if not deleted:
raise HTTPException(
status_code=401,
status_code=401,
detail="Invalid session token",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
)
@@ -365,7 +373,7 @@ def revoke_session(authorization: Optional[str] = Header(None)):
return Response(
status_code=204,
headers={
"X-Request-Id": req_id,
"X-Request-Id": req_id,
"X-Instance-Id": INSTANCE_ID
},
)
@@ -383,7 +391,7 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = INSTANCE_ID
user = get_current_user_from_token(authorization)
member_ids_str = [str(mid) for mid in body.member_ids]
if user["id"] not in member_ids_str:
@@ -433,7 +441,7 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = INSTANCE_ID
user = get_current_user_from_token(authorization)
if limit < 1 or limit > 100:
raise HTTPException(status_code=400, detail="Invalid limit")
@@ -446,21 +454,18 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response
ORDER BY c.created_at ASC, c.id ASC
""", (user["id"],))
rows = cur.fetchall()
conn.close()
user_chats = []
for r in rows:
conn = get_db_connection()
with conn.cursor() as cur:
user_chats = []
for r in rows:
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],))
m_rows = cur.fetchall()
conn.close()
user_chats.append({
"id": r['id'],
"title": r['title'],
"member_ids": [m['user_id'] for m in m_rows],
"created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at']
})
user_chats.append({
"id": str(r['id']),
"title": r['title'],
"member_ids": [str(m['user_id']) for m in m_rows],
"created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at']
})
conn.close()
start_idx = 0
if cursor:
@@ -483,7 +488,7 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response
if start_idx + limit < len(user_chats):
last = page_items[-1]
next_cursor = f"{last['created_at']}_{last['id']}"
return {"items": page_items, "next_cursor": next_cursor}
@app.get("/api/v1/chats/{chat_id}")
@@ -492,7 +497,7 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = INSTANCE_ID
user = get_current_user_from_token(authorization)
chat_id_str = str(chat_id)
conn = get_db_connection()
with conn.cursor() as cur:
@@ -511,16 +516,16 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H
raise HTTPException(status_code=404, detail="Chat not found")
return {
"id": chat['id'],
"id": str(chat['id']),
"title": chat['title'],
"member_ids": [m['user_id'] for m in members],
"member_ids": [str(m['user_id']) for m in members],
"created_at": chat['created_at'].isoformat().replace("+00:00", "Z") if isinstance(chat['created_at'], datetime) else chat['created_at']
}
def validate_image_and_get_dimensions(data: bytes, content_type: str):
if len(data) > 10 * 1024 * 1024:
raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)")
width, height = 0, 0
if content_type == "image/png":
if not data.startswith(b'\x89PNG\r\n\x1a\n'):
@@ -544,7 +549,13 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str):
if 0xc0 <= marker_type <= 0xc3:
height, width = struct.unpack('>HH', data[i+5:i+9])
break
i += 2 + struct.unpack('>H', data[i+2:i+4])[0]
if marker_type in {0xd8, 0xd9} or (0xd0 <= marker_type <= 0xd7):
i += 2
continue
seg_len = struct.unpack('>H', data[i+2:i+4])[0]
if seg_len < 2:
raise HTTPException(status_code=415, detail="Invalid JPEG segment")
i += 2 + seg_len
else:
i += 1
else:
@@ -556,7 +567,7 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str):
if width * height > 40_000_000:
raise HTTPException(status_code=413, detail="Image exceeds 40 million pixels limit")
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(
chat_id: UUID,
filename: str = Query(..., min_length=1, max_length=255),
@@ -579,7 +590,7 @@ async def upload_attachment(
conn.close()
if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename) or '/' in filename or '\\' in filename:
raise HTTPException(400, "Invalid filename")
raise HTTPException(status_code=400, detail="Invalid filename")
content_type = request.headers.get("content-type")
if content_type not in ["image/png", "image/jpeg"]:
@@ -608,30 +619,14 @@ async def upload_attachment(
size_bytes = len(body_bytes)
sha256_hash = hashlib.sha256(body_bytes).hexdigest()
attachment_id = str(uuid.uuid4())
job_id = str(uuid.uuid4())
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
s3_key = f"attachments/{chat_id_str}/{attachment_id}/{filename}"
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, variants, error_code, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, 'pending', '[]', NULL, %s, %s)
""", (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now))
conn.commit()
except Exception:
conn.rollback()
conn.close()
raise HTTPException(status_code=500, detail="Internal error initializing attachment")
finally:
if not conn.closed:
conn.close()
staging_key = f"staging/{chat_id_str}/{attachment_id}/original"
try:
s3_client.put_object(
Bucket=S3_BUCKET,
Key=s3_key,
Key=staging_key,
Body=body_bytes,
ContentType=content_type
)
@@ -642,16 +637,36 @@ async def upload_attachment(
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments SET status = 'ready', updated_at = %s WHERE id = %s
""", (now, attachment_id))
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type,
size_bytes, sha256, status, variants, error_code,
pipeline_version, attempt, created_at, updated_at
)
VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', '[]', NULL, 1, 0, %s, %s)
""", (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now))
conn.commit()
except Exception:
conn.rollback()
raise HTTPException(status_code=500, detail="Internal error finalizing attachment status")
conn.close()
raise HTTPException(status_code=500, detail="Internal error saving attachment metadata")
finally:
if not conn.closed:
conn.close()
job_payload = {
"schema_version": 1,
"job_id": job_id,
"type": "attachment.process.v1",
"attachment_id": attachment_id,
"pipeline_version": 1,
"attempt": 1,
"created_at": now
}
try:
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(job_payload)})
except Exception:
raise HTTPException(status_code=503, detail="Broker unavailable")
return {
"id": attachment_id,
"chat_id": chat_id_str,
@@ -660,7 +675,7 @@ async def upload_attachment(
"content_type": content_type,
"size_bytes": size_bytes,
"sha256": sha256_hash,
"status": "ready",
"status": "queued",
"variants": [],
"error_code": None,
"created_at": now,
@@ -704,7 +719,12 @@ def get_attachment(attachment_id: UUID, response: Response, authorization: Optio
}
@app.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(attachment_id: UUID, response: Response, authorization: Optional[str] = Header(None)):
def download_attachment(
attachment_id: UUID,
variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"),
response: Response = None,
authorization: Optional[str] = Header(None)
):
req_id = generate_request_id()
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = INSTANCE_ID
@@ -725,9 +745,19 @@ def download_attachment(attachment_id: UUID, response: Response, authorization:
raise HTTPException(status_code=404, detail="Attachment not found or inaccessible")
if row['status'] != 'ready':
raise HTTPException(status_code=409, detail="Attachment not ready")
raise HTTPException(status_code=409, detail=f"Attachment is in status '{row['status']}'")
if variant == "original":
s3_key = f"attachments/{row['chat_id']}/{row['id']}/original"
media_type = row['content_type']
else:
variants_list = row['variants'] or []
matched = [v for v in variants_list if v.get("name") == variant]
if not matched:
raise HTTPException(status_code=404, detail=f"Variant '{variant}' not found")
s3_key = f"attachments/{row['chat_id']}/{row['id']}/{variant}"
media_type = matched[0]["content_type"]
s3_key = f"attachments/{row['chat_id']}/{row['id']}/{row['filename']}"
try:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key)
data = obj['Body'].read()
@@ -736,7 +766,7 @@ def download_attachment(attachment_id: UUID, response: Response, authorization:
return Response(
content=data,
media_type=row['content_type'],
media_type=media_type,
headers={
"X-Request-Id": req_id,
"X-Instance-Id": INSTANCE_ID,
@@ -745,9 +775,9 @@ def download_attachment(attachment_id: UUID, response: Response, authorization:
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def create_message(
chat_id: UUID,
body: CreateMessage,
response: Response,
chat_id: UUID,
body: CreateMessage,
response: Response,
authorization: Optional[str] = Header(None),
idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key")
):
@@ -755,14 +785,14 @@ def create_message(
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = INSTANCE_ID
user = get_current_user_from_token(authorization)
if not idempotency_key or not re.match(r"^[A-Za-z0-9._:-]{1,128}$", idempotency_key):
raise HTTPException(status_code=400, detail="Missing or invalid Idempotency-Key")
chat_id_str = str(chat_id)
method = "POST"
path = f"/api/v1/chats/{chat_id_str}/messages"
body_json_str = json.dumps(body.model_dump(mode='json'), sort_keys=True)
request_hash = hashlib.sha256(body_json_str.encode("utf-8")).hexdigest()
@@ -794,7 +824,7 @@ def create_message(
msg_id = str(uuid.uuid4())
created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
msg_response = {
"id": msg_id,
"chat_id": chat_id_str,
@@ -809,7 +839,7 @@ def create_message(
VALUES (%s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (user_id, method, path, key) DO NOTHING
""", (user["id"], method, path, idempotency_key, request_hash, 201, json.dumps(msg_response)))
if cur.rowcount == 0:
cur.execute("""
SELECT request_hash, response_status, response_body
@@ -822,7 +852,7 @@ def create_message(
if row["request_hash"] != request_hash:
raise HTTPException(status_code=409, detail="Idempotency key conflict with different payload")
return row["response_body"]
cur.execute(
@@ -832,7 +862,7 @@ def create_message(
conn.commit()
except HTTPException:
raise
except Exception as exc:
except Exception:
conn.rollback()
conn.close()
raise HTTPException(status_code=500, detail="Internal error processing message")
@@ -890,16 +920,12 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
formatted_msgs = []
for m in page_items:
raw_atts = m.get('attachment_ids')
atts = []
if raw_atts:
for aid in raw_atts:
if aid:
atts.append(str(aid))
atts = [str(aid) for aid in raw_atts if aid] if raw_atts else []
formatted_msgs.append({
"id": m['id'],
"chat_id": m['chat_id'],
"sender_id": m['sender_id'],
"id": str(m['id']),
"chat_id": str(m['chat_id']),
"sender_id": str(m['sender_id']),
"text": m['text'],
"created_at": m['created_at'].isoformat().replace("+00:00", "Z") if isinstance(m['created_at'], datetime) else m['created_at'],
"attachment_ids": atts
+76 -39
View File
@@ -23,7 +23,7 @@ services:
redis:
image: redis:7.2.4-alpine
command: redis-server --save 60 1 --loglevel warning
command: redis-server --appendonly yes --appendfsync everysec --loglevel warning
volumes:
- redisdata:/data
deploy:
@@ -51,6 +51,44 @@ services:
condition: service_healthy
restart: "no"
s3:
image: chrislusf/seaweedfs:latest
command: ["server", "-s3", "-dir=/data", "-s3.port=8333"]
environment:
- AWS_ACCESS_KEY_ID=appuser
- AWS_SECRET_ACCESS_KEY=appsecretpassword
volumes:
- s3data:/data
deploy:
resources:
limits:
cpus: '0.50'
memory: 512M
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz || exit 1"]
interval: 5s
timeout: 3s
retries: 5
start_period: 10s
s3-init:
image: python:3.11.7-slim-bookworm
volumes:
- ./scripts:/scripts
- ./requirements.txt:/requirements.txt
environment:
- S3_ENDPOINT=http://s3:8333
- S3_BUCKET=messenger-attachments
- S3_ROOT_ACCESS_KEY=appuser
- S3_ROOT_SECRET_KEY=appsecretpassword
command: >
sh -c "pip install --no-cache-dir -r /requirements.txt &&
python /scripts/bootstrap_s3.py"
depends_on:
s3:
condition: service_healthy
restart: "no"
api-1:
build: .
stop_grace_period: 15s
@@ -64,6 +102,7 @@ services:
- S3_BUCKET=messenger-attachments
- S3_ACCESS_KEY_ID=appuser
- S3_SECRET_ACCESS_KEY=appsecretpassword
- BROKER_STREAM=attachment_jobs
deploy:
resources:
limits:
@@ -97,6 +136,7 @@ services:
- S3_BUCKET=messenger-attachments
- S3_ACCESS_KEY_ID=appuser
- S3_SECRET_ACCESS_KEY=appsecretpassword
- BROKER_STREAM=attachment_jobs
deploy:
resources:
limits:
@@ -117,6 +157,41 @@ services:
timeout: 3s
retries: 3
worker:
build: .
command: ["python", "worker.py"]
stop_grace_period: 15s
environment:
- DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger
- REDIS_URL=redis://redis:6379/0
- S3_ENDPOINT=http://s3:8333
- S3_BUCKET=messenger-attachments
- S3_ACCESS_KEY_ID=appuser
- S3_SECRET_ACCESS_KEY=appsecretpassword
- BROKER_STREAM=attachment_jobs
- BROKER_GROUP=attachment_workers
- JOB_MAX_ATTEMPTS=5
- MAX_IMAGE_PIXELS=40000000
deploy:
resources:
limits:
cpus: '0.50'
memory: 256M
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import os, redis; redis.Redis.from_url(os.environ['REDIS_URL']).ping()"]
interval: 5s
timeout: 3s
retries: 3
nginx:
image: nginx:1.25.4-alpine
ports:
@@ -137,44 +212,6 @@ services:
timeout: 3s
retries: 3
s3:
image: chrislusf/seaweedfs:latest
command: "mini -dir=/data"
environment:
- AWS_ACCESS_KEY_ID=appuser
- AWS_SECRET_ACCESS_KEY=appsecretpassword
- S3_BUCKET=messenger-attachments
volumes:
- s3data:/data
deploy:
resources:
limits:
cpus: '0.50'
memory: 512M
healthcheck:
test: ["CMD", "nc", "-z", "localhost", "8333"]
interval: 5s
timeout: 3s
retries: 3
s3-init:
image: python:3.11-slim
volumes:
- ./scripts:/scripts
- ./requirements.txt:/requirements.txt
environment:
- S3_ENDPOINT=http://s3:8333
- S3_BUCKET=messenger-attachments
- S3_ROOT_ACCESS_KEY=appuser
- S3_ROOT_SECRET_KEY=appsecretpassword
command: >
sh -c "pip install --no-cache-dir -r /requirements.txt &&
python /scripts/bootstrap_s3.py"
depends_on:
s3:
condition: service_healthy
restart: "no"
volumes:
pgdata:
redisdata:
-1
View File
@@ -1 +0,0 @@
PASS: anonymous GET -> 403
-7
View File
@@ -1,7 +0,0 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab4-api-1-1 mtusi2026lab4-api-1 "uvicorn app:app --h…" api-1 15 minutes ago Up 12 minutes (healthy) 8080/tcp
mtusi2026lab4-api-2-1 mtusi2026lab4-api-2 "uvicorn app:app --h…" api-2 15 minutes ago Up 23 seconds (healthy) 8080/tcp
mtusi2026lab4-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.…" nginx 15 minutes ago Up 12 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp
mtusi2026lab4-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.s…" postgres 15 minutes ago Up 12 minutes (healthy) 5432/tcp
mtusi2026lab4-redis-1 redis:7.2.4-alpine "docker-entrypoint.s…" redis 15 minutes ago Up 12 minutes (healthy) 6379/tcp
mtusi2026lab4-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh mini…" s3 15 minutes ago Up 12 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
+6
View File
@@ -0,0 +1,6 @@
{
"id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
"username": "demo_75866",
"display_name": "Demo User",
"created_at": "2026-11-16T13:43:38.059472Z"
}
+11
View File
@@ -0,0 +1,11 @@
{
"token": "grSsHp1gJpFLckF14UT44obEonSGQRvP878JhLyCaNzM6PGBH1LwrdgZIFF8Zu8U",
"token_type": "Bearer",
"expires_at": "2026-11-16T14:43:38.850596Z",
"user": {
"id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
"username": "demo_75866",
"display_name": "Demo User",
"created_at": "2026-11-16T13:43:38.059472Z"
}
}
+8
View File
@@ -0,0 +1,8 @@
{
"id": "5db90ff5-9604-4853-b002-db4144868bcb",
"title": "Demo chat",
"member_ids": [
"fdeb71d4-ee4f-47a6-8968-ffc7740e5242"
],
"created_at": "2026-11-16T13:43:39.240968Z"
}
+10
View File
@@ -0,0 +1,10 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab5-api-1-1 mtusi2026lab5-api-1 "uvicorn app:app --hтАж" api-1 30 minutes ago Up 6 minutes (healthy) 8080/tcp
mtusi2026lab5-api-2-1 mtusi2026lab5-api-2 "uvicorn app:app --hтАж" api-2 30 minutes ago Up 6 minutes (healthy) 8080/tcp
mtusi2026lab5-migrate-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" migrate 30 minutes ago Exited (0) 5 minutes ago
mtusi2026lab5-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.тАж" nginx 30 minutes ago Up 25 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp
mtusi2026lab5-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" postgres 30 minutes ago Up 30 minutes (healthy) 5432/tcp
mtusi2026lab5-redis-1 redis:7.2.4-alpine "docker-entrypoint.sтАж" redis 30 minutes ago Up 4 minutes (healthy) 6379/tcp
mtusi2026lab5-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh servтАж" s3 30 minutes ago Up 30 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
mtusi2026lab5-s3-init-1 python:3.11.7-slim-bookworm "sh -c 'pip install тАж" s3-init 30 minutes ago Exited (0) 5 minutes ago
mtusi2026lab5-worker-1 mtusi2026lab5-worker "python worker.py" worker 30 minutes ago Exited (0) Less than a second ago
+1
View File
@@ -0,0 +1 @@
{"id":"9becbd32-e413-417a-8762-79bcc195e296","chat_id":"5db90ff5-9604-4853-b002-db4144868bcb","owner_id":"fdeb71d4-ee4f-47a6-8968-ffc7740e5242","filename":"sample.png","content_type":"image/png","size_bytes":111817,"sha256":"99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75","status":"queued","variants":[],"error_code":null,"created_at":"2026-11-16T13:43:41.734766Z","updated_at":"2026-11-16T13:43:41.734766Z"}
+10
View File
@@ -0,0 +1,10 @@
{
"id": "74dc2e91-a901-4067-8993-da9d2cc4ca55",
"chat_id": "5db90ff5-9604-4853-b002-db4144868bcb",
"sender_id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
"text": "",
"created_at": "2026-11-16T13:43:42.100558Z",
"attachment_ids": [
"9becbd32-e413-417a-8762-79bcc195e296"
]
}
+7
View File
@@ -0,0 +1,7 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab5-api-1-1 mtusi2026lab5-api-1 "uvicorn app:app --hтАж" api-1 30 minutes ago Up Less than a second (health: starting) 8080/tcp
mtusi2026lab5-api-2-1 mtusi2026lab5-api-2 "uvicorn app:app --hтАж" api-2 30 minutes ago Up Less than a second (health: starting) 8080/tcp
mtusi2026lab5-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.тАж" nginx 30 minutes ago Up 25 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp
mtusi2026lab5-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" postgres 30 minutes ago Up 30 minutes (healthy) 5432/tcp
mtusi2026lab5-redis-1 redis:7.2.4-alpine "docker-entrypoint.sтАж" redis 30 minutes ago Up 4 minutes (healthy) 6379/tcp
mtusi2026lab5-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh servтАж" s3 30 minutes ago Up 30 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
+10
View File
@@ -0,0 +1,10 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab5-api-1-1 mtusi2026lab5-api-1 "uvicorn app:app --hтАж" api-1 30 minutes ago Up 10 seconds (healthy) 8080/tcp
mtusi2026lab5-api-2-1 mtusi2026lab5-api-2 "uvicorn app:app --hтАж" api-2 30 minutes ago Up 9 seconds (healthy) 8080/tcp
mtusi2026lab5-migrate-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" migrate 30 minutes ago Exited (0) 7 seconds ago
mtusi2026lab5-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.тАж" nginx 30 minutes ago Up 26 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp
mtusi2026lab5-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" postgres 30 minutes ago Up 30 minutes (healthy) 5432/tcp
mtusi2026lab5-redis-1 redis:7.2.4-alpine "docker-entrypoint.sтАж" redis 30 minutes ago Up 4 minutes (healthy) 6379/tcp
mtusi2026lab5-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh servтАж" s3 30 minutes ago Up 30 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
mtusi2026lab5-s3-init-1 python:3.11.7-slim-bookworm "sh -c 'pip install тАж" s3-init 30 minutes ago Exited (0) Less than a second ago
mtusi2026lab5-worker-1 mtusi2026lab5-worker "python worker.py" worker 30 minutes ago Up Less than a second (health: starting) 8080/tcp
+23
View File
@@ -0,0 +1,23 @@
{
"id": "9becbd32-e413-417a-8762-79bcc195e296",
"chat_id": "5db90ff5-9604-4853-b002-db4144868bcb",
"owner_id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
"filename": "sample.png",
"content_type": "image/png",
"size_bytes": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"status": "ready",
"variants": [
{
"name": "thumbnail",
"width": 256,
"height": 160,
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
"size_bytes": 4679,
"content_type": "image/jpeg"
}
],
"error_code": null,
"created_at": "2026-11-16T13:43:41.734766Z",
"updated_at": "2026-11-16T13:43:55.375481Z"
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.6 KiB

@@ -0,0 +1,23 @@
{
"id": "9becbd32-e413-417a-8762-79bcc195e296",
"chat_id": "5db90ff5-9604-4853-b002-db4144868bcb",
"owner_id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
"filename": "sample.png",
"content_type": "image/png",
"size_bytes": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"status": "ready",
"variants": [
{
"name": "thumbnail",
"width": 256,
"height": 160,
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
"size_bytes": 4679,
"content_type": "image/jpeg"
}
],
"error_code": null,
"created_at": "2026-11-16T13:43:41.734766Z",
"updated_at": "2026-11-16T13:43:55.375481Z"
}
-4
View File
@@ -1,4 +0,0 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 82, "failed_attempts": 2, "first_recovery_seconds": 1.312}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
-17
View File
@@ -1,17 +0,0 @@
test_corrupted_image_signature (__main__.TestMessengerLevel4Complete.test_corrupted_image_signature)
Ïðîâåðêà 415: áèòàÿ ñòðóêòóðà / íåâåðíûå ðàçìåðû PNG. ... ok
test_cross_chat_attachment_isolation (__main__.TestMessengerLevel4Complete.test_cross_chat_attachment_isolation)
Ïðîâåðêà 404: ïîïûòêà ïðèêðåïèòü âëîæåíèå èç ÷óæîãî ÷àòà. ... ok
test_invalid_filename_characters (__main__.TestMessengerLevel4Complete.test_invalid_filename_characters)
Ïðîâåðêà 400: íåäîïóñòèìûå ñèìâîëû (path traversal) â èìåíè ôàéëà. ... ok
test_nonexistent_attachment_id (__main__.TestMessengerLevel4Complete.test_nonexistent_attachment_id)
Ïðîâåðêà 404: ïåðåäà÷à íåñóùåñòâóþùåãî ID âëîæåíèÿ ïðè îòïðàâêå ñîîáùåíèÿ. ... ok
test_payload_too_large (__main__.TestMessengerLevel4Complete.test_payload_too_large)
Ïðîâåðêà 413: ôàéë ðàçìåðîì áîëüøå 10 MiB. ... ok
test_unsupported_media_type (__main__.TestMessengerLevel4Complete.test_unsupported_media_type)
Ïðîâåðêà 415: ïåðåäà÷à òåêñòîâîãî ôàéëà ïîä âèäîì êàðòèíêè. ... ok
----------------------------------------------------------------------
Ran 6 tests in 19.521s
OK
-3
View File
@@ -1,3 +0,0 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
-4
View File
@@ -1,4 +0,0 @@
Container mtusi2026lab4-api-1-run-9e5b0d92d11b Creating
Container mtusi2026lab4-api-1-run-9e5b0d92d11b Created
Starting reconciliation (dry_run=True)...
Reconciliation completed successfully.
-2
View File
@@ -1,2 +0,0 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
-7
View File
@@ -1,7 +0,0 @@
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
de2d9b4cdf10 mtusi2026lab4-nginx-1 2.71% 2.355MiB / 128MiB 1.84% 389kB / 408kB 0B / 12.3kB 2
bd85245e25be mtusi2026lab4-api-1-1 14.75% 74.22MiB / 256MiB 28.99% 518kB / 647kB 0B / 53.2kB 2
dd34824ed0c5 mtusi2026lab4-api-2-1 13.03% 74.76MiB / 256MiB 29.20% 750kB / 879kB 0B / 53.2kB 2
650e2ec7ac6a mtusi2026lab4-s3-1 1.02% 68.62MiB / 512MiB 13.40% 559kB / 248kB 0B / 1.09MB 8
1938313cd5da mtusi2026lab4-redis-1 0.24% 3.16MiB / 256MiB 1.23% 99.6kB / 46.7kB 24.6kB / 12.3kB 5
9348c37fb0f0 mtusi2026lab4-postgres-1 0.01% 34.39MiB / 512MiB 6.72% 679kB / 765kB 0B / 55.1MB 6
-40
View File
@@ -1,40 +0,0 @@
mtusi2026lab4-api-1-1 63.46MiB / 256MiB 0.20%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.46MiB / 256MiB 0.20%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.49MiB / 256MiB 0.39%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.49MiB / 256MiB 0.24%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 72.76MiB / 256MiB 7.76%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.53MiB / 256MiB 0.22%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.53MiB / 256MiB 0.20%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.61MiB / 256MiB 22.90%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.56MiB / 256MiB 0.20%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.57MiB / 256MiB 50.53%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.65MiB / 256MiB 14.81%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.68MiB / 256MiB 0.22%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.68MiB / 256MiB 0.30%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.71MiB / 256MiB 0.21%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.71MiB / 256MiB 0.20%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 73.85MiB / 256MiB 14.64%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.75MiB / 256MiB 0.22%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.75MiB / 256MiB 0.24%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.8MiB / 256MiB 27.08%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
mtusi2026lab4-api-1-1 63.78MiB / 256MiB 0.24%
mtusi2026lab4-api-2-1 0B / 0B 0.00%
+6 -2
View File
@@ -49,13 +49,17 @@ CREATE TABLE IF NOT EXISTS attachments (
content_type VARCHAR(50) NOT NULL,
size_bytes BIGINT NOT NULL,
sha256 VARCHAR(64) NOT NULL,
status VARCHAR(32) NOT NULL DEFAULT 'ready',
status VARCHAR(32) NOT NULL DEFAULT 'queued',
variants JSONB NOT NULL DEFAULT '[]',
error_code VARCHAR(64),
pipeline_version INT NOT NULL DEFAULT 1,
attempt INT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id);
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}';
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}';
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0;
+2 -1
View File
@@ -11,4 +11,5 @@ annotated-types==0.6.0
psycopg2-binary==2.9.9
bcrypt==4.1.2
redis==5.0.1
boto3==1.34.0
boto3==1.34.0
Pillow==10.2.0
+4 -4
View File
@@ -8,8 +8,8 @@ import re
ROOT = Path(__file__).resolve().parents[1]
def main():
lab = json.loads((ROOT/'lab.json').read_text())['number']
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
lab = json.loads((ROOT/'lab.json').read_text(encoding='utf-8'))['number']
spec = json.loads((ROOT/'contracts/openapi.json').read_text(encoding='utf-8'))
assert spec['openapi']=='3.1.0'
assert spec['info']['version']=='1.%d.0'%lab
operation_ids = set()
@@ -41,11 +41,11 @@ def main():
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
ast.parse(file.read_text(),filename=str(file))
ast.parse(file.read_text(encoding='utf-8'),filename=str(file))
for file in ROOT.rglob('*.json'):
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
continue
json.loads(file.read_text())
json.loads(file.read_text(encoding='utf-8'))
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
assert (ROOT/required).is_file(), 'missing '+required
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
+1 -1
View File
@@ -14,7 +14,7 @@ import uuid
ROOT = Path(__file__).resolve().parents[1]
LAB = json.loads((ROOT / 'lab.json').read_text())['number']
SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text())
SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8'))
class Failure(AssertionError):
pass
+1 -1
View File
@@ -55,7 +55,7 @@ def main():
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
# No shell interpolation and no Docker commands in this test.
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
proc = subprocess.Popen(['sh', str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
started = time.monotonic()
successes,failures,after_instances = [],0,[]
try:
+300
View File
@@ -0,0 +1,300 @@
import os
import io
import json
import time
import signal
import hashlib
import logging
from datetime import datetime, timezone
import psycopg2
import psycopg2.extras
import redis
import boto3
from botocore.client import Config
from PIL import Image, ImageOps, UnidentifiedImageError
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s"
)
logger = logging.getLogger("attachment-worker")
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
s3_client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
config=Config(signature_version="s3v4"),
region_name="us-east-1"
)
running = True
def handle_sigterm(signum, frame):
global running
logger.info("Received shutdown signal, finishing current job...")
running = False
signal.signal(signal.SIGTERM, handle_sigterm)
signal.signal(signal.SIGINT, handle_sigterm)
def get_db_connection():
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
def ensure_consumer_group():
while running:
try:
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
logger.info("Created consumer group '%s' on stream '%s'", GROUP_NAME, STREAM_NAME)
return
except redis.exceptions.ResponseError as e:
if "BUSYGROUP" in str(e):
logger.info("Consumer group '%s' already exists", GROUP_NAME)
return
raise
except Exception as e:
logger.warning("Waiting for Redis broker: %s", type(e).__name__)
time.sleep(1)
def build_thumbnail_jpeg(raw_bytes: bytes) -> tuple[bytes, int, int, str]:
with Image.open(io.BytesIO(raw_bytes)) as img:
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.split()[3])
img = background
elif img.mode != "RGB":
img = img.convert("RGB")
orig_w, orig_h = img.size
if orig_w <= 256 and orig_h <= 256:
target_w, target_h = orig_w, orig_h
else:
scale = min(256.0 / orig_w, 256.0 / orig_h)
target_w = max(1, int(orig_w * scale))
target_h = max(1, int(orig_h * scale))
if (target_w, target_h) != (orig_w, orig_h):
img = img.resize((target_w, target_h), Image.Resampling.LANCZOS)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85, optimize=True)
thumb_bytes = buf.getvalue()
thumb_sha256 = hashlib.sha256(thumb_bytes).hexdigest()
return thumb_bytes, target_w, target_h, thumb_sha256
def mark_failed(attachment_id: str, error_code: str):
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments
SET status = 'failed', error_code = %s, updated_at = %s
WHERE id = %s AND status != 'ready'
""", (error_code, now, attachment_id))
conn.commit()
finally:
conn.close()
def process_job(message_id: str, fields: dict):
try:
job = json.loads(fields.get("payload", "{}"))
except Exception:
logger.error("Malformed job JSON in message %s, acknowledging to drop", message_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
attachment_id = job.get("attachment_id")
job_id = job.get("job_id")
pipeline_version = int(job.get("pipeline_version", 1))
attempt = int(job.get("attempt", 1))
if not attachment_id:
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,))
att = cur.fetchone()
if not att:
conn.commit()
logger.warning("Attachment %s not found in DB, acking job %s", attachment_id, job_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
conn.commit()
logger.info("Attachment %s already ready (or newer pipeline), no-op ack for job %s", attachment_id, job_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
if att["status"] == "failed":
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
cur.execute("""
UPDATE attachments
SET status = 'processing', attempt = %s, updated_at = %s
WHERE id = %s
""", (attempt, now, attachment_id))
conn.commit()
finally:
conn.close()
chat_id = str(att["chat_id"])
staging_key = f"staging/{chat_id}/{attachment_id}/original"
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
final_thumb_key = f"attachments/{chat_id}/{attachment_id}/thumbnail"
try:
try:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
raw_bytes = obj["Body"].read()
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
raise ValueError("Staging checksum mismatch")
thumb_bytes, thumb_w, thumb_h, thumb_sha256 = build_thumbnail_jpeg(raw_bytes)
s3_client.put_object(
Bucket=S3_BUCKET,
Key=final_original_key,
Body=raw_bytes,
ContentType=att["content_type"]
)
s3_client.put_object(
Bucket=S3_BUCKET,
Key=final_thumb_key,
Body=thumb_bytes,
ContentType="image/jpeg"
)
variants = [
{
"name": "thumbnail",
"content_type": "image/jpeg",
"size_bytes": len(thumb_bytes),
"width": thumb_w,
"height": thumb_h,
"sha256": thumb_sha256
}
]
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments
SET status = 'ready',
variants = %s::jsonb,
error_code = NULL,
updated_at = %s
WHERE id = %s AND pipeline_version <= %s
""", (json.dumps(variants), ready_at, attachment_id, pipeline_version))
conn.commit()
finally:
conn.close()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
logger.info("Job %s completed: attachment %s is ready (thumbnail %dx%d)", job_id, attachment_id, thumb_w, thumb_h)
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
pass
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError) as perm_err:
logger.error("Permanent image error for %s: %s", attachment_id, type(perm_err).__name__)
mark_failed(attachment_id, "invalid_image")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except Exception as transient_err:
logger.warning(
"Transient error processing attachment %s (attempt %d/%d): %s",
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(transient_err).__name__
)
if attempt >= JOB_MAX_ATTEMPTS:
mark_failed(attachment_id, "retry_exhausted")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
else:
next_job = dict(job)
next_job["attempt"] = attempt + 1
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
def main():
ensure_consumer_group()
logger.info("Worker '%s' started listening on stream '%s'", CONSUMER_NAME, STREAM_NAME)
while running:
try:
pending = redis_client.xreadgroup(
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: "0"}, count=10
)
messages = pending[0][1] if pending else []
if not messages:
break
for msg_id, fields in messages:
if not running:
break
process_job(msg_id, fields)
except Exception as e:
logger.warning("Error reading pending messages: %s", type(e).__name__)
time.sleep(1)
while running:
try:
resp = redis_client.xreadgroup(
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=1, block=2000
)
if not resp:
continue
for _, messages in resp:
for msg_id, fields in messages:
if not running:
break
process_job(msg_id, fields)
except Exception as e:
logger.warning("Broker read error, retrying: %s", type(e).__name__)
time.sleep(1)
logger.info("Worker stopped cleanly")
if __name__ == "__main__":
main()