forked from mtusi2026/mtusi2026lab5
grade3
This commit is contained in:
+1
-1
@@ -10,7 +10,7 @@ RUN useradd -u 10001 -m appuser
|
||||
WORKDIR /home/appuser
|
||||
|
||||
COPY --from=builder /install /usr/local
|
||||
COPY app.py .
|
||||
COPY app.py worker.py ./
|
||||
|
||||
USER appuser
|
||||
EXPOSE 8080
|
||||
|
||||
@@ -24,7 +24,7 @@ import redis
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger — lab4", version="1.4.0")
|
||||
app = FastAPI(title="MTUSI Messenger — lab5", version="1.5.0")
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
@@ -37,8 +37,10 @@ SECRET_HMAC_KEY = os.environ["SECRET_HMAC_KEY"]
|
||||
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "minioadmin")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "minioadminpassword")
|
||||
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
||||
|
||||
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
|
||||
|
||||
s3_client = boto3.client(
|
||||
"s3",
|
||||
@@ -50,8 +52,7 @@ s3_client = boto3.client(
|
||||
)
|
||||
|
||||
def get_db_connection():
|
||||
conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
return conn
|
||||
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
|
||||
def generate_request_id() -> str:
|
||||
return uuid.uuid4().hex
|
||||
@@ -91,14 +92,18 @@ def decode_cursor(cursor: str, expected_chat_id: str) -> tuple:
|
||||
if len(parts) != 4:
|
||||
raise ValueError()
|
||||
chat_id, created_at, msg_id, signature = parts
|
||||
|
||||
|
||||
if chat_id != expected_chat_id:
|
||||
raise HTTPException(status_code=400, detail="Cursor belongs to another chat")
|
||||
|
||||
expected_sig = hmac.new(SECRET_HMAC_KEY.encode(), f"{chat_id}|{created_at}|{msg_id}".encode(), hashlib.sha256).hexdigest()[:16]
|
||||
|
||||
expected_sig = hmac.new(
|
||||
SECRET_HMAC_KEY.encode(),
|
||||
f"{chat_id}|{created_at}|{msg_id}".encode(),
|
||||
hashlib.sha256
|
||||
).hexdigest()[:16]
|
||||
if not hmac.compare_digest(signature, expected_sig):
|
||||
raise ValueError()
|
||||
|
||||
|
||||
return created_at, msg_id
|
||||
except HTTPException:
|
||||
raise
|
||||
@@ -175,7 +180,7 @@ class CreateUser(BaseModel):
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
title: str = Field(..., min_length=1, max_length=100)
|
||||
member_ids: List[uuid.UUID] = Field(..., min_items=1, max_items=100)
|
||||
member_ids: List[uuid.UUID] = Field(..., min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
def not_blank(cls, v):
|
||||
@@ -186,7 +191,7 @@ class CreateChat(BaseModel):
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
text: str = Field(default="", max_length=4000)
|
||||
attachment_ids: List[uuid.UUID] = Field(default=[], max_items=10)
|
||||
attachment_ids: List[uuid.UUID] = Field(default=[], max_length=10)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_content(self) -> 'CreateMessage':
|
||||
@@ -220,7 +225,7 @@ def register_user(body: CreateUser, response: Response):
|
||||
req_id = generate_request_id()
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
|
||||
|
||||
password_hash = bcrypt.hashpw(body.password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')
|
||||
user_id = str(uuid.uuid4())
|
||||
created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
@@ -235,7 +240,10 @@ def register_user(body: CreateUser, response: Response):
|
||||
conn.commit()
|
||||
except psycopg2.errors.UniqueViolation:
|
||||
conn.rollback()
|
||||
raise HTTPException(status_code=409, detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}})
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}}
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -254,18 +262,18 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
|
||||
|
||||
if not authorization or not authorization.startswith("Basic "):
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
status_code=401,
|
||||
detail="Missing or invalid basic auth",
|
||||
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
encoded = authorization.split(" ")[1]
|
||||
decoded = base64.b64decode(encoded).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
except Exception:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
status_code=401,
|
||||
detail="Invalid credentials format",
|
||||
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
)
|
||||
@@ -274,11 +282,11 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM users WHERE username = %s", (username,))
|
||||
user = cur.fetchone()
|
||||
|
||||
conn.close()
|
||||
|
||||
if not user or not bcrypt.checkpw(password.encode('utf-8'), user['password_hash'].encode('utf-8')):
|
||||
conn.close()
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
status_code=401,
|
||||
detail="Invalid credentials",
|
||||
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
)
|
||||
@@ -293,7 +301,7 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
|
||||
"token_type": "Bearer",
|
||||
"expires_at": expires_at_str,
|
||||
"user": {
|
||||
"id": user['id'],
|
||||
"id": str(user['id']),
|
||||
"username": user['username'],
|
||||
"display_name": user['display_name'],
|
||||
"created_at": user['created_at'].isoformat().replace("+00:00", "Z") if isinstance(user['created_at'], datetime) else user['created_at']
|
||||
@@ -303,7 +311,7 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
|
||||
def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> dict:
|
||||
if not authorization or not authorization.startswith("Bearer "):
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
status_code=401,
|
||||
detail="Missing or invalid bearer token",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
)
|
||||
@@ -316,7 +324,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
|
||||
|
||||
if not user_id:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
status_code=401,
|
||||
detail="Invalid or expired session token",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
)
|
||||
@@ -329,7 +337,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
|
||||
|
||||
if not row:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
status_code=401,
|
||||
detail="User not found",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
)
|
||||
@@ -344,10 +352,10 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def revoke_session(authorization: Optional[str] = Header(None)):
|
||||
req_id = generate_request_id()
|
||||
|
||||
|
||||
if not authorization or not authorization.startswith("Bearer "):
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
status_code=401,
|
||||
detail="Unauthorized",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
)
|
||||
@@ -357,7 +365,7 @@ def revoke_session(authorization: Optional[str] = Header(None)):
|
||||
|
||||
if not deleted:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
status_code=401,
|
||||
detail="Invalid session token",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
)
|
||||
@@ -365,7 +373,7 @@ def revoke_session(authorization: Optional[str] = Header(None)):
|
||||
return Response(
|
||||
status_code=204,
|
||||
headers={
|
||||
"X-Request-Id": req_id,
|
||||
"X-Request-Id": req_id,
|
||||
"X-Instance-Id": INSTANCE_ID
|
||||
},
|
||||
)
|
||||
@@ -383,7 +391,7 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
user = get_current_user_from_token(authorization)
|
||||
|
||||
|
||||
member_ids_str = [str(mid) for mid in body.member_ids]
|
||||
|
||||
if user["id"] not in member_ids_str:
|
||||
@@ -433,7 +441,7 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
user = get_current_user_from_token(authorization)
|
||||
|
||||
|
||||
if limit < 1 or limit > 100:
|
||||
raise HTTPException(status_code=400, detail="Invalid limit")
|
||||
|
||||
@@ -446,21 +454,18 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response
|
||||
ORDER BY c.created_at ASC, c.id ASC
|
||||
""", (user["id"],))
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
|
||||
user_chats = []
|
||||
for r in rows:
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
user_chats = []
|
||||
for r in rows:
|
||||
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],))
|
||||
m_rows = cur.fetchall()
|
||||
conn.close()
|
||||
user_chats.append({
|
||||
"id": r['id'],
|
||||
"title": r['title'],
|
||||
"member_ids": [m['user_id'] for m in m_rows],
|
||||
"created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at']
|
||||
})
|
||||
user_chats.append({
|
||||
"id": str(r['id']),
|
||||
"title": r['title'],
|
||||
"member_ids": [str(m['user_id']) for m in m_rows],
|
||||
"created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at']
|
||||
})
|
||||
conn.close()
|
||||
|
||||
start_idx = 0
|
||||
if cursor:
|
||||
@@ -483,7 +488,7 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response
|
||||
if start_idx + limit < len(user_chats):
|
||||
last = page_items[-1]
|
||||
next_cursor = f"{last['created_at']}_{last['id']}"
|
||||
|
||||
|
||||
return {"items": page_items, "next_cursor": next_cursor}
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
@@ -492,7 +497,7 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
user = get_current_user_from_token(authorization)
|
||||
|
||||
|
||||
chat_id_str = str(chat_id)
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
@@ -511,16 +516,16 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H
|
||||
raise HTTPException(status_code=404, detail="Chat not found")
|
||||
|
||||
return {
|
||||
"id": chat['id'],
|
||||
"id": str(chat['id']),
|
||||
"title": chat['title'],
|
||||
"member_ids": [m['user_id'] for m in members],
|
||||
"member_ids": [str(m['user_id']) for m in members],
|
||||
"created_at": chat['created_at'].isoformat().replace("+00:00", "Z") if isinstance(chat['created_at'], datetime) else chat['created_at']
|
||||
}
|
||||
|
||||
def validate_image_and_get_dimensions(data: bytes, content_type: str):
|
||||
if len(data) > 10 * 1024 * 1024:
|
||||
raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)")
|
||||
|
||||
|
||||
width, height = 0, 0
|
||||
if content_type == "image/png":
|
||||
if not data.startswith(b'\x89PNG\r\n\x1a\n'):
|
||||
@@ -544,7 +549,13 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str):
|
||||
if 0xc0 <= marker_type <= 0xc3:
|
||||
height, width = struct.unpack('>HH', data[i+5:i+9])
|
||||
break
|
||||
i += 2 + struct.unpack('>H', data[i+2:i+4])[0]
|
||||
if marker_type in {0xd8, 0xd9} or (0xd0 <= marker_type <= 0xd7):
|
||||
i += 2
|
||||
continue
|
||||
seg_len = struct.unpack('>H', data[i+2:i+4])[0]
|
||||
if seg_len < 2:
|
||||
raise HTTPException(status_code=415, detail="Invalid JPEG segment")
|
||||
i += 2 + seg_len
|
||||
else:
|
||||
i += 1
|
||||
else:
|
||||
@@ -556,7 +567,7 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str):
|
||||
if width * height > 40_000_000:
|
||||
raise HTTPException(status_code=413, detail="Image exceeds 40 million pixels limit")
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(
|
||||
chat_id: UUID,
|
||||
filename: str = Query(..., min_length=1, max_length=255),
|
||||
@@ -579,7 +590,7 @@ async def upload_attachment(
|
||||
conn.close()
|
||||
|
||||
if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename) or '/' in filename or '\\' in filename:
|
||||
raise HTTPException(400, "Invalid filename")
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
|
||||
content_type = request.headers.get("content-type")
|
||||
if content_type not in ["image/png", "image/jpeg"]:
|
||||
@@ -608,30 +619,14 @@ async def upload_attachment(
|
||||
size_bytes = len(body_bytes)
|
||||
sha256_hash = hashlib.sha256(body_bytes).hexdigest()
|
||||
attachment_id = str(uuid.uuid4())
|
||||
job_id = str(uuid.uuid4())
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
s3_key = f"attachments/{chat_id_str}/{attachment_id}/{filename}"
|
||||
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, variants, error_code, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, 'pending', '[]', NULL, %s, %s)
|
||||
""", (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now))
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
conn.close()
|
||||
raise HTTPException(status_code=500, detail="Internal error initializing attachment")
|
||||
finally:
|
||||
if not conn.closed:
|
||||
conn.close()
|
||||
|
||||
staging_key = f"staging/{chat_id_str}/{attachment_id}/original"
|
||||
try:
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=s3_key,
|
||||
Key=staging_key,
|
||||
Body=body_bytes,
|
||||
ContentType=content_type
|
||||
)
|
||||
@@ -642,16 +637,36 @@ async def upload_attachment(
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
UPDATE attachments SET status = 'ready', updated_at = %s WHERE id = %s
|
||||
""", (now, attachment_id))
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type,
|
||||
size_bytes, sha256, status, variants, error_code,
|
||||
pipeline_version, attempt, created_at, updated_at
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', '[]', NULL, 1, 0, %s, %s)
|
||||
""", (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now))
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise HTTPException(status_code=500, detail="Internal error finalizing attachment status")
|
||||
conn.close()
|
||||
raise HTTPException(status_code=500, detail="Internal error saving attachment metadata")
|
||||
finally:
|
||||
if not conn.closed:
|
||||
conn.close()
|
||||
|
||||
job_payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": job_id,
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": attachment_id,
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": now
|
||||
}
|
||||
try:
|
||||
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(job_payload)})
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="Broker unavailable")
|
||||
|
||||
return {
|
||||
"id": attachment_id,
|
||||
"chat_id": chat_id_str,
|
||||
@@ -660,7 +675,7 @@ async def upload_attachment(
|
||||
"content_type": content_type,
|
||||
"size_bytes": size_bytes,
|
||||
"sha256": sha256_hash,
|
||||
"status": "ready",
|
||||
"status": "queued",
|
||||
"variants": [],
|
||||
"error_code": None,
|
||||
"created_at": now,
|
||||
@@ -704,7 +719,12 @@ def get_attachment(attachment_id: UUID, response: Response, authorization: Optio
|
||||
}
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download_attachment(attachment_id: UUID, response: Response, authorization: Optional[str] = Header(None)):
|
||||
def download_attachment(
|
||||
attachment_id: UUID,
|
||||
variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"),
|
||||
response: Response = None,
|
||||
authorization: Optional[str] = Header(None)
|
||||
):
|
||||
req_id = generate_request_id()
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
@@ -725,9 +745,19 @@ def download_attachment(attachment_id: UUID, response: Response, authorization:
|
||||
raise HTTPException(status_code=404, detail="Attachment not found or inaccessible")
|
||||
|
||||
if row['status'] != 'ready':
|
||||
raise HTTPException(status_code=409, detail="Attachment not ready")
|
||||
raise HTTPException(status_code=409, detail=f"Attachment is in status '{row['status']}'")
|
||||
|
||||
if variant == "original":
|
||||
s3_key = f"attachments/{row['chat_id']}/{row['id']}/original"
|
||||
media_type = row['content_type']
|
||||
else:
|
||||
variants_list = row['variants'] or []
|
||||
matched = [v for v in variants_list if v.get("name") == variant]
|
||||
if not matched:
|
||||
raise HTTPException(status_code=404, detail=f"Variant '{variant}' not found")
|
||||
s3_key = f"attachments/{row['chat_id']}/{row['id']}/{variant}"
|
||||
media_type = matched[0]["content_type"]
|
||||
|
||||
s3_key = f"attachments/{row['chat_id']}/{row['id']}/{row['filename']}"
|
||||
try:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key)
|
||||
data = obj['Body'].read()
|
||||
@@ -736,7 +766,7 @@ def download_attachment(attachment_id: UUID, response: Response, authorization:
|
||||
|
||||
return Response(
|
||||
content=data,
|
||||
media_type=row['content_type'],
|
||||
media_type=media_type,
|
||||
headers={
|
||||
"X-Request-Id": req_id,
|
||||
"X-Instance-Id": INSTANCE_ID,
|
||||
@@ -745,9 +775,9 @@ def download_attachment(attachment_id: UUID, response: Response, authorization:
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def create_message(
|
||||
chat_id: UUID,
|
||||
body: CreateMessage,
|
||||
response: Response,
|
||||
chat_id: UUID,
|
||||
body: CreateMessage,
|
||||
response: Response,
|
||||
authorization: Optional[str] = Header(None),
|
||||
idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key")
|
||||
):
|
||||
@@ -755,14 +785,14 @@ def create_message(
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
user = get_current_user_from_token(authorization)
|
||||
|
||||
|
||||
if not idempotency_key or not re.match(r"^[A-Za-z0-9._:-]{1,128}$", idempotency_key):
|
||||
raise HTTPException(status_code=400, detail="Missing or invalid Idempotency-Key")
|
||||
|
||||
chat_id_str = str(chat_id)
|
||||
method = "POST"
|
||||
path = f"/api/v1/chats/{chat_id_str}/messages"
|
||||
|
||||
|
||||
body_json_str = json.dumps(body.model_dump(mode='json'), sort_keys=True)
|
||||
request_hash = hashlib.sha256(body_json_str.encode("utf-8")).hexdigest()
|
||||
|
||||
@@ -794,7 +824,7 @@ def create_message(
|
||||
|
||||
msg_id = str(uuid.uuid4())
|
||||
created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
msg_response = {
|
||||
"id": msg_id,
|
||||
"chat_id": chat_id_str,
|
||||
@@ -809,7 +839,7 @@ def create_message(
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (user_id, method, path, key) DO NOTHING
|
||||
""", (user["id"], method, path, idempotency_key, request_hash, 201, json.dumps(msg_response)))
|
||||
|
||||
|
||||
if cur.rowcount == 0:
|
||||
cur.execute("""
|
||||
SELECT request_hash, response_status, response_body
|
||||
@@ -822,7 +852,7 @@ def create_message(
|
||||
|
||||
if row["request_hash"] != request_hash:
|
||||
raise HTTPException(status_code=409, detail="Idempotency key conflict with different payload")
|
||||
|
||||
|
||||
return row["response_body"]
|
||||
|
||||
cur.execute(
|
||||
@@ -832,7 +862,7 @@ def create_message(
|
||||
conn.commit()
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
conn.close()
|
||||
raise HTTPException(status_code=500, detail="Internal error processing message")
|
||||
@@ -890,16 +920,12 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
|
||||
formatted_msgs = []
|
||||
for m in page_items:
|
||||
raw_atts = m.get('attachment_ids')
|
||||
atts = []
|
||||
if raw_atts:
|
||||
for aid in raw_atts:
|
||||
if aid:
|
||||
atts.append(str(aid))
|
||||
|
||||
atts = [str(aid) for aid in raw_atts if aid] if raw_atts else []
|
||||
|
||||
formatted_msgs.append({
|
||||
"id": m['id'],
|
||||
"chat_id": m['chat_id'],
|
||||
"sender_id": m['sender_id'],
|
||||
"id": str(m['id']),
|
||||
"chat_id": str(m['chat_id']),
|
||||
"sender_id": str(m['sender_id']),
|
||||
"text": m['text'],
|
||||
"created_at": m['created_at'].isoformat().replace("+00:00", "Z") if isinstance(m['created_at'], datetime) else m['created_at'],
|
||||
"attachment_ids": atts
|
||||
|
||||
+76
-39
@@ -23,7 +23,7 @@ services:
|
||||
|
||||
redis:
|
||||
image: redis:7.2.4-alpine
|
||||
command: redis-server --save 60 1 --loglevel warning
|
||||
command: redis-server --appendonly yes --appendfsync everysec --loglevel warning
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
deploy:
|
||||
@@ -51,6 +51,44 @@ services:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:latest
|
||||
command: ["server", "-s3", "-dir=/data", "-s3.port=8333"]
|
||||
environment:
|
||||
- AWS_ACCESS_KEY_ID=appuser
|
||||
- AWS_SECRET_ACCESS_KEY=appsecretpassword
|
||||
volumes:
|
||||
- s3data:/data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.50'
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
|
||||
s3-init:
|
||||
image: python:3.11.7-slim-bookworm
|
||||
volumes:
|
||||
- ./scripts:/scripts
|
||||
- ./requirements.txt:/requirements.txt
|
||||
environment:
|
||||
- S3_ENDPOINT=http://s3:8333
|
||||
- S3_BUCKET=messenger-attachments
|
||||
- S3_ROOT_ACCESS_KEY=appuser
|
||||
- S3_ROOT_SECRET_KEY=appsecretpassword
|
||||
command: >
|
||||
sh -c "pip install --no-cache-dir -r /requirements.txt &&
|
||||
python /scripts/bootstrap_s3.py"
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api-1:
|
||||
build: .
|
||||
stop_grace_period: 15s
|
||||
@@ -64,6 +102,7 @@ services:
|
||||
- S3_BUCKET=messenger-attachments
|
||||
- S3_ACCESS_KEY_ID=appuser
|
||||
- S3_SECRET_ACCESS_KEY=appsecretpassword
|
||||
- BROKER_STREAM=attachment_jobs
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
@@ -97,6 +136,7 @@ services:
|
||||
- S3_BUCKET=messenger-attachments
|
||||
- S3_ACCESS_KEY_ID=appuser
|
||||
- S3_SECRET_ACCESS_KEY=appsecretpassword
|
||||
- BROKER_STREAM=attachment_jobs
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
@@ -117,6 +157,41 @@ services:
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
|
||||
worker:
|
||||
build: .
|
||||
command: ["python", "worker.py"]
|
||||
stop_grace_period: 15s
|
||||
environment:
|
||||
- DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger
|
||||
- REDIS_URL=redis://redis:6379/0
|
||||
- S3_ENDPOINT=http://s3:8333
|
||||
- S3_BUCKET=messenger-attachments
|
||||
- S3_ACCESS_KEY_ID=appuser
|
||||
- S3_SECRET_ACCESS_KEY=appsecretpassword
|
||||
- BROKER_STREAM=attachment_jobs
|
||||
- BROKER_GROUP=attachment_workers
|
||||
- JOB_MAX_ATTEMPTS=5
|
||||
- MAX_IMAGE_PIXELS=40000000
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.50'
|
||||
memory: 256M
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os, redis; redis.Redis.from_url(os.environ['REDIS_URL']).ping()"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
|
||||
nginx:
|
||||
image: nginx:1.25.4-alpine
|
||||
ports:
|
||||
@@ -137,44 +212,6 @@ services:
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:latest
|
||||
command: "mini -dir=/data"
|
||||
environment:
|
||||
- AWS_ACCESS_KEY_ID=appuser
|
||||
- AWS_SECRET_ACCESS_KEY=appsecretpassword
|
||||
- S3_BUCKET=messenger-attachments
|
||||
volumes:
|
||||
- s3data:/data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.50'
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD", "nc", "-z", "localhost", "8333"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
|
||||
s3-init:
|
||||
image: python:3.11-slim
|
||||
volumes:
|
||||
- ./scripts:/scripts
|
||||
- ./requirements.txt:/requirements.txt
|
||||
environment:
|
||||
- S3_ENDPOINT=http://s3:8333
|
||||
- S3_BUCKET=messenger-attachments
|
||||
- S3_ROOT_ACCESS_KEY=appuser
|
||||
- S3_ROOT_SECRET_KEY=appsecretpassword
|
||||
command: >
|
||||
sh -c "pip install --no-cache-dir -r /requirements.txt &&
|
||||
python /scripts/bootstrap_s3.py"
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
PASS: anonymous GET -> 403
|
||||
@@ -1,7 +0,0 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab4-api-1-1 mtusi2026lab4-api-1 "uvicorn app:app --h…" api-1 15 minutes ago Up 12 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab4-api-2-1 mtusi2026lab4-api-2 "uvicorn app:app --h…" api-2 15 minutes ago Up 23 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab4-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.…" nginx 15 minutes ago Up 12 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp
|
||||
mtusi2026lab4-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.s…" postgres 15 minutes ago Up 12 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab4-redis-1 redis:7.2.4-alpine "docker-entrypoint.s…" redis 15 minutes ago Up 12 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab4-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh mini…" s3 15 minutes ago Up 12 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
|
||||
"username": "demo_75866",
|
||||
"display_name": "Demo User",
|
||||
"created_at": "2026-11-16T13:43:38.059472Z"
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"token": "grSsHp1gJpFLckF14UT44obEonSGQRvP878JhLyCaNzM6PGBH1LwrdgZIFF8Zu8U",
|
||||
"token_type": "Bearer",
|
||||
"expires_at": "2026-11-16T14:43:38.850596Z",
|
||||
"user": {
|
||||
"id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
|
||||
"username": "demo_75866",
|
||||
"display_name": "Demo User",
|
||||
"created_at": "2026-11-16T13:43:38.059472Z"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"id": "5db90ff5-9604-4853-b002-db4144868bcb",
|
||||
"title": "Demo chat",
|
||||
"member_ids": [
|
||||
"fdeb71d4-ee4f-47a6-8968-ffc7740e5242"
|
||||
],
|
||||
"created_at": "2026-11-16T13:43:39.240968Z"
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab5-api-1-1 mtusi2026lab5-api-1 "uvicorn app:app --hтАж" api-1 30 minutes ago Up 6 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab5-api-2-1 mtusi2026lab5-api-2 "uvicorn app:app --hтАж" api-2 30 minutes ago Up 6 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab5-migrate-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" migrate 30 minutes ago Exited (0) 5 minutes ago
|
||||
mtusi2026lab5-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.тАж" nginx 30 minutes ago Up 25 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp
|
||||
mtusi2026lab5-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" postgres 30 minutes ago Up 30 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab5-redis-1 redis:7.2.4-alpine "docker-entrypoint.sтАж" redis 30 minutes ago Up 4 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab5-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh servтАж" s3 30 minutes ago Up 30 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
mtusi2026lab5-s3-init-1 python:3.11.7-slim-bookworm "sh -c 'pip install тАж" s3-init 30 minutes ago Exited (0) 5 minutes ago
|
||||
mtusi2026lab5-worker-1 mtusi2026lab5-worker "python worker.py" worker 30 minutes ago Exited (0) Less than a second ago
|
||||
@@ -0,0 +1 @@
|
||||
{"id":"9becbd32-e413-417a-8762-79bcc195e296","chat_id":"5db90ff5-9604-4853-b002-db4144868bcb","owner_id":"fdeb71d4-ee4f-47a6-8968-ffc7740e5242","filename":"sample.png","content_type":"image/png","size_bytes":111817,"sha256":"99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75","status":"queued","variants":[],"error_code":null,"created_at":"2026-11-16T13:43:41.734766Z","updated_at":"2026-11-16T13:43:41.734766Z"}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"id": "74dc2e91-a901-4067-8993-da9d2cc4ca55",
|
||||
"chat_id": "5db90ff5-9604-4853-b002-db4144868bcb",
|
||||
"sender_id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
|
||||
"text": "",
|
||||
"created_at": "2026-11-16T13:43:42.100558Z",
|
||||
"attachment_ids": [
|
||||
"9becbd32-e413-417a-8762-79bcc195e296"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab5-api-1-1 mtusi2026lab5-api-1 "uvicorn app:app --hтАж" api-1 30 minutes ago Up Less than a second (health: starting) 8080/tcp
|
||||
mtusi2026lab5-api-2-1 mtusi2026lab5-api-2 "uvicorn app:app --hтАж" api-2 30 minutes ago Up Less than a second (health: starting) 8080/tcp
|
||||
mtusi2026lab5-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.тАж" nginx 30 minutes ago Up 25 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp
|
||||
mtusi2026lab5-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" postgres 30 minutes ago Up 30 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab5-redis-1 redis:7.2.4-alpine "docker-entrypoint.sтАж" redis 30 minutes ago Up 4 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab5-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh servтАж" s3 30 minutes ago Up 30 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
@@ -0,0 +1,10 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab5-api-1-1 mtusi2026lab5-api-1 "uvicorn app:app --hтАж" api-1 30 minutes ago Up 10 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab5-api-2-1 mtusi2026lab5-api-2 "uvicorn app:app --hтАж" api-2 30 minutes ago Up 9 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab5-migrate-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" migrate 30 minutes ago Exited (0) 7 seconds ago
|
||||
mtusi2026lab5-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.тАж" nginx 30 minutes ago Up 26 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp
|
||||
mtusi2026lab5-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.sтАж" postgres 30 minutes ago Up 30 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab5-redis-1 redis:7.2.4-alpine "docker-entrypoint.sтАж" redis 30 minutes ago Up 4 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab5-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh servтАж" s3 30 minutes ago Up 30 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
mtusi2026lab5-s3-init-1 python:3.11.7-slim-bookworm "sh -c 'pip install тАж" s3-init 30 minutes ago Exited (0) Less than a second ago
|
||||
mtusi2026lab5-worker-1 mtusi2026lab5-worker "python worker.py" worker 30 minutes ago Up Less than a second (health: starting) 8080/tcp
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "9becbd32-e413-417a-8762-79bcc195e296",
|
||||
"chat_id": "5db90ff5-9604-4853-b002-db4144868bcb",
|
||||
"owner_id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
|
||||
"filename": "sample.png",
|
||||
"content_type": "image/png",
|
||||
"size_bytes": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"status": "ready",
|
||||
"variants": [
|
||||
{
|
||||
"name": "thumbnail",
|
||||
"width": 256,
|
||||
"height": 160,
|
||||
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
|
||||
"size_bytes": 4679,
|
||||
"content_type": "image/jpeg"
|
||||
}
|
||||
],
|
||||
"error_code": null,
|
||||
"created_at": "2026-11-16T13:43:41.734766Z",
|
||||
"updated_at": "2026-11-16T13:43:55.375481Z"
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 4.6 KiB |
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "9becbd32-e413-417a-8762-79bcc195e296",
|
||||
"chat_id": "5db90ff5-9604-4853-b002-db4144868bcb",
|
||||
"owner_id": "fdeb71d4-ee4f-47a6-8968-ffc7740e5242",
|
||||
"filename": "sample.png",
|
||||
"content_type": "image/png",
|
||||
"size_bytes": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"status": "ready",
|
||||
"variants": [
|
||||
{
|
||||
"name": "thumbnail",
|
||||
"width": 256,
|
||||
"height": 160,
|
||||
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
|
||||
"size_bytes": 4679,
|
||||
"content_type": "image/jpeg"
|
||||
}
|
||||
],
|
||||
"error_code": null,
|
||||
"created_at": "2026-11-16T13:43:41.734766Z",
|
||||
"updated_at": "2026-11-16T13:43:55.375481Z"
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 82, "failed_attempts": 2, "first_recovery_seconds": 1.312}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -1,17 +0,0 @@
|
||||
test_corrupted_image_signature (__main__.TestMessengerLevel4Complete.test_corrupted_image_signature)
|
||||
Ïðîâåðêà 415: áèòàÿ ñòðóêòóðà / íåâåðíûå ðàçìåðû PNG. ... ok
|
||||
test_cross_chat_attachment_isolation (__main__.TestMessengerLevel4Complete.test_cross_chat_attachment_isolation)
|
||||
Ïðîâåðêà 404: ïîïûòêà ïðèêðåïèòü âëîæåíèå èç ÷óæîãî ÷àòà. ... ok
|
||||
test_invalid_filename_characters (__main__.TestMessengerLevel4Complete.test_invalid_filename_characters)
|
||||
Ïðîâåðêà 400: íåäîïóñòèìûå ñèìâîëû (path traversal) â èìåíè ôàéëà. ... ok
|
||||
test_nonexistent_attachment_id (__main__.TestMessengerLevel4Complete.test_nonexistent_attachment_id)
|
||||
Ïðîâåðêà 404: ïåðåäà÷à íåñóùåñòâóþùåãî ID âëîæåíèÿ ïðè îòïðàâêå ñîîáùåíèÿ. ... ok
|
||||
test_payload_too_large (__main__.TestMessengerLevel4Complete.test_payload_too_large)
|
||||
Ïðîâåðêà 413: ôàéë ðàçìåðîì áîëüøå 10 MiB. ... ok
|
||||
test_unsupported_media_type (__main__.TestMessengerLevel4Complete.test_unsupported_media_type)
|
||||
Ïðîâåðêà 415: ïåðåäà÷à òåêñòîâîãî ôàéëà ïîä âèäîì êàðòèíêè. ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 6 tests in 19.521s
|
||||
|
||||
OK
|
||||
@@ -1,3 +0,0 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -1,4 +0,0 @@
|
||||
Container mtusi2026lab4-api-1-run-9e5b0d92d11b Creating
|
||||
Container mtusi2026lab4-api-1-run-9e5b0d92d11b Created
|
||||
Starting reconciliation (dry_run=True)...
|
||||
Reconciliation completed successfully.
|
||||
@@ -1,2 +0,0 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -1,7 +0,0 @@
|
||||
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
|
||||
de2d9b4cdf10 mtusi2026lab4-nginx-1 2.71% 2.355MiB / 128MiB 1.84% 389kB / 408kB 0B / 12.3kB 2
|
||||
bd85245e25be mtusi2026lab4-api-1-1 14.75% 74.22MiB / 256MiB 28.99% 518kB / 647kB 0B / 53.2kB 2
|
||||
dd34824ed0c5 mtusi2026lab4-api-2-1 13.03% 74.76MiB / 256MiB 29.20% 750kB / 879kB 0B / 53.2kB 2
|
||||
650e2ec7ac6a mtusi2026lab4-s3-1 1.02% 68.62MiB / 512MiB 13.40% 559kB / 248kB 0B / 1.09MB 8
|
||||
1938313cd5da mtusi2026lab4-redis-1 0.24% 3.16MiB / 256MiB 1.23% 99.6kB / 46.7kB 24.6kB / 12.3kB 5
|
||||
9348c37fb0f0 mtusi2026lab4-postgres-1 0.01% 34.39MiB / 512MiB 6.72% 679kB / 765kB 0B / 55.1MB 6
|
||||
@@ -1,40 +0,0 @@
|
||||
mtusi2026lab4-api-1-1 63.46MiB / 256MiB 0.20%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.46MiB / 256MiB 0.20%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.49MiB / 256MiB 0.39%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.49MiB / 256MiB 0.24%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 72.76MiB / 256MiB 7.76%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.53MiB / 256MiB 0.22%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.53MiB / 256MiB 0.20%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.61MiB / 256MiB 22.90%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.56MiB / 256MiB 0.20%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.57MiB / 256MiB 50.53%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.65MiB / 256MiB 14.81%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.68MiB / 256MiB 0.22%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.68MiB / 256MiB 0.30%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.71MiB / 256MiB 0.21%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.71MiB / 256MiB 0.20%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 73.85MiB / 256MiB 14.64%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.75MiB / 256MiB 0.22%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.75MiB / 256MiB 0.24%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.8MiB / 256MiB 27.08%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
mtusi2026lab4-api-1-1 63.78MiB / 256MiB 0.24%
|
||||
mtusi2026lab4-api-2-1 0B / 0B 0.00%
|
||||
+6
-2
@@ -49,13 +49,17 @@ CREATE TABLE IF NOT EXISTS attachments (
|
||||
content_type VARCHAR(50) NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 VARCHAR(64) NOT NULL,
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'ready',
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'queued',
|
||||
variants JSONB NOT NULL DEFAULT '[]',
|
||||
error_code VARCHAR(64),
|
||||
pipeline_version INT NOT NULL DEFAULT 1,
|
||||
attempt INT NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id);
|
||||
|
||||
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}';
|
||||
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}';
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1;
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0;
|
||||
+2
-1
@@ -11,4 +11,5 @@ annotated-types==0.6.0
|
||||
psycopg2-binary==2.9.9
|
||||
bcrypt==4.1.2
|
||||
redis==5.0.1
|
||||
boto3==1.34.0
|
||||
boto3==1.34.0
|
||||
Pillow==10.2.0
|
||||
@@ -8,8 +8,8 @@ import re
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
def main():
|
||||
lab = json.loads((ROOT/'lab.json').read_text())['number']
|
||||
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
|
||||
lab = json.loads((ROOT/'lab.json').read_text(encoding='utf-8'))['number']
|
||||
spec = json.loads((ROOT/'contracts/openapi.json').read_text(encoding='utf-8'))
|
||||
assert spec['openapi']=='3.1.0'
|
||||
assert spec['info']['version']=='1.%d.0'%lab
|
||||
operation_ids = set()
|
||||
@@ -41,11 +41,11 @@ def main():
|
||||
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
|
||||
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
|
||||
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
|
||||
ast.parse(file.read_text(),filename=str(file))
|
||||
ast.parse(file.read_text(encoding='utf-8'),filename=str(file))
|
||||
for file in ROOT.rglob('*.json'):
|
||||
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
|
||||
continue
|
||||
json.loads(file.read_text())
|
||||
json.loads(file.read_text(encoding='utf-8'))
|
||||
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
|
||||
assert (ROOT/required).is_file(), 'missing '+required
|
||||
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LAB = json.loads((ROOT / 'lab.json').read_text())['number']
|
||||
SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text())
|
||||
SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8'))
|
||||
|
||||
class Failure(AssertionError):
|
||||
pass
|
||||
|
||||
+1
-1
@@ -55,7 +55,7 @@ def main():
|
||||
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
|
||||
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
|
||||
# No shell interpolation and no Docker commands in this test.
|
||||
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
proc = subprocess.Popen(['sh', str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
started = time.monotonic()
|
||||
successes,failures,after_instances = [],0,[]
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
import os
|
||||
import io
|
||||
import json
|
||||
import time
|
||||
import signal
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
import redis
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s"
|
||||
)
|
||||
logger = logging.getLogger("attachment-worker")
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
||||
|
||||
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
|
||||
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
|
||||
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
|
||||
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
||||
|
||||
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
|
||||
s3_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_SECRET_KEY,
|
||||
config=Config(signature_version="s3v4"),
|
||||
region_name="us-east-1"
|
||||
)
|
||||
|
||||
running = True
|
||||
|
||||
def handle_sigterm(signum, frame):
|
||||
global running
|
||||
logger.info("Received shutdown signal, finishing current job...")
|
||||
running = False
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_sigterm)
|
||||
signal.signal(signal.SIGINT, handle_sigterm)
|
||||
|
||||
|
||||
def get_db_connection():
|
||||
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
|
||||
|
||||
def ensure_consumer_group():
|
||||
while running:
|
||||
try:
|
||||
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
|
||||
logger.info("Created consumer group '%s' on stream '%s'", GROUP_NAME, STREAM_NAME)
|
||||
return
|
||||
except redis.exceptions.ResponseError as e:
|
||||
if "BUSYGROUP" in str(e):
|
||||
logger.info("Consumer group '%s' already exists", GROUP_NAME)
|
||||
return
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.warning("Waiting for Redis broker: %s", type(e).__name__)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def build_thumbnail_jpeg(raw_bytes: bytes) -> tuple[bytes, int, int, str]:
|
||||
with Image.open(io.BytesIO(raw_bytes)) as img:
|
||||
img = ImageOps.exif_transpose(img)
|
||||
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.split()[3])
|
||||
img = background
|
||||
elif img.mode != "RGB":
|
||||
img = img.convert("RGB")
|
||||
|
||||
orig_w, orig_h = img.size
|
||||
|
||||
if orig_w <= 256 and orig_h <= 256:
|
||||
target_w, target_h = orig_w, orig_h
|
||||
else:
|
||||
scale = min(256.0 / orig_w, 256.0 / orig_h)
|
||||
target_w = max(1, int(orig_w * scale))
|
||||
target_h = max(1, int(orig_h * scale))
|
||||
|
||||
if (target_w, target_h) != (orig_w, orig_h):
|
||||
img = img.resize((target_w, target_h), Image.Resampling.LANCZOS)
|
||||
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
thumb_bytes = buf.getvalue()
|
||||
thumb_sha256 = hashlib.sha256(thumb_bytes).hexdigest()
|
||||
return thumb_bytes, target_w, target_h, thumb_sha256
|
||||
|
||||
|
||||
def mark_failed(attachment_id: str, error_code: str):
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = %s, updated_at = %s
|
||||
WHERE id = %s AND status != 'ready'
|
||||
""", (error_code, now, attachment_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def process_job(message_id: str, fields: dict):
|
||||
try:
|
||||
job = json.loads(fields.get("payload", "{}"))
|
||||
except Exception:
|
||||
logger.error("Malformed job JSON in message %s, acknowledging to drop", message_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
attachment_id = job.get("attachment_id")
|
||||
job_id = job.get("job_id")
|
||||
pipeline_version = int(job.get("pipeline_version", 1))
|
||||
attempt = int(job.get("attempt", 1))
|
||||
|
||||
if not attachment_id:
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,))
|
||||
att = cur.fetchone()
|
||||
|
||||
if not att:
|
||||
conn.commit()
|
||||
logger.warning("Attachment %s not found in DB, acking job %s", attachment_id, job_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
|
||||
conn.commit()
|
||||
logger.info("Attachment %s already ready (or newer pipeline), no-op ack for job %s", attachment_id, job_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
if att["status"] == "failed":
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
cur.execute("""
|
||||
UPDATE attachments
|
||||
SET status = 'processing', attempt = %s, updated_at = %s
|
||||
WHERE id = %s
|
||||
""", (attempt, now, attachment_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
chat_id = str(att["chat_id"])
|
||||
staging_key = f"staging/{chat_id}/{attachment_id}/original"
|
||||
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
|
||||
final_thumb_key = f"attachments/{chat_id}/{attachment_id}/thumbnail"
|
||||
|
||||
try:
|
||||
try:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
|
||||
raw_bytes = obj["Body"].read()
|
||||
|
||||
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
|
||||
raise ValueError("Staging checksum mismatch")
|
||||
|
||||
thumb_bytes, thumb_w, thumb_h, thumb_sha256 = build_thumbnail_jpeg(raw_bytes)
|
||||
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=final_original_key,
|
||||
Body=raw_bytes,
|
||||
ContentType=att["content_type"]
|
||||
)
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=final_thumb_key,
|
||||
Body=thumb_bytes,
|
||||
ContentType="image/jpeg"
|
||||
)
|
||||
|
||||
variants = [
|
||||
{
|
||||
"name": "thumbnail",
|
||||
"content_type": "image/jpeg",
|
||||
"size_bytes": len(thumb_bytes),
|
||||
"width": thumb_w,
|
||||
"height": thumb_h,
|
||||
"sha256": thumb_sha256
|
||||
}
|
||||
]
|
||||
|
||||
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
UPDATE attachments
|
||||
SET status = 'ready',
|
||||
variants = %s::jsonb,
|
||||
error_code = NULL,
|
||||
updated_at = %s
|
||||
WHERE id = %s AND pipeline_version <= %s
|
||||
""", (json.dumps(variants), ready_at, attachment_id, pipeline_version))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
logger.info("Job %s completed: attachment %s is ready (thumbnail %dx%d)", job_id, attachment_id, thumb_w, thumb_h)
|
||||
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError) as perm_err:
|
||||
logger.error("Permanent image error for %s: %s", attachment_id, type(perm_err).__name__)
|
||||
mark_failed(attachment_id, "invalid_image")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except Exception as transient_err:
|
||||
logger.warning(
|
||||
"Transient error processing attachment %s (attempt %d/%d): %s",
|
||||
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(transient_err).__name__
|
||||
)
|
||||
if attempt >= JOB_MAX_ATTEMPTS:
|
||||
mark_failed(attachment_id, "retry_exhausted")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
else:
|
||||
next_job = dict(job)
|
||||
next_job["attempt"] = attempt + 1
|
||||
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
|
||||
def main():
|
||||
ensure_consumer_group()
|
||||
logger.info("Worker '%s' started listening on stream '%s'", CONSUMER_NAME, STREAM_NAME)
|
||||
|
||||
while running:
|
||||
try:
|
||||
pending = redis_client.xreadgroup(
|
||||
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: "0"}, count=10
|
||||
)
|
||||
messages = pending[0][1] if pending else []
|
||||
if not messages:
|
||||
break
|
||||
for msg_id, fields in messages:
|
||||
if not running:
|
||||
break
|
||||
process_job(msg_id, fields)
|
||||
except Exception as e:
|
||||
logger.warning("Error reading pending messages: %s", type(e).__name__)
|
||||
time.sleep(1)
|
||||
|
||||
while running:
|
||||
try:
|
||||
resp = redis_client.xreadgroup(
|
||||
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=1, block=2000
|
||||
)
|
||||
if not resp:
|
||||
continue
|
||||
for _, messages in resp:
|
||||
for msg_id, fields in messages:
|
||||
if not running:
|
||||
break
|
||||
process_job(msg_id, fields)
|
||||
except Exception as e:
|
||||
logger.warning("Broker read error, retrying: %s", type(e).__name__)
|
||||
time.sleep(1)
|
||||
|
||||
logger.info("Worker stopped cleanly")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user