Implement lab6 observability with OpenTelemetry and Grafana

This commit is contained in:
ferraronp
2026-11-17 19:14:43 +03:00
parent 470d9d6f1e
commit e636d4cc76
68 changed files with 5863 additions and 47 deletions
+1
View File
@@ -0,0 +1 @@
"""Lab 3 messenger API."""
+4
View File
@@ -0,0 +1,4 @@
import os
import uvicorn
if __name__ == "__main__":
uvicorn.run("app.main:app", host="0.0.0.0", port=int(os.getenv("APP_PORT", "8080")), workers=1, access_log=False, timeout_graceful_shutdown=8)
+193
View File
@@ -0,0 +1,193 @@
"""Messenger HTTP handlers; state lives only in PostgreSQL/Redis."""
import hashlib
import json
import re
import uuid
import psycopg
from argon2 import PasswordHasher
from fastapi import APIRouter, Request
from .auth import current_user, hasher, login, logout, public_user
from .core import APIError, invalid, iso, make_cursor, parse_limit, parse_uuid, read_cursor
from .models import CreateChat, CreateMessage, CreateUser
from .storage import db
router = APIRouter()
KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII)
def actor(req: Request) -> dict:
return getattr(req.state, "actor", None) or current_user(req)
def chat_detail(conn, chat: dict) -> dict:
members = conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position", (chat["id"],)).fetchall()
return {"id": str(chat["id"]), "title": bytes(chat["title"]).decode("utf-8"), "created_at": iso(chat["created_at"]), "member_ids": [str(x["user_id"]) for x in members]}
def message_detail(conn, row: dict) -> dict:
linked = conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (row["id"],)).fetchall()
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": bytes(row["text"]).decode("utf-8"), "created_at": iso(row["created_at"]), "attachment_ids": [str(x["attachment_id"]) for x in linked]}
def require_chat(conn, chat_id: str, actor_id: str) -> dict:
row = conn.execute("SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id WHERE c.id=%s AND cm.user_id=%s", (chat_id, actor_id)).fetchone()
if row is None:
raise APIError(404, "not_found", "Object not found")
return row
@router.get("/health/live")
def live():
return {"status": "ok"}
@router.get("/health/ready")
def ready():
from .storage import ready as storage_ready
if not storage_ready():
raise APIError(503, "unavailable", "Service not ready")
return {"status": "ok"}
@router.post("/api/v1/users", status_code=201)
def register(body: CreateUser):
hashed = hasher.hash(body.password)
ident = str(uuid.uuid4())
try:
with db() as conn:
row = conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (ident, body.username, body.display_name.encode("utf-8"), hashed)).fetchone()
return public_user(row)
except psycopg.errors.UniqueViolation:
raise APIError(409, "conflict", "Username already exists") from None
@router.post("/api/v1/auth/sessions", status_code=201)
def sessions(req: Request):
return login(req)
@router.delete("/api/v1/auth/sessions/current", status_code=204)
def sessions_delete(req: Request):
logout(req)
return None
@router.get("/api/v1/users/me")
def users_me(req: Request):
return actor(req)
@router.post("/api/v1/chats", status_code=201)
def chats_create(req: Request, body: CreateChat):
ident = actor(req)["id"]
if ident not in body.member_ids:
raise invalid("Current user must belong to the chat")
chat_id = str(uuid.uuid4())
try:
with db() as conn:
with conn.transaction():
# Validate all members; FK constraints are the final authority for races.
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s::uuid[])", (body.member_ids,)).fetchall()
if len(found) != len(body.member_ids):
raise invalid("Unknown member")
row = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (chat_id, body.title.encode("utf-8"))).fetchone()
for pos, member in enumerate(body.member_ids):
conn.execute("INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)", (chat_id, member, pos))
result = chat_detail(conn, row)
return result
except psycopg.errors.ForeignKeyViolation:
raise invalid("Unknown member") from None
@router.get("/api/v1/chats")
def chats_list(req: Request, limit: str | None = None, cursor: str | None = None):
who = actor(req)["id"]
n = parse_limit(limit)
mark = read_cursor(cursor, "chats", who)
with db() as conn:
if mark:
existing = conn.execute("SELECT 1 FROM chats c JOIN chat_members cm ON c.id=cm.chat_id WHERE c.id=%s AND c.created_at=%s AND cm.user_id=%s", (mark[1], mark[0], who)).fetchone()
if not existing:
raise invalid("Invalid cursor")
rows = conn.execute("""
SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id
WHERE cm.user_id=%s AND (%s::timestamptz IS NULL OR (c.created_at,c.id)>(%s::timestamptz,%s::uuid))
ORDER BY c.created_at,c.id LIMIT %s
""", (who, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
items = [chat_detail(conn, row) for row in rows[:n]]
more = len(rows) > n
last = rows[n-1] if more else None
return {"items": items, "next_cursor": make_cursor("chats", who, last["created_at"], str(last["id"])) if last else None}
@router.get("/api/v1/chats/{chat_id}")
def chats_get(req: Request, chat_id: str):
who = actor(req)["id"]
ident = parse_uuid(chat_id)
with db() as conn:
row = require_chat(conn, ident, who)
return chat_detail(conn, row)
@router.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def messages_create(req: Request, chat_id: str, body: CreateMessage):
who = actor(req)["id"]
ident = parse_uuid(chat_id)
key = req.headers.get("Idempotency-Key")
if not key or not KEY_RE.fullmatch(key):
raise invalid("Invalid Idempotency-Key")
# Lab3 has only text; canonical JSON avoids unstable formatting/field ordering.
fingerprint = hashlib.sha256(json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
lock_scope = f"{who}:POST:/api/v1/chats/{ident}/messages:{key}"
with db() as conn:
with conn.transaction():
require_chat(conn, ident, who)
# Transaction-scoped advisory lock is GLOBAL across API replicas.
conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (lock_scope,))
previous = conn.execute("""
SELECT idem.body_sha256, m.* FROM message_idempotency idem
JOIN messages m ON m.id=idem.message_id
WHERE idem.actor_id=%s AND idem.chat_id=%s AND idem.method='POST' AND idem.idem_key=%s
""", (who, ident, key)).fetchone()
if previous:
if previous["body_sha256"] != fingerprint:
raise APIError(409, "conflict", "Idempotency key used with another body")
return message_detail(conn, previous)
# 404 for foreign/nonexistent IDs; check BEFORE any message insert.
if body.attachment_ids:
owned = conn.execute("SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])", (ident, body.attachment_ids)).fetchall()
if len(owned) != len(body.attachment_ids):
raise APIError(404, "not_found", "Attachment not found")
if any(item["status"] == "failed" for item in owned):
raise APIError(409, "conflict", "Failed attachment cannot be linked")
message_id = str(uuid.uuid4())
row = conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (message_id, ident, who, body.text.encode("utf-8"))).fetchone()
for pos, attachment_id in enumerate(body.attachment_ids):
conn.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, attachment_id, pos))
conn.execute("INSERT INTO message_idempotency(actor_id,chat_id,method,idem_key,body_sha256,message_id) VALUES (%s,%s,'POST',%s,%s,%s)", (who, ident, key, fingerprint, message_id))
return message_detail(conn, row)
@router.get("/api/v1/chats/{chat_id}/messages")
def messages_list(req: Request, chat_id: str, limit: str | None = None, cursor: str | None = None):
who = actor(req)["id"]
ident = parse_uuid(chat_id)
n = parse_limit(limit)
mark = read_cursor(cursor, "messages", ident)
with db() as conn:
require_chat(conn, ident, who)
if mark:
existing = conn.execute("SELECT 1 FROM messages WHERE id=%s AND chat_id=%s AND created_at=%s", (mark[1], ident, mark[0])).fetchone()
if not existing:
raise invalid("Invalid cursor")
rows = conn.execute("""
SELECT * FROM messages WHERE chat_id=%s
AND (%s::timestamptz IS NULL OR (created_at,id)>(%s::timestamptz,%s::uuid))
ORDER BY created_at,id LIMIT %s
""", (ident, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
with db() as conn:
items = [message_detail(conn, row) for row in rows[:n]]
last = rows[n-1] if len(rows)>n else None
return {"items": items, "next_cursor": make_cursor("messages", ident, last["created_at"], str(last["id"])) if last else None}
+240
View File
@@ -0,0 +1,240 @@
"""Lab5 attachments: synchronous durable admission, asynchronous image pipeline.
202 means verified bytes were staged to private S3 and metadata + a job + an
outbox event were committed in ONE PostgreSQL transaction. No broker availability
is required for admission; the dispatcher replays unsent intents.
"""
import hashlib
import os
import tempfile
import threading
import uuid
from contextlib import closing
import boto3
from boto3.s3.transfer import TransferConfig
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError
from fastapi import APIRouter, Query, Request
from fastapi.responses import StreamingResponse
from PIL import Image, ImageFile, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from .auth import current_user
from .core import APIError, invalid, iso, parse_uuid
from .storage import db
from . import telemetry as ot
router = APIRouter()
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
SPOOL_MEMORY_BYTES = 1024 * 1024
S3_BUFFER_BYTES = 256 * 1024
DECODE_LIMIT = threading.BoundedSemaphore(1)
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
ImageFile.LOAD_TRUNCATED_IMAGES = False
def s3_client():
return ot.S3Tracing(boto3.client(
"s3", endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
region_name="us-east-1", use_ssl=False,
config=Config(s3={"addressing_style": "path"}, connect_timeout=3,
read_timeout=10, retries={"max_attempts": 1}),
))
def s3_error(exc):
return APIError(503, "unavailable", "Object storage unavailable")
def format_attachment(row, variants=()):
return {
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]), "filename": row["filename"],
"content_type": row["content_type"], "size_bytes": row["size_bytes"],
"sha256": row["sha256"], "status": row["status"],
"variants": [{k: item[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
for item in variants] if row["status"] == "ready" else [],
"error_code": row["error_code"] if row["status"] == "failed" else None,
"created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]),
}
def require_member(conn, chat_id, user_id):
if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone():
raise APIError(404, "not_found", "Object not found")
def attachment_row(conn, attachment_id, user_id):
row = conn.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON cm.chat_id=a.chat_id
WHERE a.id=%s AND cm.user_id=%s
""", (attachment_id, user_id)).fetchone()
if not row:
raise APIError(404, "not_found", "Object not found")
return row
def validate_filename(filename: str):
if not (1 <= len(filename) <= 255) or any(c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename):
raise invalid("Invalid filename")
def validate_image(fileobj, mime: str):
"""Validate MIME, magic, dimensions and complete decode; worker does the resize."""
try:
fileobj.seek(0)
with Image.open(fileobj) as im:
if im.format not in ("PNG", "JPEG"):
raise APIError(415, "unsupported_media_type", "Unsupported image encoding")
actual = "image/png" if im.format == "PNG" else "image/jpeg"
if actual != mime:
raise APIError(415, "unsupported_media_type", "Content-Type does not match image")
if im.width * im.height > MAX_PIXELS:
raise APIError(413, "payload_too_large", "Image dimensions too large")
im.verify()
fileobj.seek(0)
with DECODE_LIMIT:
with Image.open(fileobj) as im:
im.load()
except APIError:
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise APIError(415, "unsupported_media_type", "Invalid image") from None
finally:
fileobj.seek(0)
def _admit_upload(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest):
from .jobs import job_payload
stage_key = f"staging/{attachment_id}/original"
key = f"attachments/{attachment_id}/original"
backup = f"recovery/{attachment_id}/original"
job_id = str(uuid.uuid4())
outbox_id = str(uuid.uuid4())
client = s3_client()
try:
spool.seek(0)
client.upload_fileobj(spool, BUCKET, stage_key,
ExtraArgs={"ContentType": mime},
Config=TransferConfig(use_threads=False, multipart_threshold=16*1024*1024,
io_chunksize=S3_BUFFER_BYTES))
except (BotoCoreError, ClientError, EndpointConnectionError, OSError) as exc:
raise s3_error(exc) from exc
# Atomic DB metadata + intent. If DB commit is ambiguous, do not delete the
# staged object: an accepted task may already exist; operator cleanup handles it.
with db() as conn:
with conn.transaction():
require_member(conn, chat_id, user_id)
# Small educational admission bound; pre-existing queued tasks remain safe.
pending = conn.execute("SELECT count(*) AS c FROM processing_jobs WHERE status IN ('queued','processing')").fetchone()["c"]
if pending >= int(os.getenv("JOB_BACKLOG_LIMIT", "1000")):
raise APIError(429, "rate_limited", "Processing queue is full")
row = conn.execute("""
INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename,
content_type,size_bytes,sha256,status,staging_key,pipeline_version)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,1) RETURNING *
""", (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest, stage_key)).fetchone()
conn.execute("""
INSERT INTO processing_jobs(id,attachment_id,pipeline_version,status,attempt)
VALUES (%s,%s,1,'queued',1)
""", (job_id, attachment_id))
payload = job_payload(job_id, attachment_id, 1, 1)
conn.execute("""
INSERT INTO job_outbox(id,job_id,payload) VALUES (%s,%s,%s::jsonb)
""", (outbox_id, job_id, __import__('json').dumps(payload)))
ot.event("upload.accepted", job_id=job_id)
return format_attachment(row) # snapshot = queued, even if worker is very fast
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)):
from .api import actor, require_chat
who = actor(req)["id"]
chat_id = parse_uuid(chat_id)
validate_filename(filename)
mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if mime not in ("image/png", "image/jpeg"):
raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
with db() as conn:
require_chat(conn, chat_id, who)
announced = req.headers.get("content-length")
if announced:
try:
if int(announced) > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
if int(announced) < 0:
raise invalid("Invalid Content-Length")
except ValueError:
raise invalid("Invalid Content-Length") from None
size = 0
digest = hashlib.sha256()
with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool:
async for chunk in req.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
digest.update(chunk)
spool.write(chunk)
if not size:
raise APIError(415, "unsupported_media_type", "Empty image")
await run_in_threadpool(validate_image, spool, mime)
return await run_in_threadpool(_admit_upload, spool, attachment_id=str(uuid.uuid4()),
chat_id=chat_id, user_id=who, filename=filename,
mime=mime, size=size, digest=digest.hexdigest())
@router.get("/api/v1/attachments/{attachment_id}")
def get_attachment(req: Request, attachment_id: str):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
variants = conn.execute("""
SELECT name,content_type,size_bytes,width,height,sha256
FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s
ORDER BY CASE name WHEN 'thumbnail' THEN 0 WHEN 'crop' THEN 1 ELSE 2 END
""", (row["id"], row["pipeline_version"])).fetchall() if row["status"] == "ready" else []
return format_attachment(row, variants)
@router.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(req: Request, attachment_id: str, variant: str = "original"):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
if row["status"] != "ready":
raise APIError(409, "conflict", "Attachment is not ready")
if variant == "original":
key, mime, size = row["object_key"], row["content_type"], row["size_bytes"]
else:
v = conn.execute("""
SELECT * FROM attachment_variants
WHERE attachment_id=%s AND pipeline_version=%s AND name=%s
""", (row["id"], row["pipeline_version"], variant)).fetchone()
if not v:
raise APIError(404, "not_found", "Object not found")
key, mime, size = v["object_key"], v["content_type"], v["size_bytes"]
client = s3_client()
try:
response = client.get_object(Bucket=BUCKET, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"):
raise APIError(503, "unavailable", "Stored object missing") from exc
raise s3_error(exc) from exc
except BotoCoreError as exc:
raise s3_error(exc) from exc
def iterator():
with closing(response["Body"]) as body:
while chunk := body.read(S3_BUFFER_BYTES):
yield chunk
return StreamingResponse(iterator(), media_type=mime, headers={"Content-Length": str(size)})
+104
View File
@@ -0,0 +1,104 @@
"""HTTP Basic only on login; bearer sessions shared by all API replicas."""
import base64
import binascii
import hashlib
import re
import secrets
import uuid
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError, VerificationError
from fastapi import Request
from redis.exceptions import RedisError
from .core import APIError, SESSION_TTL, iso
from .storage import cache, db
hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
PASSWORD_RE = re.compile(r"^[a-z0-9_]{3,32}$")
def public_user(row: dict):
return {"id": str(row["id"]), "username": row["username"], "display_name": bytes(row["display_name"]).decode("utf-8"), "created_at": iso(row["created_at"])}
def unauthorized() -> APIError:
return APIError(401, "unauthorized", "Unauthorized")
def challenge(req: Request) -> str:
return 'Basic realm="messenger", charset="UTF-8"' if req.url.path == "/api/v1/auth/sessions" else 'Bearer realm="messenger"'
def bearer_token(req: Request) -> str:
value = req.headers.get("Authorization", "")
if not value.startswith("Bearer ") or len(value) <= 7 or " " in value[7:] or len(value) > 512:
raise unauthorized()
try:
value[7:].encode("ascii")
except UnicodeEncodeError:
raise unauthorized() from None
return value[7:]
def session_key(token: str) -> str:
return "session:" + hashlib.sha256(token.encode("ascii")).hexdigest()
def current_user(req: Request) -> dict:
token = bearer_token(req)
try:
raw = cache().get(session_key(token))
except (RedisError, UnicodeError) as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
if not raw:
raise unauthorized()
try:
ident = str(uuid.UUID(raw))
except (ValueError, TypeError):
try:
cache().delete(session_key(token))
except RedisError as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
raise unauthorized() from None
with db() as conn:
row = conn.execute("SELECT * FROM users WHERE id = %s", (ident,)).fetchone()
if row is None:
raise unauthorized()
return public_user(row)
def login(req: Request):
auth = req.headers.get("Authorization", "")
if not auth.startswith("Basic "):
raise unauthorized()
try:
decoded = base64.b64decode(auth[6:].encode("ascii"), validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
if not PASSWORD_RE.fullmatch(username):
raise ValueError()
except (ValueError, UnicodeError, binascii.Error):
raise unauthorized() from None
with db() as conn:
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
if not row:
raise unauthorized()
try:
hasher.verify(row["password_hash"], password)
except (VerifyMismatchError, VerificationError):
raise unauthorized() from None
token = secrets.token_urlsafe(32)
try:
cache().set(session_key(token), str(row["id"]), ex=SESSION_TTL)
except RedisError as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
return {"token": token, "token_type": "Bearer", "expires_at": iso(datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)), "user": public_user(row)}
def logout(req: Request):
current_user(req)
try:
cache().delete(session_key(bearer_token(req)))
except RedisError as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
+79
View File
@@ -0,0 +1,79 @@
"""Configuration, HTTP errors and shared representation helpers."""
import base64
import json
import os
import uuid
from datetime import datetime, timezone
from fastapi import Request
INSTANCE_ID = os.getenv("API_INSTANCE_ID", uuid.uuid4().hex[:12])
SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
IDEMPOTENCY_TTL = max(86400, int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")))
class APIError(Exception):
def __init__(self, status: int, code: str, message: str):
super().__init__(message)
self.status = status
self.code = code
self.message = message
def iso(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def invalid(message: str = "Invalid request") -> APIError:
return APIError(400, "invalid_request", message)
def parse_uuid(value: str) -> str:
try:
return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError):
raise invalid("Invalid UUID") from None
def parse_limit(raw: str | None) -> int:
if raw is None:
return 50
try:
if not raw.isascii() or not raw.isdecimal():
raise ValueError()
value = int(raw)
except (ValueError, AttributeError):
raise invalid("Invalid limit") from None
if not 1 <= value <= 100:
raise invalid("Invalid limit")
return value
def make_cursor(kind: str, scope: str, created: datetime, ident: str) -> str:
b = json.dumps({"kind": kind, "scope": scope, "created_at": iso(created), "id": ident}, separators=(",", ":")).encode()
return base64.urlsafe_b64encode(b).decode().rstrip("=")
def read_cursor(raw: str | None, kind: str, scope: str):
if raw is None:
return None
try:
if not raw or len(raw) > 2048 or any(ch not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" for ch in raw):
raise ValueError()
data = json.loads(base64.b64decode(raw + "=" * (-len(raw) % 4), altchars=b"-_", validate=True))
if not isinstance(data, dict) or data.get("kind") != kind or data.get("scope") != scope:
raise ValueError()
created = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
if created.tzinfo is None:
raise ValueError()
ident = str(uuid.UUID(data["id"]))
return created, ident
except (ValueError, TypeError, KeyError, UnicodeError, json.JSONDecodeError):
raise invalid("Invalid cursor") from None
def request_id(req: Request) -> str:
return getattr(req.state, "request_id", uuid.uuid4().hex)
def error_body(req: Request, code: str, message: str):
return {"error": {"code": code, "message": message, "request_id": request_id(req)}}
+124
View File
@@ -0,0 +1,124 @@
"""Sandboxed-per-task image work. Process is killed after JOB_TIMEOUT_SECONDS.
Outputs only JSON variant metadata. Original bytes are copied from durable staging,
never transformed. Pillow reopens/exif-transposes pixels for derivatives, stripping
metadata on re-encode. No API tokens or arbitrary user paths enter this command.
"""
import hashlib
import io
import json
import os
import sys
from PIL import Image, ImageOps, UnidentifiedImageError
from botocore.exceptions import ClientError, BotoCoreError
from .attachments import BUCKET, MAX_BYTES, MAX_PIXELS, s3_client
from . import telemetry as ot
from opentelemetry.trace import SpanKind
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
def rgb_image(src):
orient = ImageOps.exif_transpose(src)
orient.load()
if orient.width * orient.height > MAX_PIXELS:
raise ValueError("too many decoded pixels")
# Compose transparency on white, strip EXIF/ICC and do not enlarge.
rgba = orient.convert("RGBA")
white = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
white.alpha_composite(rgba)
return white.convert("RGB")
def transform(raw):
with Image.open(io.BytesIO(raw)) as image:
if image.format not in ("PNG", "JPEG"):
raise ValueError("invalid encoding")
rgb = rgb_image(image)
t = rgb.copy()
t.thumbnail((256, 256), resample=Image.Resampling.LANCZOS)
# Center crop fit scales up small images when needed.
crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
optim = rgb.copy()
optim.thumbnail((1280, 1280), resample=Image.Resampling.LANCZOS)
return [
("thumbnail", t, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
("crop", crop, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
("optimized", optim, "WEBP", "image/webp", {"quality": 78, "method": 4}),
]
def run(attachment_id, pipeline_version):
import psycopg
from psycopg.rows import dict_row
with ot.span("db.load_staging_metadata", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5) as conn:
row = conn.execute("""
SELECT * FROM attachments WHERE id=%s AND pipeline_version=%s
""", (attachment_id, pipeline_version)).fetchone()
if not row or not row["staging_key"]:
raise ValueError("staging metadata not found")
client = s3_client()
body = client.get_object(Bucket=BUCKET, Key=row["staging_key"])["Body"]
try:
raw = body.read(MAX_BYTES + 1)
finally:
body.close()
if not raw or len(raw) > MAX_BYTES or len(raw) != row["size_bytes"]:
raise ValueError("staging size mismatch")
if hashlib.sha256(raw).hexdigest() != row["sha256"]:
raise ValueError("staging checksum mismatch")
# Recompute every result if a prior run was interrupted. Keys are deterministic
# for (attachment_id,pipeline_version,variant), so duplicates don't multiply objects.
output = []
for name, image, fmt, mime, options in transform(raw):
buf = io.BytesIO()
image.save(buf, format=fmt, **options)
encoded = buf.getvalue()
key = f"variants/{attachment_id}/v{pipeline_version}/{name}"
client.put_object(Bucket=BUCKET, Key=key, Body=encoded, ContentType=mime)
output.append({"name": name, "content_type": mime,
"size_bytes": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(),
"width": image.width, "height": image.height, "object_key": key})
# Only after derivatives exist, publish unchanged primary and recovery copy.
client.copy_object(Bucket=BUCKET, Key=row["object_key"],
CopySource={"Bucket": BUCKET, "Key": row["staging_key"]},
MetadataDirective="REPLACE", ContentType=row["content_type"])
client.copy_object(Bucket=BUCKET, Key=row["backup_key"],
CopySource={"Bucket": BUCKET, "Key": row["object_key"]},
MetadataDirective="REPLACE", ContentType=row["content_type"])
return output
def main():
import uuid
if len(sys.argv) != 3:
raise SystemExit(2)
try:
ident = str(uuid.UUID(sys.argv[1]))
version = int(sys.argv[2])
if version < 1:
raise ValueError("invalid version")
ot.init("worker", child=True)
with ot.span("image.transform", context=ot.extract(json.loads(os.getenv("LAB6_PARENT_CONTEXT", "{}")))):
out = run(ident, version)
ot.event("image.transformed")
print(json.dumps({"variants": out}, ensure_ascii=False))
except (ValueError, UnidentifiedImageError, Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
print("Permanent image processing failure", file=sys.stderr)
raise SystemExit(20) from None
except (BotoCoreError, ClientError, OSError):
print("Transient object storage failure", file=sys.stderr)
raise SystemExit(30) from None
if __name__ == "__main__":
try:
main()
finally:
ot.flush()
+470
View File
@@ -0,0 +1,470 @@
"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases.
There is no exactly-once transport guarantee. We guarantee eventual recovery of
committed intent and idempotent DB results under at-least-once redelivery.
Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`.
"""
import argparse
from contextlib import contextmanager
import json
import logging
import os
import random
import subprocess
import sys
import time
import threading
from . import telemetry as ot
from opentelemetry.trace import SpanKind, Status, StatusCode
import uuid
from datetime import datetime, timezone, timedelta
import pika
import psycopg
from pika.exceptions import AMQPError
from psycopg.rows import dict_row
log = logging.getLogger("lab5.jobs")
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
QUEUE = "attachment-processing.v1"
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
LEASE_SECONDS = JOB_TIMEOUT + 45
@contextmanager
def connect_db():
# No credentials, SQL text, or parameters in spans.
with ot.dependency_span('db.job.transaction', kind=SpanKind.CLIENT, attrs={'db.system.name':'postgresql'}):
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5, options='-c statement_timeout=5000') as conn:
yield conn
def job_payload(job_id, attachment_id, version, attempt):
return {
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
"attachment_id": str(attachment_id), "pipeline_version": version,
"attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(),
**ot.inject(),
}
@contextmanager
def connection():
params = pika.ConnectionParameters(
host=os.getenv("BROKER_HOST", "broker"), port=5672,
credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]),
heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120,
socket_timeout=5, connection_attempts=3, retry_delay=1)
mq = pika.BlockingConnection(params)
try:
yield mq
finally:
try:
if mq.is_open:
mq.close()
except AMQPError:
pass
def setup_channel(conn):
ch = conn.channel()
ch.queue_declare(queue=QUEUE, durable=True)
return ch
def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None):
payload = job_payload(job_id, attachment_id, version, attempt)
if not payload.get('traceparent'):
# Lease rescue/replay has no active consumer context. Preserve the first
# acceptance context rather than inventing a new disconnected trace.
previous = db.execute("SELECT payload FROM job_outbox WHERE job_id=%s ORDER BY created_at,id LIMIT 1", (job_id,)).fetchone()
if previous:
payload.update({k: previous['payload'][k] for k in ('traceparent','tracestate') if k in previous['payload']})
db.execute("""
INSERT INTO job_outbox(id,job_id,payload,available_at)
VALUES (%s,%s,%s::jsonb,COALESCE(%s,now()))
""", (str(uuid.uuid4()), job_id, json.dumps(payload), available_at))
def cleanup_abandoned_staging():
"""Best effort cleanup of retained stages; never delete current processing work."""
from .attachments import BUCKET, s3_client
client = s3_client()
retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
with connect_db() as db:
rows = db.execute("""
SELECT id,staging_key FROM attachments
WHERE staging_key IS NOT NULL AND
((status='ready' AND updated_at < now()-interval '300 seconds') OR
(status='failed' AND updated_at < now()-(%s*interval '1 second')))
LIMIT 50
""", (retention,)).fetchall()
for row in rows:
with connect_db() as db:
with db.transaction():
# Row-level lock serializes against operator replay.
current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone()
if not current or current["staging_key"] != row["staging_key"]:
continue
age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds()
if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention):
continue
client.delete_object(Bucket=BUCKET, Key=row["staging_key"])
db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"]))
# Also prune staging uploads whose DB COMMIT failed, after a long retention.
from botocore.exceptions import ClientError
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"):
for obj in page.get("Contents", []):
if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention:
continue
key = obj["Key"]
with connect_db() as db:
with db.transaction():
referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
if not referenced:
client.delete_object(Bucket=BUCKET, Key=key)
def rescue_expired_jobs(conn):
"""Requeue leased jobs whose worker died (durably, no manual task recreation)."""
with conn.transaction():
rows = conn.execute("""
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt
FROM processing_jobs j
WHERE j.status='processing' AND j.leased_until < now()
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if row["attempt"] >= MAX_ATTEMPTS:
conn.execute("""UPDATE processing_jobs
SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now()
WHERE id=%s""", (row["id"],))
conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now()
WHERE id=%s""", (row["attachment_id"],))
else:
attempt = row["attempt"] + 1
conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,
lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now()
WHERE id=%s""", (attempt, row["id"]))
conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],))
queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt)
log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt)
def sample_backlog():
# Query durable *queued* jobs, including delayed retries and unpublished outbox.
# No broker membership scan; oldest age is wall clock since initial acceptance.
while True:
try:
with psycopg.connect(os.environ['DATABASE_URL'], row_factory=dict_row, connect_timeout=3, options='-c statement_timeout=2000') as db:
row = db.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM processing_jobs WHERE status='queued'").fetchone()
ot.backlog_depth, ot.backlog_age = row['depth'], float(row['age'])
except psycopg.Error:
ot.backlog_depth = ot.backlog_age = None
time.sleep(3)
def dispatch_loop():
"""Publish outbox rows with publisher confirm, mark only AFTER confirm."""
threading.Thread(target=sample_backlog, daemon=True).start()
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.confirm_delivery()
last_clean = time.monotonic()
last_queue_sample = 0
while True:
# Periodic lease recovery works even if original message was consumed.
with connect_db() as conn:
rescue_expired_jobs(conn)
with conn.transaction():
rows = conn.execute("""
SELECT id,payload FROM job_outbox
WHERE published_at IS NULL AND available_at<=now()
ORDER BY available_at,created_at
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once":
# Persist one-time fault before hard exit; outbox still
# unmarked and dispatcher will publish on restart.
with connect_db() as faultdb:
inserted = faultdb.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish')
ON CONFLICT DO NOTHING RETURNING job_id
""", (row["payload"]["job_id"],)).fetchone()
if inserted:
os._exit(78)
payload = row['payload']
with ot.span('rabbitmq.publish', kind=SpanKind.PRODUCER, context=ot.extract(payload),
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE,'job.attempt':payload['attempt']}) as producer:
# Propagate actual producer span, while DB retains the original intent.
raw = json.dumps({**payload, **ot.inject()}, separators=(",", ":"))
ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"),
mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2))
conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],))
ot.event('job.published', job_id=payload['job_id'], attempt=payload['attempt'])
if time.monotonic() - last_queue_sample > 3:
q = ch.queue_declare(queue=QUEUE, passive=True).method
ot.broker_ready, ot.broker_consumers = q.message_count, q.consumer_count
last_queue_sample = time.monotonic()
if time.monotonic() - last_clean > 60:
try:
cleanup_abandoned_staging()
except Exception:
log.warning("staging janitor failed; retry later")
last_clean = time.monotonic()
if not rows:
mq.sleep(1)
except (AMQPError, OSError, psycopg.Error) as exc:
ot.broker_ready = ot.broker_consumers = None
log.warning("dispatcher reconnect after %s", type(exc).__name__)
time.sleep(2)
def claim_job(db, message):
job_id = message["job_id"]
with db.transaction():
job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone()
if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]:
return "ignore", None
if job["status"] == "processing":
return "busy", None
if job["next_run_at"] > datetime.now(timezone.utc):
return "busy", None
lease = str(uuid.uuid4())
db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s,
leased_until=now()+(%s*interval '1 second'),updated_at=now()
WHERE id=%s""", (lease, LEASE_SECONDS, job_id))
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],))
return "claimed", {"job": job, "lease": lease}
def publish_result(db, *, job, lease, variants):
"""Final result is committed atomically; stale leases/versions cannot publish."""
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]:
log.warning("stale worker result ignored %s", job["id"])
return False
for v in variants:
db.execute("""
INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET
object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,
size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256,
width=EXCLUDED.width,height=EXCLUDED.height
""", (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"],
v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"]))
db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now()
WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"]))
db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL,
error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],))
return True
def fail_job(db, job, lease, *, permanent=False, code="processing_failed"):
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease:
return "stale"
if permanent or job["attempt"] >= MAX_ATTEMPTS:
code = code if permanent else "retry_exhausted"
db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL,
leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"]))
db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"]))
return "failed"
else:
attempt = job["attempt"] + 1
delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0)
run_at = datetime.now(timezone.utc) + timedelta(seconds=delay)
db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s,
lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"]))
db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],))
queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at)
log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay)
return "retry"
def cleanup_staging(job):
from .attachments import BUCKET, s3_client
from botocore.exceptions import BotoCoreError, ClientError
try:
with connect_db() as conn:
row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone()
if row and row["status"] == "ready" and row["staging_key"]:
s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"])
with connect_db() as conn:
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s",
(job["attachment_id"], row["staging_key"]))
except (BotoCoreError, ClientError, psycopg.Error, OSError):
log.warning("staging cleanup deferred for %s", job["id"])
def process_message(raw):
try:
carrier = json.loads(raw)
if not isinstance(carrier, dict): carrier = {}
except (ValueError, TypeError):
carrier = {}
with ot.span('rabbitmq.consume', kind=SpanKind.CONSUMER, context=ot.extract(carrier),
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE}) as consumer:
return process_message_inner(raw, consumer)
def process_message_inner(raw, consumer):
try:
msg = json.loads(raw)
assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1"
uuid.UUID(msg["job_id"])
uuid.UUID(msg["attachment_id"])
assert type(msg["attempt"]) is int and msg["attempt"] >= 1
except (ValueError, KeyError, AssertionError, TypeError):
log.error("invalid internal broker message discarded")
return "ack" # no secrets printed, does not touch DB
with connect_db() as db:
status, state = claim_job(db, msg)
if status == "ignore":
return "ack"
if status == "busy":
return "requeue"
job, lease = state["job"], state["lease"]
consumer.set_attribute('job.attempt',job['attempt'])
consumer.set_attribute('job.id',str(job['id']))
ot.event('job.claimed', job_id=str(job['id']), attempt=job['attempt'])
start = time.monotonic()
ot.worker_busy = 1
def done(outcome):
attrs = {'job_type':'attachment.process.v1','outcome':outcome}
ot.count('lab_jobs',attrs)
ot.observe('lab_job_duration',time.monotonic()-start,attrs)
if outcome == 'retry':
ot.count('lab_job_retries',{'job_type':'attachment.process.v1'})
if outcome in ('retry','failed','requeue'):
consumer.set_status(Status(StatusCode.ERROR))
consumer.set_attribute('job.outcome',outcome)
ot.event('job.'+outcome, level=logging.ERROR if outcome in ('failed','retry','requeue') else logging.INFO, job_id=str(job['id']), attempt=job['attempt'])
try:
if os.getenv('LAB6_FAULT_RETRY_ONCE') == 'once':
with connect_db() as db:
inserted = db.execute("INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'lab6_retry') ON CONFLICT DO NOTHING RETURNING job_id", (job['id'],)).fetchone()
if inserted:
raise RuntimeError('injected transient retry')
cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])]
with ot.span('image.subprocess'):
env = {**os.environ, 'LAB6_PARENT_CONTEXT': json.dumps(ot.inject())}
proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False, env=env)
if proc.returncode != 0:
with connect_db() as db:
outcome = fail_job(db, job, lease, permanent=(proc.returncode == 20),
code="invalid_image" if proc.returncode == 20 else "processing_failed")
done(outcome)
return "ack" # next attempt is already safely in DB outbox
result = json.loads(proc.stdout)
variants = result["variants"]
if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]:
raise ValueError("incomplete variants")
if os.getenv("LAB5_FAULT_AFTER_S3") == "once":
# One-time operator-only injected hard death, after all S3 writes and
# before DB result commit/ack; SQL PK prevents endless failure loop.
with connect_db() as db:
with db.transaction():
inserted = db.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3')
ON CONFLICT DO NOTHING RETURNING job_id
""", (job["id"],)).fetchone()
if inserted:
os._exit(77)
with ot.span("db.publish_result", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
with connect_db() as db:
ok = publish_result(db, job=job, lease=lease, variants=variants)
if ok:
ot.observe('lab_ready_duration',(datetime.now(timezone.utc)-job['created_at']).total_seconds(), {'job_type':'attachment.process.v1'})
cleanup_staging(job)
done('ready' if ok else 'stale')
return "ack"
except subprocess.TimeoutExpired:
with connect_db() as db:
outcome = fail_job(db, job, lease, code="processing_timeout")
done(outcome)
return "ack"
except Exception:
log.exception("job processing failure (internal details not sent to API)")
# Leave lease for dispatcher recovery if even DB unavailable.
try:
with connect_db() as db:
outcome = fail_job(db, job, lease)
done(outcome)
return "ack"
except psycopg.Error:
done("requeue")
return "requeue"
finally:
ot.worker_busy = 0
def worker_loop():
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1"))))
def callback(channel, method, props, body):
outcome = process_message(body)
if outcome == "ack":
channel.basic_ack(method.delivery_tag)
else:
channel.basic_nack(method.delivery_tag, requeue=True)
time.sleep(2) # prevent tight redelivery spin on an active lease
ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False)
log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT)
ch.start_consuming()
except (AMQPError, OSError, psycopg.Error) as exc:
log.warning("worker reconnect after %s", type(exc).__name__)
time.sleep(2)
def replay_failed(ident):
with connect_db() as db:
with db.transaction():
row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone()
if not row or row["status"] != "failed":
raise RuntimeError("Only failed jobs may be explicitly replayed")
attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone()
if not attachment or not attachment["staging_key"]:
raise RuntimeError("Retained staging is required for replay")
db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(),
lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],))
db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,))
queue_outbox(db, row["id"], ident, row["pipeline_version"], 1)
db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"]))
print("Replayed failed job", ident)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"])
parser.add_argument("--id", help="UUID for explicit operator replay/status")
args = parser.parse_args()
if args.mode == "dispatch":
ot.init("dispatcher")
dispatch_loop()
elif args.mode == "worker":
ot.init("worker")
worker_loop()
elif args.mode in ("replay", "status"):
if not args.id:
parser.error("--id is required")
ident = str(uuid.UUID(args.id))
if args.mode == "replay":
replay_failed(ident)
else:
with connect_db() as db:
print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str))
if __name__ == "__main__":
main()
+117
View File
@@ -0,0 +1,117 @@
"""FastAPI process: one stateless replica. No auto-migration on startup."""
import logging
import uuid
import time
from starlette.routing import Match
from opentelemetry.trace import SpanKind, Status, StatusCode
from . import telemetry as ot
from contextlib import asynccontextmanager
import psycopg
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.concurrency import run_in_threadpool
from redis.exceptions import RedisError
from .api import router
from .attachments import router as attachment_router
from .auth import challenge, current_user
from .core import APIError, INSTANCE_ID, error_body, invalid, request_id
from .storage import start, stop
logging.basicConfig(level="INFO")
logger = logging.getLogger("lab5")
@asynccontextmanager
async def lifespan(app: FastAPI):
ot.init("api")
start()
try:
yield
finally:
stop()
ot.flush()
app = FastAPI(title="MTUCI Messenger Lab 6", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan)
app.include_router(router)
app.include_router(attachment_router)
PROTECTED = ("/api/v1/chats", "/api/v1/attachments", "/api/v1/users/me", "/api/v1/auth/sessions/current")
JSON_POST = ("/api/v1/users", "/api/v1/chats")
@app.middleware("http")
async def request_context(req: Request, call_next):
# Match the registered template even when early authentication fails.
route = next((r.path for r in app.routes if r.matches(req.scope)[0] == Match.FULL), 'unmatched')
method = req.method if req.method in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER'
attrs = {'method': method, 'route': route}
start_at = time.monotonic()
if route in ("/health/live", "/health/ready"):
response = await handle_request(req, call_next)
attrs["status_class"] = str(response.status_code//100)+"xx"
ot.count("lab_http_requests", attrs)
ot.observe("lab_http_duration", time.monotonic()-start_at, attrs)
return response
with ot.span(method+' '+route, kind=SpanKind.SERVER, attrs={'http.request.method':method,'http.route':route}) as span:
response = await handle_request(req, call_next)
attrs['status_class'] = str(response.status_code//100)+'xx'
span.set_attribute('http.response.status_code', response.status_code)
if response.status_code >= 500:
span.set_status(Status(StatusCode.ERROR))
ot.count('lab_http_requests', attrs)
ot.observe('lab_http_duration', time.monotonic()-start_at, attrs)
ot.event('http.completed', level=logging.ERROR if response.status_code >= 500 else logging.INFO,
request_id=request_id(req), route=route, method=method, status_code=response.status_code)
if ot.trace_id():
response.headers['X-Trace-Id'] = ot.trace_id()
return response
async def handle_request(req: Request, call_next):
req.state.request_id = uuid.uuid4().hex
try:
path = req.url.path
# Authenticate before parsing the request body or resource-specific fields.
if any(path == p or path.startswith(p + "/") for p in PROTECTED):
req.state.actor = await run_in_threadpool(current_user, req)
if req.method == "POST" and (path in JSON_POST or (path.startswith("/api/v1/chats/") and path.endswith("/messages"))):
if req.headers.get("content-type", "").split(";", 1)[0].strip().lower() != "application/json":
raise APIError(415, "unsupported_media_type", "Content-Type must be application/json")
if req.method == "POST" and path == "/api/v1/auth/sessions" and await req.body():
raise invalid("Login body is not allowed")
response = await call_next(req)
except APIError as exc:
response = api_error(req, exc)
except (RedisError, psycopg.OperationalError, psycopg.InterfaceError):
response = api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
except Exception:
logger.exception("unexpected request failure id=%s", request_id(req))
response = api_error(req, APIError(500, "internal_error", "Internal server error"))
response.headers["X-Request-Id"] = request_id(req)
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
def api_error(req: Request, exc: APIError) -> JSONResponse:
headers = {"WWW-Authenticate": challenge(req)} if exc.status == 401 else {}
return JSONResponse(status_code=exc.status, content=error_body(req, exc.code, exc.message), headers=headers)
@app.exception_handler(APIError)
async def api_error_handler(req: Request, exc: APIError):
return api_error(req, exc)
@app.exception_handler(RequestValidationError)
async def validation_error_handler(req: Request, exc: RequestValidationError):
return api_error(req, invalid())
@app.exception_handler(Exception)
async def unexpected_error_handler(req: Request, exc: Exception):
logger.exception("handler failure id=%s", request_id(req))
if isinstance(exc, (psycopg.OperationalError, psycopg.InterfaceError, RedisError)):
return api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
return api_error(req, APIError(500, "internal_error", "Internal server error"))
+23
View File
@@ -0,0 +1,23 @@
"""Run once before API instances; transactional and advisory-lock coordinated migrations."""
import os
from pathlib import Path
import psycopg
MIGRATIONS = Path(__file__).resolve().parents[1] / "db" / "migrations"
def main():
with psycopg.connect(os.environ["DATABASE_URL"], autocommit=False) as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(62132103)")
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())")
for path in sorted(MIGRATIONS.glob("[0-9]*.sql")):
applied = conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone()
if applied:
continue
conn.execute(path.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (path.name,))
print("Applied", path.name, flush=True)
print("Migrations up to date", flush=True)
if __name__ == "__main__":
main()
+77
View File
@@ -0,0 +1,77 @@
"""HTTP request models; all strings use Unicode code point limits."""
import re
import uuid
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
class CreateUser(StrictModel):
username: str
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_valid(cls, value: str) -> str:
if not USERNAME_RE.fullmatch(value):
raise ValueError("Invalid username")
return value
@field_validator("display_name", "password")
@classmethod
def non_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("Blank value")
value.encode("utf-8", errors="strict")
return value
class CreateChat(StrictModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def non_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("Blank title")
value.encode("utf-8", errors="strict")
return value
@field_validator("member_ids")
@classmethod
def uuid_members(cls, value: list[str]) -> list[str]:
for member in value:
if not isinstance(member, str) or str(uuid.UUID(member)) != member:
raise ValueError("Invalid member UUID")
if len(value) != len(set(value)):
raise ValueError("Duplicate member")
return value
class CreateMessage(StrictModel):
text: str = Field(min_length=0, max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@field_validator("text")
@classmethod
def non_blank(cls, value: str) -> str:
value.encode("utf-8", errors="strict")
return value
@field_validator("attachment_ids")
@classmethod
def valid_attachments(cls, value: list[str]) -> list[str]:
if len(value) != len(set(value)):
raise ValueError("Duplicate attachment IDs")
for item in value:
if not isinstance(item, str) or str(uuid.UUID(item)) != item:
raise ValueError("Invalid attachment UUID")
return value
@model_validator(mode="after")
def nonempty_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Empty message")
return self
+25
View File
@@ -0,0 +1,25 @@
"""Offline image conversion tests inside the application image; no S3 or DB."""
import io
from PIL import Image
from .image_worker import transform
def main():
source = Image.new('RGB',(320,200), (255,0,0))
buf=io.BytesIO();source.save(buf,'PNG')
result=transform(buf.getvalue())
items={name:(img,fmt,mime,opts) for name,img,fmt,mime,opts in result}
assert set(items)=={'thumbnail','crop','optimized'}
assert items['thumbnail'][0].size==(256,160)
assert items['crop'][0].size==(128,128)
assert items['optimized'][0].size==(320,200)
for name,(im,fmt,mime,opts) in items.items():
data=io.BytesIO();im.save(data,format=fmt,**opts)
with Image.open(io.BytesIO(data.getvalue())) as decoded:
assert decoded.size==im.size
assert decoded.format==fmt
assert 'exif' not in decoded.info
print('PASS: PNG 320x200 -> JPEG thumb 256x160, crop 128x128, optimized WebP, metadata stripped')
if __name__=='__main__':
main()
+132
View File
@@ -0,0 +1,132 @@
"""Lab4 grade-5 track 2: repeatable DB/S3 reconciliation.
Dry-run is default. Apply never drops metadata when BOTH object copies are gone:
without a trustworthy copy or external backup, deletion would destroy evidence.
"""
import argparse
import json
import sys
import time
from datetime import datetime, timezone
import psycopg
from psycopg.rows import dict_row
from botocore.exceptions import BotoCoreError, ClientError
from .attachments import BUCKET, s3_client
def inventory(client):
result = {}
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET):
for obj in page.get("Contents", []):
# Only reconciliation-owned prefixes; never mutate arbitrary S3 keys.
key = obj["Key"]
if key.startswith(("attachments/", "recovery/")):
result[key] = obj["LastModified"]
return result
def get_db_rows(conn):
return conn.execute("SELECT id, object_key, backup_key, status FROM attachments").fetchall()
def current_reference(conn, key):
return conn.execute(
"SELECT 1 FROM attachments WHERE object_key=%s OR backup_key=%s", (key, key)
).fetchone() is not None
def reconcile(*, apply=False, grace_seconds=300, now=None, client=None, connect=None):
if grace_seconds < 300 and apply:
raise ValueError("Apply requires >=300 s grace period to protect in-flight uploads")
started = time.monotonic()
now = now or datetime.now(timezone.utc)
client = client or s3_client()
connect = connect or (lambda: psycopg.connect(__import__('os').environ['DATABASE_URL'], row_factory=dict_row))
objects = inventory(client)
stats = {
"mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds,
"s3_keys_scanned": len(objects), "db_rows_scanned": 0,
"orphan_keys": [], "orphan_keys_young": [],
"missing_primary_repairable": [], "missing_backup_repairable": [],
"missing_both_unrecoverable": [], "actions": [], "errors": [],
}
with connect() as conn:
rows = get_db_rows(conn)
stats["db_rows_scanned"] = len(rows)
references = {key for row in rows for key in (row["object_key"], row["backup_key"])}
for row in rows:
if row["status"] != "ready":
continue # queued/processing have durable staging, not final variants yet
primary, backup = row["object_key"], row["backup_key"]
have_primary, have_backup = primary in objects, backup in objects
rid = str(row["id"])
if have_primary and have_backup:
continue
if not have_primary and not have_backup:
stats["missing_both_unrecoverable"].append(rid)
continue
dest, source, typ = (primary, backup, "missing_primary_repairable") if not have_primary else (backup, primary, "missing_backup_repairable")
stats[typ].append(rid)
if apply:
# Defensive recheck. Object may have reappeared after inventory.
try:
client.head_object(Bucket=BUCKET, Key=dest)
stats["actions"].append({"action": "already-restored", "key": dest})
continue
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
if code not in ("404", "NotFound", "NoSuchKey"):
raise
try:
client.copy_object(Bucket=BUCKET, Key=dest,
CopySource={"Bucket": BUCKET, "Key": source})
stats["actions"].append({"action": "restore", "key": dest, "from": source})
except (BotoCoreError, ClientError) as exc:
stats["errors"].append({"action": "restore", "key": dest, "error_type": type(exc).__name__})
for key, modified in objects.items():
if key in references:
continue
age = (now - modified).total_seconds()
if age < grace_seconds:
stats["orphan_keys_young"].append(key)
continue
stats["orphan_keys"].append(key)
if apply:
# Query again immediately before destructive action; never delete
# an object now referenced by a committed metadata row.
if current_reference(conn, key):
stats["actions"].append({"action": "skip-now-referenced", "key": key})
continue
try:
client.delete_object(Bucket=BUCKET, Key=key)
stats["actions"].append({"action": "delete-old-orphan", "key": key})
except (BotoCoreError, ClientError) as exc:
stats["errors"].append({"action": "delete-old-orphan", "key": key, "error_type": type(exc).__name__})
stats["scan_elapsed_seconds"] = round(time.monotonic() - started, 3)
return stats
def main():
parser = argparse.ArgumentParser(description="Reconcile S3 attachments vs PostgreSQL (dry-run default)")
parser.add_argument("--apply", action="store_true", help="Make safe changes; orphans older than 300 seconds only")
parser.add_argument("--grace-seconds", type=int, default=300)
parser.add_argument("--output", help="Write JSON summary (no credentials)")
args = parser.parse_args()
try:
result = reconcile(apply=args.apply, grace_seconds=args.grace_seconds)
except Exception as exc:
# No exception string: botocore errors may contain signed credentials/URLs.
print(json.dumps({"status":"failed", "error_type":type(exc).__name__}), file=sys.stderr)
raise SystemExit(2) from None
rendered = json.dumps(result, ensure_ascii=False, indent=2)
print(rendered)
if args.output:
with open(args.output, "w", encoding="utf-8") as fh:
fh.write(rendered + "\n")
if result["errors"] or result["missing_both_unrecoverable"]:
raise SystemExit(1)
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
"""Operator-only fault injection in a LOCAL TEST stack; never exposed as HTTP."""
import argparse
import json
import os
import uuid
import psycopg
from psycopg.rows import dict_row
from .attachments import BUCKET, s3_client
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["delete-primary", "delete-backup", "create-orphan"])
p.add_argument("--id", help="Attachment UUID for the delete actions")
args = p.parse_args()
if os.environ.get("LAB4_ALLOW_FAULT_INJECTION") != "yes":
raise SystemExit("Fault injection disabled; use LAB4_ALLOW_FAULT_INJECTION=yes in isolated test")
client = s3_client()
if args.action == "create-orphan":
key = f"attachments/{uuid.uuid4()}/original"
client.put_object(Bucket=BUCKET, Key=key, Body=b"synthetic orphan")
print(json.dumps({"fault":"orphan-created", "object_key":key}))
return
if not args.id:
raise SystemExit("--id is required")
ident = str(uuid.UUID(args.id))
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) as conn:
row = conn.execute("SELECT object_key,backup_key FROM attachments WHERE id=%s", (ident,)).fetchone()
if not row:
raise SystemExit("Attachment does not exist")
key = row["object_key"] if args.action == "delete-primary" else row["backup_key"]
client.delete_object(Bucket=BUCKET, Key=key)
print(json.dumps({"fault":args.action, "object_key":key}))
if __name__ == "__main__":
main()
+61
View File
@@ -0,0 +1,61 @@
"""Fast isolated tests for reconciliation safety; run in application image."""
import uuid
from datetime import datetime, timezone, timedelta
from .reconcile import reconcile
from .attachments import BUCKET
NOW = datetime.now(timezone.utc)
class FakeConn:
def __init__(self, rows): self.rows=rows
def __enter__(self): return self
def __exit__(self,*a): return False
def execute(self, sql, args=None):
self.sql=sql
self.args=args
return self
def fetchall(self): return self.rows
def fetchone(self):
key=self.args[0]
return {'id': 1} if any(key in (row['object_key'],row['backup_key']) for row in self.rows) else None
class FakeS3:
def __init__(self,objects): self.objects=objects
def get_paginator(self,op): return self
def paginate(self,**kwargs):
yield {'Contents':[{'Key':k,'LastModified':t} for k,t in self.objects.items()]}
def head_object(self,**kw):
if kw['Key'] not in self.objects:
from botocore.exceptions import ClientError
raise ClientError({'Error':{'Code':'404'}},'HeadObject')
def copy_object(self,**kw):
assert kw['CopySource']['Key'] in self.objects
self.objects[kw['Key']]=NOW
def delete_object(self,**kw):
self.objects.pop(kw['Key'],None)
def test():
i=str(uuid.uuid4())
primary=f'attachments/{i}/original';backup=f'recovery/{i}/original'
young=f'attachments/{uuid.uuid4()}/original'
old=f'attachments/{uuid.uuid4()}/original'
rows=[{'id':i,'object_key':primary,'backup_key':backup,'status':'ready'}]
fake=FakeS3({backup: NOW-timedelta(days=1),young:NOW-timedelta(seconds=10),old:NOW-timedelta(days=1)})
kwargs={'now':NOW,'client':fake,'connect':lambda:FakeConn(rows)}
first=reconcile(apply=False,**kwargs)
assert first['missing_primary_repairable']==[i]
assert old in first['orphan_keys'] and young in first['orphan_keys_young']
assert primary not in fake.objects and old in fake.objects, 'Dry run mutated objects'
second=reconcile(apply=True,**kwargs)
assert primary in fake.objects and old not in fake.objects and young in fake.objects
assert any(x['action']=='restore' for x in second['actions'])
third=reconcile(apply=True,**kwargs)
assert not third['actions'], 'Re-run unexpectedly mutated objects'
assert rows[0]['object_key']==primary, 'Metadata destroyed'
try: reconcile(apply=True, grace_seconds=0, **kwargs)
except ValueError: pass
else: raise AssertionError('Unsafe zero-grace apply accepted')
print('PASS: dry-run, orphan grace, recover primary from backup, apply idempotence')
if __name__ == '__main__': test()
+35
View File
@@ -0,0 +1,35 @@
"""Idempotently create the private S3 bucket with bootstrap-only credentials."""
import os
import time
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError, BotoCoreError
def main():
bucket = os.environ["S3_BUCKET"]
client = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ADMIN_ACCESS_KEY"],
aws_secret_access_key=os.environ["S3_ADMIN_SECRET_KEY"],
region_name="us-east-1",
config=Config(s3={"addressing_style": "path"},
connect_timeout=2, read_timeout=4, retries={"max_attempts": 1}))
for attempt in range(70):
try:
try:
client.head_bucket(Bucket=bucket)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code", "")
if code not in ("404", "NoSuchBucket", "NotFound"):
raise
client.create_bucket(Bucket=bucket)
# PRIVATE: no public bucket policy, ACL or anonymous identity.
print(f"Private bucket {bucket} ready", flush=True)
return
except (ClientError, BotoCoreError, OSError) as exc:
if attempt == 69:
raise RuntimeError("S3 bootstrap failed; inspect S3 container") from exc
time.sleep(1)
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
"""One-off creation of private SeaweedFS S3 identities; secrets stay in Docker volume."""
import json
import os
from pathlib import Path
def main():
bucket = os.environ["S3_BUCKET"]
identities = [
{"name": "bucket-bootstrap", "credentials": [{"accessKey": os.environ["S3_ADMIN_ACCESS_KEY"],
"secretKey": os.environ["S3_ADMIN_SECRET_KEY"]}],
"actions": ["Admin", "Read", "Write", "List"]},
{"name": "messenger-api", "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY"],
"secretKey": os.environ["S3_SECRET_KEY"]}],
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
]
if len({entry["credentials"][0]["accessKey"] for entry in identities}) != 2:
raise ValueError("Admin and API credentials must differ")
path = Path("/s3-config/config.json")
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".tmp")
old_umask = os.umask(0o077)
try:
tmp.write_text(json.dumps({"identities": identities}, separators=(",", ":")), encoding="utf-8")
os.chmod(tmp, 0o644) # readable by the SeaweedFS runtime UID; Docker volume stays private
os.replace(tmp, path)
finally:
os.umask(old_umask)
print("S3 identities configured (no secrets printed)", flush=True)
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
"""Verify bucket is private at S3 boundary (not merely hidden behind HTTP API)."""
import json
import os
import boto3
from botocore import UNSIGNED
from botocore.config import Config
from botocore.exceptions import ClientError
from .attachments import BUCKET, s3_client
def main():
client = s3_client()
client.head_bucket(Bucket=BUCKET) # restricted app identity works
unauth = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
region_name="us-east-1", config=Config(signature_version=UNSIGNED,
s3={"addressing_style":"path"}))
try:
unauth.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status not in (401,403):
raise RuntimeError("Unexpected anonymous S3 response") from None
else:
raise RuntimeError("Private bucket leaked object list to anonymous caller")
listing = client.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
keys = [entry["Key"] for entry in listing.get("Contents", [])]
if keys:
try:
unauth.get_object(Bucket=BUCKET, Key=keys[0])
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status not in (401, 403):
raise RuntimeError("Unexpected anonymous object GET response") from None
else:
raise RuntimeError("Private S3 object downloadable anonymously")
print(json.dumps({"private_bucket": True, "anonymous_listing": "denied",
"anonymous_object_get": "denied" if keys else "not_tested_empty_bucket"}))
if __name__ == "__main__":
main()
+67
View File
@@ -0,0 +1,67 @@
"""Shared external state: psycopg connection pool and Redis session store."""
import os
from contextlib import contextmanager
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from redis import Redis
from redis.exceptions import RedisError
from .core import APIError
from . import telemetry as ot
from opentelemetry.trace import SpanKind
pool: ConnectionPool | None = None
redis_client: Redis | None = None
def start() -> None:
global pool, redis_client
url = os.environ["DATABASE_URL"]
size = int(os.getenv("DB_POOL_MAX", "8"))
pool = ConnectionPool(conninfo=url, min_size=1, max_size=size, timeout=3, kwargs={"row_factory": dict_row, "connect_timeout": 5, "options": "-c statement_timeout=5000"}, open=False)
pool.open(wait=True, timeout=30)
redis_client = Redis.from_url(os.environ.get("REDIS_URL", "redis://redis:6379/0"), socket_connect_timeout=2, socket_timeout=2, decode_responses=True)
redis_client.ping()
def stop() -> None:
global pool, redis_client
if pool is not None:
pool.close()
pool = None
if redis_client is not None:
redis_client.close()
redis_client = None
@contextmanager
def db():
if pool is None:
raise APIError(503, "unavailable", "Database unavailable")
try:
with ot.dependency_span("db.transaction", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
with pool.connection() as conn:
yield conn
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, PoolTimeout) as exc:
raise APIError(503, "unavailable", "Database unavailable") from exc
def cache() -> Redis:
if redis_client is None:
raise APIError(503, "unavailable", "Session store unavailable")
return redis_client
def ready() -> bool:
try:
with db() as conn:
conn.execute("SELECT 1").fetchone()
cache().ping()
# S3 is needed for the mandatory synchronous upload/download path.
from .attachments import s3_client, BUCKET
s3_client().head_bucket(Bucket=BUCKET)
return True
except Exception:
return False
+204
View File
@@ -0,0 +1,204 @@
"""Explicit, allowlisted OTLP instrumentation; no automatic headers, SQL or bodies.
SDK exporters run asynchronously with bounded queues; telemetry is best effort.
"""
import logging
import os
import socket
import time
from contextlib import contextmanager, nullcontext
from pathlib import Path
from opentelemetry import metrics, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
PROP = TraceContextTextMapPropagator()
tracer = trace.get_tracer('messenger.lab6')
log = logging.getLogger('messenger.telemetry')
log.setLevel(logging.INFO)
log.propagate = False
providers = []
started = False
m = None
instruments = {}
worker_busy = 0
backlog_depth = None
backlog_age = None
broker_ready = None
broker_consumers = None
QUEUE = 'attachment-processing.v1'
BUCKETS = (0.005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 15, 30, 60, 120, 300, 600)
def init(role, *, child=False):
global started, tracer, m
if started or os.getenv('OTEL_ENABLED', 'no') != 'yes':
return
started = True
resource = Resource.create({'service.name': 'messenger-' + role,
'service.version': '1.6.0', 'service.instance.id': os.getenv('API_INSTANCE_ID', socket.gethostname()),
'deployment.environment.name': 'lab'})
base = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
tp = TracerProvider(resource=resource)
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=base+'/v1/traces', timeout=2),
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
trace.set_tracer_provider(tp)
tracer = tp.get_tracer('messenger.lab6')
lp = LoggerProvider(resource=resource)
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=base+'/v1/logs', timeout=2),
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
log.addHandler(LoggingHandler(logger_provider=lp))
providers.extend([tp, lp])
if child:
return
reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=base+'/v1/metrics', timeout=2),
export_interval_millis=3000, export_timeout_millis=2000)
mp = MeterProvider(resource=resource, metric_readers=[reader], views=[View(instrument_name=n,
aggregation=ExplicitBucketHistogramAggregation(boundaries=BUCKETS)) for n in
('lab_http_duration','lab_job_duration','lab_ready_duration')])
metrics.set_meter_provider(mp)
providers.append(mp)
m = mp.get_meter('messenger.lab6')
for name in ('lab_http_requests','lab_jobs','lab_job_retries'):
instruments[name] = m.create_counter(name)
for name in ('lab_http_duration','lab_job_duration','lab_ready_duration'):
instruments[name] = m.create_histogram(name, unit='s')
if role == 'api':
for name, key in [('lab_db_pool_used','used'),('lab_db_pool_waiting','requests_waiting'),('lab_db_pool_capacity','pool_max')]:
m.create_observable_gauge(name, callbacks=[lambda _, key=key: pool_stats(key)])
if role == 'worker':
m.create_observable_gauge('lab_worker_busy', callbacks=[lambda _: [metrics.Observation(worker_busy)]])
m.create_observable_gauge('lab_worker_capacity', callbacks=[lambda _: [metrics.Observation(1)]])
if role == 'dispatcher':
m.create_observable_gauge('lab_broker_ready', callbacks=[lambda _: queue_observation(broker_ready)])
m.create_observable_gauge('lab_broker_consumers', callbacks=[lambda _: queue_observation(broker_consumers)])
m.create_observable_gauge('lab_queue_depth', callbacks=[lambda _: queue_observation(backlog_depth)])
m.create_observable_gauge('lab_queue_oldest_age', unit='s', callbacks=[lambda _: queue_observation(backlog_age)])
m.create_observable_gauge('lab_container_memory', unit='By', callbacks=[lambda _: container_observation('memory')])
m.create_observable_gauge('lab_container_cpu_cores', callbacks=[lambda _: container_observation('cpu')])
def pool_stats(key):
from . import storage
if storage.pool is None:
return []
stats = storage.pool.get_stats()
val = stats.get('pool_size',0)-stats.get('pool_available',0) if key=='used' else stats.get(key,0)
return [metrics.Observation(max(0,val))]
def queue_observation(value):
return [] if value is None else [metrics.Observation(value, {'queue': QUEUE})]
_cpu_previous = None
def container_observation(kind):
global _cpu_previous
try:
if kind == 'memory':
return [metrics.Observation(int(Path('/sys/fs/cgroup/memory.current').read_text()))]
cpu = int(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())['usage_usec']) / 1e6
now = time.monotonic()
old = _cpu_previous
_cpu_previous = (now,cpu)
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
except (OSError, KeyError, ValueError):
# cgroup v1 fallback, process tree (not full container accounting).
import psutil
try:
p = psutil.Process()
processes = [p]+p.children(recursive=True)
if kind == 'memory':
value = 0
for proc in processes:
try: value += proc.memory_info().rss
except psutil.Error: pass
return [metrics.Observation(value)]
# cgroup v1 (Docker Linux) if available, CPU usage in nanoseconds.
for path in ('/sys/fs/cgroup/cpuacct/cpuacct.usage','/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage'):
if Path(path).is_file():
cpu = int(Path(path).read_text()) / 1e9
now = time.monotonic()
old = _cpu_previous
_cpu_previous = (now,cpu)
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
except (psutil.Error,OSError,ValueError):
pass
return []
def count(name, attrs=None):
if name in instruments: instruments[name].add(1, attrs or {})
def observe(name, value, attrs=None):
if name in instruments: instruments[name].record(max(0,value), attrs or {})
def event(name, *, level=logging.INFO, **attrs):
# Callers pass fixed event names and technical ID/status only, never payloads.
if started:
log.log(level, name, extra={'event':name, **attrs})
@contextmanager
def span(name, *, kind=SpanKind.INTERNAL, context=None, attrs=None):
with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attrs,
record_exception=False, set_status_on_exception=False) as s:
try:
yield s
except BaseException as exc:
s.set_status(Status(StatusCode.ERROR))
s.set_attribute('error.type', type(exc).__name__)
event('operation.error', level=logging.ERROR, operation=name, error_type=type(exc).__name__)
raise
def inject():
result = {}
PROP.inject(result)
return result
def extract(carrier):
# Ignore baggage and all unrelated fields.
return PROP.extract({k:v for k,v in carrier.items() if k in ('traceparent','tracestate') and isinstance(v,str)})
def dependency_span(name, **kwargs):
return span(name, **kwargs) if trace_id() else nullcontext()
def trace_id():
ctx = trace.get_current_span().get_span_context()
return f'{ctx.trace_id:032x}' if ctx.is_valid else ''
def flush():
for p in providers:
p.force_flush(timeout_millis=2500)
class S3Tracing:
"""Only operation name is exported; bucket/key/bytes/signed URLs never exported."""
def __init__(self, client): self.client = client
def __getattr__(self, name):
f = getattr(self.client,name)
if name not in ('upload_fileobj','get_object','put_object','copy_object','delete_object','head_object','head_bucket'):
return f
def call(*a, **kw):
with dependency_span('s3.'+name, kind=SpanKind.CLIENT, attrs={'storage.system':'s3'}):
return f(*a, **kw)
return call