forked from mtusi2026/mtusi2026lab6
Implement lab6 observability with OpenTelemetry and Grafana
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Lab 3 messenger API."""
|
||||
@@ -0,0 +1,4 @@
|
||||
import os
|
||||
import uvicorn
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run("app.main:app", host="0.0.0.0", port=int(os.getenv("APP_PORT", "8080")), workers=1, access_log=False, timeout_graceful_shutdown=8)
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
"""Messenger HTTP handlers; state lives only in PostgreSQL/Redis."""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import psycopg
|
||||
from argon2 import PasswordHasher
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from .auth import current_user, hasher, login, logout, public_user
|
||||
from .core import APIError, invalid, iso, make_cursor, parse_limit, parse_uuid, read_cursor
|
||||
from .models import CreateChat, CreateMessage, CreateUser
|
||||
from .storage import db
|
||||
|
||||
router = APIRouter()
|
||||
KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII)
|
||||
|
||||
|
||||
def actor(req: Request) -> dict:
|
||||
return getattr(req.state, "actor", None) or current_user(req)
|
||||
|
||||
|
||||
def chat_detail(conn, chat: dict) -> dict:
|
||||
members = conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position", (chat["id"],)).fetchall()
|
||||
return {"id": str(chat["id"]), "title": bytes(chat["title"]).decode("utf-8"), "created_at": iso(chat["created_at"]), "member_ids": [str(x["user_id"]) for x in members]}
|
||||
|
||||
|
||||
def message_detail(conn, row: dict) -> dict:
|
||||
linked = conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (row["id"],)).fetchall()
|
||||
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": bytes(row["text"]).decode("utf-8"), "created_at": iso(row["created_at"]), "attachment_ids": [str(x["attachment_id"]) for x in linked]}
|
||||
|
||||
|
||||
def require_chat(conn, chat_id: str, actor_id: str) -> dict:
|
||||
row = conn.execute("SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id WHERE c.id=%s AND cm.user_id=%s", (chat_id, actor_id)).fetchone()
|
||||
if row is None:
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/health/ready")
|
||||
def ready():
|
||||
from .storage import ready as storage_ready
|
||||
if not storage_ready():
|
||||
raise APIError(503, "unavailable", "Service not ready")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.post("/api/v1/users", status_code=201)
|
||||
def register(body: CreateUser):
|
||||
hashed = hasher.hash(body.password)
|
||||
ident = str(uuid.uuid4())
|
||||
try:
|
||||
with db() as conn:
|
||||
row = conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (ident, body.username, body.display_name.encode("utf-8"), hashed)).fetchone()
|
||||
return public_user(row)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise APIError(409, "conflict", "Username already exists") from None
|
||||
|
||||
|
||||
@router.post("/api/v1/auth/sessions", status_code=201)
|
||||
def sessions(req: Request):
|
||||
return login(req)
|
||||
|
||||
|
||||
@router.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def sessions_delete(req: Request):
|
||||
logout(req)
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/api/v1/users/me")
|
||||
def users_me(req: Request):
|
||||
return actor(req)
|
||||
|
||||
|
||||
@router.post("/api/v1/chats", status_code=201)
|
||||
def chats_create(req: Request, body: CreateChat):
|
||||
ident = actor(req)["id"]
|
||||
if ident not in body.member_ids:
|
||||
raise invalid("Current user must belong to the chat")
|
||||
chat_id = str(uuid.uuid4())
|
||||
try:
|
||||
with db() as conn:
|
||||
with conn.transaction():
|
||||
# Validate all members; FK constraints are the final authority for races.
|
||||
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s::uuid[])", (body.member_ids,)).fetchall()
|
||||
if len(found) != len(body.member_ids):
|
||||
raise invalid("Unknown member")
|
||||
row = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (chat_id, body.title.encode("utf-8"))).fetchone()
|
||||
for pos, member in enumerate(body.member_ids):
|
||||
conn.execute("INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)", (chat_id, member, pos))
|
||||
result = chat_detail(conn, row)
|
||||
return result
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise invalid("Unknown member") from None
|
||||
|
||||
|
||||
@router.get("/api/v1/chats")
|
||||
def chats_list(req: Request, limit: str | None = None, cursor: str | None = None):
|
||||
who = actor(req)["id"]
|
||||
n = parse_limit(limit)
|
||||
mark = read_cursor(cursor, "chats", who)
|
||||
with db() as conn:
|
||||
if mark:
|
||||
existing = conn.execute("SELECT 1 FROM chats c JOIN chat_members cm ON c.id=cm.chat_id WHERE c.id=%s AND c.created_at=%s AND cm.user_id=%s", (mark[1], mark[0], who)).fetchone()
|
||||
if not existing:
|
||||
raise invalid("Invalid cursor")
|
||||
rows = conn.execute("""
|
||||
SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id
|
||||
WHERE cm.user_id=%s AND (%s::timestamptz IS NULL OR (c.created_at,c.id)>(%s::timestamptz,%s::uuid))
|
||||
ORDER BY c.created_at,c.id LIMIT %s
|
||||
""", (who, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
|
||||
items = [chat_detail(conn, row) for row in rows[:n]]
|
||||
more = len(rows) > n
|
||||
last = rows[n-1] if more else None
|
||||
return {"items": items, "next_cursor": make_cursor("chats", who, last["created_at"], str(last["id"])) if last else None}
|
||||
|
||||
|
||||
@router.get("/api/v1/chats/{chat_id}")
|
||||
def chats_get(req: Request, chat_id: str):
|
||||
who = actor(req)["id"]
|
||||
ident = parse_uuid(chat_id)
|
||||
with db() as conn:
|
||||
row = require_chat(conn, ident, who)
|
||||
return chat_detail(conn, row)
|
||||
|
||||
|
||||
@router.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def messages_create(req: Request, chat_id: str, body: CreateMessage):
|
||||
who = actor(req)["id"]
|
||||
ident = parse_uuid(chat_id)
|
||||
key = req.headers.get("Idempotency-Key")
|
||||
if not key or not KEY_RE.fullmatch(key):
|
||||
raise invalid("Invalid Idempotency-Key")
|
||||
# Lab3 has only text; canonical JSON avoids unstable formatting/field ordering.
|
||||
fingerprint = hashlib.sha256(json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
||||
lock_scope = f"{who}:POST:/api/v1/chats/{ident}/messages:{key}"
|
||||
with db() as conn:
|
||||
with conn.transaction():
|
||||
require_chat(conn, ident, who)
|
||||
# Transaction-scoped advisory lock is GLOBAL across API replicas.
|
||||
conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (lock_scope,))
|
||||
previous = conn.execute("""
|
||||
SELECT idem.body_sha256, m.* FROM message_idempotency idem
|
||||
JOIN messages m ON m.id=idem.message_id
|
||||
WHERE idem.actor_id=%s AND idem.chat_id=%s AND idem.method='POST' AND idem.idem_key=%s
|
||||
""", (who, ident, key)).fetchone()
|
||||
if previous:
|
||||
if previous["body_sha256"] != fingerprint:
|
||||
raise APIError(409, "conflict", "Idempotency key used with another body")
|
||||
return message_detail(conn, previous)
|
||||
# 404 for foreign/nonexistent IDs; check BEFORE any message insert.
|
||||
if body.attachment_ids:
|
||||
owned = conn.execute("SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])", (ident, body.attachment_ids)).fetchall()
|
||||
if len(owned) != len(body.attachment_ids):
|
||||
raise APIError(404, "not_found", "Attachment not found")
|
||||
if any(item["status"] == "failed" for item in owned):
|
||||
raise APIError(409, "conflict", "Failed attachment cannot be linked")
|
||||
message_id = str(uuid.uuid4())
|
||||
row = conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (message_id, ident, who, body.text.encode("utf-8"))).fetchone()
|
||||
for pos, attachment_id in enumerate(body.attachment_ids):
|
||||
conn.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, attachment_id, pos))
|
||||
conn.execute("INSERT INTO message_idempotency(actor_id,chat_id,method,idem_key,body_sha256,message_id) VALUES (%s,%s,'POST',%s,%s,%s)", (who, ident, key, fingerprint, message_id))
|
||||
return message_detail(conn, row)
|
||||
|
||||
|
||||
@router.get("/api/v1/chats/{chat_id}/messages")
|
||||
def messages_list(req: Request, chat_id: str, limit: str | None = None, cursor: str | None = None):
|
||||
who = actor(req)["id"]
|
||||
ident = parse_uuid(chat_id)
|
||||
n = parse_limit(limit)
|
||||
mark = read_cursor(cursor, "messages", ident)
|
||||
with db() as conn:
|
||||
require_chat(conn, ident, who)
|
||||
if mark:
|
||||
existing = conn.execute("SELECT 1 FROM messages WHERE id=%s AND chat_id=%s AND created_at=%s", (mark[1], ident, mark[0])).fetchone()
|
||||
if not existing:
|
||||
raise invalid("Invalid cursor")
|
||||
rows = conn.execute("""
|
||||
SELECT * FROM messages WHERE chat_id=%s
|
||||
AND (%s::timestamptz IS NULL OR (created_at,id)>(%s::timestamptz,%s::uuid))
|
||||
ORDER BY created_at,id LIMIT %s
|
||||
""", (ident, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
|
||||
with db() as conn:
|
||||
items = [message_detail(conn, row) for row in rows[:n]]
|
||||
last = rows[n-1] if len(rows)>n else None
|
||||
return {"items": items, "next_cursor": make_cursor("messages", ident, last["created_at"], str(last["id"])) if last else None}
|
||||
@@ -0,0 +1,240 @@
|
||||
"""Lab5 attachments: synchronous durable admission, asynchronous image pipeline.
|
||||
|
||||
202 means verified bytes were staged to private S3 and metadata + a job + an
|
||||
outbox event were committed in ONE PostgreSQL transaction. No broker availability
|
||||
is required for admission; the dispatcher replays unsent intents.
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
from contextlib import closing
|
||||
|
||||
import boto3
|
||||
from boto3.s3.transfer import TransferConfig
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from PIL import Image, ImageFile, UnidentifiedImageError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .auth import current_user
|
||||
from .core import APIError, invalid, iso, parse_uuid
|
||||
from .storage import db
|
||||
from . import telemetry as ot
|
||||
|
||||
router = APIRouter()
|
||||
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
SPOOL_MEMORY_BYTES = 1024 * 1024
|
||||
S3_BUFFER_BYTES = 256 * 1024
|
||||
DECODE_LIMIT = threading.BoundedSemaphore(1)
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
|
||||
|
||||
def s3_client():
|
||||
return ot.S3Tracing(boto3.client(
|
||||
"s3", endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
|
||||
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
|
||||
region_name="us-east-1", use_ssl=False,
|
||||
config=Config(s3={"addressing_style": "path"}, connect_timeout=3,
|
||||
read_timeout=10, retries={"max_attempts": 1}),
|
||||
))
|
||||
|
||||
|
||||
def s3_error(exc):
|
||||
return APIError(503, "unavailable", "Object storage unavailable")
|
||||
|
||||
|
||||
def format_attachment(row, variants=()):
|
||||
return {
|
||||
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]), "filename": row["filename"],
|
||||
"content_type": row["content_type"], "size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"], "status": row["status"],
|
||||
"variants": [{k: item[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
|
||||
for item in variants] if row["status"] == "ready" else [],
|
||||
"error_code": row["error_code"] if row["status"] == "failed" else None,
|
||||
"created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
def require_member(conn, chat_id, user_id):
|
||||
if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone():
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
|
||||
|
||||
def attachment_row(conn, attachment_id, user_id):
|
||||
row = conn.execute("""
|
||||
SELECT a.* FROM attachments a
|
||||
JOIN chat_members cm ON cm.chat_id=a.chat_id
|
||||
WHERE a.id=%s AND cm.user_id=%s
|
||||
""", (attachment_id, user_id)).fetchone()
|
||||
if not row:
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
return row
|
||||
|
||||
|
||||
def validate_filename(filename: str):
|
||||
if not (1 <= len(filename) <= 255) or any(c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename):
|
||||
raise invalid("Invalid filename")
|
||||
|
||||
|
||||
def validate_image(fileobj, mime: str):
|
||||
"""Validate MIME, magic, dimensions and complete decode; worker does the resize."""
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
with Image.open(fileobj) as im:
|
||||
if im.format not in ("PNG", "JPEG"):
|
||||
raise APIError(415, "unsupported_media_type", "Unsupported image encoding")
|
||||
actual = "image/png" if im.format == "PNG" else "image/jpeg"
|
||||
if actual != mime:
|
||||
raise APIError(415, "unsupported_media_type", "Content-Type does not match image")
|
||||
if im.width * im.height > MAX_PIXELS:
|
||||
raise APIError(413, "payload_too_large", "Image dimensions too large")
|
||||
im.verify()
|
||||
fileobj.seek(0)
|
||||
with DECODE_LIMIT:
|
||||
with Image.open(fileobj) as im:
|
||||
im.load()
|
||||
except APIError:
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
raise APIError(415, "unsupported_media_type", "Invalid image") from None
|
||||
finally:
|
||||
fileobj.seek(0)
|
||||
|
||||
|
||||
def _admit_upload(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest):
|
||||
from .jobs import job_payload
|
||||
stage_key = f"staging/{attachment_id}/original"
|
||||
key = f"attachments/{attachment_id}/original"
|
||||
backup = f"recovery/{attachment_id}/original"
|
||||
job_id = str(uuid.uuid4())
|
||||
outbox_id = str(uuid.uuid4())
|
||||
client = s3_client()
|
||||
try:
|
||||
spool.seek(0)
|
||||
client.upload_fileobj(spool, BUCKET, stage_key,
|
||||
ExtraArgs={"ContentType": mime},
|
||||
Config=TransferConfig(use_threads=False, multipart_threshold=16*1024*1024,
|
||||
io_chunksize=S3_BUFFER_BYTES))
|
||||
except (BotoCoreError, ClientError, EndpointConnectionError, OSError) as exc:
|
||||
raise s3_error(exc) from exc
|
||||
|
||||
# Atomic DB metadata + intent. If DB commit is ambiguous, do not delete the
|
||||
# staged object: an accepted task may already exist; operator cleanup handles it.
|
||||
with db() as conn:
|
||||
with conn.transaction():
|
||||
require_member(conn, chat_id, user_id)
|
||||
# Small educational admission bound; pre-existing queued tasks remain safe.
|
||||
pending = conn.execute("SELECT count(*) AS c FROM processing_jobs WHERE status IN ('queued','processing')").fetchone()["c"]
|
||||
if pending >= int(os.getenv("JOB_BACKLOG_LIMIT", "1000")):
|
||||
raise APIError(429, "rate_limited", "Processing queue is full")
|
||||
row = conn.execute("""
|
||||
INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename,
|
||||
content_type,size_bytes,sha256,status,staging_key,pipeline_version)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,1) RETURNING *
|
||||
""", (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest, stage_key)).fetchone()
|
||||
conn.execute("""
|
||||
INSERT INTO processing_jobs(id,attachment_id,pipeline_version,status,attempt)
|
||||
VALUES (%s,%s,1,'queued',1)
|
||||
""", (job_id, attachment_id))
|
||||
payload = job_payload(job_id, attachment_id, 1, 1)
|
||||
conn.execute("""
|
||||
INSERT INTO job_outbox(id,job_id,payload) VALUES (%s,%s,%s::jsonb)
|
||||
""", (outbox_id, job_id, __import__('json').dumps(payload)))
|
||||
ot.event("upload.accepted", job_id=job_id)
|
||||
return format_attachment(row) # snapshot = queued, even if worker is very fast
|
||||
|
||||
|
||||
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)):
|
||||
from .api import actor, require_chat
|
||||
who = actor(req)["id"]
|
||||
chat_id = parse_uuid(chat_id)
|
||||
validate_filename(filename)
|
||||
mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if mime not in ("image/png", "image/jpeg"):
|
||||
raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
|
||||
with db() as conn:
|
||||
require_chat(conn, chat_id, who)
|
||||
announced = req.headers.get("content-length")
|
||||
if announced:
|
||||
try:
|
||||
if int(announced) > MAX_BYTES:
|
||||
raise APIError(413, "payload_too_large", "Image too large")
|
||||
if int(announced) < 0:
|
||||
raise invalid("Invalid Content-Length")
|
||||
except ValueError:
|
||||
raise invalid("Invalid Content-Length") from None
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool:
|
||||
async for chunk in req.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise APIError(413, "payload_too_large", "Image too large")
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
if not size:
|
||||
raise APIError(415, "unsupported_media_type", "Empty image")
|
||||
await run_in_threadpool(validate_image, spool, mime)
|
||||
return await run_in_threadpool(_admit_upload, spool, attachment_id=str(uuid.uuid4()),
|
||||
chat_id=chat_id, user_id=who, filename=filename,
|
||||
mime=mime, size=size, digest=digest.hexdigest())
|
||||
|
||||
|
||||
@router.get("/api/v1/attachments/{attachment_id}")
|
||||
def get_attachment(req: Request, attachment_id: str):
|
||||
who = getattr(req.state, "actor", None) or current_user(req)
|
||||
with db() as conn:
|
||||
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
|
||||
variants = conn.execute("""
|
||||
SELECT name,content_type,size_bytes,width,height,sha256
|
||||
FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s
|
||||
ORDER BY CASE name WHEN 'thumbnail' THEN 0 WHEN 'crop' THEN 1 ELSE 2 END
|
||||
""", (row["id"], row["pipeline_version"])).fetchall() if row["status"] == "ready" else []
|
||||
return format_attachment(row, variants)
|
||||
|
||||
|
||||
@router.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download_attachment(req: Request, attachment_id: str, variant: str = "original"):
|
||||
who = getattr(req.state, "actor", None) or current_user(req)
|
||||
with db() as conn:
|
||||
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
|
||||
if row["status"] != "ready":
|
||||
raise APIError(409, "conflict", "Attachment is not ready")
|
||||
if variant == "original":
|
||||
key, mime, size = row["object_key"], row["content_type"], row["size_bytes"]
|
||||
else:
|
||||
v = conn.execute("""
|
||||
SELECT * FROM attachment_variants
|
||||
WHERE attachment_id=%s AND pipeline_version=%s AND name=%s
|
||||
""", (row["id"], row["pipeline_version"], variant)).fetchone()
|
||||
if not v:
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
key, mime, size = v["object_key"], v["content_type"], v["size_bytes"]
|
||||
client = s3_client()
|
||||
try:
|
||||
response = client.get_object(Bucket=BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"):
|
||||
raise APIError(503, "unavailable", "Stored object missing") from exc
|
||||
raise s3_error(exc) from exc
|
||||
except BotoCoreError as exc:
|
||||
raise s3_error(exc) from exc
|
||||
|
||||
def iterator():
|
||||
with closing(response["Body"]) as body:
|
||||
while chunk := body.read(S3_BUFFER_BYTES):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(iterator(), media_type=mime, headers={"Content-Length": str(size)})
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
"""HTTP Basic only on login; bearer sessions shared by all API replicas."""
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError, VerificationError
|
||||
from fastapi import Request
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .core import APIError, SESSION_TTL, iso
|
||||
from .storage import cache, db
|
||||
|
||||
hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
PASSWORD_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
|
||||
|
||||
def public_user(row: dict):
|
||||
return {"id": str(row["id"]), "username": row["username"], "display_name": bytes(row["display_name"]).decode("utf-8"), "created_at": iso(row["created_at"])}
|
||||
|
||||
|
||||
def unauthorized() -> APIError:
|
||||
return APIError(401, "unauthorized", "Unauthorized")
|
||||
|
||||
|
||||
def challenge(req: Request) -> str:
|
||||
return 'Basic realm="messenger", charset="UTF-8"' if req.url.path == "/api/v1/auth/sessions" else 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def bearer_token(req: Request) -> str:
|
||||
value = req.headers.get("Authorization", "")
|
||||
if not value.startswith("Bearer ") or len(value) <= 7 or " " in value[7:] or len(value) > 512:
|
||||
raise unauthorized()
|
||||
try:
|
||||
value[7:].encode("ascii")
|
||||
except UnicodeEncodeError:
|
||||
raise unauthorized() from None
|
||||
return value[7:]
|
||||
|
||||
|
||||
def session_key(token: str) -> str:
|
||||
return "session:" + hashlib.sha256(token.encode("ascii")).hexdigest()
|
||||
|
||||
|
||||
def current_user(req: Request) -> dict:
|
||||
token = bearer_token(req)
|
||||
try:
|
||||
raw = cache().get(session_key(token))
|
||||
except (RedisError, UnicodeError) as exc:
|
||||
raise APIError(503, "unavailable", "Session store unavailable") from exc
|
||||
if not raw:
|
||||
raise unauthorized()
|
||||
try:
|
||||
ident = str(uuid.UUID(raw))
|
||||
except (ValueError, TypeError):
|
||||
try:
|
||||
cache().delete(session_key(token))
|
||||
except RedisError as exc:
|
||||
raise APIError(503, "unavailable", "Session store unavailable") from exc
|
||||
raise unauthorized() from None
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE id = %s", (ident,)).fetchone()
|
||||
if row is None:
|
||||
raise unauthorized()
|
||||
return public_user(row)
|
||||
|
||||
|
||||
def login(req: Request):
|
||||
auth = req.headers.get("Authorization", "")
|
||||
if not auth.startswith("Basic "):
|
||||
raise unauthorized()
|
||||
try:
|
||||
decoded = base64.b64decode(auth[6:].encode("ascii"), validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
if not PASSWORD_RE.fullmatch(username):
|
||||
raise ValueError()
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise unauthorized() from None
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
try:
|
||||
hasher.verify(row["password_hash"], password)
|
||||
except (VerifyMismatchError, VerificationError):
|
||||
raise unauthorized() from None
|
||||
token = secrets.token_urlsafe(32)
|
||||
try:
|
||||
cache().set(session_key(token), str(row["id"]), ex=SESSION_TTL)
|
||||
except RedisError as exc:
|
||||
raise APIError(503, "unavailable", "Session store unavailable") from exc
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": iso(datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)), "user": public_user(row)}
|
||||
|
||||
|
||||
def logout(req: Request):
|
||||
current_user(req)
|
||||
try:
|
||||
cache().delete(session_key(bearer_token(req)))
|
||||
except RedisError as exc:
|
||||
raise APIError(503, "unavailable", "Session store unavailable") from exc
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
"""Configuration, HTTP errors and shared representation helpers."""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
INSTANCE_ID = os.getenv("API_INSTANCE_ID", uuid.uuid4().hex[:12])
|
||||
SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
IDEMPOTENCY_TTL = max(86400, int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")))
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
def iso(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def invalid(message: str = "Invalid request") -> APIError:
|
||||
return APIError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def parse_uuid(value: str) -> str:
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise invalid("Invalid UUID") from None
|
||||
|
||||
|
||||
def parse_limit(raw: str | None) -> int:
|
||||
if raw is None:
|
||||
return 50
|
||||
try:
|
||||
if not raw.isascii() or not raw.isdecimal():
|
||||
raise ValueError()
|
||||
value = int(raw)
|
||||
except (ValueError, AttributeError):
|
||||
raise invalid("Invalid limit") from None
|
||||
if not 1 <= value <= 100:
|
||||
raise invalid("Invalid limit")
|
||||
return value
|
||||
|
||||
|
||||
def make_cursor(kind: str, scope: str, created: datetime, ident: str) -> str:
|
||||
b = json.dumps({"kind": kind, "scope": scope, "created_at": iso(created), "id": ident}, separators=(",", ":")).encode()
|
||||
return base64.urlsafe_b64encode(b).decode().rstrip("=")
|
||||
|
||||
|
||||
def read_cursor(raw: str | None, kind: str, scope: str):
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
if not raw or len(raw) > 2048 or any(ch not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" for ch in raw):
|
||||
raise ValueError()
|
||||
data = json.loads(base64.b64decode(raw + "=" * (-len(raw) % 4), altchars=b"-_", validate=True))
|
||||
if not isinstance(data, dict) or data.get("kind") != kind or data.get("scope") != scope:
|
||||
raise ValueError()
|
||||
created = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
|
||||
if created.tzinfo is None:
|
||||
raise ValueError()
|
||||
ident = str(uuid.UUID(data["id"]))
|
||||
return created, ident
|
||||
except (ValueError, TypeError, KeyError, UnicodeError, json.JSONDecodeError):
|
||||
raise invalid("Invalid cursor") from None
|
||||
|
||||
|
||||
def request_id(req: Request) -> str:
|
||||
return getattr(req.state, "request_id", uuid.uuid4().hex)
|
||||
|
||||
|
||||
def error_body(req: Request, code: str, message: str):
|
||||
return {"error": {"code": code, "message": message, "request_id": request_id(req)}}
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Sandboxed-per-task image work. Process is killed after JOB_TIMEOUT_SECONDS.
|
||||
|
||||
Outputs only JSON variant metadata. Original bytes are copied from durable staging,
|
||||
never transformed. Pillow reopens/exif-transposes pixels for derivatives, stripping
|
||||
metadata on re-encode. No API tokens or arbitrary user paths enter this command.
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
|
||||
from .attachments import BUCKET, MAX_BYTES, MAX_PIXELS, s3_client
|
||||
from . import telemetry as ot
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
|
||||
|
||||
def rgb_image(src):
|
||||
orient = ImageOps.exif_transpose(src)
|
||||
orient.load()
|
||||
if orient.width * orient.height > MAX_PIXELS:
|
||||
raise ValueError("too many decoded pixels")
|
||||
# Compose transparency on white, strip EXIF/ICC and do not enlarge.
|
||||
rgba = orient.convert("RGBA")
|
||||
white = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
|
||||
white.alpha_composite(rgba)
|
||||
return white.convert("RGB")
|
||||
|
||||
|
||||
def transform(raw):
|
||||
with Image.open(io.BytesIO(raw)) as image:
|
||||
if image.format not in ("PNG", "JPEG"):
|
||||
raise ValueError("invalid encoding")
|
||||
rgb = rgb_image(image)
|
||||
t = rgb.copy()
|
||||
t.thumbnail((256, 256), resample=Image.Resampling.LANCZOS)
|
||||
# Center crop fit scales up small images when needed.
|
||||
crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
|
||||
optim = rgb.copy()
|
||||
optim.thumbnail((1280, 1280), resample=Image.Resampling.LANCZOS)
|
||||
return [
|
||||
("thumbnail", t, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
|
||||
("crop", crop, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
|
||||
("optimized", optim, "WEBP", "image/webp", {"quality": 78, "method": 4}),
|
||||
]
|
||||
|
||||
|
||||
def run(attachment_id, pipeline_version):
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
with ot.span("db.load_staging_metadata", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5) as conn:
|
||||
row = conn.execute("""
|
||||
SELECT * FROM attachments WHERE id=%s AND pipeline_version=%s
|
||||
""", (attachment_id, pipeline_version)).fetchone()
|
||||
if not row or not row["staging_key"]:
|
||||
raise ValueError("staging metadata not found")
|
||||
|
||||
client = s3_client()
|
||||
body = client.get_object(Bucket=BUCKET, Key=row["staging_key"])["Body"]
|
||||
try:
|
||||
raw = body.read(MAX_BYTES + 1)
|
||||
finally:
|
||||
body.close()
|
||||
if not raw or len(raw) > MAX_BYTES or len(raw) != row["size_bytes"]:
|
||||
raise ValueError("staging size mismatch")
|
||||
if hashlib.sha256(raw).hexdigest() != row["sha256"]:
|
||||
raise ValueError("staging checksum mismatch")
|
||||
|
||||
# Recompute every result if a prior run was interrupted. Keys are deterministic
|
||||
# for (attachment_id,pipeline_version,variant), so duplicates don't multiply objects.
|
||||
output = []
|
||||
for name, image, fmt, mime, options in transform(raw):
|
||||
buf = io.BytesIO()
|
||||
image.save(buf, format=fmt, **options)
|
||||
encoded = buf.getvalue()
|
||||
key = f"variants/{attachment_id}/v{pipeline_version}/{name}"
|
||||
client.put_object(Bucket=BUCKET, Key=key, Body=encoded, ContentType=mime)
|
||||
output.append({"name": name, "content_type": mime,
|
||||
"size_bytes": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(),
|
||||
"width": image.width, "height": image.height, "object_key": key})
|
||||
|
||||
# Only after derivatives exist, publish unchanged primary and recovery copy.
|
||||
client.copy_object(Bucket=BUCKET, Key=row["object_key"],
|
||||
CopySource={"Bucket": BUCKET, "Key": row["staging_key"]},
|
||||
MetadataDirective="REPLACE", ContentType=row["content_type"])
|
||||
client.copy_object(Bucket=BUCKET, Key=row["backup_key"],
|
||||
CopySource={"Bucket": BUCKET, "Key": row["object_key"]},
|
||||
MetadataDirective="REPLACE", ContentType=row["content_type"])
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
import uuid
|
||||
if len(sys.argv) != 3:
|
||||
raise SystemExit(2)
|
||||
try:
|
||||
ident = str(uuid.UUID(sys.argv[1]))
|
||||
version = int(sys.argv[2])
|
||||
if version < 1:
|
||||
raise ValueError("invalid version")
|
||||
ot.init("worker", child=True)
|
||||
with ot.span("image.transform", context=ot.extract(json.loads(os.getenv("LAB6_PARENT_CONTEXT", "{}")))):
|
||||
out = run(ident, version)
|
||||
ot.event("image.transformed")
|
||||
print(json.dumps({"variants": out}, ensure_ascii=False))
|
||||
except (ValueError, UnidentifiedImageError, Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
|
||||
print("Permanent image processing failure", file=sys.stderr)
|
||||
raise SystemExit(20) from None
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
print("Transient object storage failure", file=sys.stderr)
|
||||
raise SystemExit(30) from None
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
ot.flush()
|
||||
+470
@@ -0,0 +1,470 @@
|
||||
"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases.
|
||||
|
||||
There is no exactly-once transport guarantee. We guarantee eventual recovery of
|
||||
committed intent and idempotent DB results under at-least-once redelivery.
|
||||
Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
from . import telemetry as ot
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from pika.exceptions import AMQPError
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
log = logging.getLogger("lab5.jobs")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
QUEUE = "attachment-processing.v1"
|
||||
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
|
||||
LEASE_SECONDS = JOB_TIMEOUT + 45
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connect_db():
|
||||
# No credentials, SQL text, or parameters in spans.
|
||||
with ot.dependency_span('db.job.transaction', kind=SpanKind.CLIENT, attrs={'db.system.name':'postgresql'}):
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5, options='-c statement_timeout=5000') as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
def job_payload(job_id, attachment_id, version, attempt):
|
||||
return {
|
||||
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
|
||||
"attachment_id": str(attachment_id), "pipeline_version": version,
|
||||
"attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(),
|
||||
**ot.inject(),
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection():
|
||||
params = pika.ConnectionParameters(
|
||||
host=os.getenv("BROKER_HOST", "broker"), port=5672,
|
||||
credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]),
|
||||
heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120,
|
||||
socket_timeout=5, connection_attempts=3, retry_delay=1)
|
||||
mq = pika.BlockingConnection(params)
|
||||
try:
|
||||
yield mq
|
||||
finally:
|
||||
try:
|
||||
if mq.is_open:
|
||||
mq.close()
|
||||
except AMQPError:
|
||||
pass
|
||||
|
||||
|
||||
def setup_channel(conn):
|
||||
ch = conn.channel()
|
||||
ch.queue_declare(queue=QUEUE, durable=True)
|
||||
return ch
|
||||
|
||||
|
||||
def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None):
|
||||
payload = job_payload(job_id, attachment_id, version, attempt)
|
||||
if not payload.get('traceparent'):
|
||||
# Lease rescue/replay has no active consumer context. Preserve the first
|
||||
# acceptance context rather than inventing a new disconnected trace.
|
||||
previous = db.execute("SELECT payload FROM job_outbox WHERE job_id=%s ORDER BY created_at,id LIMIT 1", (job_id,)).fetchone()
|
||||
if previous:
|
||||
payload.update({k: previous['payload'][k] for k in ('traceparent','tracestate') if k in previous['payload']})
|
||||
db.execute("""
|
||||
INSERT INTO job_outbox(id,job_id,payload,available_at)
|
||||
VALUES (%s,%s,%s::jsonb,COALESCE(%s,now()))
|
||||
""", (str(uuid.uuid4()), job_id, json.dumps(payload), available_at))
|
||||
|
||||
|
||||
def cleanup_abandoned_staging():
|
||||
"""Best effort cleanup of retained stages; never delete current processing work."""
|
||||
from .attachments import BUCKET, s3_client
|
||||
client = s3_client()
|
||||
retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
|
||||
with connect_db() as db:
|
||||
rows = db.execute("""
|
||||
SELECT id,staging_key FROM attachments
|
||||
WHERE staging_key IS NOT NULL AND
|
||||
((status='ready' AND updated_at < now()-interval '300 seconds') OR
|
||||
(status='failed' AND updated_at < now()-(%s*interval '1 second')))
|
||||
LIMIT 50
|
||||
""", (retention,)).fetchall()
|
||||
for row in rows:
|
||||
with connect_db() as db:
|
||||
with db.transaction():
|
||||
# Row-level lock serializes against operator replay.
|
||||
current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone()
|
||||
if not current or current["staging_key"] != row["staging_key"]:
|
||||
continue
|
||||
age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds()
|
||||
if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention):
|
||||
continue
|
||||
client.delete_object(Bucket=BUCKET, Key=row["staging_key"])
|
||||
db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"]))
|
||||
# Also prune staging uploads whose DB COMMIT failed, after a long retention.
|
||||
from botocore.exceptions import ClientError
|
||||
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"):
|
||||
for obj in page.get("Contents", []):
|
||||
if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention:
|
||||
continue
|
||||
key = obj["Key"]
|
||||
with connect_db() as db:
|
||||
with db.transaction():
|
||||
referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
|
||||
if not referenced:
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
|
||||
def rescue_expired_jobs(conn):
|
||||
"""Requeue leased jobs whose worker died (durably, no manual task recreation)."""
|
||||
with conn.transaction():
|
||||
rows = conn.execute("""
|
||||
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt
|
||||
FROM processing_jobs j
|
||||
WHERE j.status='processing' AND j.leased_until < now()
|
||||
FOR UPDATE SKIP LOCKED LIMIT 20
|
||||
""").fetchall()
|
||||
for row in rows:
|
||||
if row["attempt"] >= MAX_ATTEMPTS:
|
||||
conn.execute("""UPDATE processing_jobs
|
||||
SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now()
|
||||
WHERE id=%s""", (row["id"],))
|
||||
conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now()
|
||||
WHERE id=%s""", (row["attachment_id"],))
|
||||
else:
|
||||
attempt = row["attempt"] + 1
|
||||
conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,
|
||||
lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now()
|
||||
WHERE id=%s""", (attempt, row["id"]))
|
||||
conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],))
|
||||
queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt)
|
||||
log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt)
|
||||
|
||||
|
||||
def sample_backlog():
|
||||
# Query durable *queued* jobs, including delayed retries and unpublished outbox.
|
||||
# No broker membership scan; oldest age is wall clock since initial acceptance.
|
||||
while True:
|
||||
try:
|
||||
with psycopg.connect(os.environ['DATABASE_URL'], row_factory=dict_row, connect_timeout=3, options='-c statement_timeout=2000') as db:
|
||||
row = db.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM processing_jobs WHERE status='queued'").fetchone()
|
||||
ot.backlog_depth, ot.backlog_age = row['depth'], float(row['age'])
|
||||
except psycopg.Error:
|
||||
ot.backlog_depth = ot.backlog_age = None
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def dispatch_loop():
|
||||
"""Publish outbox rows with publisher confirm, mark only AFTER confirm."""
|
||||
threading.Thread(target=sample_backlog, daemon=True).start()
|
||||
while True:
|
||||
try:
|
||||
with connection() as mq:
|
||||
ch = setup_channel(mq)
|
||||
ch.confirm_delivery()
|
||||
last_clean = time.monotonic()
|
||||
last_queue_sample = 0
|
||||
while True:
|
||||
# Periodic lease recovery works even if original message was consumed.
|
||||
with connect_db() as conn:
|
||||
rescue_expired_jobs(conn)
|
||||
with conn.transaction():
|
||||
rows = conn.execute("""
|
||||
SELECT id,payload FROM job_outbox
|
||||
WHERE published_at IS NULL AND available_at<=now()
|
||||
ORDER BY available_at,created_at
|
||||
FOR UPDATE SKIP LOCKED LIMIT 20
|
||||
""").fetchall()
|
||||
for row in rows:
|
||||
if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once":
|
||||
# Persist one-time fault before hard exit; outbox still
|
||||
# unmarked and dispatcher will publish on restart.
|
||||
with connect_db() as faultdb:
|
||||
inserted = faultdb.execute("""
|
||||
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish')
|
||||
ON CONFLICT DO NOTHING RETURNING job_id
|
||||
""", (row["payload"]["job_id"],)).fetchone()
|
||||
if inserted:
|
||||
os._exit(78)
|
||||
payload = row['payload']
|
||||
with ot.span('rabbitmq.publish', kind=SpanKind.PRODUCER, context=ot.extract(payload),
|
||||
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE,'job.attempt':payload['attempt']}) as producer:
|
||||
# Propagate actual producer span, while DB retains the original intent.
|
||||
raw = json.dumps({**payload, **ot.inject()}, separators=(",", ":"))
|
||||
ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"),
|
||||
mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2))
|
||||
conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],))
|
||||
ot.event('job.published', job_id=payload['job_id'], attempt=payload['attempt'])
|
||||
if time.monotonic() - last_queue_sample > 3:
|
||||
q = ch.queue_declare(queue=QUEUE, passive=True).method
|
||||
ot.broker_ready, ot.broker_consumers = q.message_count, q.consumer_count
|
||||
last_queue_sample = time.monotonic()
|
||||
if time.monotonic() - last_clean > 60:
|
||||
try:
|
||||
cleanup_abandoned_staging()
|
||||
except Exception:
|
||||
log.warning("staging janitor failed; retry later")
|
||||
last_clean = time.monotonic()
|
||||
if not rows:
|
||||
mq.sleep(1)
|
||||
except (AMQPError, OSError, psycopg.Error) as exc:
|
||||
ot.broker_ready = ot.broker_consumers = None
|
||||
log.warning("dispatcher reconnect after %s", type(exc).__name__)
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def claim_job(db, message):
|
||||
job_id = message["job_id"]
|
||||
with db.transaction():
|
||||
job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone()
|
||||
if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]:
|
||||
return "ignore", None
|
||||
if job["status"] == "processing":
|
||||
return "busy", None
|
||||
if job["next_run_at"] > datetime.now(timezone.utc):
|
||||
return "busy", None
|
||||
lease = str(uuid.uuid4())
|
||||
db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s,
|
||||
leased_until=now()+(%s*interval '1 second'),updated_at=now()
|
||||
WHERE id=%s""", (lease, LEASE_SECONDS, job_id))
|
||||
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],))
|
||||
return "claimed", {"job": job, "lease": lease}
|
||||
|
||||
|
||||
def publish_result(db, *, job, lease, variants):
|
||||
"""Final result is committed atomically; stale leases/versions cannot publish."""
|
||||
with db.transaction():
|
||||
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
|
||||
if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]:
|
||||
log.warning("stale worker result ignored %s", job["id"])
|
||||
return False
|
||||
for v in variants:
|
||||
db.execute("""
|
||||
INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
||||
ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET
|
||||
object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,
|
||||
size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256,
|
||||
width=EXCLUDED.width,height=EXCLUDED.height
|
||||
""", (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"],
|
||||
v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"]))
|
||||
db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now()
|
||||
WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"]))
|
||||
db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL,
|
||||
error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],))
|
||||
return True
|
||||
|
||||
|
||||
def fail_job(db, job, lease, *, permanent=False, code="processing_failed"):
|
||||
with db.transaction():
|
||||
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
|
||||
if current["status"] != "processing" or str(current["lease_token"]) != lease:
|
||||
return "stale"
|
||||
if permanent or job["attempt"] >= MAX_ATTEMPTS:
|
||||
code = code if permanent else "retry_exhausted"
|
||||
db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL,
|
||||
leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"]))
|
||||
db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"]))
|
||||
return "failed"
|
||||
else:
|
||||
attempt = job["attempt"] + 1
|
||||
delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0)
|
||||
run_at = datetime.now(timezone.utc) + timedelta(seconds=delay)
|
||||
db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s,
|
||||
lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"]))
|
||||
db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],))
|
||||
queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at)
|
||||
log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay)
|
||||
return "retry"
|
||||
|
||||
|
||||
def cleanup_staging(job):
|
||||
from .attachments import BUCKET, s3_client
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
try:
|
||||
with connect_db() as conn:
|
||||
row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone()
|
||||
if row and row["status"] == "ready" and row["staging_key"]:
|
||||
s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"])
|
||||
with connect_db() as conn:
|
||||
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s",
|
||||
(job["attachment_id"], row["staging_key"]))
|
||||
except (BotoCoreError, ClientError, psycopg.Error, OSError):
|
||||
log.warning("staging cleanup deferred for %s", job["id"])
|
||||
|
||||
|
||||
def process_message(raw):
|
||||
try:
|
||||
carrier = json.loads(raw)
|
||||
if not isinstance(carrier, dict): carrier = {}
|
||||
except (ValueError, TypeError):
|
||||
carrier = {}
|
||||
with ot.span('rabbitmq.consume', kind=SpanKind.CONSUMER, context=ot.extract(carrier),
|
||||
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE}) as consumer:
|
||||
return process_message_inner(raw, consumer)
|
||||
|
||||
|
||||
def process_message_inner(raw, consumer):
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1"
|
||||
uuid.UUID(msg["job_id"])
|
||||
uuid.UUID(msg["attachment_id"])
|
||||
assert type(msg["attempt"]) is int and msg["attempt"] >= 1
|
||||
except (ValueError, KeyError, AssertionError, TypeError):
|
||||
log.error("invalid internal broker message discarded")
|
||||
return "ack" # no secrets printed, does not touch DB
|
||||
with connect_db() as db:
|
||||
status, state = claim_job(db, msg)
|
||||
if status == "ignore":
|
||||
return "ack"
|
||||
if status == "busy":
|
||||
return "requeue"
|
||||
job, lease = state["job"], state["lease"]
|
||||
consumer.set_attribute('job.attempt',job['attempt'])
|
||||
consumer.set_attribute('job.id',str(job['id']))
|
||||
ot.event('job.claimed', job_id=str(job['id']), attempt=job['attempt'])
|
||||
start = time.monotonic()
|
||||
ot.worker_busy = 1
|
||||
def done(outcome):
|
||||
attrs = {'job_type':'attachment.process.v1','outcome':outcome}
|
||||
ot.count('lab_jobs',attrs)
|
||||
ot.observe('lab_job_duration',time.monotonic()-start,attrs)
|
||||
if outcome == 'retry':
|
||||
ot.count('lab_job_retries',{'job_type':'attachment.process.v1'})
|
||||
if outcome in ('retry','failed','requeue'):
|
||||
consumer.set_status(Status(StatusCode.ERROR))
|
||||
consumer.set_attribute('job.outcome',outcome)
|
||||
ot.event('job.'+outcome, level=logging.ERROR if outcome in ('failed','retry','requeue') else logging.INFO, job_id=str(job['id']), attempt=job['attempt'])
|
||||
try:
|
||||
if os.getenv('LAB6_FAULT_RETRY_ONCE') == 'once':
|
||||
with connect_db() as db:
|
||||
inserted = db.execute("INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'lab6_retry') ON CONFLICT DO NOTHING RETURNING job_id", (job['id'],)).fetchone()
|
||||
if inserted:
|
||||
raise RuntimeError('injected transient retry')
|
||||
cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])]
|
||||
with ot.span('image.subprocess'):
|
||||
env = {**os.environ, 'LAB6_PARENT_CONTEXT': json.dumps(ot.inject())}
|
||||
proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False, env=env)
|
||||
if proc.returncode != 0:
|
||||
with connect_db() as db:
|
||||
outcome = fail_job(db, job, lease, permanent=(proc.returncode == 20),
|
||||
code="invalid_image" if proc.returncode == 20 else "processing_failed")
|
||||
done(outcome)
|
||||
return "ack" # next attempt is already safely in DB outbox
|
||||
result = json.loads(proc.stdout)
|
||||
variants = result["variants"]
|
||||
if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]:
|
||||
raise ValueError("incomplete variants")
|
||||
if os.getenv("LAB5_FAULT_AFTER_S3") == "once":
|
||||
# One-time operator-only injected hard death, after all S3 writes and
|
||||
# before DB result commit/ack; SQL PK prevents endless failure loop.
|
||||
with connect_db() as db:
|
||||
with db.transaction():
|
||||
inserted = db.execute("""
|
||||
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3')
|
||||
ON CONFLICT DO NOTHING RETURNING job_id
|
||||
""", (job["id"],)).fetchone()
|
||||
if inserted:
|
||||
os._exit(77)
|
||||
with ot.span("db.publish_result", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
|
||||
with connect_db() as db:
|
||||
ok = publish_result(db, job=job, lease=lease, variants=variants)
|
||||
if ok:
|
||||
ot.observe('lab_ready_duration',(datetime.now(timezone.utc)-job['created_at']).total_seconds(), {'job_type':'attachment.process.v1'})
|
||||
cleanup_staging(job)
|
||||
done('ready' if ok else 'stale')
|
||||
return "ack"
|
||||
except subprocess.TimeoutExpired:
|
||||
with connect_db() as db:
|
||||
outcome = fail_job(db, job, lease, code="processing_timeout")
|
||||
done(outcome)
|
||||
return "ack"
|
||||
except Exception:
|
||||
log.exception("job processing failure (internal details not sent to API)")
|
||||
# Leave lease for dispatcher recovery if even DB unavailable.
|
||||
try:
|
||||
with connect_db() as db:
|
||||
outcome = fail_job(db, job, lease)
|
||||
done(outcome)
|
||||
return "ack"
|
||||
except psycopg.Error:
|
||||
done("requeue")
|
||||
return "requeue"
|
||||
finally:
|
||||
ot.worker_busy = 0
|
||||
|
||||
|
||||
def worker_loop():
|
||||
while True:
|
||||
try:
|
||||
with connection() as mq:
|
||||
ch = setup_channel(mq)
|
||||
ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1"))))
|
||||
|
||||
def callback(channel, method, props, body):
|
||||
outcome = process_message(body)
|
||||
if outcome == "ack":
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
else:
|
||||
channel.basic_nack(method.delivery_tag, requeue=True)
|
||||
time.sleep(2) # prevent tight redelivery spin on an active lease
|
||||
|
||||
ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False)
|
||||
log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT)
|
||||
ch.start_consuming()
|
||||
except (AMQPError, OSError, psycopg.Error) as exc:
|
||||
log.warning("worker reconnect after %s", type(exc).__name__)
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def replay_failed(ident):
|
||||
with connect_db() as db:
|
||||
with db.transaction():
|
||||
row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone()
|
||||
if not row or row["status"] != "failed":
|
||||
raise RuntimeError("Only failed jobs may be explicitly replayed")
|
||||
attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone()
|
||||
if not attachment or not attachment["staging_key"]:
|
||||
raise RuntimeError("Retained staging is required for replay")
|
||||
db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(),
|
||||
lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],))
|
||||
db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,))
|
||||
queue_outbox(db, row["id"], ident, row["pipeline_version"], 1)
|
||||
db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"]))
|
||||
print("Replayed failed job", ident)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"])
|
||||
parser.add_argument("--id", help="UUID for explicit operator replay/status")
|
||||
args = parser.parse_args()
|
||||
if args.mode == "dispatch":
|
||||
ot.init("dispatcher")
|
||||
dispatch_loop()
|
||||
elif args.mode == "worker":
|
||||
ot.init("worker")
|
||||
worker_loop()
|
||||
elif args.mode in ("replay", "status"):
|
||||
if not args.id:
|
||||
parser.error("--id is required")
|
||||
ident = str(uuid.UUID(args.id))
|
||||
if args.mode == "replay":
|
||||
replay_failed(ident)
|
||||
else:
|
||||
with connect_db() as db:
|
||||
print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""FastAPI process: one stateless replica. No auto-migration on startup."""
|
||||
import logging
|
||||
import uuid
|
||||
import time
|
||||
from starlette.routing import Match
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from . import telemetry as ot
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import psycopg
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .api import router
|
||||
from .attachments import router as attachment_router
|
||||
from .auth import challenge, current_user
|
||||
from .core import APIError, INSTANCE_ID, error_body, invalid, request_id
|
||||
from .storage import start, stop
|
||||
|
||||
logging.basicConfig(level="INFO")
|
||||
logger = logging.getLogger("lab5")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
ot.init("api")
|
||||
start()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
stop()
|
||||
ot.flush()
|
||||
|
||||
app = FastAPI(title="MTUCI Messenger Lab 6", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan)
|
||||
app.include_router(router)
|
||||
app.include_router(attachment_router)
|
||||
|
||||
PROTECTED = ("/api/v1/chats", "/api/v1/attachments", "/api/v1/users/me", "/api/v1/auth/sessions/current")
|
||||
JSON_POST = ("/api/v1/users", "/api/v1/chats")
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_context(req: Request, call_next):
|
||||
# Match the registered template even when early authentication fails.
|
||||
route = next((r.path for r in app.routes if r.matches(req.scope)[0] == Match.FULL), 'unmatched')
|
||||
method = req.method if req.method in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER'
|
||||
attrs = {'method': method, 'route': route}
|
||||
start_at = time.monotonic()
|
||||
if route in ("/health/live", "/health/ready"):
|
||||
response = await handle_request(req, call_next)
|
||||
attrs["status_class"] = str(response.status_code//100)+"xx"
|
||||
ot.count("lab_http_requests", attrs)
|
||||
ot.observe("lab_http_duration", time.monotonic()-start_at, attrs)
|
||||
return response
|
||||
with ot.span(method+' '+route, kind=SpanKind.SERVER, attrs={'http.request.method':method,'http.route':route}) as span:
|
||||
response = await handle_request(req, call_next)
|
||||
attrs['status_class'] = str(response.status_code//100)+'xx'
|
||||
span.set_attribute('http.response.status_code', response.status_code)
|
||||
if response.status_code >= 500:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
ot.count('lab_http_requests', attrs)
|
||||
ot.observe('lab_http_duration', time.monotonic()-start_at, attrs)
|
||||
ot.event('http.completed', level=logging.ERROR if response.status_code >= 500 else logging.INFO,
|
||||
request_id=request_id(req), route=route, method=method, status_code=response.status_code)
|
||||
if ot.trace_id():
|
||||
response.headers['X-Trace-Id'] = ot.trace_id()
|
||||
return response
|
||||
|
||||
|
||||
async def handle_request(req: Request, call_next):
|
||||
req.state.request_id = uuid.uuid4().hex
|
||||
try:
|
||||
path = req.url.path
|
||||
# Authenticate before parsing the request body or resource-specific fields.
|
||||
if any(path == p or path.startswith(p + "/") for p in PROTECTED):
|
||||
req.state.actor = await run_in_threadpool(current_user, req)
|
||||
if req.method == "POST" and (path in JSON_POST or (path.startswith("/api/v1/chats/") and path.endswith("/messages"))):
|
||||
if req.headers.get("content-type", "").split(";", 1)[0].strip().lower() != "application/json":
|
||||
raise APIError(415, "unsupported_media_type", "Content-Type must be application/json")
|
||||
if req.method == "POST" and path == "/api/v1/auth/sessions" and await req.body():
|
||||
raise invalid("Login body is not allowed")
|
||||
response = await call_next(req)
|
||||
except APIError as exc:
|
||||
response = api_error(req, exc)
|
||||
except (RedisError, psycopg.OperationalError, psycopg.InterfaceError):
|
||||
response = api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
|
||||
except Exception:
|
||||
logger.exception("unexpected request failure id=%s", request_id(req))
|
||||
response = api_error(req, APIError(500, "internal_error", "Internal server error"))
|
||||
response.headers["X-Request-Id"] = request_id(req)
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
def api_error(req: Request, exc: APIError) -> JSONResponse:
|
||||
headers = {"WWW-Authenticate": challenge(req)} if exc.status == 401 else {}
|
||||
return JSONResponse(status_code=exc.status, content=error_body(req, exc.code, exc.message), headers=headers)
|
||||
|
||||
|
||||
@app.exception_handler(APIError)
|
||||
async def api_error_handler(req: Request, exc: APIError):
|
||||
return api_error(req, exc)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(req: Request, exc: RequestValidationError):
|
||||
return api_error(req, invalid())
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error_handler(req: Request, exc: Exception):
|
||||
logger.exception("handler failure id=%s", request_id(req))
|
||||
if isinstance(exc, (psycopg.OperationalError, psycopg.InterfaceError, RedisError)):
|
||||
return api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
|
||||
return api_error(req, APIError(500, "internal_error", "Internal server error"))
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Run once before API instances; transactional and advisory-lock coordinated migrations."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
|
||||
MIGRATIONS = Path(__file__).resolve().parents[1] / "db" / "migrations"
|
||||
|
||||
def main():
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], autocommit=False) as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(62132103)")
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())")
|
||||
for path in sorted(MIGRATIONS.glob("[0-9]*.sql")):
|
||||
applied = conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone()
|
||||
if applied:
|
||||
continue
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (path.name,))
|
||||
print("Applied", path.name, flush=True)
|
||||
print("Migrations up to date", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,77 @@
|
||||
"""HTTP request models; all strings use Unicode code point limits."""
|
||||
import re
|
||||
import uuid
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
class CreateUser(StrictModel):
|
||||
username: str
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def username_valid(cls, value: str) -> str:
|
||||
if not USERNAME_RE.fullmatch(value):
|
||||
raise ValueError("Invalid username")
|
||||
return value
|
||||
|
||||
@field_validator("display_name", "password")
|
||||
@classmethod
|
||||
def non_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("Blank value")
|
||||
value.encode("utf-8", errors="strict")
|
||||
return value
|
||||
|
||||
class CreateChat(StrictModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def non_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("Blank title")
|
||||
value.encode("utf-8", errors="strict")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def uuid_members(cls, value: list[str]) -> list[str]:
|
||||
for member in value:
|
||||
if not isinstance(member, str) or str(uuid.UUID(member)) != member:
|
||||
raise ValueError("Invalid member UUID")
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("Duplicate member")
|
||||
return value
|
||||
|
||||
class CreateMessage(StrictModel):
|
||||
text: str = Field(min_length=0, max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("text")
|
||||
@classmethod
|
||||
def non_blank(cls, value: str) -> str:
|
||||
value.encode("utf-8", errors="strict")
|
||||
return value
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def valid_attachments(cls, value: list[str]) -> list[str]:
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("Duplicate attachment IDs")
|
||||
for item in value:
|
||||
if not isinstance(item, str) or str(uuid.UUID(item)) != item:
|
||||
raise ValueError("Invalid attachment UUID")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def nonempty_message(self):
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError("Empty message")
|
||||
return self
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Offline image conversion tests inside the application image; no S3 or DB."""
|
||||
import io
|
||||
from PIL import Image
|
||||
from .image_worker import transform
|
||||
|
||||
|
||||
def main():
|
||||
source = Image.new('RGB',(320,200), (255,0,0))
|
||||
buf=io.BytesIO();source.save(buf,'PNG')
|
||||
result=transform(buf.getvalue())
|
||||
items={name:(img,fmt,mime,opts) for name,img,fmt,mime,opts in result}
|
||||
assert set(items)=={'thumbnail','crop','optimized'}
|
||||
assert items['thumbnail'][0].size==(256,160)
|
||||
assert items['crop'][0].size==(128,128)
|
||||
assert items['optimized'][0].size==(320,200)
|
||||
for name,(im,fmt,mime,opts) in items.items():
|
||||
data=io.BytesIO();im.save(data,format=fmt,**opts)
|
||||
with Image.open(io.BytesIO(data.getvalue())) as decoded:
|
||||
assert decoded.size==im.size
|
||||
assert decoded.format==fmt
|
||||
assert 'exif' not in decoded.info
|
||||
print('PASS: PNG 320x200 -> JPEG thumb 256x160, crop 128x128, optimized WebP, metadata stripped')
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Lab4 grade-5 track 2: repeatable DB/S3 reconciliation.
|
||||
|
||||
Dry-run is default. Apply never drops metadata when BOTH object copies are gone:
|
||||
without a trustworthy copy or external backup, deletion would destroy evidence.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .attachments import BUCKET, s3_client
|
||||
|
||||
|
||||
def inventory(client):
|
||||
result = {}
|
||||
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET):
|
||||
for obj in page.get("Contents", []):
|
||||
# Only reconciliation-owned prefixes; never mutate arbitrary S3 keys.
|
||||
key = obj["Key"]
|
||||
if key.startswith(("attachments/", "recovery/")):
|
||||
result[key] = obj["LastModified"]
|
||||
return result
|
||||
|
||||
|
||||
def get_db_rows(conn):
|
||||
return conn.execute("SELECT id, object_key, backup_key, status FROM attachments").fetchall()
|
||||
|
||||
|
||||
def current_reference(conn, key):
|
||||
return conn.execute(
|
||||
"SELECT 1 FROM attachments WHERE object_key=%s OR backup_key=%s", (key, key)
|
||||
).fetchone() is not None
|
||||
|
||||
|
||||
def reconcile(*, apply=False, grace_seconds=300, now=None, client=None, connect=None):
|
||||
if grace_seconds < 300 and apply:
|
||||
raise ValueError("Apply requires >=300 s grace period to protect in-flight uploads")
|
||||
started = time.monotonic()
|
||||
now = now or datetime.now(timezone.utc)
|
||||
client = client or s3_client()
|
||||
connect = connect or (lambda: psycopg.connect(__import__('os').environ['DATABASE_URL'], row_factory=dict_row))
|
||||
objects = inventory(client)
|
||||
stats = {
|
||||
"mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds,
|
||||
"s3_keys_scanned": len(objects), "db_rows_scanned": 0,
|
||||
"orphan_keys": [], "orphan_keys_young": [],
|
||||
"missing_primary_repairable": [], "missing_backup_repairable": [],
|
||||
"missing_both_unrecoverable": [], "actions": [], "errors": [],
|
||||
}
|
||||
with connect() as conn:
|
||||
rows = get_db_rows(conn)
|
||||
stats["db_rows_scanned"] = len(rows)
|
||||
references = {key for row in rows for key in (row["object_key"], row["backup_key"])}
|
||||
for row in rows:
|
||||
if row["status"] != "ready":
|
||||
continue # queued/processing have durable staging, not final variants yet
|
||||
primary, backup = row["object_key"], row["backup_key"]
|
||||
have_primary, have_backup = primary in objects, backup in objects
|
||||
rid = str(row["id"])
|
||||
if have_primary and have_backup:
|
||||
continue
|
||||
if not have_primary and not have_backup:
|
||||
stats["missing_both_unrecoverable"].append(rid)
|
||||
continue
|
||||
dest, source, typ = (primary, backup, "missing_primary_repairable") if not have_primary else (backup, primary, "missing_backup_repairable")
|
||||
stats[typ].append(rid)
|
||||
if apply:
|
||||
# Defensive recheck. Object may have reappeared after inventory.
|
||||
try:
|
||||
client.head_object(Bucket=BUCKET, Key=dest)
|
||||
stats["actions"].append({"action": "already-restored", "key": dest})
|
||||
continue
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code")
|
||||
if code not in ("404", "NotFound", "NoSuchKey"):
|
||||
raise
|
||||
try:
|
||||
client.copy_object(Bucket=BUCKET, Key=dest,
|
||||
CopySource={"Bucket": BUCKET, "Key": source})
|
||||
stats["actions"].append({"action": "restore", "key": dest, "from": source})
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
stats["errors"].append({"action": "restore", "key": dest, "error_type": type(exc).__name__})
|
||||
for key, modified in objects.items():
|
||||
if key in references:
|
||||
continue
|
||||
age = (now - modified).total_seconds()
|
||||
if age < grace_seconds:
|
||||
stats["orphan_keys_young"].append(key)
|
||||
continue
|
||||
stats["orphan_keys"].append(key)
|
||||
if apply:
|
||||
# Query again immediately before destructive action; never delete
|
||||
# an object now referenced by a committed metadata row.
|
||||
if current_reference(conn, key):
|
||||
stats["actions"].append({"action": "skip-now-referenced", "key": key})
|
||||
continue
|
||||
try:
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
stats["actions"].append({"action": "delete-old-orphan", "key": key})
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
stats["errors"].append({"action": "delete-old-orphan", "key": key, "error_type": type(exc).__name__})
|
||||
stats["scan_elapsed_seconds"] = round(time.monotonic() - started, 3)
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Reconcile S3 attachments vs PostgreSQL (dry-run default)")
|
||||
parser.add_argument("--apply", action="store_true", help="Make safe changes; orphans older than 300 seconds only")
|
||||
parser.add_argument("--grace-seconds", type=int, default=300)
|
||||
parser.add_argument("--output", help="Write JSON summary (no credentials)")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = reconcile(apply=args.apply, grace_seconds=args.grace_seconds)
|
||||
except Exception as exc:
|
||||
# No exception string: botocore errors may contain signed credentials/URLs.
|
||||
print(json.dumps({"status":"failed", "error_type":type(exc).__name__}), file=sys.stderr)
|
||||
raise SystemExit(2) from None
|
||||
rendered = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
print(rendered)
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
fh.write(rendered + "\n")
|
||||
if result["errors"] or result["missing_both_unrecoverable"]:
|
||||
raise SystemExit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Operator-only fault injection in a LOCAL TEST stack; never exposed as HTTP."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from .attachments import BUCKET, s3_client
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("action", choices=["delete-primary", "delete-backup", "create-orphan"])
|
||||
p.add_argument("--id", help="Attachment UUID for the delete actions")
|
||||
args = p.parse_args()
|
||||
if os.environ.get("LAB4_ALLOW_FAULT_INJECTION") != "yes":
|
||||
raise SystemExit("Fault injection disabled; use LAB4_ALLOW_FAULT_INJECTION=yes in isolated test")
|
||||
client = s3_client()
|
||||
if args.action == "create-orphan":
|
||||
key = f"attachments/{uuid.uuid4()}/original"
|
||||
client.put_object(Bucket=BUCKET, Key=key, Body=b"synthetic orphan")
|
||||
print(json.dumps({"fault":"orphan-created", "object_key":key}))
|
||||
return
|
||||
if not args.id:
|
||||
raise SystemExit("--id is required")
|
||||
ident = str(uuid.UUID(args.id))
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) as conn:
|
||||
row = conn.execute("SELECT object_key,backup_key FROM attachments WHERE id=%s", (ident,)).fetchone()
|
||||
if not row:
|
||||
raise SystemExit("Attachment does not exist")
|
||||
key = row["object_key"] if args.action == "delete-primary" else row["backup_key"]
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
print(json.dumps({"fault":args.action, "object_key":key}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Fast isolated tests for reconciliation safety; run in application image."""
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from .reconcile import reconcile
|
||||
from .attachments import BUCKET
|
||||
|
||||
NOW = datetime.now(timezone.utc)
|
||||
|
||||
class FakeConn:
|
||||
def __init__(self, rows): self.rows=rows
|
||||
def __enter__(self): return self
|
||||
def __exit__(self,*a): return False
|
||||
def execute(self, sql, args=None):
|
||||
self.sql=sql
|
||||
self.args=args
|
||||
return self
|
||||
def fetchall(self): return self.rows
|
||||
def fetchone(self):
|
||||
key=self.args[0]
|
||||
return {'id': 1} if any(key in (row['object_key'],row['backup_key']) for row in self.rows) else None
|
||||
|
||||
class FakeS3:
|
||||
def __init__(self,objects): self.objects=objects
|
||||
def get_paginator(self,op): return self
|
||||
def paginate(self,**kwargs):
|
||||
yield {'Contents':[{'Key':k,'LastModified':t} for k,t in self.objects.items()]}
|
||||
def head_object(self,**kw):
|
||||
if kw['Key'] not in self.objects:
|
||||
from botocore.exceptions import ClientError
|
||||
raise ClientError({'Error':{'Code':'404'}},'HeadObject')
|
||||
def copy_object(self,**kw):
|
||||
assert kw['CopySource']['Key'] in self.objects
|
||||
self.objects[kw['Key']]=NOW
|
||||
def delete_object(self,**kw):
|
||||
self.objects.pop(kw['Key'],None)
|
||||
|
||||
|
||||
def test():
|
||||
i=str(uuid.uuid4())
|
||||
primary=f'attachments/{i}/original';backup=f'recovery/{i}/original'
|
||||
young=f'attachments/{uuid.uuid4()}/original'
|
||||
old=f'attachments/{uuid.uuid4()}/original'
|
||||
rows=[{'id':i,'object_key':primary,'backup_key':backup,'status':'ready'}]
|
||||
fake=FakeS3({backup: NOW-timedelta(days=1),young:NOW-timedelta(seconds=10),old:NOW-timedelta(days=1)})
|
||||
kwargs={'now':NOW,'client':fake,'connect':lambda:FakeConn(rows)}
|
||||
first=reconcile(apply=False,**kwargs)
|
||||
assert first['missing_primary_repairable']==[i]
|
||||
assert old in first['orphan_keys'] and young in first['orphan_keys_young']
|
||||
assert primary not in fake.objects and old in fake.objects, 'Dry run mutated objects'
|
||||
second=reconcile(apply=True,**kwargs)
|
||||
assert primary in fake.objects and old not in fake.objects and young in fake.objects
|
||||
assert any(x['action']=='restore' for x in second['actions'])
|
||||
third=reconcile(apply=True,**kwargs)
|
||||
assert not third['actions'], 'Re-run unexpectedly mutated objects'
|
||||
assert rows[0]['object_key']==primary, 'Metadata destroyed'
|
||||
try: reconcile(apply=True, grace_seconds=0, **kwargs)
|
||||
except ValueError: pass
|
||||
else: raise AssertionError('Unsafe zero-grace apply accepted')
|
||||
print('PASS: dry-run, orphan grace, recover primary from backup, apply idempotence')
|
||||
|
||||
if __name__ == '__main__': test()
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Idempotently create the private S3 bucket with bootstrap-only credentials."""
|
||||
import os
|
||||
import time
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
|
||||
|
||||
def main():
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
client = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
aws_access_key_id=os.environ["S3_ADMIN_ACCESS_KEY"],
|
||||
aws_secret_access_key=os.environ["S3_ADMIN_SECRET_KEY"],
|
||||
region_name="us-east-1",
|
||||
config=Config(s3={"addressing_style": "path"},
|
||||
connect_timeout=2, read_timeout=4, retries={"max_attempts": 1}))
|
||||
for attempt in range(70):
|
||||
try:
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code not in ("404", "NoSuchBucket", "NotFound"):
|
||||
raise
|
||||
client.create_bucket(Bucket=bucket)
|
||||
# PRIVATE: no public bucket policy, ACL or anonymous identity.
|
||||
print(f"Private bucket {bucket} ready", flush=True)
|
||||
return
|
||||
except (ClientError, BotoCoreError, OSError) as exc:
|
||||
if attempt == 69:
|
||||
raise RuntimeError("S3 bootstrap failed; inspect S3 container") from exc
|
||||
time.sleep(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
"""One-off creation of private SeaweedFS S3 identities; secrets stay in Docker volume."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
identities = [
|
||||
{"name": "bucket-bootstrap", "credentials": [{"accessKey": os.environ["S3_ADMIN_ACCESS_KEY"],
|
||||
"secretKey": os.environ["S3_ADMIN_SECRET_KEY"]}],
|
||||
"actions": ["Admin", "Read", "Write", "List"]},
|
||||
{"name": "messenger-api", "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY"],
|
||||
"secretKey": os.environ["S3_SECRET_KEY"]}],
|
||||
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
|
||||
]
|
||||
if len({entry["credentials"][0]["accessKey"] for entry in identities}) != 2:
|
||||
raise ValueError("Admin and API credentials must differ")
|
||||
path = Path("/s3-config/config.json")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
old_umask = os.umask(0o077)
|
||||
try:
|
||||
tmp.write_text(json.dumps({"identities": identities}, separators=(",", ":")), encoding="utf-8")
|
||||
os.chmod(tmp, 0o644) # readable by the SeaweedFS runtime UID; Docker volume stays private
|
||||
os.replace(tmp, path)
|
||||
finally:
|
||||
os.umask(old_umask)
|
||||
print("S3 identities configured (no secrets printed)", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Verify bucket is private at S3 boundary (not merely hidden behind HTTP API)."""
|
||||
import json
|
||||
import os
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from .attachments import BUCKET, s3_client
|
||||
|
||||
|
||||
def main():
|
||||
client = s3_client()
|
||||
client.head_bucket(Bucket=BUCKET) # restricted app identity works
|
||||
unauth = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
region_name="us-east-1", config=Config(signature_version=UNSIGNED,
|
||||
s3={"addressing_style":"path"}))
|
||||
try:
|
||||
unauth.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status not in (401,403):
|
||||
raise RuntimeError("Unexpected anonymous S3 response") from None
|
||||
else:
|
||||
raise RuntimeError("Private bucket leaked object list to anonymous caller")
|
||||
listing = client.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
|
||||
keys = [entry["Key"] for entry in listing.get("Contents", [])]
|
||||
if keys:
|
||||
try:
|
||||
unauth.get_object(Bucket=BUCKET, Key=keys[0])
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status not in (401, 403):
|
||||
raise RuntimeError("Unexpected anonymous object GET response") from None
|
||||
else:
|
||||
raise RuntimeError("Private S3 object downloadable anonymously")
|
||||
print(json.dumps({"private_bucket": True, "anonymous_listing": "denied",
|
||||
"anonymous_object_get": "denied" if keys else "not_tested_empty_bucket"}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Shared external state: psycopg connection pool and Redis session store."""
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
from redis import Redis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .core import APIError
|
||||
from . import telemetry as ot
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
pool: ConnectionPool | None = None
|
||||
redis_client: Redis | None = None
|
||||
|
||||
|
||||
def start() -> None:
|
||||
global pool, redis_client
|
||||
url = os.environ["DATABASE_URL"]
|
||||
size = int(os.getenv("DB_POOL_MAX", "8"))
|
||||
pool = ConnectionPool(conninfo=url, min_size=1, max_size=size, timeout=3, kwargs={"row_factory": dict_row, "connect_timeout": 5, "options": "-c statement_timeout=5000"}, open=False)
|
||||
pool.open(wait=True, timeout=30)
|
||||
redis_client = Redis.from_url(os.environ.get("REDIS_URL", "redis://redis:6379/0"), socket_connect_timeout=2, socket_timeout=2, decode_responses=True)
|
||||
redis_client.ping()
|
||||
|
||||
|
||||
def stop() -> None:
|
||||
global pool, redis_client
|
||||
if pool is not None:
|
||||
pool.close()
|
||||
pool = None
|
||||
if redis_client is not None:
|
||||
redis_client.close()
|
||||
redis_client = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def db():
|
||||
if pool is None:
|
||||
raise APIError(503, "unavailable", "Database unavailable")
|
||||
try:
|
||||
with ot.dependency_span("db.transaction", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
|
||||
with pool.connection() as conn:
|
||||
yield conn
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, PoolTimeout) as exc:
|
||||
raise APIError(503, "unavailable", "Database unavailable") from exc
|
||||
|
||||
|
||||
def cache() -> Redis:
|
||||
if redis_client is None:
|
||||
raise APIError(503, "unavailable", "Session store unavailable")
|
||||
return redis_client
|
||||
|
||||
|
||||
def ready() -> bool:
|
||||
try:
|
||||
with db() as conn:
|
||||
conn.execute("SELECT 1").fetchone()
|
||||
cache().ping()
|
||||
# S3 is needed for the mandatory synchronous upload/download path.
|
||||
from .attachments import s3_client, BUCKET
|
||||
s3_client().head_bucket(Bucket=BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Explicit, allowlisted OTLP instrumentation; no automatic headers, SQL or bodies.
|
||||
SDK exporters run asynchronously with bounded queues; telemetry is best effort.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from pathlib import Path
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
|
||||
PROP = TraceContextTextMapPropagator()
|
||||
tracer = trace.get_tracer('messenger.lab6')
|
||||
log = logging.getLogger('messenger.telemetry')
|
||||
log.setLevel(logging.INFO)
|
||||
log.propagate = False
|
||||
providers = []
|
||||
started = False
|
||||
m = None
|
||||
instruments = {}
|
||||
worker_busy = 0
|
||||
backlog_depth = None
|
||||
backlog_age = None
|
||||
broker_ready = None
|
||||
broker_consumers = None
|
||||
QUEUE = 'attachment-processing.v1'
|
||||
BUCKETS = (0.005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 15, 30, 60, 120, 300, 600)
|
||||
|
||||
|
||||
def init(role, *, child=False):
|
||||
global started, tracer, m
|
||||
if started or os.getenv('OTEL_ENABLED', 'no') != 'yes':
|
||||
return
|
||||
started = True
|
||||
resource = Resource.create({'service.name': 'messenger-' + role,
|
||||
'service.version': '1.6.0', 'service.instance.id': os.getenv('API_INSTANCE_ID', socket.gethostname()),
|
||||
'deployment.environment.name': 'lab'})
|
||||
base = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
|
||||
tp = TracerProvider(resource=resource)
|
||||
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=base+'/v1/traces', timeout=2),
|
||||
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
|
||||
trace.set_tracer_provider(tp)
|
||||
tracer = tp.get_tracer('messenger.lab6')
|
||||
lp = LoggerProvider(resource=resource)
|
||||
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=base+'/v1/logs', timeout=2),
|
||||
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
|
||||
log.addHandler(LoggingHandler(logger_provider=lp))
|
||||
providers.extend([tp, lp])
|
||||
if child:
|
||||
return
|
||||
reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=base+'/v1/metrics', timeout=2),
|
||||
export_interval_millis=3000, export_timeout_millis=2000)
|
||||
mp = MeterProvider(resource=resource, metric_readers=[reader], views=[View(instrument_name=n,
|
||||
aggregation=ExplicitBucketHistogramAggregation(boundaries=BUCKETS)) for n in
|
||||
('lab_http_duration','lab_job_duration','lab_ready_duration')])
|
||||
metrics.set_meter_provider(mp)
|
||||
providers.append(mp)
|
||||
m = mp.get_meter('messenger.lab6')
|
||||
for name in ('lab_http_requests','lab_jobs','lab_job_retries'):
|
||||
instruments[name] = m.create_counter(name)
|
||||
for name in ('lab_http_duration','lab_job_duration','lab_ready_duration'):
|
||||
instruments[name] = m.create_histogram(name, unit='s')
|
||||
if role == 'api':
|
||||
for name, key in [('lab_db_pool_used','used'),('lab_db_pool_waiting','requests_waiting'),('lab_db_pool_capacity','pool_max')]:
|
||||
m.create_observable_gauge(name, callbacks=[lambda _, key=key: pool_stats(key)])
|
||||
if role == 'worker':
|
||||
m.create_observable_gauge('lab_worker_busy', callbacks=[lambda _: [metrics.Observation(worker_busy)]])
|
||||
m.create_observable_gauge('lab_worker_capacity', callbacks=[lambda _: [metrics.Observation(1)]])
|
||||
if role == 'dispatcher':
|
||||
m.create_observable_gauge('lab_broker_ready', callbacks=[lambda _: queue_observation(broker_ready)])
|
||||
m.create_observable_gauge('lab_broker_consumers', callbacks=[lambda _: queue_observation(broker_consumers)])
|
||||
m.create_observable_gauge('lab_queue_depth', callbacks=[lambda _: queue_observation(backlog_depth)])
|
||||
m.create_observable_gauge('lab_queue_oldest_age', unit='s', callbacks=[lambda _: queue_observation(backlog_age)])
|
||||
m.create_observable_gauge('lab_container_memory', unit='By', callbacks=[lambda _: container_observation('memory')])
|
||||
m.create_observable_gauge('lab_container_cpu_cores', callbacks=[lambda _: container_observation('cpu')])
|
||||
|
||||
|
||||
def pool_stats(key):
|
||||
from . import storage
|
||||
if storage.pool is None:
|
||||
return []
|
||||
stats = storage.pool.get_stats()
|
||||
val = stats.get('pool_size',0)-stats.get('pool_available',0) if key=='used' else stats.get(key,0)
|
||||
return [metrics.Observation(max(0,val))]
|
||||
|
||||
|
||||
def queue_observation(value):
|
||||
return [] if value is None else [metrics.Observation(value, {'queue': QUEUE})]
|
||||
|
||||
|
||||
_cpu_previous = None
|
||||
|
||||
def container_observation(kind):
|
||||
global _cpu_previous
|
||||
try:
|
||||
if kind == 'memory':
|
||||
return [metrics.Observation(int(Path('/sys/fs/cgroup/memory.current').read_text()))]
|
||||
cpu = int(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())['usage_usec']) / 1e6
|
||||
now = time.monotonic()
|
||||
old = _cpu_previous
|
||||
_cpu_previous = (now,cpu)
|
||||
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
|
||||
except (OSError, KeyError, ValueError):
|
||||
# cgroup v1 fallback, process tree (not full container accounting).
|
||||
import psutil
|
||||
try:
|
||||
p = psutil.Process()
|
||||
processes = [p]+p.children(recursive=True)
|
||||
if kind == 'memory':
|
||||
value = 0
|
||||
for proc in processes:
|
||||
try: value += proc.memory_info().rss
|
||||
except psutil.Error: pass
|
||||
return [metrics.Observation(value)]
|
||||
# cgroup v1 (Docker Linux) if available, CPU usage in nanoseconds.
|
||||
for path in ('/sys/fs/cgroup/cpuacct/cpuacct.usage','/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage'):
|
||||
if Path(path).is_file():
|
||||
cpu = int(Path(path).read_text()) / 1e9
|
||||
now = time.monotonic()
|
||||
old = _cpu_previous
|
||||
_cpu_previous = (now,cpu)
|
||||
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
|
||||
except (psutil.Error,OSError,ValueError):
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def count(name, attrs=None):
|
||||
if name in instruments: instruments[name].add(1, attrs or {})
|
||||
|
||||
|
||||
def observe(name, value, attrs=None):
|
||||
if name in instruments: instruments[name].record(max(0,value), attrs or {})
|
||||
|
||||
|
||||
def event(name, *, level=logging.INFO, **attrs):
|
||||
# Callers pass fixed event names and technical ID/status only, never payloads.
|
||||
if started:
|
||||
log.log(level, name, extra={'event':name, **attrs})
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name, *, kind=SpanKind.INTERNAL, context=None, attrs=None):
|
||||
with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attrs,
|
||||
record_exception=False, set_status_on_exception=False) as s:
|
||||
try:
|
||||
yield s
|
||||
except BaseException as exc:
|
||||
s.set_status(Status(StatusCode.ERROR))
|
||||
s.set_attribute('error.type', type(exc).__name__)
|
||||
event('operation.error', level=logging.ERROR, operation=name, error_type=type(exc).__name__)
|
||||
raise
|
||||
|
||||
|
||||
def inject():
|
||||
result = {}
|
||||
PROP.inject(result)
|
||||
return result
|
||||
|
||||
|
||||
def extract(carrier):
|
||||
# Ignore baggage and all unrelated fields.
|
||||
return PROP.extract({k:v for k,v in carrier.items() if k in ('traceparent','tracestate') and isinstance(v,str)})
|
||||
|
||||
|
||||
def dependency_span(name, **kwargs):
|
||||
return span(name, **kwargs) if trace_id() else nullcontext()
|
||||
|
||||
|
||||
def trace_id():
|
||||
ctx = trace.get_current_span().get_span_context()
|
||||
return f'{ctx.trace_id:032x}' if ctx.is_valid else ''
|
||||
|
||||
|
||||
def flush():
|
||||
for p in providers:
|
||||
p.force_flush(timeout_millis=2500)
|
||||
|
||||
|
||||
class S3Tracing:
|
||||
"""Only operation name is exported; bucket/key/bytes/signed URLs never exported."""
|
||||
def __init__(self, client): self.client = client
|
||||
def __getattr__(self, name):
|
||||
f = getattr(self.client,name)
|
||||
if name not in ('upload_fileobj','get_object','put_object','copy_object','delete_object','head_object','head_bucket'):
|
||||
return f
|
||||
def call(*a, **kw):
|
||||
with dependency_span('s3.'+name, kind=SpanKind.CLIENT, attrs={'storage.system':'s3'}):
|
||||
return f(*a, **kw)
|
||||
return call
|
||||
Reference in New Issue
Block a user