503 lines
20 KiB
Python
503 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Lab 4 PostgreSQL messenger, adapted from the author's lab 1."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import binascii
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import hashlib
|
|
import secrets
|
|
from pathlib import Path
|
|
from contextlib import asynccontextmanager
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
|
import redis
|
|
from argon2 import PasswordHasher
|
|
from argon2.exceptions import VerificationError
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Optional, Tuple
|
|
|
|
import uvicorn
|
|
from fastapi import Depends, FastAPI, Header, Query, Request
|
|
from fastapi.exceptions import RequestValidationError
|
|
from fastapi.responses import JSONResponse, Response
|
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
|
|
|
|
|
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
|
LOG = logging.getLogger("messenger")
|
|
INSTANCE_ID = os.environ.get("INSTANCE_ID", str(uuid.uuid4()))
|
|
|
|
|
|
def env_int(name: str, default: int, minimum: int = 0) -> int:
|
|
try:
|
|
value = int(os.environ.get(name, str(default)))
|
|
except ValueError as exc:
|
|
raise SystemExit("%s must be an integer" % name) from exc
|
|
if value < minimum:
|
|
raise SystemExit("%s must be at least %s" % (name, minimum))
|
|
return value
|
|
|
|
|
|
class ApiError(Exception):
|
|
def __init__(self, status: int, code: str, message: str) -> None:
|
|
super().__init__(message)
|
|
self.status = status
|
|
self.code = code
|
|
self.message = message
|
|
|
|
|
|
class StrictModel(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
class Health(StrictModel):
|
|
status: str = Field(pattern="^ok$")
|
|
|
|
|
|
class User(StrictModel):
|
|
id: uuid.UUID
|
|
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
|
display_name: str = Field(min_length=1, max_length=100)
|
|
created_at: datetime
|
|
|
|
|
|
class CreateUser(StrictModel):
|
|
password: str = Field(min_length=12, max_length=128)
|
|
|
|
@field_validator("password")
|
|
@classmethod
|
|
def non_blank_password(cls, value: str) -> str:
|
|
if not value.strip():
|
|
raise ValueError("password must not be blank")
|
|
return value
|
|
|
|
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
|
display_name: str = Field(min_length=1, max_length=100)
|
|
|
|
@field_validator("display_name")
|
|
@classmethod
|
|
def non_blank_display_name(cls, value: str) -> str:
|
|
if not value.strip():
|
|
raise ValueError("display_name must not be blank")
|
|
return value
|
|
|
|
|
|
class Chat(StrictModel):
|
|
id: uuid.UUID
|
|
title: str = Field(min_length=1, max_length=100)
|
|
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
|
|
created_at: datetime
|
|
|
|
|
|
class CreateChat(StrictModel):
|
|
title: str = Field(min_length=1, max_length=100)
|
|
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
|
|
|
|
@field_validator("title")
|
|
@classmethod
|
|
def non_blank_title(cls, value: str) -> str:
|
|
if not value.strip():
|
|
raise ValueError("title must not be blank")
|
|
return value
|
|
|
|
@field_validator("member_ids")
|
|
@classmethod
|
|
def unique_members(cls, value: list[uuid.UUID]) -> list[uuid.UUID]:
|
|
if len(set(value)) != len(value):
|
|
raise ValueError("member_ids must be unique")
|
|
return value
|
|
|
|
|
|
class Message(StrictModel):
|
|
id: uuid.UUID
|
|
chat_id: uuid.UUID
|
|
sender_id: uuid.UUID
|
|
text: str = Field(max_length=4000)
|
|
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
|
|
created_at: datetime
|
|
|
|
|
|
class CreateMessage(StrictModel):
|
|
text: str = Field(max_length=4000)
|
|
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
|
|
|
|
@model_validator(mode="after")
|
|
def valid_message(self):
|
|
if not self.text.strip() and not self.attachment_ids:
|
|
raise ValueError("message must not be empty")
|
|
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
|
raise ValueError("duplicate attachment")
|
|
return self
|
|
|
|
|
|
class ChatPage(StrictModel):
|
|
items: list[Chat]
|
|
next_cursor: Optional[str]
|
|
|
|
|
|
class MessagePage(StrictModel):
|
|
items: list[Message]
|
|
next_cursor: Optional[str]
|
|
|
|
|
|
class ErrorDetails(StrictModel):
|
|
code: str
|
|
message: str
|
|
request_id: str
|
|
|
|
|
|
class ErrorResponse(StrictModel):
|
|
error: ErrorDetails
|
|
|
|
|
|
def valid_uuid(value: Any) -> bool:
|
|
if not isinstance(value, str):
|
|
return False
|
|
try:
|
|
return str(uuid.UUID(value)) == value.lower()
|
|
except (ValueError, AttributeError):
|
|
return False
|
|
|
|
|
|
def make_cursor(scope: str, item: dict[str, Any]) -> str:
|
|
raw = json.dumps(
|
|
{"v": 1, "scope": scope, "created_at": item["created_at"], "id": item["id"]},
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
|
|
|
|
|
def read_cursor(value: str, scope: str) -> Tuple[str, str]:
|
|
try:
|
|
raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
|
|
data = json.loads(raw)
|
|
if set(data) != {"v", "scope", "created_at", "id"}:
|
|
raise ValueError
|
|
if data["v"] != 1 or data["scope"] != scope or not valid_uuid(data["id"]):
|
|
raise ValueError
|
|
datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
|
|
return data["created_at"], data["id"]
|
|
except (ValueError, TypeError, AttributeError, UnicodeError, json.JSONDecodeError, binascii.Error):
|
|
raise ApiError(400, "invalid_request", "Invalid cursor") from None
|
|
|
|
|
|
|
|
def secret(name):
|
|
return Path(os.environ[name + "_FILE"]).read_text().strip()
|
|
|
|
POOL = ConnectionPool(secret("DATABASE_URL"), min_size=1, max_size=8, timeout=3,
|
|
max_waiting=32, open=False, check=ConnectionPool.check_connection,
|
|
kwargs={"row_factory": dict_row, "connect_timeout": 3,
|
|
"options": "-c statement_timeout=3000 -c lock_timeout=2000"})
|
|
CACHE = redis.Redis.from_url(secret("CACHE_URL"), socket_connect_timeout=0.2,
|
|
socket_timeout=0.2, decode_responses=True)
|
|
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
|
DUMMY_HASH = HASHER.hash(secrets.token_urlsafe(32))
|
|
TTL = env_int("SESSION_TTL_SECONDS", 3600, 1)
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app):
|
|
POOL.open()
|
|
try:
|
|
yield
|
|
finally:
|
|
POOL.close()
|
|
CACHE.close()
|
|
|
|
app = FastAPI(title="MTUSI Messenger — Lab 4", version="4.0.0", lifespan=lifespan)
|
|
|
|
|
|
def wire(row):
|
|
return {k: (v.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
|
if isinstance(v, datetime) else str(v) if isinstance(v, uuid.UUID) else v)
|
|
for k, v in row.items()}
|
|
|
|
|
|
def query(sql, params=()):
|
|
with POOL.connection() as conn:
|
|
return [wire(row) for row in conn.execute(sql, params).fetchall()]
|
|
|
|
|
|
def profile(user_id):
|
|
key = "user:" + user_id
|
|
try:
|
|
value = CACHE.get(key)
|
|
if value:
|
|
return json.loads(value)
|
|
except (redis.RedisError, ValueError):
|
|
pass
|
|
rows = query("SELECT id, username, display_name, created_at FROM users WHERE id=%s", (user_id,))
|
|
if not rows:
|
|
raise ApiError(401, "unauthorized", "Invalid session")
|
|
user = rows[0]
|
|
try:
|
|
CACHE.setex(key, 60, json.dumps(user))
|
|
except redis.RedisError:
|
|
pass
|
|
return user
|
|
|
|
|
|
def token_hash(token):
|
|
return hashlib.sha256(token.encode()).hexdigest()
|
|
|
|
|
|
def bearer(authorization):
|
|
if not authorization or not authorization.startswith("Bearer "):
|
|
raise ApiError(401, "unauthorized", "Bearer session required")
|
|
token = authorization[7:]
|
|
if not re.fullmatch(r"[A-Za-z0-9_-]{43}", token):
|
|
raise ApiError(401, "unauthorized", "Invalid session")
|
|
return token_hash(token)
|
|
|
|
|
|
def current_user(authorization: Optional[str] = Header(default=None)):
|
|
digest = bearer(authorization)
|
|
rows = query("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()", (digest,))
|
|
if not rows:
|
|
raise ApiError(401, "unauthorized", "Invalid session")
|
|
return profile(rows[0]["user_id"])
|
|
|
|
|
|
def visible_chat(chat_id, actor_id):
|
|
if not valid_uuid(chat_id):
|
|
raise ApiError(400, "invalid_request", "Invalid chat id")
|
|
rows = query("SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s", (chat_id, actor_id))
|
|
if not rows:
|
|
raise ApiError(404, "not_found", "Chat not found")
|
|
return chat_members(rows)[0]
|
|
|
|
|
|
def chat_members(rows):
|
|
if rows:
|
|
members = query("SELECT chat_id, user_id FROM members WHERE chat_id=ANY(%s::uuid[]) ORDER BY user_id", ([r["id"] for r in rows],))
|
|
grouped = {r["id"]: [] for r in rows}
|
|
for m in members:
|
|
grouped[m["chat_id"]].append(m["user_id"])
|
|
for row in rows:
|
|
row["member_ids"] = grouped[row["id"]]
|
|
return rows
|
|
|
|
|
|
def page_args(request, scope, limit, cursor):
|
|
if set(request.query_params).difference({"limit", "cursor"}):
|
|
raise ApiError(400, "invalid_request", "Invalid query parameters")
|
|
return limit, read_cursor(cursor, scope) if cursor is not None else None
|
|
|
|
|
|
def page(rows, limit, scope):
|
|
return {"items": rows[:limit], "next_cursor": make_cursor(scope, rows[limit-1]) if len(rows)>limit else None}
|
|
def request_id(request: Request) -> str:
|
|
return getattr(request.state, "request_id", str(uuid.uuid4()))
|
|
|
|
|
|
def error_response(request: Request, status: int, code: str, message: str) -> JSONResponse:
|
|
rid = request_id(request)
|
|
response = JSONResponse(
|
|
status_code=status,
|
|
content={"error": {"code": code, "message": message, "request_id": rid}},
|
|
)
|
|
response.headers["X-Request-Id"] = rid
|
|
response.headers["X-Instance-Id"] = INSTANCE_ID
|
|
if status == 401:
|
|
response.headers["WWW-Authenticate"] = ('Basic realm="messenger", charset="UTF-8"'
|
|
if request.url.path == "/api/v1/auth/sessions" else 'Bearer realm="messenger"')
|
|
return response
|
|
|
|
|
|
@app.middleware("http")
|
|
async def request_envelope(request: Request, call_next: Any) -> JSONResponse:
|
|
request.state.request_id = str(uuid.uuid4())
|
|
try:
|
|
response = await call_next(request)
|
|
except Exception:
|
|
LOG.error("request_id=%s unexpected error", request_id(request))
|
|
response = error_response(request, 500, "internal_error", "Internal server error")
|
|
response.headers["X-Request-Id"] = request_id(request)
|
|
response.headers["X-Instance-Id"] = INSTANCE_ID
|
|
return response
|
|
|
|
|
|
@app.exception_handler(ApiError)
|
|
async def api_error_handler(request: Request, error: ApiError) -> JSONResponse:
|
|
return error_response(request, error.status, error.code, error.message)
|
|
|
|
|
|
@app.exception_handler(RequestValidationError)
|
|
async def validation_error_handler(request: Request, _error: RequestValidationError) -> JSONResponse:
|
|
return error_response(request, 400, "invalid_request", "Request validation failed")
|
|
|
|
|
|
@app.exception_handler(StarletteHTTPException)
|
|
async def http_error_handler(request: Request, error: StarletteHTTPException) -> JSONResponse:
|
|
if error.status_code == 404:
|
|
return error_response(request, 404, "not_found", "Resource not found")
|
|
return error_response(request, error.status_code, "invalid_request", "Request failed")
|
|
|
|
|
|
async def require_json(request: Request) -> None:
|
|
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
|
if content_type != "application/json":
|
|
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
|
|
|
|
|
|
|
|
@app.exception_handler(psycopg.Error)
|
|
@app.exception_handler(PoolTimeout)
|
|
@app.exception_handler(TooManyRequests)
|
|
async def database_error(request, error):
|
|
return error_response(request, 503, "unavailable", "Database temporarily unavailable")
|
|
|
|
@app.get("/health/live", response_model=Health)
|
|
def health_live():
|
|
return {"status": "ok"}
|
|
|
|
@app.get("/health/ready", response_model=Health)
|
|
def health_ready():
|
|
query("SELECT 1 FROM schema_migrations LIMIT 1")
|
|
storage.client.head_bucket(Bucket=storage.BUCKET)
|
|
return {"status": "ok"}
|
|
|
|
@app.post("/api/v1/users", response_model=User, status_code=201, dependencies=[Depends(require_json)])
|
|
def create_user(payload: CreateUser):
|
|
password_hash = HASHER.hash(payload.password)
|
|
try:
|
|
return query("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at",
|
|
(uuid.uuid4(), payload.username, payload.display_name, password_hash))[0]
|
|
except psycopg.errors.UniqueViolation:
|
|
raise ApiError(409, "conflict", "Username already exists") from None
|
|
|
|
@app.post("/api/v1/auth/sessions", status_code=201)
|
|
async def login(request: Request, authorization: Optional[str] = Header(default=None)):
|
|
# CPU/SQL work runs in FastAPI's worker pool, not in the event loop.
|
|
from starlette.concurrency import run_in_threadpool
|
|
if await request.body():
|
|
raise ApiError(400, "invalid_request", "Session request must not have a body")
|
|
return await run_in_threadpool(login_sync, authorization)
|
|
|
|
|
|
def login_sync(authorization):
|
|
try:
|
|
if not authorization or not authorization.startswith("Basic "):
|
|
raise ValueError
|
|
username, password = base64.b64decode(authorization[6:], validate=True).decode("utf-8").split(":", 1)
|
|
if not USERNAME_RE.fullmatch(username) or not 12 <= len(password) <= 128:
|
|
raise ValueError
|
|
except (ValueError, UnicodeError, binascii.Error):
|
|
raise ApiError(401, "unauthorized", "Invalid credentials") from None
|
|
users = query("SELECT * FROM users WHERE username=%s", (username,))
|
|
try:
|
|
HASHER.verify(users[0]["password_hash"] if users else DUMMY_HASH, password)
|
|
except VerificationError:
|
|
raise ApiError(401, "unauthorized", "Invalid credentials") from None
|
|
if not users:
|
|
raise ApiError(401, "unauthorized", "Invalid credentials")
|
|
token = secrets.token_urlsafe(32)
|
|
expiry = query("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at",
|
|
(token_hash(token), users[0]["id"], TTL))[0]["expires_at"]
|
|
user = users[0]
|
|
del user["password_hash"]
|
|
return {"token": token, "token_type": "Bearer", "expires_at": expiry, "user": user}
|
|
|
|
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
|
def logout(actor=Depends(current_user), authorization: Optional[str] = Header(default=None)):
|
|
query("DELETE FROM sessions WHERE token_hash=%s RETURNING user_id", (bearer(authorization),))
|
|
return Response(status_code=204)
|
|
|
|
@app.get("/api/v1/users/me", response_model=User)
|
|
def get_me(actor=Depends(current_user)):
|
|
return actor
|
|
|
|
@app.post("/api/v1/chats", response_model=Chat, status_code=201, dependencies=[Depends(require_json)])
|
|
def create_chat(payload: CreateChat, actor=Depends(current_user)):
|
|
members = [str(m) for m in payload.member_ids]
|
|
if actor["id"] not in members:
|
|
raise ApiError(400, "invalid_request", "The current user must be a member")
|
|
try:
|
|
with POOL.connection() as conn:
|
|
chat = wire(conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid.uuid4(),payload.title)).fetchone())
|
|
with conn.cursor() as cur:
|
|
cur.executemany("INSERT INTO members(chat_id,user_id) VALUES (%s,%s)", [(chat["id"],m) for m in members])
|
|
chat["member_ids"] = members
|
|
return chat
|
|
except psycopg.errors.ForeignKeyViolation:
|
|
raise ApiError(400, "invalid_request", "All chat members must exist") from None
|
|
|
|
@app.get("/api/v1/chats", response_model=ChatPage)
|
|
def list_chats(request: Request, limit: int = Query(50,ge=1,le=100), cursor: Optional[str] = Query(None,min_length=1), actor=Depends(current_user)):
|
|
scope = "chats:" + actor["id"]
|
|
limit, after = page_args(request,scope,limit,cursor)
|
|
sql = "SELECT c.* FROM chats c JOIN members m ON c.id=m.chat_id WHERE m.user_id=%s"
|
|
params = [actor["id"]]
|
|
if after:
|
|
sql += " AND (c.created_at,c.id) > (%s,%s)"
|
|
params.extend(after)
|
|
rows = query(sql + " ORDER BY c.created_at,c.id LIMIT %s", params + [limit+1])
|
|
return page(chat_members(rows),limit,scope)
|
|
|
|
@app.get("/api/v1/chats/{chat_id}", response_model=Chat)
|
|
def get_chat(chat_id: str, actor=Depends(current_user)):
|
|
return visible_chat(chat_id,actor["id"])
|
|
|
|
@app.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201, dependencies=[Depends(require_json)])
|
|
def create_message(chat_id: str, payload: CreateMessage, actor=Depends(current_user),
|
|
idempotency_key: Optional[str] = Header(default=None)):
|
|
visible_chat(chat_id, actor["id"])
|
|
if not idempotency_key or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key):
|
|
raise ApiError(400, "invalid_request", "Invalid Idempotency-Key")
|
|
ids = [str(i) for i in payload.attachment_ids]
|
|
body = json.dumps({"text": payload.text, "attachment_ids": ids}, sort_keys=True)
|
|
scope = actor["id"] + ":" + chat_id + ":" + idempotency_key
|
|
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
|
|
with POOL.connection() as conn:
|
|
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
|
previous = conn.execute("SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()",
|
|
(actor["id"], chat_id, idempotency_key)).fetchone()
|
|
if previous:
|
|
if previous["body"] != body:
|
|
raise ApiError(409, "conflict", "Key already used for a different message")
|
|
return previous["response"]
|
|
if ids:
|
|
found = conn.execute("SELECT id FROM attachments WHERE id=ANY(%s::uuid[]) AND chat_id=%s AND status='ready'", (ids, chat_id)).fetchall()
|
|
if len(found) != len(ids):
|
|
raise ApiError(404, "not_found", "Attachment not found")
|
|
msg = wire(conn.execute("INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES (%s,%s,%s,%s,%s::uuid[]) RETURNING *",
|
|
(uuid.uuid4(),chat_id,actor["id"],payload.text,ids)).fetchone())
|
|
msg["attachment_ids"] = ids
|
|
from psycopg.types.json import Jsonb
|
|
conn.execute("""INSERT INTO idempotency(user_id,chat_id,key,body,response) VALUES (%s,%s,%s,%s,%s)
|
|
ON CONFLICT(user_id,chat_id,key) DO UPDATE SET body=EXCLUDED.body,response=EXCLUDED.response,expires_at=clock_timestamp()+interval '24 hours'""",
|
|
(actor["id"],chat_id,idempotency_key,body,Jsonb(msg)))
|
|
return msg
|
|
|
|
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
|
|
def list_messages(request: Request, chat_id: str, limit: int = Query(50,ge=1,le=100), cursor: Optional[str] = Query(None,min_length=1), actor=Depends(current_user)):
|
|
visible_chat(chat_id,actor["id"])
|
|
scope = "messages:" + chat_id
|
|
limit,after = page_args(request,scope,limit,cursor)
|
|
sql = "SELECT * FROM messages WHERE chat_id=%s"
|
|
params = [chat_id]
|
|
if after:
|
|
sql += " AND (created_at,id) > (%s,%s)"
|
|
params.extend(after)
|
|
return page(query(sql+" ORDER BY created_at,id LIMIT %s",params+[limit+1]),limit,scope)
|
|
|
|
import storage
|
|
from attachments import install
|
|
install(app, POOL, current_user, visible_chat, query, wire, ApiError)
|
|
|
|
@app.exception_handler(storage.StorageError)
|
|
@app.exception_handler(storage.ClientError)
|
|
async def storage_error(request, error):
|
|
return error_response(request, 503, "unavailable", "Storage temporarily unavailable")
|
|
|
|
if __name__ == "__main__":
|
|
uvicorn.run(app, host=os.environ.get("APP_HOST","0.0.0.0"), port=env_int("APP_PORT",8080,1))
|