forked from mtusi2026/mtusi2026lab4
Стенд: PostgreSQL 16 + две API-реплики за HAProxy + SeaweedFS S3 в приватном bucket. Метаданные вложений в PostgreSQL, байты в S3, object key назначает сервер. Лимиты 10 MiB и 40 млн пикселей, MIME по магическим байтам, проверка filename. Добавлены smoke/негативные тесты, хуки restart/stop-one, сценарий отказа S3 и отчёт с доказательствами.
1075 lines
42 KiB
Python
1075 lines
42 KiB
Python
"""HTTP API implementation for laboratory 4."""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sys
|
|
import traceback
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from urllib.parse import parse_qs, quote, unquote, urlsplit
|
|
|
|
import psycopg
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from psycopg.errors import UniqueViolation
|
|
from psycopg.types.json import Jsonb
|
|
|
|
from .config import Config
|
|
from .db import create_pool, migrate, wait_for_database
|
|
from .images import ImageTooLarge, InvalidImage, inspect_image
|
|
from .security import (
|
|
decode_cursor,
|
|
dummy_password_hash,
|
|
encode_cursor,
|
|
hash_password,
|
|
new_session_token,
|
|
token_hash,
|
|
verify_password,
|
|
)
|
|
from .storage import Storage, is_missing_object
|
|
|
|
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
|
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
|
JSON_BODY_LIMIT = 1024 * 1024
|
|
DEFAULT_PAGE_SIZE = 50
|
|
MAX_PAGE_SIZE = 100
|
|
|
|
ERROR_MESSAGES = {
|
|
400: "Invalid request",
|
|
401: "Unauthorized",
|
|
404: "Not found",
|
|
409: "Conflict",
|
|
413: "Payload too large",
|
|
415: "Unsupported media type",
|
|
429: "Too many requests",
|
|
500: "Internal server error",
|
|
503: "Service unavailable",
|
|
}
|
|
|
|
|
|
class ApiError(Exception):
|
|
def __init__(self, status, code, message=None):
|
|
super().__init__(message or ERROR_MESSAGES.get(status, "Error"))
|
|
self.status = status
|
|
self.code = code
|
|
self.message = message or ERROR_MESSAGES.get(status, "Error")
|
|
|
|
|
|
def utcnow():
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def format_timestamp(value):
|
|
if value.tzinfo is None:
|
|
value = value.replace(tzinfo=timezone.utc)
|
|
value = value.astimezone(timezone.utc)
|
|
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
|
|
|
|
|
def parse_timestamp(value):
|
|
if not isinstance(value, str) or not value.endswith("Z"):
|
|
raise ValueError("invalid timestamp")
|
|
parsed = datetime.fromisoformat(value[:-1] + "+00:00")
|
|
return parsed.astimezone(timezone.utc)
|
|
|
|
|
|
def user_json(row):
|
|
return {
|
|
"id": str(row["id"]),
|
|
"username": row["username"],
|
|
"display_name": row["display_name"],
|
|
"created_at": format_timestamp(row["created_at"]),
|
|
}
|
|
|
|
|
|
def chat_json(row):
|
|
members = row.get("member_ids") or []
|
|
return {
|
|
"id": str(row["id"]),
|
|
"title": row["title"],
|
|
"member_ids": sorted(str(member) for member in members),
|
|
"created_at": format_timestamp(row["created_at"]),
|
|
}
|
|
|
|
|
|
def message_json(row):
|
|
attachments = row.get("attachment_ids") or []
|
|
if attachments == [None]:
|
|
attachments = []
|
|
return {
|
|
"id": str(row["id"]),
|
|
"chat_id": str(row["chat_id"]),
|
|
"sender_id": str(row["sender_id"]),
|
|
"text": row["text"],
|
|
"created_at": format_timestamp(row["created_at"]),
|
|
"attachment_ids": [str(item) for item in attachments],
|
|
}
|
|
|
|
|
|
def attachment_json(row):
|
|
variants = row.get("variants") or []
|
|
if isinstance(variants, str):
|
|
variants = json.loads(variants)
|
|
return {
|
|
"id": str(row["id"]),
|
|
"chat_id": str(row["chat_id"]),
|
|
"owner_id": str(row["owner_id"]),
|
|
"filename": row["filename"],
|
|
"content_type": row["content_type"],
|
|
"size_bytes": int(row["size_bytes"]),
|
|
"sha256": row["sha256"],
|
|
"status": row["status"],
|
|
"variants": variants,
|
|
"error_code": row["error_code"],
|
|
"created_at": format_timestamp(row["created_at"]),
|
|
"updated_at": format_timestamp(row["updated_at"]),
|
|
}
|
|
|
|
|
|
class App:
|
|
def __init__(self, config, pool, storage):
|
|
self.config = config
|
|
self.pool = pool
|
|
self.storage = storage
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
protocol_version = "HTTP/1.1"
|
|
server_version = "MessengerLab4/1.0"
|
|
sys_version = ""
|
|
|
|
def log_message(self, fmt, *args):
|
|
sys.stderr.write("%s %s\n" % (self.address_string(), fmt % args))
|
|
|
|
@property
|
|
def app(self):
|
|
return self.server.app
|
|
|
|
def do_GET(self):
|
|
self._handle()
|
|
|
|
def do_POST(self):
|
|
self._handle()
|
|
|
|
def do_DELETE(self):
|
|
self._handle()
|
|
|
|
def do_PUT(self):
|
|
self._handle()
|
|
|
|
def do_PATCH(self):
|
|
self._handle()
|
|
|
|
def _handle(self):
|
|
self.request_id = uuid.uuid4().hex
|
|
self._response_started = False
|
|
self._body_consumed = False
|
|
self._path = urlsplit(self.path).path
|
|
try:
|
|
self._dispatch()
|
|
except ApiError as error:
|
|
self._send_error(error.status, error.code, error.message)
|
|
except (BrokenPipeError, ConnectionResetError):
|
|
self.close_connection = True
|
|
except Exception:
|
|
traceback.print_exc()
|
|
self._send_error(500, "internal_error", ERROR_MESSAGES[500])
|
|
|
|
def _common_headers(self):
|
|
self.send_header("X-Request-Id", self.request_id)
|
|
self.send_header("X-Instance-Id", self.app.config.instance_id)
|
|
|
|
def _send_json(self, status, payload, challenge=None):
|
|
if self._response_started:
|
|
self.close_connection = True
|
|
return
|
|
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self._common_headers()
|
|
if challenge:
|
|
self.send_header("WWW-Authenticate", challenge)
|
|
if self.close_connection:
|
|
self.send_header("Connection", "close")
|
|
self.end_headers()
|
|
self._response_started = True
|
|
self.wfile.write(body)
|
|
|
|
def _send_empty(self, status):
|
|
if self._response_started:
|
|
self.close_connection = True
|
|
return
|
|
self.send_response(status)
|
|
self.send_header("Content-Length", "0")
|
|
self._common_headers()
|
|
if self.close_connection:
|
|
self.send_header("Connection", "close")
|
|
self.end_headers()
|
|
self._response_started = True
|
|
|
|
def _send_error(self, status, code, message):
|
|
if self._response_started:
|
|
self.close_connection = True
|
|
return
|
|
if self.command in ("POST", "PUT", "PATCH") and not self._body_consumed:
|
|
self.close_connection = True
|
|
challenge = None
|
|
if status == 401:
|
|
if self.command == "POST" and self._path == "/api/v1/auth/sessions":
|
|
challenge = 'Basic realm="messenger", charset="UTF-8"'
|
|
else:
|
|
challenge = 'Bearer realm="messenger"'
|
|
payload = {
|
|
"error": {
|
|
"code": code,
|
|
"message": message or ERROR_MESSAGES.get(status, "Error"),
|
|
"request_id": self.request_id,
|
|
}
|
|
}
|
|
self._send_json(status, payload, challenge)
|
|
|
|
def _dispatch(self):
|
|
parsed = urlsplit(self.path)
|
|
self._path = parsed.path
|
|
self._query = parse_qs(parsed.query, keep_blank_values=True)
|
|
method = self.command
|
|
path = parsed.path
|
|
|
|
if path == "/health/live" and method == "GET":
|
|
return self._health_live()
|
|
if path == "/health/ready" and method == "GET":
|
|
return self._health_ready()
|
|
if path == "/api/v1/users" and method == "POST":
|
|
return self._register_user()
|
|
if path == "/api/v1/users/me" and method == "GET":
|
|
return self._current_user_endpoint()
|
|
if path == "/api/v1/auth/sessions" and method == "POST":
|
|
return self._create_session()
|
|
if path == "/api/v1/auth/sessions/current" and method == "DELETE":
|
|
return self._delete_session()
|
|
if path == "/api/v1/chats" and method == "POST":
|
|
return self._create_chat()
|
|
if path == "/api/v1/chats" and method == "GET":
|
|
return self._list_chats()
|
|
|
|
parts = [unquote(part) for part in path.strip("/").split("/") if part]
|
|
if len(parts) == 4 and parts[:3] == ["api", "v1", "chats"] and method == "GET":
|
|
return self._get_chat(self._parse_uuid(parts[3]))
|
|
if len(parts) == 5 and parts[:3] == ["api", "v1", "chats"] and parts[4] == "messages":
|
|
chat_id = self._parse_uuid(parts[3])
|
|
if method == "GET":
|
|
return self._list_messages(chat_id)
|
|
if method == "POST":
|
|
return self._create_message(chat_id)
|
|
if len(parts) == 5 and parts[:3] == ["api", "v1", "chats"] and parts[4] == "attachments":
|
|
if method == "POST":
|
|
return self._upload_attachment(self._parse_uuid(parts[3]))
|
|
if len(parts) == 4 and parts[:3] == ["api", "v1", "attachments"] and method == "GET":
|
|
return self._get_attachment(self._parse_uuid(parts[3]))
|
|
if len(parts) == 5 and parts[:3] == ["api", "v1", "attachments"] and parts[4] == "content":
|
|
if method == "GET":
|
|
return self._download_attachment(self._parse_uuid(parts[3]))
|
|
|
|
raise ApiError(404, "not_found", "Endpoint not found")
|
|
|
|
# ------------------------------------------------------------------
|
|
# request helpers
|
|
# ------------------------------------------------------------------
|
|
def _read_body(self, max_bytes):
|
|
transfer_encoding = self.headers.get("Transfer-Encoding", "").lower()
|
|
if "chunked" in transfer_encoding:
|
|
data = self._read_chunked(max_bytes)
|
|
self._body_consumed = True
|
|
return data
|
|
|
|
raw_length = self.headers.get("Content-Length")
|
|
if raw_length is None:
|
|
self._body_consumed = True
|
|
return b""
|
|
try:
|
|
length = int(raw_length)
|
|
except (TypeError, ValueError):
|
|
raise ApiError(400, "invalid_request", "Invalid Content-Length") from None
|
|
if length < 0:
|
|
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
|
if length > max_bytes:
|
|
self.close_connection = True
|
|
raise ApiError(413, "payload_too_large", "Request body is too large")
|
|
data = self.rfile.read(length)
|
|
if len(data) != length:
|
|
raise ApiError(400, "invalid_request", "Incomplete request body")
|
|
self._body_consumed = True
|
|
return data
|
|
|
|
def _read_chunked(self, max_bytes):
|
|
data = bytearray()
|
|
while True:
|
|
line = self.rfile.readline(1024)
|
|
if not line:
|
|
raise ApiError(400, "invalid_request", "Incomplete chunked body")
|
|
size_text = line.strip().split(b";", 1)[0]
|
|
try:
|
|
size = int(size_text, 16)
|
|
except ValueError:
|
|
raise ApiError(400, "invalid_request", "Invalid chunk size") from None
|
|
if size == 0:
|
|
while True:
|
|
trailer = self.rfile.readline(1024)
|
|
if trailer in (b"\r\n", b"\n", b""):
|
|
break
|
|
break
|
|
if len(data) + size > max_bytes:
|
|
self.close_connection = True
|
|
raise ApiError(413, "payload_too_large", "Request body is too large")
|
|
chunk = self.rfile.read(size)
|
|
if len(chunk) != size:
|
|
raise ApiError(400, "invalid_request", "Incomplete chunk")
|
|
data.extend(chunk)
|
|
if self.rfile.read(2) != b"\r\n":
|
|
raise ApiError(400, "invalid_request", "Invalid chunk terminator")
|
|
return bytes(data)
|
|
|
|
def _json_body(self):
|
|
if self.headers.get_content_type() != "application/json":
|
|
raise ApiError(400, "invalid_request", "Expected application/json")
|
|
data = self._read_body(JSON_BODY_LIMIT)
|
|
if not data:
|
|
raise ApiError(400, "invalid_request", "JSON body is required")
|
|
try:
|
|
value = json.loads(data.decode("utf-8"))
|
|
except (ValueError, UnicodeError):
|
|
raise ApiError(400, "invalid_request", "Malformed JSON") from None
|
|
if not isinstance(value, dict):
|
|
raise ApiError(400, "invalid_request", "JSON object is required")
|
|
return value
|
|
|
|
def _check_fields(self, body, allowed):
|
|
if set(body) - set(allowed):
|
|
raise ApiError(400, "invalid_request", "Unexpected JSON field")
|
|
|
|
def _query_value(self, name, required=False):
|
|
values = self._query.get(name)
|
|
if not values:
|
|
if required:
|
|
raise ApiError(400, "invalid_request", "Missing query parameter: " + name)
|
|
return None
|
|
if len(values) != 1:
|
|
raise ApiError(400, "invalid_request", "Repeated query parameter: " + name)
|
|
return values[0]
|
|
|
|
def _parse_limit(self):
|
|
raw = self._query_value("limit")
|
|
if raw is None:
|
|
return DEFAULT_PAGE_SIZE
|
|
if not re.fullmatch(r"[0-9]{1,3}", raw):
|
|
raise ApiError(400, "invalid_request", "Invalid limit")
|
|
value = int(raw)
|
|
if value < 1 or value > MAX_PAGE_SIZE:
|
|
raise ApiError(400, "invalid_request", "Invalid limit")
|
|
return value
|
|
|
|
def _parse_cursor(self):
|
|
raw = self._query_value("cursor")
|
|
if raw is None:
|
|
return None
|
|
try:
|
|
return decode_cursor(self.app.config.cursor_secret, raw)
|
|
except (ValueError, TypeError, KeyError):
|
|
raise ApiError(400, "invalid_request", "Invalid cursor") from None
|
|
|
|
@staticmethod
|
|
def _parse_uuid(value):
|
|
try:
|
|
return uuid.UUID(value)
|
|
except (ValueError, TypeError, AttributeError):
|
|
raise ApiError(400, "invalid_request", "Invalid UUID") from None
|
|
|
|
@staticmethod
|
|
def _require_string(value, field, minimum=0, maximum=None):
|
|
if not isinstance(value, str):
|
|
raise ApiError(400, "invalid_request", "Invalid " + field)
|
|
if len(value) < minimum or (maximum is not None and len(value) > maximum):
|
|
raise ApiError(400, "invalid_request", "Invalid " + field)
|
|
return value
|
|
|
|
# ------------------------------------------------------------------
|
|
# authentication
|
|
# ------------------------------------------------------------------
|
|
def _bearer_token(self):
|
|
header = self.headers.get("Authorization", "")
|
|
scheme, separator, value = header.partition(" ")
|
|
if not separator or scheme.lower() != "bearer" or not value.strip():
|
|
raise ApiError(401, "unauthorized", "Bearer token is required")
|
|
return value.strip()
|
|
|
|
def _current_user(self):
|
|
token = self._bearer_token()
|
|
with self.app.pool.connection() as connection:
|
|
row = connection.execute(
|
|
"""
|
|
SELECT u.id, u.username, u.display_name, u.created_at
|
|
FROM sessions s
|
|
JOIN users u ON u.id = s.user_id
|
|
WHERE s.token_hash = %s AND s.expires_at > now()
|
|
""",
|
|
(token_hash(token),),
|
|
).fetchone()
|
|
if row is None:
|
|
raise ApiError(401, "unauthorized", "Invalid or expired session")
|
|
return row
|
|
|
|
@staticmethod
|
|
def _require_member(connection, chat_id, user_id):
|
|
row = connection.execute(
|
|
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
|
(chat_id, user_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Chat not found")
|
|
|
|
# ------------------------------------------------------------------
|
|
# health and users
|
|
# ------------------------------------------------------------------
|
|
def _health_live(self):
|
|
self._send_json(200, {"status": "ok"})
|
|
|
|
def _health_ready(self):
|
|
# Readiness covers the whole mandatory synchronous path: PostgreSQL for
|
|
# metadata/sessions and S3 for attachment bytes. A storage outage makes
|
|
# replicas unready, so the load balancer stops routing to them; this is a
|
|
# deliberate, documented choice (see REPORT.md) and keeps the public
|
|
# persistence check meaningful.
|
|
try:
|
|
with self.app.pool.connection() as connection:
|
|
connection.execute("SELECT 1")
|
|
except Exception:
|
|
raise ApiError(503, "unavailable", "Service is not ready") from None
|
|
if not self.app.storage.ready():
|
|
raise ApiError(503, "unavailable", "Service is not ready")
|
|
self._send_json(200, {"status": "ok"})
|
|
|
|
def _register_user(self):
|
|
body = self._json_body()
|
|
self._check_fields(body, {"username", "display_name", "password"})
|
|
for field in ("username", "display_name", "password"):
|
|
if field not in body:
|
|
raise ApiError(400, "invalid_request", "Missing field: " + field)
|
|
|
|
username = self._require_string(body["username"], "username", 3, 32)
|
|
if USERNAME_RE.fullmatch(username) is None:
|
|
raise ApiError(400, "invalid_request", "Invalid username")
|
|
display_name = self._require_string(body["display_name"], "display_name", 1, 100)
|
|
if not display_name.strip():
|
|
raise ApiError(400, "invalid_request", "Invalid display_name")
|
|
password = self._require_string(body["password"], "password", 12, 128)
|
|
|
|
user_id = uuid.uuid4()
|
|
created_at = utcnow()
|
|
password_hash = hash_password(password)
|
|
try:
|
|
with self.app.pool.connection() as connection:
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
|
VALUES (%s, %s, %s, %s, %s)
|
|
""",
|
|
(user_id, username, display_name, password_hash, created_at),
|
|
)
|
|
except UniqueViolation:
|
|
raise ApiError(409, "conflict", "Username is already registered") from None
|
|
self._send_json(
|
|
201,
|
|
user_json(
|
|
{
|
|
"id": user_id,
|
|
"username": username,
|
|
"display_name": display_name,
|
|
"created_at": created_at,
|
|
}
|
|
),
|
|
)
|
|
|
|
def _create_session(self):
|
|
if self.headers.get("Content-Length") not in (None, "0") or self.headers.get("Transfer-Encoding"):
|
|
self.close_connection = True
|
|
raise ApiError(400, "invalid_request", "Session creation takes no body")
|
|
header = self.headers.get("Authorization", "")
|
|
scheme, separator, encoded = header.partition(" ")
|
|
if not separator or scheme.lower() != "basic" or not encoded.strip():
|
|
raise ApiError(401, "unauthorized", "Basic credentials are required")
|
|
try:
|
|
decoded = base64.b64decode(encoded.strip(), validate=True).decode("utf-8")
|
|
username, password = decoded.split(":", 1)
|
|
except (ValueError, UnicodeError):
|
|
raise ApiError(401, "unauthorized", "Invalid Basic credentials") from None
|
|
if not username or not password:
|
|
raise ApiError(401, "unauthorized", "Invalid Basic credentials")
|
|
|
|
with self.app.pool.connection() as connection:
|
|
row = connection.execute(
|
|
"""
|
|
SELECT id, username, display_name, created_at, password_hash
|
|
FROM users WHERE username = %s
|
|
""",
|
|
(username,),
|
|
).fetchone()
|
|
if row is None:
|
|
verify_password(password, dummy_password_hash())
|
|
raise ApiError(401, "unauthorized", "Invalid credentials")
|
|
if not verify_password(password, row["password_hash"]):
|
|
raise ApiError(401, "unauthorized", "Invalid credentials")
|
|
|
|
token = new_session_token()
|
|
expires_at = utcnow() + timedelta(seconds=self.app.config.session_ttl_seconds)
|
|
with self.app.pool.connection() as connection:
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO sessions (token_hash, user_id, expires_at, created_at)
|
|
VALUES (%s, %s, %s, %s)
|
|
""",
|
|
(token_hash(token), row["id"], expires_at, utcnow()),
|
|
)
|
|
self._send_json(
|
|
201,
|
|
{
|
|
"token": token,
|
|
"token_type": "Bearer",
|
|
"expires_at": format_timestamp(expires_at),
|
|
"user": user_json(row),
|
|
},
|
|
)
|
|
|
|
# Separate name from _current_user(), which is the bearer authentication helper.
|
|
def _current_user_endpoint(self):
|
|
user = self._current_user()
|
|
self._send_json(200, user_json(user))
|
|
|
|
def _delete_session(self):
|
|
token = self._bearer_token()
|
|
with self.app.pool.connection() as connection:
|
|
cursor = connection.execute(
|
|
"DELETE FROM sessions WHERE token_hash = %s", (token_hash(token),)
|
|
)
|
|
deleted = cursor.rowcount
|
|
if deleted == 0:
|
|
raise ApiError(401, "unauthorized", "Invalid or expired session")
|
|
self._send_empty(204)
|
|
|
|
# ------------------------------------------------------------------
|
|
# chats
|
|
# ------------------------------------------------------------------
|
|
def _create_chat(self):
|
|
user = self._current_user()
|
|
body = self._json_body()
|
|
self._check_fields(body, {"title", "member_ids"})
|
|
if "title" not in body or "member_ids" not in body:
|
|
raise ApiError(400, "invalid_request", "title and member_ids are required")
|
|
title = self._require_string(body["title"], "title", 1, 100)
|
|
if not title.strip():
|
|
raise ApiError(400, "invalid_request", "Invalid title")
|
|
raw_members = body["member_ids"]
|
|
if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100:
|
|
raise ApiError(400, "invalid_request", "Invalid member_ids")
|
|
members = []
|
|
for value in raw_members:
|
|
if not isinstance(value, str):
|
|
raise ApiError(400, "invalid_request", "Invalid member_ids")
|
|
members.append(str(self._parse_uuid(value)))
|
|
if len(set(members)) != len(members):
|
|
raise ApiError(400, "invalid_request", "Duplicate member_ids")
|
|
if str(user["id"]) not in members:
|
|
raise ApiError(400, "invalid_request", "Caller must be a chat member")
|
|
|
|
chat_id = uuid.uuid4()
|
|
created_at = utcnow()
|
|
member_uuids = [uuid.UUID(value) for value in members]
|
|
with self.app.pool.connection() as connection:
|
|
with connection.transaction():
|
|
found = connection.execute(
|
|
"SELECT id FROM users WHERE id = ANY(%s)", (member_uuids,)
|
|
).fetchall()
|
|
if len(found) != len(members):
|
|
raise ApiError(400, "invalid_request", "Unknown chat member")
|
|
connection.execute(
|
|
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
|
|
(chat_id, title, created_at),
|
|
)
|
|
with connection.cursor() as cursor:
|
|
cursor.executemany(
|
|
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
|
[(chat_id, member_id) for member_id in member_uuids],
|
|
)
|
|
self._send_json(
|
|
201,
|
|
chat_json(
|
|
{
|
|
"id": chat_id,
|
|
"title": title,
|
|
"member_ids": member_uuids,
|
|
"created_at": created_at,
|
|
}
|
|
),
|
|
)
|
|
|
|
def _list_chats(self):
|
|
user = self._current_user()
|
|
limit = self._parse_limit()
|
|
payload = self._parse_cursor()
|
|
condition = ""
|
|
parameters = [user["id"]]
|
|
if payload is not None:
|
|
if payload.get("kind") != "chats":
|
|
raise ApiError(400, "invalid_request", "Invalid cursor")
|
|
try:
|
|
created_at = parse_timestamp(payload["created_at"])
|
|
chat_id = uuid.UUID(payload["id"])
|
|
except (KeyError, ValueError, TypeError):
|
|
raise ApiError(400, "invalid_request", "Invalid cursor") from None
|
|
condition = " AND (c.created_at, c.id) > (%s, %s)"
|
|
parameters.extend([created_at, chat_id])
|
|
parameters.append(limit + 1)
|
|
sql = (
|
|
"""
|
|
SELECT c.id, c.title, c.created_at,
|
|
(SELECT array_agg(cm2.user_id ORDER BY cm2.user_id)
|
|
FROM chat_members cm2 WHERE cm2.chat_id = c.id) AS member_ids
|
|
FROM chats c
|
|
JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = %s
|
|
WHERE TRUE
|
|
"""
|
|
+ condition
|
|
+ " ORDER BY c.created_at, c.id LIMIT %s"
|
|
)
|
|
with self.app.pool.connection() as connection:
|
|
rows = connection.execute(sql, parameters).fetchall()
|
|
has_more = len(rows) > limit
|
|
rows = rows[:limit]
|
|
next_cursor = None
|
|
if has_more and rows:
|
|
last = rows[-1]
|
|
next_cursor = encode_cursor(
|
|
self.app.config.cursor_secret,
|
|
{
|
|
"kind": "chats",
|
|
"created_at": format_timestamp(last["created_at"]),
|
|
"id": str(last["id"]),
|
|
},
|
|
)
|
|
self._send_json(
|
|
200, {"items": [chat_json(row) for row in rows], "next_cursor": next_cursor}
|
|
)
|
|
|
|
def _get_chat(self, chat_id):
|
|
user = self._current_user()
|
|
with self.app.pool.connection() as connection:
|
|
self._require_member(connection, chat_id, user["id"])
|
|
row = connection.execute(
|
|
"""
|
|
SELECT c.id, c.title, c.created_at,
|
|
(SELECT array_agg(cm2.user_id ORDER BY cm2.user_id)
|
|
FROM chat_members cm2 WHERE cm2.chat_id = c.id) AS member_ids
|
|
FROM chats c WHERE c.id = %s
|
|
""",
|
|
(chat_id,),
|
|
).fetchone()
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Chat not found")
|
|
self._send_json(200, chat_json(row))
|
|
|
|
# ------------------------------------------------------------------
|
|
# messages
|
|
# ------------------------------------------------------------------
|
|
@staticmethod
|
|
def _fetch_message(connection, message_id):
|
|
row = connection.execute(
|
|
"""
|
|
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
|
COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position)
|
|
FROM message_attachments ma
|
|
WHERE ma.message_id = m.id), '{}') AS attachment_ids
|
|
FROM messages m WHERE m.id = %s
|
|
""",
|
|
(message_id,),
|
|
).fetchone()
|
|
return None if row is None else message_json(row)
|
|
|
|
def _create_message(self, chat_id):
|
|
user = self._current_user()
|
|
with self.app.pool.connection() as connection:
|
|
self._require_member(connection, chat_id, user["id"])
|
|
|
|
idempotency_key = self.headers.get("Idempotency-Key")
|
|
if not idempotency_key or IDEMPOTENCY_RE.fullmatch(idempotency_key) is None:
|
|
raise ApiError(400, "invalid_request", "Valid Idempotency-Key is required")
|
|
|
|
body = self._json_body()
|
|
self._check_fields(body, {"text", "attachment_ids"})
|
|
if "text" not in body:
|
|
raise ApiError(400, "invalid_request", "text is required")
|
|
text = self._require_string(body["text"], "text", 0, 4000)
|
|
raw_attachments = body.get("attachment_ids", [])
|
|
if not isinstance(raw_attachments, list) or len(raw_attachments) > 10:
|
|
raise ApiError(400, "invalid_request", "Invalid attachment_ids")
|
|
attachment_ids = []
|
|
for value in raw_attachments:
|
|
if not isinstance(value, str):
|
|
raise ApiError(400, "invalid_request", "Invalid attachment_ids")
|
|
attachment_ids.append(str(self._parse_uuid(value)))
|
|
if len(set(attachment_ids)) != len(attachment_ids):
|
|
raise ApiError(400, "invalid_request", "Duplicate attachment_ids")
|
|
if not attachment_ids and not text.strip():
|
|
raise ApiError(400, "invalid_request", "Empty message")
|
|
|
|
request_hash = hashlib.sha256(
|
|
json.dumps(
|
|
{"text": text, "attachment_ids": attachment_ids},
|
|
sort_keys=True,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
).hexdigest()
|
|
scope = "POST|%s|%s|%s" % (user["id"], chat_id, idempotency_key)
|
|
lock_id = int.from_bytes(
|
|
hashlib.sha256(scope.encode("utf-8")).digest()[:8], "big", signed=True
|
|
)
|
|
|
|
with self.app.pool.connection() as connection:
|
|
with connection.transaction():
|
|
connection.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
|
|
existing = connection.execute(
|
|
"""
|
|
SELECT request_hash, message_id FROM idempotency
|
|
WHERE user_id = %s AND chat_id = %s AND key = %s
|
|
""",
|
|
(user["id"], chat_id, idempotency_key),
|
|
).fetchone()
|
|
if existing is not None:
|
|
if existing["request_hash"] != request_hash:
|
|
raise ApiError(
|
|
409, "conflict", "Idempotency key was used with a different request"
|
|
)
|
|
message = self._fetch_message(connection, existing["message_id"])
|
|
if message is None:
|
|
raise ApiError(409, "conflict", "Idempotent result is unavailable")
|
|
else:
|
|
if attachment_ids:
|
|
rows = connection.execute(
|
|
"""
|
|
SELECT id, chat_id, status FROM attachments
|
|
WHERE id = ANY(%s)
|
|
""",
|
|
([uuid.UUID(value) for value in attachment_ids],),
|
|
).fetchall()
|
|
if len(rows) != len(attachment_ids):
|
|
raise ApiError(404, "not_found", "Attachment not found")
|
|
by_id = {str(row["id"]): row for row in rows}
|
|
for attachment_id in attachment_ids:
|
|
row = by_id[attachment_id]
|
|
if str(row["chat_id"]) != str(chat_id):
|
|
raise ApiError(404, "not_found", "Attachment not found")
|
|
if row["status"] == "failed":
|
|
raise ApiError(409, "conflict", "Attachment failed processing")
|
|
|
|
message_id = uuid.uuid4()
|
|
created_at = utcnow()
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
|
VALUES (%s, %s, %s, %s, %s)
|
|
""",
|
|
(message_id, chat_id, user["id"], text, created_at),
|
|
)
|
|
if attachment_ids:
|
|
with connection.cursor() as cursor:
|
|
cursor.executemany(
|
|
"""
|
|
INSERT INTO message_attachments
|
|
(message_id, attachment_id, position)
|
|
VALUES (%s, %s, %s)
|
|
""",
|
|
[
|
|
(message_id, uuid.UUID(value), position)
|
|
for position, value in enumerate(attachment_ids)
|
|
],
|
|
)
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO idempotency
|
|
(user_id, chat_id, key, request_hash, message_id, created_at)
|
|
VALUES (%s, %s, %s, %s, %s, %s)
|
|
""",
|
|
(
|
|
user["id"],
|
|
chat_id,
|
|
idempotency_key,
|
|
request_hash,
|
|
message_id,
|
|
created_at,
|
|
),
|
|
)
|
|
message = self._fetch_message(connection, message_id)
|
|
self._send_json(201, message)
|
|
|
|
def _list_messages(self, chat_id):
|
|
user = self._current_user()
|
|
with self.app.pool.connection() as connection:
|
|
self._require_member(connection, chat_id, user["id"])
|
|
limit = self._parse_limit()
|
|
payload = self._parse_cursor()
|
|
condition = ""
|
|
parameters = [chat_id]
|
|
if payload is not None:
|
|
if payload.get("kind") != "messages" or str(payload.get("chat_id")) != str(chat_id):
|
|
raise ApiError(400, "invalid_request", "Cursor belongs to another resource")
|
|
try:
|
|
created_at = parse_timestamp(payload["created_at"])
|
|
message_id = uuid.UUID(payload["id"])
|
|
except (KeyError, ValueError, TypeError):
|
|
raise ApiError(400, "invalid_request", "Invalid cursor") from None
|
|
condition = " AND (m.created_at, m.id) > (%s, %s)"
|
|
parameters.extend([created_at, message_id])
|
|
parameters.append(limit + 1)
|
|
sql = (
|
|
"""
|
|
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
|
COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position)
|
|
FROM message_attachments ma
|
|
WHERE ma.message_id = m.id), '{}') AS attachment_ids
|
|
FROM messages m
|
|
WHERE m.chat_id = %s
|
|
"""
|
|
+ condition
|
|
+ " ORDER BY m.created_at, m.id LIMIT %s"
|
|
)
|
|
with self.app.pool.connection() as connection:
|
|
rows = connection.execute(sql, parameters).fetchall()
|
|
has_more = len(rows) > limit
|
|
rows = rows[:limit]
|
|
next_cursor = None
|
|
if has_more and rows:
|
|
last = rows[-1]
|
|
next_cursor = encode_cursor(
|
|
self.app.config.cursor_secret,
|
|
{
|
|
"kind": "messages",
|
|
"chat_id": str(chat_id),
|
|
"created_at": format_timestamp(last["created_at"]),
|
|
"id": str(last["id"]),
|
|
},
|
|
)
|
|
self._send_json(
|
|
200, {"items": [message_json(row) for row in rows], "next_cursor": next_cursor}
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# attachments
|
|
# ------------------------------------------------------------------
|
|
@staticmethod
|
|
def _valid_filename(filename):
|
|
if not filename or len(filename) > 255:
|
|
return False
|
|
for character in filename:
|
|
if character in ("/", "\\") or ord(character) < 32 or ord(character) == 127:
|
|
return False
|
|
return True
|
|
|
|
def _upload_attachment(self, chat_id):
|
|
user = self._current_user()
|
|
try:
|
|
with self.app.pool.connection() as connection:
|
|
self._require_member(connection, chat_id, user["id"])
|
|
content_type = self.headers.get_content_type()
|
|
if content_type not in ("image/png", "image/jpeg"):
|
|
self.close_connection = True
|
|
raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are accepted")
|
|
filename = self._query_value("filename", required=True)
|
|
if not self._valid_filename(filename):
|
|
self.close_connection = True
|
|
raise ApiError(400, "invalid_request", "Invalid filename")
|
|
data = self._read_body(self.app.config.max_upload_bytes)
|
|
except ApiError:
|
|
if not self._body_consumed:
|
|
self.close_connection = True
|
|
raise
|
|
|
|
if not data:
|
|
self.close_connection = True
|
|
raise ApiError(415, "unsupported_media_type", "Empty image")
|
|
try:
|
|
actual_content_type, _width, _height = inspect_image(
|
|
data, content_type, self.app.config.max_image_pixels
|
|
)
|
|
except ImageTooLarge:
|
|
self.close_connection = True
|
|
raise ApiError(413, "payload_too_large", "Image has too many pixels") from None
|
|
except InvalidImage:
|
|
self.close_connection = True
|
|
raise ApiError(415, "unsupported_media_type", "Invalid or mismatched image") from None
|
|
|
|
attachment_id = uuid.uuid4()
|
|
object_key = "chats/{}/{}/original".format(chat_id, attachment_id)
|
|
created_at = utcnow()
|
|
sha256 = hashlib.sha256(data).hexdigest()
|
|
try:
|
|
self.app.storage.put(object_key, data, actual_content_type)
|
|
except (BotoCoreError, ClientError):
|
|
raise ApiError(503, "unavailable", "Object storage is unavailable") from None
|
|
|
|
try:
|
|
with self.app.pool.connection() as connection:
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO attachments
|
|
(id, chat_id, owner_id, filename, content_type, size_bytes,
|
|
sha256, object_key, status, variants, error_code,
|
|
created_at, updated_at)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'ready', %s, NULL, %s, %s)
|
|
""",
|
|
(
|
|
attachment_id,
|
|
chat_id,
|
|
user["id"],
|
|
filename,
|
|
actual_content_type,
|
|
len(data),
|
|
sha256,
|
|
object_key,
|
|
Jsonb([]),
|
|
created_at,
|
|
created_at,
|
|
),
|
|
)
|
|
except psycopg.Error:
|
|
try:
|
|
self.app.storage.delete(object_key)
|
|
except (BotoCoreError, ClientError):
|
|
traceback.print_exc()
|
|
raise ApiError(503, "unavailable", "Database is unavailable") from None
|
|
|
|
self._send_json(
|
|
201,
|
|
attachment_json(
|
|
{
|
|
"id": attachment_id,
|
|
"chat_id": chat_id,
|
|
"owner_id": user["id"],
|
|
"filename": filename,
|
|
"content_type": actual_content_type,
|
|
"size_bytes": len(data),
|
|
"sha256": sha256,
|
|
"status": "ready",
|
|
"variants": [],
|
|
"error_code": None,
|
|
"created_at": created_at,
|
|
"updated_at": created_at,
|
|
}
|
|
),
|
|
)
|
|
|
|
def _get_attachment(self, attachment_id):
|
|
user = self._current_user()
|
|
with self.app.pool.connection() as connection:
|
|
row = connection.execute(
|
|
"""
|
|
SELECT a.* FROM attachments a
|
|
JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s
|
|
WHERE a.id = %s
|
|
""",
|
|
(user["id"], attachment_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Attachment not found")
|
|
self._send_json(200, attachment_json(row))
|
|
|
|
def _download_attachment(self, attachment_id):
|
|
user = self._current_user()
|
|
variant = self._query_value("variant")
|
|
if variant is None:
|
|
variant = "original"
|
|
if variant != "original":
|
|
raise ApiError(400, "invalid_request", "Unsupported variant")
|
|
with self.app.pool.connection() as connection:
|
|
row = connection.execute(
|
|
"""
|
|
SELECT a.* FROM attachments a
|
|
JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s
|
|
WHERE a.id = %s
|
|
""",
|
|
(user["id"], attachment_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Attachment not found")
|
|
if row["status"] != "ready":
|
|
raise ApiError(409, "conflict", "Attachment is not ready")
|
|
|
|
try:
|
|
response = self.app.storage.get(row["object_key"])
|
|
except (BotoCoreError, ClientError) as error:
|
|
if is_missing_object(error):
|
|
raise ApiError(404, "not_found", "Attachment content not found") from None
|
|
raise ApiError(503, "unavailable", "Object storage is unavailable") from None
|
|
|
|
body = response["Body"]
|
|
length = int(response.get("ContentLength") or row["size_bytes"])
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", row["content_type"])
|
|
self.send_header("Content-Length", str(length))
|
|
self.send_header(
|
|
"Content-Disposition",
|
|
"inline; filename*=UTF-8''" + quote(row["filename"], safe=""),
|
|
)
|
|
self._common_headers()
|
|
self.end_headers()
|
|
self._response_started = True
|
|
try:
|
|
for chunk in body.iter_chunks(chunk_size=65536):
|
|
if chunk:
|
|
self.wfile.write(chunk)
|
|
except (BrokenPipeError, ConnectionResetError):
|
|
self.close_connection = True
|
|
finally:
|
|
body.close()
|
|
|
|
|
|
class ApiServer(ThreadingHTTPServer):
|
|
daemon_threads = True
|
|
|
|
def handle_error(self, request, client_address):
|
|
error = sys.exc_info()[1]
|
|
if isinstance(error, (BrokenPipeError, ConnectionResetError)):
|
|
return
|
|
super().handle_error(request, client_address)
|
|
|
|
|
|
def make_server(app):
|
|
server = ApiServer((app.config.host, app.config.port), Handler)
|
|
server.app = app
|
|
return server
|
|
|
|
|
|
def main():
|
|
config = Config.from_env()
|
|
wait_for_database(config.database_url)
|
|
pool = create_pool(config.database_url)
|
|
pool.open(wait=True, timeout=30)
|
|
try:
|
|
migrate(pool)
|
|
storage = Storage(
|
|
endpoint=config.s3_endpoint,
|
|
region=config.s3_region,
|
|
bucket=config.s3_bucket,
|
|
access_key_id=config.s3_access_key_id,
|
|
secret_access_key=config.s3_secret_access_key,
|
|
)
|
|
app = App(config, pool, storage)
|
|
server = make_server(app)
|
|
sys.stderr.write(
|
|
"messenger lab4 listening on %s:%s as %s\n"
|
|
% (config.host, config.port, config.instance_id)
|
|
)
|
|
server.serve_forever()
|
|
finally:
|
|
pool.close()
|