Files
mtusi2026lab4/app/server.py
drunkdiente 935307d412 Реализовать ЛР4: S3-хранилище и вложения на оценку 3
Стенд: PostgreSQL 16 + две API-реплики за HAProxy + SeaweedFS S3 в приватном bucket. Метаданные вложений в PostgreSQL, байты в S3, object key назначает сервер. Лимиты 10 MiB и 40 млн пикселей, MIME по магическим байтам, проверка filename. Добавлены smoke/негативные тесты, хуки restart/stop-one, сценарий отказа S3 и отчёт с доказательствами.
2026-11-08 14:29:02 +03:00

1075 lines
42 KiB
Python

"""HTTP API implementation for laboratory 4."""
import base64
import hashlib
import json
import re
import sys
import traceback
import uuid
from datetime import datetime, timedelta, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, quote, unquote, urlsplit
import psycopg
from botocore.exceptions import BotoCoreError, ClientError
from psycopg.errors import UniqueViolation
from psycopg.types.json import Jsonb
from .config import Config
from .db import create_pool, migrate, wait_for_database
from .images import ImageTooLarge, InvalidImage, inspect_image
from .security import (
decode_cursor,
dummy_password_hash,
encode_cursor,
hash_password,
new_session_token,
token_hash,
verify_password,
)
from .storage import Storage, is_missing_object
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
JSON_BODY_LIMIT = 1024 * 1024
DEFAULT_PAGE_SIZE = 50
MAX_PAGE_SIZE = 100
ERROR_MESSAGES = {
400: "Invalid request",
401: "Unauthorized",
404: "Not found",
409: "Conflict",
413: "Payload too large",
415: "Unsupported media type",
429: "Too many requests",
500: "Internal server error",
503: "Service unavailable",
}
class ApiError(Exception):
def __init__(self, status, code, message=None):
super().__init__(message or ERROR_MESSAGES.get(status, "Error"))
self.status = status
self.code = code
self.message = message or ERROR_MESSAGES.get(status, "Error")
def utcnow():
return datetime.now(timezone.utc)
def format_timestamp(value):
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
value = value.astimezone(timezone.utc)
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
def parse_timestamp(value):
if not isinstance(value, str) or not value.endswith("Z"):
raise ValueError("invalid timestamp")
parsed = datetime.fromisoformat(value[:-1] + "+00:00")
return parsed.astimezone(timezone.utc)
def user_json(row):
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": format_timestamp(row["created_at"]),
}
def chat_json(row):
members = row.get("member_ids") or []
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": sorted(str(member) for member in members),
"created_at": format_timestamp(row["created_at"]),
}
def message_json(row):
attachments = row.get("attachment_ids") or []
if attachments == [None]:
attachments = []
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": format_timestamp(row["created_at"]),
"attachment_ids": [str(item) for item in attachments],
}
def attachment_json(row):
variants = row.get("variants") or []
if isinstance(variants, str):
variants = json.loads(variants)
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": int(row["size_bytes"]),
"sha256": row["sha256"],
"status": row["status"],
"variants": variants,
"error_code": row["error_code"],
"created_at": format_timestamp(row["created_at"]),
"updated_at": format_timestamp(row["updated_at"]),
}
class App:
def __init__(self, config, pool, storage):
self.config = config
self.pool = pool
self.storage = storage
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "MessengerLab4/1.0"
sys_version = ""
def log_message(self, fmt, *args):
sys.stderr.write("%s %s\n" % (self.address_string(), fmt % args))
@property
def app(self):
return self.server.app
def do_GET(self):
self._handle()
def do_POST(self):
self._handle()
def do_DELETE(self):
self._handle()
def do_PUT(self):
self._handle()
def do_PATCH(self):
self._handle()
def _handle(self):
self.request_id = uuid.uuid4().hex
self._response_started = False
self._body_consumed = False
self._path = urlsplit(self.path).path
try:
self._dispatch()
except ApiError as error:
self._send_error(error.status, error.code, error.message)
except (BrokenPipeError, ConnectionResetError):
self.close_connection = True
except Exception:
traceback.print_exc()
self._send_error(500, "internal_error", ERROR_MESSAGES[500])
def _common_headers(self):
self.send_header("X-Request-Id", self.request_id)
self.send_header("X-Instance-Id", self.app.config.instance_id)
def _send_json(self, status, payload, challenge=None):
if self._response_started:
self.close_connection = True
return
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self._common_headers()
if challenge:
self.send_header("WWW-Authenticate", challenge)
if self.close_connection:
self.send_header("Connection", "close")
self.end_headers()
self._response_started = True
self.wfile.write(body)
def _send_empty(self, status):
if self._response_started:
self.close_connection = True
return
self.send_response(status)
self.send_header("Content-Length", "0")
self._common_headers()
if self.close_connection:
self.send_header("Connection", "close")
self.end_headers()
self._response_started = True
def _send_error(self, status, code, message):
if self._response_started:
self.close_connection = True
return
if self.command in ("POST", "PUT", "PATCH") and not self._body_consumed:
self.close_connection = True
challenge = None
if status == 401:
if self.command == "POST" and self._path == "/api/v1/auth/sessions":
challenge = 'Basic realm="messenger", charset="UTF-8"'
else:
challenge = 'Bearer realm="messenger"'
payload = {
"error": {
"code": code,
"message": message or ERROR_MESSAGES.get(status, "Error"),
"request_id": self.request_id,
}
}
self._send_json(status, payload, challenge)
def _dispatch(self):
parsed = urlsplit(self.path)
self._path = parsed.path
self._query = parse_qs(parsed.query, keep_blank_values=True)
method = self.command
path = parsed.path
if path == "/health/live" and method == "GET":
return self._health_live()
if path == "/health/ready" and method == "GET":
return self._health_ready()
if path == "/api/v1/users" and method == "POST":
return self._register_user()
if path == "/api/v1/users/me" and method == "GET":
return self._current_user_endpoint()
if path == "/api/v1/auth/sessions" and method == "POST":
return self._create_session()
if path == "/api/v1/auth/sessions/current" and method == "DELETE":
return self._delete_session()
if path == "/api/v1/chats" and method == "POST":
return self._create_chat()
if path == "/api/v1/chats" and method == "GET":
return self._list_chats()
parts = [unquote(part) for part in path.strip("/").split("/") if part]
if len(parts) == 4 and parts[:3] == ["api", "v1", "chats"] and method == "GET":
return self._get_chat(self._parse_uuid(parts[3]))
if len(parts) == 5 and parts[:3] == ["api", "v1", "chats"] and parts[4] == "messages":
chat_id = self._parse_uuid(parts[3])
if method == "GET":
return self._list_messages(chat_id)
if method == "POST":
return self._create_message(chat_id)
if len(parts) == 5 and parts[:3] == ["api", "v1", "chats"] and parts[4] == "attachments":
if method == "POST":
return self._upload_attachment(self._parse_uuid(parts[3]))
if len(parts) == 4 and parts[:3] == ["api", "v1", "attachments"] and method == "GET":
return self._get_attachment(self._parse_uuid(parts[3]))
if len(parts) == 5 and parts[:3] == ["api", "v1", "attachments"] and parts[4] == "content":
if method == "GET":
return self._download_attachment(self._parse_uuid(parts[3]))
raise ApiError(404, "not_found", "Endpoint not found")
# ------------------------------------------------------------------
# request helpers
# ------------------------------------------------------------------
def _read_body(self, max_bytes):
transfer_encoding = self.headers.get("Transfer-Encoding", "").lower()
if "chunked" in transfer_encoding:
data = self._read_chunked(max_bytes)
self._body_consumed = True
return data
raw_length = self.headers.get("Content-Length")
if raw_length is None:
self._body_consumed = True
return b""
try:
length = int(raw_length)
except (TypeError, ValueError):
raise ApiError(400, "invalid_request", "Invalid Content-Length") from None
if length < 0:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if length > max_bytes:
self.close_connection = True
raise ApiError(413, "payload_too_large", "Request body is too large")
data = self.rfile.read(length)
if len(data) != length:
raise ApiError(400, "invalid_request", "Incomplete request body")
self._body_consumed = True
return data
def _read_chunked(self, max_bytes):
data = bytearray()
while True:
line = self.rfile.readline(1024)
if not line:
raise ApiError(400, "invalid_request", "Incomplete chunked body")
size_text = line.strip().split(b";", 1)[0]
try:
size = int(size_text, 16)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid chunk size") from None
if size == 0:
while True:
trailer = self.rfile.readline(1024)
if trailer in (b"\r\n", b"\n", b""):
break
break
if len(data) + size > max_bytes:
self.close_connection = True
raise ApiError(413, "payload_too_large", "Request body is too large")
chunk = self.rfile.read(size)
if len(chunk) != size:
raise ApiError(400, "invalid_request", "Incomplete chunk")
data.extend(chunk)
if self.rfile.read(2) != b"\r\n":
raise ApiError(400, "invalid_request", "Invalid chunk terminator")
return bytes(data)
def _json_body(self):
if self.headers.get_content_type() != "application/json":
raise ApiError(400, "invalid_request", "Expected application/json")
data = self._read_body(JSON_BODY_LIMIT)
if not data:
raise ApiError(400, "invalid_request", "JSON body is required")
try:
value = json.loads(data.decode("utf-8"))
except (ValueError, UnicodeError):
raise ApiError(400, "invalid_request", "Malformed JSON") from None
if not isinstance(value, dict):
raise ApiError(400, "invalid_request", "JSON object is required")
return value
def _check_fields(self, body, allowed):
if set(body) - set(allowed):
raise ApiError(400, "invalid_request", "Unexpected JSON field")
def _query_value(self, name, required=False):
values = self._query.get(name)
if not values:
if required:
raise ApiError(400, "invalid_request", "Missing query parameter: " + name)
return None
if len(values) != 1:
raise ApiError(400, "invalid_request", "Repeated query parameter: " + name)
return values[0]
def _parse_limit(self):
raw = self._query_value("limit")
if raw is None:
return DEFAULT_PAGE_SIZE
if not re.fullmatch(r"[0-9]{1,3}", raw):
raise ApiError(400, "invalid_request", "Invalid limit")
value = int(raw)
if value < 1 or value > MAX_PAGE_SIZE:
raise ApiError(400, "invalid_request", "Invalid limit")
return value
def _parse_cursor(self):
raw = self._query_value("cursor")
if raw is None:
return None
try:
return decode_cursor(self.app.config.cursor_secret, raw)
except (ValueError, TypeError, KeyError):
raise ApiError(400, "invalid_request", "Invalid cursor") from None
@staticmethod
def _parse_uuid(value):
try:
return uuid.UUID(value)
except (ValueError, TypeError, AttributeError):
raise ApiError(400, "invalid_request", "Invalid UUID") from None
@staticmethod
def _require_string(value, field, minimum=0, maximum=None):
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "Invalid " + field)
if len(value) < minimum or (maximum is not None and len(value) > maximum):
raise ApiError(400, "invalid_request", "Invalid " + field)
return value
# ------------------------------------------------------------------
# authentication
# ------------------------------------------------------------------
def _bearer_token(self):
header = self.headers.get("Authorization", "")
scheme, separator, value = header.partition(" ")
if not separator or scheme.lower() != "bearer" or not value.strip():
raise ApiError(401, "unauthorized", "Bearer token is required")
return value.strip()
def _current_user(self):
token = self._bearer_token()
with self.app.pool.connection() as connection:
row = connection.execute(
"""
SELECT u.id, u.username, u.display_name, u.created_at
FROM sessions s
JOIN users u ON u.id = s.user_id
WHERE s.token_hash = %s AND s.expires_at > now()
""",
(token_hash(token),),
).fetchone()
if row is None:
raise ApiError(401, "unauthorized", "Invalid or expired session")
return row
@staticmethod
def _require_member(connection, chat_id, user_id):
row = connection.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Chat not found")
# ------------------------------------------------------------------
# health and users
# ------------------------------------------------------------------
def _health_live(self):
self._send_json(200, {"status": "ok"})
def _health_ready(self):
# Readiness covers the whole mandatory synchronous path: PostgreSQL for
# metadata/sessions and S3 for attachment bytes. A storage outage makes
# replicas unready, so the load balancer stops routing to them; this is a
# deliberate, documented choice (see REPORT.md) and keeps the public
# persistence check meaningful.
try:
with self.app.pool.connection() as connection:
connection.execute("SELECT 1")
except Exception:
raise ApiError(503, "unavailable", "Service is not ready") from None
if not self.app.storage.ready():
raise ApiError(503, "unavailable", "Service is not ready")
self._send_json(200, {"status": "ok"})
def _register_user(self):
body = self._json_body()
self._check_fields(body, {"username", "display_name", "password"})
for field in ("username", "display_name", "password"):
if field not in body:
raise ApiError(400, "invalid_request", "Missing field: " + field)
username = self._require_string(body["username"], "username", 3, 32)
if USERNAME_RE.fullmatch(username) is None:
raise ApiError(400, "invalid_request", "Invalid username")
display_name = self._require_string(body["display_name"], "display_name", 1, 100)
if not display_name.strip():
raise ApiError(400, "invalid_request", "Invalid display_name")
password = self._require_string(body["password"], "password", 12, 128)
user_id = uuid.uuid4()
created_at = utcnow()
password_hash = hash_password(password)
try:
with self.app.pool.connection() as connection:
connection.execute(
"""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (%s, %s, %s, %s, %s)
""",
(user_id, username, display_name, password_hash, created_at),
)
except UniqueViolation:
raise ApiError(409, "conflict", "Username is already registered") from None
self._send_json(
201,
user_json(
{
"id": user_id,
"username": username,
"display_name": display_name,
"created_at": created_at,
}
),
)
def _create_session(self):
if self.headers.get("Content-Length") not in (None, "0") or self.headers.get("Transfer-Encoding"):
self.close_connection = True
raise ApiError(400, "invalid_request", "Session creation takes no body")
header = self.headers.get("Authorization", "")
scheme, separator, encoded = header.partition(" ")
if not separator or scheme.lower() != "basic" or not encoded.strip():
raise ApiError(401, "unauthorized", "Basic credentials are required")
try:
decoded = base64.b64decode(encoded.strip(), validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, UnicodeError):
raise ApiError(401, "unauthorized", "Invalid Basic credentials") from None
if not username or not password:
raise ApiError(401, "unauthorized", "Invalid Basic credentials")
with self.app.pool.connection() as connection:
row = connection.execute(
"""
SELECT id, username, display_name, created_at, password_hash
FROM users WHERE username = %s
""",
(username,),
).fetchone()
if row is None:
verify_password(password, dummy_password_hash())
raise ApiError(401, "unauthorized", "Invalid credentials")
if not verify_password(password, row["password_hash"]):
raise ApiError(401, "unauthorized", "Invalid credentials")
token = new_session_token()
expires_at = utcnow() + timedelta(seconds=self.app.config.session_ttl_seconds)
with self.app.pool.connection() as connection:
connection.execute(
"""
INSERT INTO sessions (token_hash, user_id, expires_at, created_at)
VALUES (%s, %s, %s, %s)
""",
(token_hash(token), row["id"], expires_at, utcnow()),
)
self._send_json(
201,
{
"token": token,
"token_type": "Bearer",
"expires_at": format_timestamp(expires_at),
"user": user_json(row),
},
)
# Separate name from _current_user(), which is the bearer authentication helper.
def _current_user_endpoint(self):
user = self._current_user()
self._send_json(200, user_json(user))
def _delete_session(self):
token = self._bearer_token()
with self.app.pool.connection() as connection:
cursor = connection.execute(
"DELETE FROM sessions WHERE token_hash = %s", (token_hash(token),)
)
deleted = cursor.rowcount
if deleted == 0:
raise ApiError(401, "unauthorized", "Invalid or expired session")
self._send_empty(204)
# ------------------------------------------------------------------
# chats
# ------------------------------------------------------------------
def _create_chat(self):
user = self._current_user()
body = self._json_body()
self._check_fields(body, {"title", "member_ids"})
if "title" not in body or "member_ids" not in body:
raise ApiError(400, "invalid_request", "title and member_ids are required")
title = self._require_string(body["title"], "title", 1, 100)
if not title.strip():
raise ApiError(400, "invalid_request", "Invalid title")
raw_members = body["member_ids"]
if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100:
raise ApiError(400, "invalid_request", "Invalid member_ids")
members = []
for value in raw_members:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "Invalid member_ids")
members.append(str(self._parse_uuid(value)))
if len(set(members)) != len(members):
raise ApiError(400, "invalid_request", "Duplicate member_ids")
if str(user["id"]) not in members:
raise ApiError(400, "invalid_request", "Caller must be a chat member")
chat_id = uuid.uuid4()
created_at = utcnow()
member_uuids = [uuid.UUID(value) for value in members]
with self.app.pool.connection() as connection:
with connection.transaction():
found = connection.execute(
"SELECT id FROM users WHERE id = ANY(%s)", (member_uuids,)
).fetchall()
if len(found) != len(members):
raise ApiError(400, "invalid_request", "Unknown chat member")
connection.execute(
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
(chat_id, title, created_at),
)
with connection.cursor() as cursor:
cursor.executemany(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
[(chat_id, member_id) for member_id in member_uuids],
)
self._send_json(
201,
chat_json(
{
"id": chat_id,
"title": title,
"member_ids": member_uuids,
"created_at": created_at,
}
),
)
def _list_chats(self):
user = self._current_user()
limit = self._parse_limit()
payload = self._parse_cursor()
condition = ""
parameters = [user["id"]]
if payload is not None:
if payload.get("kind") != "chats":
raise ApiError(400, "invalid_request", "Invalid cursor")
try:
created_at = parse_timestamp(payload["created_at"])
chat_id = uuid.UUID(payload["id"])
except (KeyError, ValueError, TypeError):
raise ApiError(400, "invalid_request", "Invalid cursor") from None
condition = " AND (c.created_at, c.id) > (%s, %s)"
parameters.extend([created_at, chat_id])
parameters.append(limit + 1)
sql = (
"""
SELECT c.id, c.title, c.created_at,
(SELECT array_agg(cm2.user_id ORDER BY cm2.user_id)
FROM chat_members cm2 WHERE cm2.chat_id = c.id) AS member_ids
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = %s
WHERE TRUE
"""
+ condition
+ " ORDER BY c.created_at, c.id LIMIT %s"
)
with self.app.pool.connection() as connection:
rows = connection.execute(sql, parameters).fetchall()
has_more = len(rows) > limit
rows = rows[:limit]
next_cursor = None
if has_more and rows:
last = rows[-1]
next_cursor = encode_cursor(
self.app.config.cursor_secret,
{
"kind": "chats",
"created_at": format_timestamp(last["created_at"]),
"id": str(last["id"]),
},
)
self._send_json(
200, {"items": [chat_json(row) for row in rows], "next_cursor": next_cursor}
)
def _get_chat(self, chat_id):
user = self._current_user()
with self.app.pool.connection() as connection:
self._require_member(connection, chat_id, user["id"])
row = connection.execute(
"""
SELECT c.id, c.title, c.created_at,
(SELECT array_agg(cm2.user_id ORDER BY cm2.user_id)
FROM chat_members cm2 WHERE cm2.chat_id = c.id) AS member_ids
FROM chats c WHERE c.id = %s
""",
(chat_id,),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(200, chat_json(row))
# ------------------------------------------------------------------
# messages
# ------------------------------------------------------------------
@staticmethod
def _fetch_message(connection, message_id):
row = connection.execute(
"""
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position)
FROM message_attachments ma
WHERE ma.message_id = m.id), '{}') AS attachment_ids
FROM messages m WHERE m.id = %s
""",
(message_id,),
).fetchone()
return None if row is None else message_json(row)
def _create_message(self, chat_id):
user = self._current_user()
with self.app.pool.connection() as connection:
self._require_member(connection, chat_id, user["id"])
idempotency_key = self.headers.get("Idempotency-Key")
if not idempotency_key or IDEMPOTENCY_RE.fullmatch(idempotency_key) is None:
raise ApiError(400, "invalid_request", "Valid Idempotency-Key is required")
body = self._json_body()
self._check_fields(body, {"text", "attachment_ids"})
if "text" not in body:
raise ApiError(400, "invalid_request", "text is required")
text = self._require_string(body["text"], "text", 0, 4000)
raw_attachments = body.get("attachment_ids", [])
if not isinstance(raw_attachments, list) or len(raw_attachments) > 10:
raise ApiError(400, "invalid_request", "Invalid attachment_ids")
attachment_ids = []
for value in raw_attachments:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "Invalid attachment_ids")
attachment_ids.append(str(self._parse_uuid(value)))
if len(set(attachment_ids)) != len(attachment_ids):
raise ApiError(400, "invalid_request", "Duplicate attachment_ids")
if not attachment_ids and not text.strip():
raise ApiError(400, "invalid_request", "Empty message")
request_hash = hashlib.sha256(
json.dumps(
{"text": text, "attachment_ids": attachment_ids},
sort_keys=True,
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
).hexdigest()
scope = "POST|%s|%s|%s" % (user["id"], chat_id, idempotency_key)
lock_id = int.from_bytes(
hashlib.sha256(scope.encode("utf-8")).digest()[:8], "big", signed=True
)
with self.app.pool.connection() as connection:
with connection.transaction():
connection.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
existing = connection.execute(
"""
SELECT request_hash, message_id FROM idempotency
WHERE user_id = %s AND chat_id = %s AND key = %s
""",
(user["id"], chat_id, idempotency_key),
).fetchone()
if existing is not None:
if existing["request_hash"] != request_hash:
raise ApiError(
409, "conflict", "Idempotency key was used with a different request"
)
message = self._fetch_message(connection, existing["message_id"])
if message is None:
raise ApiError(409, "conflict", "Idempotent result is unavailable")
else:
if attachment_ids:
rows = connection.execute(
"""
SELECT id, chat_id, status FROM attachments
WHERE id = ANY(%s)
""",
([uuid.UUID(value) for value in attachment_ids],),
).fetchall()
if len(rows) != len(attachment_ids):
raise ApiError(404, "not_found", "Attachment not found")
by_id = {str(row["id"]): row for row in rows}
for attachment_id in attachment_ids:
row = by_id[attachment_id]
if str(row["chat_id"]) != str(chat_id):
raise ApiError(404, "not_found", "Attachment not found")
if row["status"] == "failed":
raise ApiError(409, "conflict", "Attachment failed processing")
message_id = uuid.uuid4()
created_at = utcnow()
connection.execute(
"""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES (%s, %s, %s, %s, %s)
""",
(message_id, chat_id, user["id"], text, created_at),
)
if attachment_ids:
with connection.cursor() as cursor:
cursor.executemany(
"""
INSERT INTO message_attachments
(message_id, attachment_id, position)
VALUES (%s, %s, %s)
""",
[
(message_id, uuid.UUID(value), position)
for position, value in enumerate(attachment_ids)
],
)
connection.execute(
"""
INSERT INTO idempotency
(user_id, chat_id, key, request_hash, message_id, created_at)
VALUES (%s, %s, %s, %s, %s, %s)
""",
(
user["id"],
chat_id,
idempotency_key,
request_hash,
message_id,
created_at,
),
)
message = self._fetch_message(connection, message_id)
self._send_json(201, message)
def _list_messages(self, chat_id):
user = self._current_user()
with self.app.pool.connection() as connection:
self._require_member(connection, chat_id, user["id"])
limit = self._parse_limit()
payload = self._parse_cursor()
condition = ""
parameters = [chat_id]
if payload is not None:
if payload.get("kind") != "messages" or str(payload.get("chat_id")) != str(chat_id):
raise ApiError(400, "invalid_request", "Cursor belongs to another resource")
try:
created_at = parse_timestamp(payload["created_at"])
message_id = uuid.UUID(payload["id"])
except (KeyError, ValueError, TypeError):
raise ApiError(400, "invalid_request", "Invalid cursor") from None
condition = " AND (m.created_at, m.id) > (%s, %s)"
parameters.extend([created_at, message_id])
parameters.append(limit + 1)
sql = (
"""
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position)
FROM message_attachments ma
WHERE ma.message_id = m.id), '{}') AS attachment_ids
FROM messages m
WHERE m.chat_id = %s
"""
+ condition
+ " ORDER BY m.created_at, m.id LIMIT %s"
)
with self.app.pool.connection() as connection:
rows = connection.execute(sql, parameters).fetchall()
has_more = len(rows) > limit
rows = rows[:limit]
next_cursor = None
if has_more and rows:
last = rows[-1]
next_cursor = encode_cursor(
self.app.config.cursor_secret,
{
"kind": "messages",
"chat_id": str(chat_id),
"created_at": format_timestamp(last["created_at"]),
"id": str(last["id"]),
},
)
self._send_json(
200, {"items": [message_json(row) for row in rows], "next_cursor": next_cursor}
)
# ------------------------------------------------------------------
# attachments
# ------------------------------------------------------------------
@staticmethod
def _valid_filename(filename):
if not filename or len(filename) > 255:
return False
for character in filename:
if character in ("/", "\\") or ord(character) < 32 or ord(character) == 127:
return False
return True
def _upload_attachment(self, chat_id):
user = self._current_user()
try:
with self.app.pool.connection() as connection:
self._require_member(connection, chat_id, user["id"])
content_type = self.headers.get_content_type()
if content_type not in ("image/png", "image/jpeg"):
self.close_connection = True
raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are accepted")
filename = self._query_value("filename", required=True)
if not self._valid_filename(filename):
self.close_connection = True
raise ApiError(400, "invalid_request", "Invalid filename")
data = self._read_body(self.app.config.max_upload_bytes)
except ApiError:
if not self._body_consumed:
self.close_connection = True
raise
if not data:
self.close_connection = True
raise ApiError(415, "unsupported_media_type", "Empty image")
try:
actual_content_type, _width, _height = inspect_image(
data, content_type, self.app.config.max_image_pixels
)
except ImageTooLarge:
self.close_connection = True
raise ApiError(413, "payload_too_large", "Image has too many pixels") from None
except InvalidImage:
self.close_connection = True
raise ApiError(415, "unsupported_media_type", "Invalid or mismatched image") from None
attachment_id = uuid.uuid4()
object_key = "chats/{}/{}/original".format(chat_id, attachment_id)
created_at = utcnow()
sha256 = hashlib.sha256(data).hexdigest()
try:
self.app.storage.put(object_key, data, actual_content_type)
except (BotoCoreError, ClientError):
raise ApiError(503, "unavailable", "Object storage is unavailable") from None
try:
with self.app.pool.connection() as connection:
connection.execute(
"""
INSERT INTO attachments
(id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, object_key, status, variants, error_code,
created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'ready', %s, NULL, %s, %s)
""",
(
attachment_id,
chat_id,
user["id"],
filename,
actual_content_type,
len(data),
sha256,
object_key,
Jsonb([]),
created_at,
created_at,
),
)
except psycopg.Error:
try:
self.app.storage.delete(object_key)
except (BotoCoreError, ClientError):
traceback.print_exc()
raise ApiError(503, "unavailable", "Database is unavailable") from None
self._send_json(
201,
attachment_json(
{
"id": attachment_id,
"chat_id": chat_id,
"owner_id": user["id"],
"filename": filename,
"content_type": actual_content_type,
"size_bytes": len(data),
"sha256": sha256,
"status": "ready",
"variants": [],
"error_code": None,
"created_at": created_at,
"updated_at": created_at,
}
),
)
def _get_attachment(self, attachment_id):
user = self._current_user()
with self.app.pool.connection() as connection:
row = connection.execute(
"""
SELECT a.* FROM attachments a
JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s
WHERE a.id = %s
""",
(user["id"], attachment_id),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Attachment not found")
self._send_json(200, attachment_json(row))
def _download_attachment(self, attachment_id):
user = self._current_user()
variant = self._query_value("variant")
if variant is None:
variant = "original"
if variant != "original":
raise ApiError(400, "invalid_request", "Unsupported variant")
with self.app.pool.connection() as connection:
row = connection.execute(
"""
SELECT a.* FROM attachments a
JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s
WHERE a.id = %s
""",
(user["id"], attachment_id),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Attachment not found")
if row["status"] != "ready":
raise ApiError(409, "conflict", "Attachment is not ready")
try:
response = self.app.storage.get(row["object_key"])
except (BotoCoreError, ClientError) as error:
if is_missing_object(error):
raise ApiError(404, "not_found", "Attachment content not found") from None
raise ApiError(503, "unavailable", "Object storage is unavailable") from None
body = response["Body"]
length = int(response.get("ContentLength") or row["size_bytes"])
self.send_response(200)
self.send_header("Content-Type", row["content_type"])
self.send_header("Content-Length", str(length))
self.send_header(
"Content-Disposition",
"inline; filename*=UTF-8''" + quote(row["filename"], safe=""),
)
self._common_headers()
self.end_headers()
self._response_started = True
try:
for chunk in body.iter_chunks(chunk_size=65536):
if chunk:
self.wfile.write(chunk)
except (BrokenPipeError, ConnectionResetError):
self.close_connection = True
finally:
body.close()
class ApiServer(ThreadingHTTPServer):
daemon_threads = True
def handle_error(self, request, client_address):
error = sys.exc_info()[1]
if isinstance(error, (BrokenPipeError, ConnectionResetError)):
return
super().handle_error(request, client_address)
def make_server(app):
server = ApiServer((app.config.host, app.config.port), Handler)
server.app = app
return server
def main():
config = Config.from_env()
wait_for_database(config.database_url)
pool = create_pool(config.database_url)
pool.open(wait=True, timeout=30)
try:
migrate(pool)
storage = Storage(
endpoint=config.s3_endpoint,
region=config.s3_region,
bucket=config.s3_bucket,
access_key_id=config.s3_access_key_id,
secret_access_key=config.s3_secret_access_key,
)
app = App(config, pool, storage)
server = make_server(app)
sys.stderr.write(
"messenger lab4 listening on %s:%s as %s\n"
% (config.host, config.port, config.instance_id)
)
server.serve_forever()
finally:
pool.close()