forked from mtusi2026/mtusi2026lab4
ЛР 4 — БПИ2303 — Щербаков — на 3
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Lab 1: a single-process, in-memory messenger."""
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Synchronous bounded binary upload and membership-authorized download."""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import threading
|
||||
import unicodedata
|
||||
from uuid import uuid4
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from .common import APIError, invalid, record, string_value, uuid_value
|
||||
from .objects import Objects
|
||||
|
||||
MAX_BYTES = min(10485760, int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")))
|
||||
MAX_PIXELS = min(40000000, int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")))
|
||||
# Pillow's hard error triggers at twice this limit, before pixel allocation.
|
||||
# The exact contract limit is also checked below before load().
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
|
||||
|
||||
def filename(query):
|
||||
if set(query) != {"filename"} or len(query["filename"]) != 1:
|
||||
raise invalid()
|
||||
value = string_value(query["filename"][0], 255)
|
||||
if any(c in "/\\" or unicodedata.category(c) == "Cc" for c in value):
|
||||
raise invalid()
|
||||
return value
|
||||
|
||||
|
||||
def image_type(data, declared):
|
||||
if declared not in ("image/png", "image/jpeg"):
|
||||
raise APIError(415, "unsupported_media_type", "Expected PNG or JPEG")
|
||||
try:
|
||||
with Image.open(io.BytesIO(data), formats=("PNG", "JPEG")) as image:
|
||||
if image.width * image.height > MAX_PIXELS:
|
||||
raise APIError(413, "payload_too_large", "Image exceeds pixel limit")
|
||||
actual = Image.MIME[image.format]
|
||||
if actual != declared:
|
||||
raise APIError(415, "unsupported_media_type", "Image MIME mismatch")
|
||||
image.verify()
|
||||
# verify() alone does not decode JPEG scan data. Reject truncation too.
|
||||
with Image.open(io.BytesIO(data), formats=("PNG", "JPEG")) as image:
|
||||
image.load()
|
||||
return actual
|
||||
except Image.DecompressionBombError:
|
||||
raise APIError(413, "payload_too_large", "Image exceeds pixel limit") from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
raise APIError(415, "unsupported_media_type", "Invalid PNG or JPEG") from None
|
||||
|
||||
|
||||
class Attachments:
|
||||
def init_attachments(self):
|
||||
self.objects = Objects()
|
||||
# One decode per replica bounds aggregate pixel memory at this grade.
|
||||
self.upload_slot = threading.BoundedSemaphore(1)
|
||||
|
||||
def upload(self, chat_id, actor, query, declared, read_bytes):
|
||||
self.check_chat(chat_id, actor)
|
||||
name = filename(query)
|
||||
if not self.upload_slot.acquire(timeout=1):
|
||||
raise APIError(503, "unavailable", "Upload capacity exhausted")
|
||||
try:
|
||||
data = read_bytes(MAX_BYTES)
|
||||
content_type = image_type(data, declared)
|
||||
attachment_id = uuid4()
|
||||
key = "originals/" + uuid4().hex
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
self.objects.put(key, data, content_type)
|
||||
# No 201 until both PUT and PostgreSQL COMMIT have completed.
|
||||
# No DELETE on ambiguous COMMIT: it could remove an acknowledged original.
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *",
|
||||
(attachment_id, uuid_value(chat_id), actor[0], name, content_type, len(data), digest, key),
|
||||
).fetchone()
|
||||
return 201, self.attachment_json(row)
|
||||
finally:
|
||||
self.upload_slot.release()
|
||||
|
||||
def attachment(self, conn, value, actor):
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a WHERE a.id=%s AND EXISTS "
|
||||
"(SELECT 1 FROM memberships m WHERE m.chat_id=a.chat_id AND m.user_id=%s)",
|
||||
(uuid_value(value), actor[0]),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
return row
|
||||
|
||||
@staticmethod
|
||||
def attachment_json(row):
|
||||
return dict(record({k: v for k, v in row.items() if k != "object_key"}),
|
||||
status="ready", variants=[], error_code=None)
|
||||
|
||||
def get_attachment(self, value, actor, query, content=False):
|
||||
with self.pool.connection() as conn:
|
||||
row = self.attachment(conn, value, actor)
|
||||
if not content:
|
||||
if query:
|
||||
raise invalid()
|
||||
return 200, self.attachment_json(row)
|
||||
if set(query) - {"variant"} or any(len(v) != 1 for v in query.values()):
|
||||
raise invalid()
|
||||
if query.get("variant", ["original"])[0] != "original":
|
||||
raise APIError(404, "not_found", "Variant not found")
|
||||
data = self.objects.get(row["object_key"], row["size_bytes"])
|
||||
if hashlib.sha256(data).hexdigest() != row["sha256"]:
|
||||
raise APIError(503, "unavailable", "Original unavailable")
|
||||
return 200, (data, row["content_type"])
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Idempotently create a private bucket. Uses locally generated lab credentials."""
|
||||
import os
|
||||
import time
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from .objects import client
|
||||
|
||||
|
||||
def main():
|
||||
s3 = client()
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
for attempt in range(60):
|
||||
try:
|
||||
s3.list_buckets()
|
||||
break
|
||||
except (BotoCoreError, ClientError):
|
||||
if attempt == 59:
|
||||
raise RuntimeError("S3 did not become available") from None
|
||||
time.sleep(1)
|
||||
try:
|
||||
s3.head_bucket(Bucket=bucket)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"]["Code"] not in ("404", "NoSuchBucket", "NotFound"):
|
||||
raise RuntimeError("Cannot inspect bucket") from None
|
||||
s3.create_bucket(Bucket=bucket)
|
||||
# S3 config has authenticated identities only; anonymous access is absent.
|
||||
s3.put_bucket_acl(Bucket=bucket, ACL="private")
|
||||
print("PASS: private bucket ready; authenticated access only")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
import re
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status, code, message):
|
||||
self.status, self.code, self.message = status, code, message
|
||||
|
||||
|
||||
def invalid():
|
||||
return APIError(400, "invalid_request", "Invalid request")
|
||||
|
||||
|
||||
def uuid_value(value):
|
||||
if not isinstance(value, str) or not re.fullmatch(
|
||||
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
|
||||
):
|
||||
raise invalid()
|
||||
return str(UUID(value))
|
||||
|
||||
|
||||
def string_value(value, maximum):
|
||||
if (not isinstance(value, str) or not 1 <= len(value) <= maximum
|
||||
or not value.strip() or "\x00" in value or any(0xD800 <= ord(c) <= 0xDFFF for c in value)):
|
||||
raise invalid()
|
||||
return value
|
||||
|
||||
|
||||
def fields(body, expected):
|
||||
if not isinstance(body, dict) or set(body) != set(expected):
|
||||
raise invalid()
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise invalid()
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def reject_constant(value):
|
||||
raise invalid()
|
||||
|
||||
|
||||
def record(row):
|
||||
return {k: (str(v) if isinstance(v, UUID) else
|
||||
v.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
if isinstance(v, datetime) else v) for k, v in row.items()}
|
||||
@@ -0,0 +1,44 @@
|
||||
"""PostgreSQL configuration and transactional, versioned migrations."""
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
|
||||
def database_url():
|
||||
filename = os.environ.get("DATABASE_URL_FILE")
|
||||
return Path(filename).read_text().strip() if filename else os.environ["DATABASE_URL"]
|
||||
|
||||
|
||||
def migrate():
|
||||
with psycopg.connect(database_url(), connect_timeout=3) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2026002)")
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(version text PRIMARY KEY, checksum text NOT NULL)")
|
||||
for path in sorted((Path(__file__).resolve().parents[1] / "migrations").glob("*.sql")):
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
checksum = hashlib.sha256(sql.encode()).hexdigest()
|
||||
old = conn.execute("SELECT checksum FROM schema_migrations WHERE version=%s", (path.name,)).fetchone()
|
||||
if old:
|
||||
if old[0] != checksum:
|
||||
raise RuntimeError("Applied migration checksum changed")
|
||||
continue
|
||||
conn.execute(sql)
|
||||
conn.execute("INSERT INTO schema_migrations VALUES (%s,%s)", (path.name, checksum))
|
||||
conn.execute("INSERT INTO settings VALUES ('cursor_key',%s) ON CONFLICT DO NOTHING", (secrets.token_hex(32),))
|
||||
|
||||
|
||||
def pool():
|
||||
return ConnectionPool(database_url(), min_size=1, max_size=8, timeout=2,
|
||||
max_waiting=32, open=True, check=ConnectionPool.check_connection,
|
||||
kwargs={"connect_timeout": 3, "row_factory": dict_row,
|
||||
"options": "-c timezone=UTC -c statement_timeout=3000 -c lock_timeout=2000"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
migrate()
|
||||
print("PASS: migrations applied")
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Container readiness probe; exits nonzero on failure."""
|
||||
import json
|
||||
import os
|
||||
from urllib.request import ProxyHandler, build_opener
|
||||
|
||||
|
||||
def main():
|
||||
host = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
if host == "0.0.0.0":
|
||||
host = "127.0.0.1"
|
||||
port = int(os.environ.get("APP_PORT", "8080"))
|
||||
with build_opener(ProxyHandler({})).open(f"http://{host}:{port}/health/ready", timeout=2) as response:
|
||||
if response.status != 200 or json.load(response) != {"status": "ok"}:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Private S3 originals; no local files and no credentials in public metadata."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .common import APIError
|
||||
|
||||
|
||||
def secret(name):
|
||||
return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def client():
|
||||
return boto3.client(
|
||||
"s3", endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:8333"),
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=secret("S3_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=secret("S3_SECRET_ACCESS_KEY"),
|
||||
config=Config(connect_timeout=1, read_timeout=3,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"}),
|
||||
)
|
||||
|
||||
|
||||
class Objects:
|
||||
def __init__(self):
|
||||
self.client = client()
|
||||
self.bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
|
||||
def call(self, method, **kwargs):
|
||||
try:
|
||||
return getattr(self.client, method)(Bucket=self.bucket, **kwargs)
|
||||
except (BotoCoreError, ClientError):
|
||||
raise APIError(503, "unavailable", "Object storage unavailable") from None
|
||||
|
||||
def ready(self):
|
||||
self.call("head_bucket")
|
||||
|
||||
def put(self, key, data, content_type):
|
||||
self.call("put_object", Key=key, Body=data, ContentType=content_type)
|
||||
|
||||
def get(self, key, expected_size):
|
||||
response = self.call("get_object", Key=key)
|
||||
try:
|
||||
# Bounded even if stored object has been replaced out of band.
|
||||
data = response["Body"].read(expected_size + 1)
|
||||
if len(data) != expected_size:
|
||||
raise APIError(503, "unavailable", "Original unavailable")
|
||||
return data
|
||||
except BotoCoreError:
|
||||
raise APIError(503, "unavailable", "Object storage unavailable") from None
|
||||
finally:
|
||||
response["Body"].close()
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Signed, resource-bound keyset cursors, inherited from lab1."""
|
||||
import base64
|
||||
import binascii
|
||||
import hmac
|
||||
import json
|
||||
import re
|
||||
from .common import invalid
|
||||
|
||||
class Pagination:
|
||||
def cursor(self, scope, record):
|
||||
payload = json.dumps([scope, record["created_at"], record["id"]], separators=(",", ":")).encode()
|
||||
signature = hmac.digest(self.cursor_key, payload, "sha256")
|
||||
return base64.urlsafe_b64encode(signature + payload).decode().rstrip("=")
|
||||
|
||||
def decode_page(self, query, scope):
|
||||
if set(query) - {"limit", "cursor"} or any(len(v) != 1 for v in query.values()):
|
||||
raise invalid()
|
||||
limit = query.get("limit", ["50"])[0]
|
||||
if not re.fullmatch(r"[0-9]{1,3}", limit) or not 1 <= int(limit) <= 100:
|
||||
raise invalid()
|
||||
limit = int(limit)
|
||||
after = None
|
||||
if "cursor" in query:
|
||||
token = query["cursor"][0]
|
||||
if not re.fullmatch(r"[A-Za-z0-9_-]{1,1024}", token):
|
||||
raise invalid()
|
||||
try:
|
||||
raw = base64.b64decode(token + "=" * (-len(token) % 4), altchars=b"-_", validate=True)
|
||||
signature, payload = raw[:32], raw[32:]
|
||||
if not hmac.compare_digest(signature, hmac.digest(self.cursor_key, payload, "sha256")):
|
||||
raise invalid()
|
||||
resource, created, record_id = json.loads(payload)
|
||||
if resource != scope:
|
||||
raise invalid()
|
||||
after = (created, record_id)
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise invalid() from None
|
||||
return limit, after
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
"""Stateless lab4 HTTP replica. Migrations run as a separate Compose job."""
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
|
||||
from .common import APIError, invalid, unique_object, reject_constant
|
||||
from .store import Store
|
||||
|
||||
|
||||
LOG = logging.getLogger("messenger")
|
||||
MAX_BODY = 64 * 1024
|
||||
SHUTDOWN_SECONDS = 5
|
||||
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
request_queue_size = 128
|
||||
|
||||
def __init__(self, address):
|
||||
self.instance_id = os.environ.get("INSTANCE_ID", "api") + "-" + uuid4().hex[:12]
|
||||
self.store = Store()
|
||||
self.stopping = threading.Event()
|
||||
self.workers = set()
|
||||
self.workers_lock = threading.Lock()
|
||||
super().__init__(address, Handler)
|
||||
self.timeout = 0.2
|
||||
|
||||
def process_request_thread(self, request, client_address):
|
||||
worker = threading.current_thread()
|
||||
with self.workers_lock:
|
||||
self.workers.add(worker)
|
||||
try:
|
||||
super().process_request_thread(request, client_address)
|
||||
finally:
|
||||
with self.workers_lock:
|
||||
self.workers.discard(worker)
|
||||
|
||||
def handle_error(self, request, client_address):
|
||||
LOG.error("connection_handler_failed")
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "Messenger"
|
||||
sys_version = ""
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.connection.settimeout(10)
|
||||
|
||||
def log_message(self, format, *args):
|
||||
# Base handler includes raw paths and possibly user-supplied text.
|
||||
pass
|
||||
|
||||
def send_error(self, code, message=None, explain=None):
|
||||
self.request_id = getattr(self, "request_id", str(uuid4()))
|
||||
self.reply(400, {"error": {"code": "invalid_request", "message": "Invalid HTTP request", "request_id": self.request_id}})
|
||||
|
||||
def read_body(self):
|
||||
if self.headers.get_content_type() != "application/json":
|
||||
raise APIError(415, "unsupported_media_type", "Expected application/json")
|
||||
lengths = self.headers.get_all("Content-Length", [])
|
||||
if self.headers.get("Transfer-Encoding") or len(lengths) != 1 or not re.fullmatch(r"[0-9]{1,10}", lengths[0]):
|
||||
raise invalid()
|
||||
size = int(lengths[0])
|
||||
if size > MAX_BODY:
|
||||
raise APIError(413, "payload_too_large", "Request exceeds 65536 bytes")
|
||||
try:
|
||||
raw = self.rfile.read(size)
|
||||
if len(raw) != size:
|
||||
raise invalid()
|
||||
return json.loads(raw.decode("utf-8"), object_pairs_hook=unique_object, parse_constant=reject_constant)
|
||||
except (ValueError, UnicodeError, RecursionError, TimeoutError):
|
||||
raise invalid() from None
|
||||
|
||||
def read_binary(self, maximum):
|
||||
lengths = self.headers.get_all("Content-Length", [])
|
||||
if self.headers.get("Transfer-Encoding") or len(lengths) != 1 or not re.fullmatch(r"[0-9]{1,20}", lengths[0]):
|
||||
raise invalid()
|
||||
size = int(lengths[0])
|
||||
if size > maximum:
|
||||
raise APIError(413, "payload_too_large", "Upload exceeds byte limit")
|
||||
if size == 0:
|
||||
raise APIError(415, "unsupported_media_type", "Empty image")
|
||||
try:
|
||||
data = self.rfile.read(size)
|
||||
if len(data) != size:
|
||||
raise invalid()
|
||||
return data
|
||||
except (TimeoutError, OSError):
|
||||
raise invalid() from None
|
||||
|
||||
def reply(self, status, body):
|
||||
binary = isinstance(body, tuple)
|
||||
encoded = body[0] if binary else b"" if status == 204 else json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", body[1] if binary else "application/json; charset=utf-8")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.send_header("X-Request-Id", self.request_id)
|
||||
self.send_header("X-Instance-Id", self.server.instance_id)
|
||||
if status == 401:
|
||||
challenge = 'Basic realm="messenger", charset="UTF-8"' if urlsplit(self.path).path == "/api/v1/auth/sessions" else 'Bearer realm="messenger"'
|
||||
self.send_header("WWW-Authenticate", challenge)
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
self.close_connection = True
|
||||
self.wfile.write(encoded)
|
||||
|
||||
def handle_api(self):
|
||||
started = time.monotonic()
|
||||
self.request_id = str(uuid4())
|
||||
status = 500
|
||||
try:
|
||||
server = self.server
|
||||
assert isinstance(server, Server)
|
||||
if server.stopping.is_set():
|
||||
raise APIError(503, "unavailable", "Server is stopping")
|
||||
parsed = urlsplit(self.path)
|
||||
path = parsed.path
|
||||
if path in ("/health/live", "/health/ready") and self.command == "GET":
|
||||
if path == "/health/ready":
|
||||
server.store.ready()
|
||||
status, body = 200, {"status": "ok"}
|
||||
else:
|
||||
store = server.store
|
||||
public = (path, self.command) in (("/api/v1/users", "POST"), ("/api/v1/auth/sessions", "POST"))
|
||||
actor = None if public else store.authenticate(self.headers.get("Authorization"))
|
||||
query = parse_qs(parsed.query, keep_blank_values=True, max_num_fields=10, errors="strict")
|
||||
upload = re.fullmatch(r"/api/v1/chats/([^/]+)/attachments", path)
|
||||
attachment = re.fullmatch(r"/api/v1/attachments/([^/]+)(/content)?", path)
|
||||
cached_body = None
|
||||
if upload and self.command == "POST":
|
||||
status, body = store.upload(upload[1], actor, query, self.headers.get_content_type(), self.read_binary)
|
||||
elif attachment and self.command == "GET":
|
||||
status, body = store.get_attachment(attachment[1], actor, query, bool(attachment[2]))
|
||||
elif path == "/api/v1/auth/sessions" and self.command == "POST":
|
||||
if self.headers.get("Transfer-Encoding") or self.headers.get("Content-Length", "0") != "0":
|
||||
raise invalid()
|
||||
status, body = store.login(self.headers.get("Authorization"))
|
||||
else:
|
||||
if self.command == "POST":
|
||||
match = re.fullmatch(r"/api/v1/chats/([^/]+)/messages", path)
|
||||
if match:
|
||||
store.check_chat(match[1], actor)
|
||||
cached_body = self.read_body()
|
||||
keys = self.headers.get_all("Idempotency-Key", [])
|
||||
key = keys[0] if len(keys) == 1 else None
|
||||
status, body = store.dispatch(self.command, path, query, actor, lambda: cached_body, key)
|
||||
except APIError as exc:
|
||||
status = exc.status
|
||||
body = {"error": {"code": exc.code, "message": exc.message, "request_id": self.request_id}}
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError):
|
||||
status = 503
|
||||
body = {"error": {"code": "unavailable", "message": "Storage unavailable", "request_id": self.request_id}}
|
||||
except ValueError:
|
||||
status = 400
|
||||
body = {"error": {"code": "invalid_request", "message": "Invalid request", "request_id": self.request_id}}
|
||||
except Exception as exc:
|
||||
LOG.error("request_failed request_id=%s exception=%s", self.request_id, type(exc).__name__)
|
||||
body = {"error": {"code": "internal_error", "message": "Internal error", "request_id": self.request_id}}
|
||||
try:
|
||||
self.reply(status, body)
|
||||
except OSError:
|
||||
LOG.info("client_disconnected request_id=%s", self.request_id)
|
||||
finally:
|
||||
LOG.info("request request_id=%s method=%s status=%d duration_ms=%.3f", self.request_id, self.command, status, (time.monotonic() - started) * 1000)
|
||||
|
||||
do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = handle_api
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
server = Server((os.environ.get("APP_HOST", "0.0.0.0"), int(os.environ.get("APP_PORT", "8080"))))
|
||||
|
||||
def stop(signum, frame):
|
||||
server.stopping.set()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
LOG.info("started host=%s port=%d env=%s", *server.server_address, os.environ.get("APP_ENV", "lab"))
|
||||
try:
|
||||
while not server.stopping.is_set():
|
||||
server.handle_request()
|
||||
finally:
|
||||
server.stopping.set()
|
||||
server.server_close()
|
||||
LOG.info("shutdown_started listener_closed=true budget_seconds=%d", SHUTDOWN_SECONDS)
|
||||
deadline = time.monotonic() + SHUTDOWN_SECONDS
|
||||
with server.workers_lock:
|
||||
workers = list(server.workers)
|
||||
for worker in workers:
|
||||
worker.join(max(0, deadline - time.monotonic()))
|
||||
server.store.pool.close()
|
||||
LOG.info("shutdown_complete unfinished_requests=%d", sum(w.is_alive() for w in workers))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
"""Shared PostgreSQL state and atomic idempotent message creation."""
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
|
||||
from .common import APIError, fields, invalid, string_value, uuid_value
|
||||
from .database import pool
|
||||
from .pagination import Pagination
|
||||
from .attachments import Attachments
|
||||
from .common import record
|
||||
|
||||
|
||||
|
||||
def password_hash(password, salt=None):
|
||||
salt = salt or secrets.token_hex(16)
|
||||
derived = hashlib.scrypt(password.encode("utf-8"), salt=bytes.fromhex(salt),
|
||||
n=16384, r=8, p=1, dklen=32)
|
||||
return "scrypt$16384$8$1$" + salt + "$" + derived.hex()
|
||||
|
||||
|
||||
def unauthorized():
|
||||
return APIError(401, "unauthorized", "Valid credentials required")
|
||||
|
||||
|
||||
class Store(Attachments, Pagination):
|
||||
def __init__(self):
|
||||
self.pool = pool()
|
||||
self.init_attachments()
|
||||
self.ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
if self.ttl <= 0:
|
||||
raise ValueError("SESSION_TTL_SECONDS must be positive")
|
||||
with self.pool.connection() as conn:
|
||||
self.cursor_key = bytes.fromhex(conn.execute("SELECT value FROM settings WHERE key='cursor_key'").fetchone()["value"])
|
||||
self.dummy_hash = password_hash("dummy-password-not-a-user")
|
||||
|
||||
def ready(self):
|
||||
with self.pool.connection() as conn:
|
||||
if not conn.execute("SELECT 1 FROM schema_migrations WHERE version='003_attachments.sql'").fetchone():
|
||||
raise APIError(503, "unavailable", "Schema not ready")
|
||||
conn.execute("SELECT 1 FROM sessions LIMIT 1")
|
||||
conn.execute("SELECT 1 FROM message_keys LIMIT 1")
|
||||
self.objects.ready()
|
||||
|
||||
def authenticate(self, header):
|
||||
if not isinstance(header, str) or not re.fullmatch(r"(?i:Bearer) [A-Za-z0-9_-]{43}", header):
|
||||
raise unauthorized()
|
||||
digest = hashlib.sha256(header.split(" ", 1)[1].encode()).hexdigest()
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp()", (digest,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
return (str(row["user_id"]), digest)
|
||||
|
||||
def login(self, header):
|
||||
try:
|
||||
scheme, encoded = header.split(" ", 1)
|
||||
if scheme.lower() != "basic":
|
||||
raise ValueError()
|
||||
username, password = base64.b64decode(encoded, validate=True).decode("utf-8").split(":", 1)
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
|
||||
raise ValueError()
|
||||
except (AttributeError, ValueError, UnicodeError):
|
||||
raise unauthorized() from None
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
expected = row["password_hash"] if row else self.dummy_hash
|
||||
actual = password_hash(password, expected.split("$")[4])
|
||||
if not hmac.compare_digest(actual, expected) or row is None:
|
||||
raise unauthorized()
|
||||
del row["password_hash"]
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode()).hexdigest()
|
||||
with self.pool.connection() as conn:
|
||||
expiry = conn.execute("INSERT INTO sessions VALUES (%s,%s,clock_timestamp()+%s*interval '1 second') RETURNING expires_at", (digest, row["id"], self.ttl)).fetchone()
|
||||
return 201, {"token": token, "token_type": "Bearer", "expires_at": record(expiry)["expires_at"], "user": record(row)}
|
||||
|
||||
def chat(self, conn, value, actor, lock=False):
|
||||
chat_id = uuid_value(value)
|
||||
row = conn.execute("SELECT c.* FROM chats c WHERE c.id=%s AND EXISTS "
|
||||
"(SELECT 1 FROM memberships m WHERE m.chat_id=c.id AND m.user_id=%s)" +
|
||||
(" FOR UPDATE OF c" if lock else ""), (chat_id, actor[0])).fetchone()
|
||||
if not row:
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
row = record(row)
|
||||
row["member_ids"] = [str(m["user_id"]) for m in conn.execute("SELECT user_id FROM memberships WHERE chat_id=%s ORDER BY ordinal", (chat_id,))]
|
||||
return row
|
||||
|
||||
def check_chat(self, value, actor):
|
||||
with self.pool.connection() as conn:
|
||||
self.chat(conn, value, actor)
|
||||
|
||||
def page(self, conn, query, scope, table, actor=None, chat_id=None):
|
||||
limit, after = self.decode_page(query, scope)
|
||||
# Table and predicate are selected internally, never supplied by a client.
|
||||
if table == "messages":
|
||||
sql = "SELECT id,chat_id,sender_id,text,created_at,attachment_ids FROM messages WHERE chat_id=%s"
|
||||
params = [chat_id]
|
||||
else:
|
||||
sql = "SELECT c.* FROM chats c WHERE EXISTS (SELECT 1 FROM memberships m WHERE m.chat_id=c.id AND m.user_id=%s)"
|
||||
params = [actor[0]]
|
||||
if after:
|
||||
sql += " AND (created_at,id)>(%s::timestamptz,%s::uuid)"
|
||||
params.extend(after)
|
||||
sql += " ORDER BY created_at,id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
rows = [record(r) for r in conn.execute(sql, params)]
|
||||
items = rows[:limit]
|
||||
if table == "chats":
|
||||
items = [self.chat(conn, r["id"], actor) for r in items]
|
||||
return {"items": items, "next_cursor": self.cursor(scope, items[-1]) if len(rows) > limit else None}
|
||||
|
||||
def dispatch(self, method, path, query, actor, read_body, idempotency_key=None):
|
||||
if path == "/api/v1/users" and method == "POST":
|
||||
body = read_body()
|
||||
fields(body, ("username", "display_name", "password"))
|
||||
username = body["username"]
|
||||
if not isinstance(username, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", username):
|
||||
raise invalid()
|
||||
display = string_value(body["display_name"], 100)
|
||||
password = string_value(body["password"], 128)
|
||||
if len(password) < 12:
|
||||
raise invalid()
|
||||
hashed = password_hash(password)
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at", (uuid4(), username, display, hashed)).fetchone()
|
||||
return 201, record(row)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise APIError(409, "conflict", "Username already exists") from None
|
||||
with self.pool.connection() as conn:
|
||||
if path == "/api/v1/auth/sessions/current" and method == "DELETE":
|
||||
deleted = conn.execute("DELETE FROM sessions WHERE token_hash=%s RETURNING user_id", (actor[1],)).fetchone()
|
||||
if not deleted:
|
||||
raise unauthorized()
|
||||
return 204, None
|
||||
if path == "/api/v1/users/me" and method == "GET":
|
||||
return 200, record(conn.execute("SELECT id,username,display_name,created_at FROM users WHERE id=%s", (actor[0],)).fetchone())
|
||||
if path == "/api/v1/chats":
|
||||
if method == "GET":
|
||||
return 200, self.page(conn, query, "chats:" + actor[0], "chats", actor=actor)
|
||||
if method == "POST":
|
||||
body = read_body()
|
||||
fields(body, ("title", "member_ids"))
|
||||
title = string_value(body["title"], 100)
|
||||
members = body["member_ids"]
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
raise invalid()
|
||||
members = [uuid_value(m) for m in members]
|
||||
if len(set(members)) != len(members) or actor[0] not in members:
|
||||
raise invalid()
|
||||
row = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid4(), title)).fetchone()
|
||||
existing = conn.execute("SELECT id FROM users WHERE id=ANY(%s::uuid[])", (members,)).fetchall()
|
||||
if len(existing) != len(members):
|
||||
raise invalid() # Rolls back the chat INSERT as well.
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany("INSERT INTO memberships VALUES (%s,%s,%s)", [(row["id"], m, i) for i, m in enumerate(members)])
|
||||
return 201, dict(record(row), member_ids=members)
|
||||
match = re.fullmatch(r"/api/v1/chats/([^/]+)(/messages)?", path)
|
||||
if match:
|
||||
chat = self.chat(conn, match[1], actor, lock=method == "POST")
|
||||
if match[2] is None and method == "GET":
|
||||
return 200, chat
|
||||
if match[2] == "/messages":
|
||||
if method == "GET":
|
||||
return 200, self.page(conn, query, "messages:" + chat["id"], "messages", chat_id=chat["id"])
|
||||
if method == "POST":
|
||||
if not isinstance(idempotency_key, str) or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key):
|
||||
raise invalid()
|
||||
body = read_body()
|
||||
fields(body, ("text", "attachment_ids") if isinstance(body, dict) and "attachment_ids" in body else ("text",))
|
||||
ids = body.get("attachment_ids", [])
|
||||
if not isinstance(ids, list) or len(ids) > 10:
|
||||
raise invalid()
|
||||
ids = [uuid_value(value) for value in ids]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise invalid()
|
||||
text = body["text"]
|
||||
if not isinstance(text, str) or len(text) > 4000 or "\x00" in text or any(0xD800 <= ord(c) <= 0xDFFF for c in text):
|
||||
raise invalid()
|
||||
if not text.strip() and not ids:
|
||||
raise invalid()
|
||||
for value in ids:
|
||||
attachment = self.attachment(conn, value, actor)
|
||||
if str(attachment["chat_id"]) != chat["id"]:
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
# The chat row is already locked across ALL API replicas.
|
||||
# READ COMMITTED sees the preceding writer's committed key.
|
||||
conn.execute("DELETE FROM message_keys WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()",
|
||||
(actor[0], chat["id"], idempotency_key))
|
||||
previous = conn.execute("SELECT m.* FROM message_keys k JOIN messages m ON m.id=k.message_id "
|
||||
"WHERE k.user_id=%s AND k.chat_id=%s AND k.key=%s",
|
||||
(actor[0], chat["id"], idempotency_key)).fetchone()
|
||||
if previous:
|
||||
if previous["text"] != text or previous["attachment_ids"] != ids:
|
||||
raise APIError(409, "conflict", "Key already used for another message")
|
||||
return 201, record(previous)
|
||||
row = conn.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at,attachment_ids) VALUES (%s,%s,%s,%s,GREATEST(clock_timestamp(), "
|
||||
"COALESCE((SELECT created_at+interval '1 microsecond' FROM messages "
|
||||
"WHERE chat_id=%s ORDER BY created_at DESC,id DESC LIMIT 1),clock_timestamp())),%s::jsonb) RETURNING *",
|
||||
(uuid4(), chat["id"], actor[0], text, chat["id"], psycopg.types.json.Jsonb(ids))).fetchone()
|
||||
conn.execute("INSERT INTO message_keys(user_id,chat_id,key,message_id) VALUES (%s,%s,%s,%s)",
|
||||
(actor[0], chat["id"], idempotency_key, row["id"]))
|
||||
return 201, record(row)
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
Reference in New Issue
Block a user