From d073767e80ba956a0460d28f2bcaed7807253b50 Mon Sep 17 00:00:00 2001 From: Katarhi Date: Thu, 5 Nov 2026 23:30:15 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=BD=D0=B0=D1=8F=204=20-=20=D0=93=D0=BE=D0=BD?= =?UTF-8?q?=D1=87=D0=B0=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B0=D1=8F.=D0=95.?= =?UTF-8?q?=D0=90.=20-=20=D0=91=D0=92=D0=A22304=20-=20=20=D0=9D=D0=B0=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 11 + README..md | 222 ++++++ compose.yaml | 124 ++++ infra/haproxy/haproxy.cfg | 31 + infra/minio/attachment-policy.json | 25 + migrations/001_initial.sql | 57 ++ migrations/002_idempotency.sql | 7 + migrations/003_attachments.sql | 22 + requirements.txt | 5 + scripts/init-secrets.py | 28 + scripts/restart.sh | 5 + scripts/stop-one.sh | 5 + server.py | 1004 ++++++++++++++++++++++++++++ 13 files changed, 1546 insertions(+) create mode 100644 Dockerfile create mode 100644 README..md create mode 100644 compose.yaml create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 infra/minio/attachment-policy.json create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 requirements.txt create mode 100644 scripts/init-secrets.py create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.sh create mode 100644 server.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7b0c333 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.12.8-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 +RUN useradd --create-home --uid 10001 appuser +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir --disable-pip-version-check -r requirements.txt +COPY server.py . +COPY migrations ./migrations +USER appuser +EXPOSE 8080 +CMD ["python","server.py"] diff --git a/README..md b/README..md new file mode 100644 index 0000000..a40347c --- /dev/null +++ b/README..md @@ -0,0 +1,222 @@ +# Лабораторная работа № 4 +## S3-хранилище и вложения + +Выполнила: Гончаревская Екатерина Александровна +Группа: БВТ2304 +Целевая оценка: 3 + +--- + +# 1. Цель работы + +Разделить метаданные вложений и их бинарное содержимое, добавить возможность отправки изображений в сообщениях и обеспечить доступность вложений при переключении между API-репликами. + +# 2. Постановка задачи + +Необходимо реализовать S3-совместимое хранилище вложений в инфраструктуре мессенджера. + +Основные требования: + +- Метаданные вложений должны храниться в PostgreSQL. +- Бинарное содержимое файлов должно храниться в отдельном приватном S3-бакете. +- Загрузка вложения должна возвращать статус готовности только после успешного сохранения файла. +- Пользователь должен иметь возможность скачать исходный файл без изменения его содержимого. +- Вложения должны быть связаны с сообщениями через `attachment_ids`. +- Доступ к вложениям должен предоставляться только участникам соответствующего чата. +- API должно проверять тип и допустимый размер загружаемых изображений. +- Доступ к вложениям должен сохраняться при переключении между API-репликами. +- После перезапуска сервисов с сохранением томов загруженные файлы и метаданные должны оставаться доступными. + +# 3. Используемые технологии + +- Python — реализация API. +- PostgreSQL — хранение метаданных и связей вложений с сообщениями. +- RustFS — S3-совместимое хранилище бинарных файлов. +- Valkey — хранение сессий. +- HAProxy — распределение запросов между API-репликами. +- Docker и Docker Compose — контейнеризация и запуск инфраструктуры. +- boto3 — взаимодействие с S3. +- Pillow — проверка изображений. +- psycopg — работа с PostgreSQL. + +# 4. Архитектура решения + +Схема взаимодействия компонентов: + +Клиент → HAProxy → API-реплики → PostgreSQL и RustFS. + +В инфраструктуре используются две API-реплики, работающие с общей базой данных и общим S3-хранилищем. + +PostgreSQL отвечает за хранение метаданных вложений и связей с сообщениями. Бинарное содержимое размещается в приватном бакете `messenger-attachments`. + +RustFS предоставляет S3-совместимый интерфейс. API обращается к хранилищу по внутреннему адресу Docker-сети. + +HAProxy распределяет входящие запросы между API-репликами. Благодаря общему хранилищу и общей базе данных пользователь может получить ранее загруженное вложение через другую API-реплику. + +Для сохранения данных при перезапуске используются постоянные Docker volumes. + +# 5. Реализация + +## 5.1. Загрузка вложений + +При получении запроса на загрузку API выполняет проверку входных данных и изображения. + +Реализованы следующие ограничения: + +- Максимальный размер файла — 10 МиБ. +- Максимальное количество пикселей — 40 миллионов. +- Поддерживаемые форматы — PNG и JPEG. +- MIME-тип проверяется по содержимому файла. +- Проверяется допустимость имени файла. + +После проверки бинарное содержимое сохраняется в S3-хранилище. Метаданные регистрируются в PostgreSQL. Вложение получает статус готовности после успешного сохранения файла. + +## 5.2. Получение вложений + +При запросе на скачивание API проверяет наличие вложения и права доступа пользователя. + +Доступ предоставляется только участникам чата, к которому относится вложение. + +После получения объекта из S3 выполняется проверка целостности данных по контрольной сумме SHA-256. + +## 5.3. Интеграция с сообщениями + +Вложения связываются с сообщениями через идентификаторы `attachment_ids`. + +Это позволяет получать сообщения вместе с информацией о прикреплённых файлах и сохранять связь между сообщениями и вложениями в PostgreSQL. + +## 5.4. Отказоустойчивость и сохранение данных + +Для обеспечения доступности вложений используются две API-реплики, общая база PostgreSQL и общее S3-хранилище. + +При остановке одной API-реплики HAProxy продолжает направлять запросы на доступную реплику. + +Постоянные тома обеспечивают сохранение данных PostgreSQL и RustFS после перезапуска контейнеров. + +# 6. Запуск проекта + +## 6.1. Требования + +Для запуска необходимы: + +- Docker Desktop с поддержкой Docker Compose; +- Python 3; +- Git; +- WSL с Linux для запуска сценариев отказоустойчивости в Windows. + +## 6.2. Подготовка + +Клонировать репозиторий: + +```bash +git clone https://gitea.privcloud.tech/Katarhani/mtusi2026lab4.git +cd mtusi2026lab4 +``` + +Создать локальные файлы секретов: + +```bash +python scripts/init-secrets.py +``` + +Секреты создаются в каталоге `secrets/`. Их содержимое не должно публиковаться в репозитории. + +Проверить конфигурацию Docker Compose: + +```bash +docker compose config --quiet +``` + +## 6.3. Запуск инфраструктуры + +Собрать образы и запустить сервисы: + +```bash +docker compose up --build -d +``` + +Проверить состояние контейнеров: + +```bash +docker compose ps +``` + +В работающей инфраструктуре должны присутствовать: + +- две API-реплики; +- HAProxy; +- PostgreSQL; +- Valkey; +- RustFS. + +## 6.4. Проверка готовности API + +Проверить endpoint готовности: + +```bash +curl http://localhost:8080/health/ready +``` + +Входящие запросы с компьютера направляются через HAProxy на локальный порт 8080. + +# 7. Проверка работоспособности + +## 7.1. Основной сценарий API + +Запустить smoke-тест: + +```bash +python tests/smoke.py --base-url http://localhost:8080 +``` + +Тест проверяет основной публичный сценарий работы API. + +## 7.2. Проверка сохранности данных + +Запустить проверку persistence: + +```bash +python tests/resilience.py --base-url http://localhost:8080 --mode persistence --action-script scripts/restart.sh +``` + +Сценарий проверяет сохранение подтверждённых данных и существующей сессии после перезапуска инфраструктуры. + +## 7.3. Проверка отказоустойчивости + +Запустить проверку failover: + +```bash +python tests/resilience.py --base-url http://localhost:8080 --mode failover --action-script scripts/stop-one.sh +``` + +Сценарий проверяет продолжение обработки запросов при остановке одной API-реплики и доступность ранее сохранённых данных через оставшуюся реплику. + +Для запуска сценариев persistence и failover в Windows используется WSL. + +# 8. Остановка проекта + +Для остановки контейнеров без удаления сохранённых данных: + +```bash +docker compose down +``` + +Для полного удаления контейнеров и томов с данными: + +```bash +docker compose down -v +``` + +**Внимание:** команда `docker compose down -v` удаляет постоянные тома, в том числе данные PostgreSQL и S3-хранилища. Использовать её следует только при необходимости полного сброса лабораторного стенда. + +# 9. Результаты + +В ходе выполнения лабораторной работы реализовано S3-совместимое хранение вложений, разделение метаданных и бинарного содержимого, интеграция вложений с сообщениями и работа с несколькими API-репликами. + +Выполнены проверки: + +- основной сценарий API — PASS; +- сохранение данных после перезапуска — PASS; +- отказоустойчивость при остановке API-реплики — PASS. + +Подробные результаты проведённых проверок и сведения об экспериментах приведены в `REPORT.md`. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..bb75d1a --- /dev/null +++ b/compose.yaml @@ -0,0 +1,124 @@ + +services: + postgres: + image: postgres:16.6-alpine + restart: unless-stopped + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL","pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + + valkey: + image: valkey/valkey:8.0.1-alpine + restart: unless-stopped + command: ["valkey-server","--appendonly","yes","--appendfsync","everysec"] + volumes: [valkey_data:/data] + healthcheck: + test: ["CMD","valkey-cli","ping"] + interval: 3s + timeout: 3s + retries: 20 + + rustfs: + image: rustfs/rustfs:1.0.4 + restart: unless-stopped + command: ["/data"] + environment: + RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_access_key_id + RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_access_key_id,s3_secret_access_key] + volumes: [rustfs_data:/data] + + rustfs-init: + image: rustfs/rc:v0.1.30 + restart: "no" + depends_on: + rustfs: + condition: service_started + secrets: [s3_access_key_id,s3_secret_access_key] + entrypoint: ["/bin/sh","-c"] + command: + - | + rc alias set local http://rustfs:9000 "$$(tr -d '\r\n' < /run/secrets/s3_access_key_id)" "$$(tr -d '\r\n' < /run/secrets/s3_secret_access_key)" + until rc bucket list local/ >/dev/null 2>&1; do sleep 2; done + rc bucket create local/messenger-attachments || true + rc bucket anonymous set private local/messenger-attachments + rc bucket anonymous get local/messenger-attachments + + migrate: + build: {context: ., dockerfile: Dockerfile} + command: ["python","server.py","migrate"] + restart: "no" + environment: + DATABASE_URL_FILE: /run/secrets/database_url + REDIS_URL_FILE: /run/secrets/redis_url + S3_ENDPOINT: http://rustfs:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url,redis_url,s3_access_key_id,s3_secret_access_key] + depends_on: + postgres: {condition: service_healthy} + valkey: {condition: service_healthy} + rustfs-init: {condition: service_completed_successfully} + + api: + build: {context: ., dockerfile: Dockerfile} + restart: unless-stopped + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + SESSION_TTL_SECONDS: 3600 + DATABASE_URL_FILE: /run/secrets/database_url + REDIS_URL_FILE: /run/secrets/redis_url + S3_ENDPOINT: http://rustfs:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: 10485760 + MAX_IMAGE_PIXELS: 40000000 + secrets: [database_url,redis_url,s3_access_key_id,s3_secret_access_key] + depends_on: + migrate: {condition: service_completed_successfully} + postgres: {condition: service_healthy} + valkey: {condition: service_healthy} + rustfs-init: {condition: service_completed_successfully} + healthcheck: + test: ["CMD","python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2)"] + interval: 2s + timeout: 2s + retries: 5 + start_period: 8s + stop_grace_period: 10s + + haproxy: + image: haproxy:3.1.2-alpine + restart: unless-stopped + ports: ["127.0.0.1:8080:8080"] + volumes: ["./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: ["CMD","haproxy","-c","-f","/usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + redis_url: {file: ./secrets/redis_url} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + +volumes: + postgres_data: + valkey_data: + rustfs_data: diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 0000000..bd2506b --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,31 @@ +global + log stdout format raw local0 + maxconn 2048 + +defaults + log global + mode http + option httplog + timeout connect 2s + timeout client 30s + timeout server 30s + timeout http-request 10s + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold nx 2s + +frontend public_api + bind :8080 + default_backend api_replicas + +backend api_replicas + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 1 slowstart 2s + server-template api 2 api:8080 check resolvers docker init-addr libc,none diff --git a/infra/minio/attachment-policy.json b/infra/minio/attachment-policy.json new file mode 100644 index 0000000..4db34dc --- /dev/null +++ b/infra/minio/attachment-policy.json @@ -0,0 +1,25 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments/*" + ] + } + ] +} \ No newline at end of file diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..986e15c --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,57 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_nonblank CHECK (length(btrim(display_name)) > 0) +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chats_title_nonblank CHECK (length(btrim(title)) > 0) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + position SMALLINT NOT NULL CHECK (position >= 0), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX chat_members_user_chat_idx + ON chat_members (user_id, chat_id); + +CREATE INDEX chats_created_id_idx + ON chats (created_at, id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT messages_text_nonblank CHECK (length(btrim(text)) > 0) +); + +CREATE INDEX messages_chat_created_id_idx + ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash CHAR(64) PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + revoked_at TIMESTAMPTZ, + CONSTRAINT sessions_token_hash_format CHECK (token_hash ~ '^[0-9a-f]{64}$') +); + +CREATE INDEX sessions_user_expiry_idx + ON sessions (user_id, expires_at); + +CREATE INDEX sessions_expiry_idx + ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..3327bcc --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,7 @@ +CREATE TABLE idempotency_keys ( + scope TEXT PRIMARY KEY, + request_hash CHAR(64) NOT NULL, + response_json JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..871bbad --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,22 @@ +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_nonblank; +ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT ARRAY[]::UUID[]; + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(32) NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 CHAR(64) NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + object_key TEXT NOT NULL UNIQUE, + status VARCHAR(16) NOT NULL CHECK (status IN ('ready')), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments(chat_id, created_at); +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + PRIMARY KEY(message_id, attachment_id) +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d18e23c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +psycopg[binary,pool]==3.2.4 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.35.90 +Pillow==11.0.0 diff --git a/scripts/init-secrets.py b/scripts/init-secrets.py new file mode 100644 index 0000000..8b07e34 --- /dev/null +++ b/scripts/init-secrets.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import argparse, os, secrets +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +def main(): + ap=argparse.ArgumentParser() + ap.add_argument('--force',action='store_true') + a=ap.parse_args() + d=ROOT/'secrets'; d.mkdir(mode=0o700,exist_ok=True) + password=secrets.token_hex(24) + values={ + 'postgres_password':password+'\n', + 'database_url':f'postgresql://messenger:{password}@postgres:5432/messenger?connect_timeout=5\n', + 'redis_url':'redis://valkey:6379/0\n', + 's3_access_key_id':'messenger'+secrets.token_hex(8)+'\n', + 's3_secret_access_key':secrets.token_urlsafe(36)+'\n', + 'minio_root_user':'minioadmin'+secrets.token_hex(4)+'\n', + 'minio_root_password':secrets.token_urlsafe(32)+'\n' + } + if not a.force and any((d/n).exists() for n in values): + raise SystemExit('Secret files already exist; use --force only to rotate all credentials.') + for n,v in values.items(): + f=d/n; f.write_text(v,encoding='utf-8') + try: os.chmod(f,0o600) + except OSError: pass + print('Local secrets created; values were not printed.') +if __name__=='__main__': main() + diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..058d61b --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +docker compose restart postgres valkey rustfs api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..9141809 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +case "$TARGET_INSTANCE_ID" in *[!a-fA-F0-9-]*|'') echo "Invalid API instance id" >&2; exit 2;; esac +docker stop --time 5 "$TARGET_INSTANCE_ID" >/dev/null diff --git a/server.py b/server.py new file mode 100644 index 0000000..3473a8f --- /dev/null +++ b/server.py @@ -0,0 +1,1004 @@ +import base64 +import datetime as dt +import hashlib +import json +import logging +import os +import re +import secrets +import signal +import threading +import uuid +import sys +import io +import mimetypes +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlsplit, quote + +import psycopg +from psycopg import rows +from psycopg.errors import UniqueViolation +from psycopg_pool import ConnectionPool, PoolTimeout +import redis +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError +from argon2 import PasswordHasher, exceptions as argon2_exceptions +from argon2.low_level import Type + +HOST = os.getenv("APP_HOST", "0.0.0.0") +PORT = int(os.getenv("APP_PORT", "8080")) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +DATABASE_URL_FILE = os.getenv("DATABASE_URL_FILE", "/run/secrets/database_url") +REDIS_URL_FILE = os.getenv("REDIS_URL_FILE", "/run/secrets/redis_url") +MAX_BODY_SIZE = 16 * 1024 * 1024 +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://minio:9000") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_FILE = os.getenv("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_access_key_id") +S3_SECRET_KEY_FILE = os.getenv("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_secret_access_key") +s3_client = None + +logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger("messenger") + +pool = None +redis_client = None +password_hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1, hash_len=32, salt_len=16, type=Type.ID) +shutting_down = False +server = None + + +def read_secret(path): + return Path(path).read_text(encoding="utf-8").strip() + + +def now_utc(): + return dt.datetime.now(dt.timezone.utc) + + +def format_time(value): + if value.tzinfo is None: + value = value.replace(tzinfo=dt.timezone.utc) + return value.astimezone(dt.timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def new_id(): + return str(uuid.uuid4()) + + +def error_body(code, message, request_id): + return {"error": {"code": code, "message": message, "request_id": request_id}} + + +def json_bytes(value): + return json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + +def valid_username(value): + return isinstance(value, str) and re.fullmatch(r"[a-z0-9_]{3,32}", value) is not None + + +def valid_nonblank_string(value, maximum): + return isinstance(value, str) and 1 <= len(value) <= maximum and bool(value.strip()) + + +def valid_uuid(value): + try: + uuid.UUID(value) + return True + except (ValueError, AttributeError, TypeError): + return False + + +def parse_json_object(raw): + try: + value = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + raise ValueError + if not isinstance(value, dict): + raise ValueError + return value + + +def encode_cursor(scope, created_at, object_id): + payload = json.dumps( + {"scope": scope, "created_at": format_time(created_at), "id": str(object_id)}, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def decode_cursor(cursor): + try: + raw = cursor.encode("ascii") + raw += b"=" * (-len(raw) % 4) + value = json.loads(base64.urlsafe_b64decode(raw).decode("utf-8")) + if not isinstance(value, dict): + raise ValueError + if not all(isinstance(value.get(key), str) and value[key] for key in ("scope", "created_at", "id")): + raise ValueError + object_id = uuid.UUID(value["id"]) + created_at = dt.datetime.fromisoformat(value["created_at"].replace("Z", "+00:00")) + if created_at.tzinfo is None: + raise ValueError + return value["scope"], created_at, str(object_id) + except (ValueError, UnicodeError, json.JSONDecodeError, KeyError, TypeError): + raise ValueError + + +def validate_keys(value, required): + return set(value) == set(required) + + +def apply_migrations(): + migration_dir = Path(__file__).resolve().parent / "migrations" + with pool.connection() as conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_xact_lock(%s)", (7202026,)) + for migration in sorted(migration_dir.glob("*.sql")): + version = migration.name + exists = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", (version,) + ).fetchone() + if exists: + continue + migration_sql = migration.read_text(encoding="utf-8") + for statement in migration_sql.split(";"): + if statement.strip(): + conn.execute(statement) + conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (version,)) + logger.info("migration applied version=%s", version) + + +def init_services(): + global pool, redis_client, s3_client + if SESSION_TTL_SECONDS < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") + database_url = read_secret(DATABASE_URL_FILE) + redis_url = read_secret(REDIS_URL_FILE) + pool = ConnectionPool( + conninfo=database_url, + min_size=1, + max_size=10, + timeout=5, + kwargs={"row_factory": rows.dict_row}, + open=True, + ) + pool.wait(timeout=30) + redis_client = redis.Redis.from_url( + redis_url, + decode_responses=True, + socket_connect_timeout=1, + socket_timeout=1, + health_check_interval=30, + ) + try: + redis_client.ping() + logger.info("session cache connected") + except redis.RedisError: + logger.warning("session cache unavailable; PostgreSQL fallback is active") + s3_client = boto3.client("s3", endpoint_url=S3_ENDPOINT, + aws_access_key_id=read_secret(S3_ACCESS_KEY_FILE), + aws_secret_access_key=read_secret(S3_SECRET_KEY_FILE), + region_name=os.getenv("S3_REGION", "us-east-1"), + config=Config(s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=10, retries={"max_attempts": 2})) + + +def best_effort_cache_delete(key): + try: + redis_client.delete(key) + except redis.RedisError: + logger.warning("session cache delete failed; PostgreSQL remains authoritative") + + +def cache_session(token_hash, user_id, expires_at): + ttl = max(1, int((expires_at - now_utc()).total_seconds())) + payload = json.dumps({"user_id": str(user_id), "expires_at": format_time(expires_at)}, separators=(",", ":")) + try: + redis_client.set("session:" + token_hash, payload, ex=ttl) + except redis.RedisError: + logger.warning("session cache write failed; PostgreSQL remains authoritative") + + +def public_user(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": format_time(row["created_at"]), + } + + +def public_chat(row, member_ids): + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(value) for value in member_ids], + "created_at": format_time(row["created_at"]), + } + + +def public_message(row): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": format_time(row["created_at"]), + "attachment_ids": [str(x) for x in (row.get("attachment_ids") or [])], + } + + +def public_attachment(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), + "filename": row["filename"], "content_type": row["content_type"], "size_bytes": row["size_bytes"], + "sha256": row["sha256"], "status": "ready", "variants": [], "error_code": None, + "created_at": format_time(row["created_at"]), "updated_at": format_time(row["updated_at"])} + + +def parse_limit_cursor(query): + limit_values = query.get("limit", ["50"]) + cursor_values = query.get("cursor", [None]) + if len(limit_values) != 1 or len(cursor_values) != 1: + raise ValueError + try: + limit = int(limit_values[0]) + except (TypeError, ValueError): + raise ValueError + if limit < 1 or limit > 100 or str(limit) != limit_values[0].lstrip("+"): + raise ValueError + cursor = cursor_values[0] + if cursor == "": + raise ValueError + return limit, cursor + + +def authenticate_basic(header): + if not header or not header.startswith("Basic "): + return None + try: + decoded = base64.b64decode(header[6:], validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeDecodeError): + return None + if not valid_username(username) or not isinstance(password, str): + return None + try: + with pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,), + ).fetchone() + if row is None: + return None + password_hasher.verify(row["password_hash"], password) + return row + except (argon2_exceptions.VerifyMismatchError, argon2_exceptions.InvalidHashError, argon2_exceptions.VerificationError): + return None + + +def authenticate_bearer(header): + if not header or not header.startswith("Bearer "): + return None + token = header[7:].strip() + if len(token) < 32 or len(token) > 256 or " " in token: + return None + token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest() + cached_user_id = None + try: + cached = redis_client.get("session:" + token_hash) + if cached: + cached_value = json.loads(cached) + cached_user_id = cached_value.get("user_id") + cached_expiry = dt.datetime.fromisoformat(cached_value["expires_at"].replace("Z", "+00:00")) + if cached_expiry <= now_utc() or not valid_uuid(cached_user_id): + cached_user_id = None + except (redis.RedisError, ValueError, TypeError, KeyError, AttributeError): + cached_user_id = None + logger.warning("session cache read failed; PostgreSQL fallback is active") + try: + with pool.connection() as conn: + if cached_user_id and valid_uuid(cached_user_id): + row = conn.execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s AND s.user_id = %s + AND s.revoked_at IS NULL AND s.expires_at > now() + """, + (token_hash, uuid.UUID(cached_user_id)), + ).fetchone() + else: + row = conn.execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > now() + """, + (token_hash,), + ).fetchone() + if row is None: + best_effort_cache_delete("session:" + token_hash) + return None + if cached_user_id is None: + cache_session(token_hash, row["id"], row["expires_at"]) + return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], + "created_at": row["created_at"], "token_hash": token_hash, "expires_at": row["expires_at"]} + except (psycopg.Error, PoolTimeout): + logger.exception("database unavailable during authentication") + raise + + +class MessengerServer(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def setup(self): + super().setup() + self.request_id = new_id() + self.response_status = 500 + self.actor = None + self.instance_id = os.getenv("INSTANCE_ID", os.getenv("HOSTNAME", "api")) + + def log_message(self, fmt, *args): + logger.info("%s %s %s request_id=%s user=%s", self.command, self.path.split("?", 1)[0], + self.response_status, self.request_id, (self.actor.get("id", "-") if self.actor else "-")) + + def send_json(self, status, payload, challenge=None): + data = json_bytes(payload) + self.response_status = status + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", self.instance_id) + if challenge: + self.send_header("WWW-Authenticate", challenge) + self.end_headers() + self.wfile.write(data) + + def send_empty(self, status): + self.response_status = status + self.send_response(status) + self.send_header("Content-Length", "0") + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", self.instance_id) + self.end_headers() + + def send_error_json(self, status, code, message, challenge=None): + self.send_json(status, error_body(code, message, self.request_id), challenge) + + def unavailable(self): + self.send_error_json(503, "unavailable", "Service temporarily unavailable") + + def read_body(self): + value = self.headers.get("Content-Length") + if value is None: + raise ValueError + try: + length = int(value) + except ValueError: + raise ValueError + if length < 0: + raise ValueError + if length > MAX_BODY_SIZE: + raise OverflowError + return self.rfile.read(length) + + def require_json(self): + if self.headers.get_content_type() != "application/json": + self.send_error_json(415, "unsupported_media_type", "Content-Type must be application/json") + return None + try: + return parse_json_object(self.read_body()) + except OverflowError: + self.send_error_json(413, "payload_too_large", "Request body is too large") + except ValueError: + self.send_error_json(400, "invalid_request", "Invalid JSON request") + return None + + def identify(self): + user = authenticate_bearer(self.headers.get("Authorization")) + if user is None: + self.send_error_json(401, "unauthorized", "Valid Bearer session required", + 'Bearer realm="messenger"') + return None + self.actor = user + return user + + def check_ready_for_write(self): + if shutting_down: + self.send_error_json(503, "unavailable", "Service is shutting down") + return False + return True + + def do_GET(self): + self.handle_method("GET") + + def do_POST(self): + self.handle_method("POST") + + def do_DELETE(self): + self.handle_method("DELETE") + + def do_PUT(self): + self.send_error_json(404, "not_found", "Object not found") + + def do_PATCH(self): + self.send_error_json(404, "not_found", "Object not found") + + def handle_method(self, method): + try: + self.route(method) + except (psycopg.Error, PoolTimeout): + logger.exception("database operation failed request_id=%s", self.request_id) + if not self.wfile.closed: + try: + self.unavailable() + except OSError: + pass + except redis.RedisError: + logger.exception("cache operation failed request_id=%s", self.request_id) + if not self.wfile.closed: + try: + self.unavailable() + except OSError: + pass + except Exception: + logger.exception("unexpected request error request_id=%s", self.request_id) + if not self.wfile.closed: + try: + self.send_error_json(500, "internal_error", "Internal server error") + except OSError: + pass + + def route(self, method): + parsed = urlsplit(self.path) + parts = [part for part in parsed.path.split("/") if part] + query = parse_qs(parsed.query, keep_blank_values=True) + + if method == "GET" and parts == ["health", "live"]: + self.send_json(200, {"status": "ok"}) + return + if method == "GET" and parts == ["health", "ready"]: + if shutting_down: + self.unavailable() + return + try: + with pool.connection() as conn: + conn.execute("SELECT 1") + s3_client.head_bucket(Bucket=S3_BUCKET) + self.send_json(200, {"status": "ok"}) + except (psycopg.Error, PoolTimeout, BotoCoreError, ClientError): + self.unavailable() + return + if shutting_down: + self.unavailable() + return + + if method == "POST" and parts == ["api", "v1", "users"]: + self.create_user() + return + if method == "POST" and parts == ["api", "v1", "auth", "sessions"]: + self.create_session() + return + if method == "DELETE" and parts == ["api", "v1", "auth", "sessions", "current"]: + self.delete_session() + return + if method == "GET" and parts == ["api", "v1", "users", "me"]: + user = self.identify() + if user is not None: + self.send_json(200, public_user(user)) + return + + if parts[:3] == ["api", "v1", "attachments"]: + if len(parts) == 4 and method == "GET" and valid_uuid(parts[3]): + self.get_attachment(parts[3], content=False) + return + if len(parts) == 5 and parts[4] == "content" and method == "GET" and valid_uuid(parts[3]): + self.get_attachment(parts[3], content=True) + return + if parts[:3] == ["api", "v1", "chats"]: + if len(parts) == 3: + if method == "POST": + self.create_chat() + return + if method == "GET": + self.list_chats(query) + return + if len(parts) == 4 and method == "GET": + if not valid_uuid(parts[3]): + if self.identify() is not None: + self.send_error_json(400, "invalid_request", "Invalid chat UUID") + return + self.get_chat(parts[3]) + return + if len(parts) == 5 and parts[4] == "attachments" and method == "POST": + self.create_attachment(parts[3], query) + return + if len(parts) == 5 and parts[4] == "messages": + if not valid_uuid(parts[3]): + if self.identify() is not None: + self.send_error_json(400, "invalid_request", "Invalid chat UUID") + return + if method == "POST": + self.create_message(parts[3]) + return + if method == "GET": + self.list_messages(parts[3], query) + return + self.send_error_json(404, "not_found", "Object not found") + + def create_user(self): + if not self.check_ready_for_write(): + return + body = self.require_json() + if body is None: + return + if not validate_keys(body, {"username", "display_name", "password"}): + self.send_error_json(400, "invalid_request", "Invalid user fields") + return + username = body["username"] + display_name = body["display_name"] + password = body["password"] + if (not valid_username(username) or not valid_nonblank_string(display_name, 100) + or not isinstance(password, str) or not 12 <= len(password) <= 128): + self.send_error_json(400, "invalid_request", "Invalid user fields") + return + password_hash = password_hasher.hash(password) + user_id = new_id() + try: + with pool.connection() as conn: + row = conn.execute( + """ + INSERT INTO users(id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (uuid.UUID(user_id), username, display_name, password_hash), + ).fetchone() + except UniqueViolation: + self.send_error_json(409, "conflict", "Username is already in use") + return + self.send_json(201, public_user(row)) + + def create_session(self): + if not self.check_ready_for_write(): + return + user = authenticate_basic(self.headers.get("Authorization")) + if user is None: + self.send_error_json(401, "unauthorized", "Invalid username or password", + 'Basic realm="messenger", charset="UTF-8"') + return + token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest() + expires_at = now_utc() + dt.timedelta(seconds=SESSION_TTL_SECONDS) + with pool.connection() as conn: + conn.execute( + "INSERT INTO sessions(token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash, user["id"], expires_at), + ) + cache_session(token_hash, user["id"], expires_at) + self.send_json(201, { + "token": token, + "token_type": "Bearer", + "expires_at": format_time(expires_at), + "user": public_user(user), + }) + + def delete_session(self): + if not self.check_ready_for_write(): + return + user = self.identify() + if user is None: + return + with pool.connection() as conn: + conn.execute("UPDATE sessions SET revoked_at = now() WHERE token_hash = %s AND revoked_at IS NULL", (user["token_hash"],)) + best_effort_cache_delete("session:" + user["token_hash"]) + self.send_empty(204) + + def create_chat(self): + if not self.check_ready_for_write(): + return + user = self.identify() + if user is None: + return + body = self.require_json() + if body is None: + return + if not validate_keys(body, {"title", "member_ids"}): + self.send_error_json(400, "invalid_request", "Invalid chat fields") + return + title = body["title"] + member_ids = body["member_ids"] + if (not valid_nonblank_string(title, 100) or not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100): + self.send_error_json(400, "invalid_request", "Invalid chat fields") + return + if any(not isinstance(value, str) or not valid_uuid(value) for value in member_ids): + self.send_error_json(400, "invalid_request", "Invalid chat members") + return + member_ids = [str(uuid.UUID(value)) for value in member_ids] + if len(member_ids) != len(set(member_ids)): + self.send_error_json(400, "invalid_request", "Invalid chat members") + return + if user["id"] not in member_ids: + self.send_error_json(400, "invalid_request", "Chat must include the current user") + return + chat_id = new_id() + try: + with pool.connection() as conn: + existing = conn.execute( + "SELECT id FROM users WHERE id = ANY(%s)", + ([uuid.UUID(value) for value in member_ids],), + ).fetchall() + if len(existing) != len(member_ids): + self.send_error_json(400, "invalid_request", "Unknown chat member") + return + row = conn.execute( + "INSERT INTO chats(id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (uuid.UUID(chat_id), title), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members(chat_id, user_id, position) VALUES (%s, %s, %s)", + [(uuid.UUID(chat_id), uuid.UUID(member_id), position) for position, member_id in enumerate(member_ids)], + ) + self.send_json(201, public_chat(row, member_ids)) + except psycopg.errors.ForeignKeyViolation: + self.send_error_json(400, "invalid_request", "Unknown chat member") + + def list_chats(self, query): + user = self.identify() + if user is None: + return + try: + limit, cursor = parse_limit_cursor(query) + except ValueError: + self.send_error_json(400, "invalid_request", "Invalid pagination parameters") + return + marker = None + if cursor is not None: + try: + marker = decode_cursor(cursor) + except ValueError: + self.send_error_json(400, "invalid_request", "Invalid cursor") + return + if marker[0] != "chats:" + user["id"]: + self.send_error_json(400, "invalid_request", "Cursor does not belong to this request") + return + sql = """ + SELECT c.id, c.title, c.created_at, + array_agg(cm_all.user_id ORDER BY cm_all.position) AS member_ids + FROM chats c + JOIN chat_members cm_all ON cm_all.chat_id = c.id + WHERE EXISTS ( + SELECT 1 FROM chat_members mine + WHERE mine.chat_id = c.id AND mine.user_id = %s + ) + """ + params = [uuid.UUID(user["id"])] + if marker is not None: + sql += " AND (c.created_at, c.id) > (%s, %s)" + params.extend([marker[1], uuid.UUID(marker[2])]) + sql += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT %s" + params.append(limit + 1) + with pool.connection() as conn: + rows = conn.execute(sql, params).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + items = [public_chat(row, row["member_ids"]) for row in rows] + next_cursor = encode_cursor("chats:" + user["id"], rows[-1]["created_at"], rows[-1]["id"]) if has_more else None + self.send_json(200, {"items": items, "next_cursor": next_cursor}) + + def get_chat(self, chat_id): + user = self.identify() + if user is None: + return + with pool.connection() as conn: + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at, + array_agg(cm_all.user_id ORDER BY cm_all.position) AS member_ids + FROM chats c + JOIN chat_members cm_all ON cm_all.chat_id = c.id + WHERE c.id = %s AND EXISTS ( + SELECT 1 FROM chat_members mine + WHERE mine.chat_id = c.id AND mine.user_id = %s + ) + GROUP BY c.id + """, + (uuid.UUID(chat_id), uuid.UUID(user["id"])), + ).fetchone() + if row is None: + self.send_error_json(404, "not_found", "Object not found") + return + self.send_json(200, public_chat(row, row["member_ids"])) + + def create_message(self, chat_id): + if not self.check_ready_for_write(): + return + user = self.identify() + if user is None: + return + key = self.headers.get("Idempotency-Key") + if not isinstance(key, str) or re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key) is None: + self.send_error_json(400, "invalid_request", "A valid Idempotency-Key is required") + return + body = self.require_json() + if body is None: + return + if set(body) - {"text", "attachment_ids"} or "text" not in body: + self.send_error_json(400, "invalid_request", "Invalid message fields") + return + message_text = body["text"] + attachment_ids = body.get("attachment_ids", []) + if (not isinstance(message_text, str) or len(message_text) > 4000 + or (not message_text.strip() and not attachment_ids)): + self.send_error_json(400, "invalid_request", "Invalid message text") + return + if (not isinstance(attachment_ids, list) or len(attachment_ids) > 10 + or any(not isinstance(x, str) or not valid_uuid(x) for x in attachment_ids)): + self.send_error_json(400, "invalid_request", "Invalid attachment_ids") + return + if len(attachment_ids) != len(set(attachment_ids)): + self.send_error_json(400, "invalid_request", "Duplicate attachment_ids") + return + attachment_ids = [str(uuid.UUID(x)) for x in attachment_ids] + request_hash = hashlib.sha256(json_bytes({"text": message_text, "attachment_ids": attachment_ids})).hexdigest() + scope = f"{user['id']}:POST:/api/v1/chats/{chat_id}/messages:{key}" + message_id = new_id() + with pool.connection() as conn: + allowed = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (uuid.UUID(chat_id), uuid.UUID(user["id"])), + ).fetchone() + if allowed is None: + self.send_error_json(404, "not_found", "Object not found") + return + conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (scope,)) + previous = conn.execute( + "SELECT request_hash, response_json FROM idempotency_keys WHERE scope = %s", + (scope,), + ).fetchone() + if previous is not None: + if previous["request_hash"] != request_hash: + self.send_error_json(409, "conflict", "Idempotency-Key was used with a different request") + return + self.send_json(201, previous["response_json"]) + return + if attachment_ids: + owned = conn.execute( + "SELECT id FROM attachments WHERE id = ANY(%s) AND chat_id = %s AND status = 'ready'", + ([uuid.UUID(x) for x in attachment_ids], uuid.UUID(chat_id))).fetchall() + if len(owned) != len(attachment_ids): + self.send_error_json(404, "not_found", "Attachment not found") + return + row = conn.execute( + """ + INSERT INTO messages(id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, text, created_at + """, + (uuid.UUID(message_id), uuid.UUID(chat_id), uuid.UUID(user["id"]), message_text), + ).fetchone() + if attachment_ids: + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO message_attachments(message_id, attachment_id) VALUES (%s, %s)", + [(uuid.UUID(message_id), uuid.UUID(x)) for x in attachment_ids], + ) + row["attachment_ids"] = attachment_ids + result = public_message(row) + conn.execute( + "INSERT INTO idempotency_keys(scope, request_hash, response_json, created_at) VALUES (%s, %s, %s, now())", + (scope, request_hash, json.dumps(result, ensure_ascii=False)), + ) + self.send_json(201, result) + + def create_attachment(self, chat_id, query): + if not self.check_ready_for_write(): + return + user = self.identify() + if user is None: + return + names = query.get("filename", []) + if len(names) != 1: + self.send_error_json(400, "invalid_request", "filename is required") + return + filename = names[0] + if (not filename or len(filename) > 255 or filename in {".", ".."} + or "/" in filename or "\\\\" in filename or any(ord(c) < 32 for c in filename)): + self.send_error_json(400, "invalid_request", "Unsafe filename") + return + with pool.connection() as conn: + member = conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", + (uuid.UUID(chat_id), uuid.UUID(user["id"]))).fetchone() + if member is None: + self.send_error_json(404, "not_found", "Object not found") + return + length = self.headers.get("Content-Length") + try: + size = int(length) + except (TypeError, ValueError): + self.send_error_json(411, "invalid_request", "Content-Length is required") + return + if size < 1 or size > MAX_UPLOAD_BYTES: + self.send_error_json(413, "payload_too_large", "Image exceeds upload limit") + return + declared = self.headers.get_content_type() + if declared not in ("image/png", "image/jpeg"): + self.send_error_json(415, "unsupported_media_type", "Only PNG and JPEG are accepted") + return + data = self.rfile.read(size) + if len(data) != size: + self.send_error_json(400, "invalid_request", "Incomplete upload") + return + try: + image = Image.open(io.BytesIO(data)) + actual = image.format + width, height = image.size + if width * height > MAX_IMAGE_PIXELS: + self.send_error_json(413, "payload_too_large", "Image dimensions exceed limit") + return + image.verify() + except (UnidentifiedImageError, OSError, Image.DecompressionBombError, Image.DecompressionBombWarning): + self.send_error_json(400, "invalid_request", "Invalid image") + return + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(actual) + if actual_type is None or actual_type != declared: + self.send_error_json(415, "unsupported_media_type", "Content-Type does not match image") + return + attachment_id = new_id() + object_key = "attachments/" + attachment_id + digest = hashlib.sha256(data).hexdigest() + try: + s3_client.put_object(Bucket=S3_BUCKET, Key=object_key, Body=data, + ContentType=actual_type, Metadata={"sha256": digest}) + with pool.connection() as conn: + row = conn.execute(""" + INSERT INTO attachments(id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, object_key, status) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'ready') + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, status, created_at, updated_at + """, (uuid.UUID(attachment_id), uuid.UUID(chat_id), uuid.UUID(user["id"]), + filename, actual_type, size, digest, object_key)).fetchone() + self.send_json(201, public_attachment(row)) + except (BotoCoreError, ClientError): + logger.exception("S3 upload failed") + self.unavailable() + except Exception: + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=object_key) + except Exception: + logger.exception("orphan cleanup failed") + raise + + def get_attachment(self, attachment_id, content=False): + user = self.identify() + if user is None: + return + with pool.connection() as conn: + row = conn.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON cm.chat_id=a.chat_id + WHERE a.id=%s AND cm.user_id=%s AND a.status='ready' + """, (uuid.UUID(attachment_id), uuid.UUID(user["id"]))).fetchone() + if row is None: + self.send_error_json(404, "not_found", "Object not found") + return + if not content: + self.send_json(200, public_attachment(row)) + return + try: + result = s3_client.get_object(Bucket=S3_BUCKET, Key=row["object_key"]) + data = result["Body"].read(MAX_UPLOAD_BYTES + 1) + if len(data) > MAX_UPLOAD_BYTES or hashlib.sha256(data).hexdigest() != row["sha256"]: + self.unavailable() + return + self.response_status = 200 + self.send_response(200) + self.send_header("Content-Type", row["content_type"]) + self.send_header("Content-Length", str(len(data))) + self.send_header("Content-Disposition", "inline; filename*=UTF-8''" + quote(row["filename"], safe="")) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", self.instance_id) + self.end_headers() + self.wfile.write(data) + except (BotoCoreError, ClientError): + logger.exception("S3 download failed") + self.unavailable() + + def list_messages(self, chat_id, query): + user = self.identify() + if user is None: + return + try: + limit, cursor = parse_limit_cursor(query) + except ValueError: + self.send_error_json(400, "invalid_request", "Invalid pagination parameters") + return + marker = None + if cursor is not None: + try: + marker = decode_cursor(cursor) + except ValueError: + self.send_error_json(400, "invalid_request", "Invalid cursor") + return + if marker[0] != "messages:" + chat_id: + self.send_error_json(400, "invalid_request", "Cursor does not belong to this chat") + return + with pool.connection() as conn: + allowed = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (uuid.UUID(chat_id), uuid.UUID(user["id"])), + ).fetchone() + if allowed is None: + self.send_error_json(404, "not_found", "Object not found") + return + sql = """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE(array_agg(ma.attachment_id) FILTER (WHERE ma.attachment_id IS NOT NULL), ARRAY[]::uuid[]) AS attachment_ids + FROM messages m + LEFT JOIN message_attachments ma ON ma.message_id = m.id + WHERE m.chat_id = %s + """ + params = [uuid.UUID(chat_id)] + if marker is not None: + sql += " AND (created_at, id) > (%s, %s)" + params.extend([marker[1], uuid.UUID(marker[2])]) + sql += " GROUP BY m.id ORDER BY m.created_at, m.id LIMIT %s" + params.append(limit + 1) + rows = conn.execute(sql, params).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + items = [public_message(row) for row in rows] + next_cursor = encode_cursor("messages:" + chat_id, rows[-1]["created_at"], rows[-1]["id"]) if has_more else None + self.send_json(200, {"items": items, "next_cursor": next_cursor}) + + +def handle_signal(signum, frame): + global shutting_down + shutting_down = True + logger.info("shutdown requested signal=%s", signum) + if server is not None: + threading.Thread(target=server.shutdown, daemon=True).start() + + +def main(): + global server + init_services() + if len(sys.argv) > 1 and sys.argv[1] == "migrate": + apply_migrations() + pool.close() + redis_client.close() + return + server = MessengerServer((HOST, PORT), Handler) + signal.signal(signal.SIGTERM, handle_signal) + signal.signal(signal.SIGINT, handle_signal) + logger.info("server started host=%s port=%s", HOST, PORT) + try: + server.serve_forever(poll_interval=0.2) + finally: + server.server_close() + if pool is not None: + pool.close() + if redis_client is not None: + redis_client.close() + logger.info("server stopped") + + +if __name__ == "__main__": + main() \ No newline at end of file