Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de2246a888 |
+6
-19
@@ -1,20 +1,7 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Copy to .env and replace local-only passwords. Never commit .env.
|
||||
POSTGRES_PASSWORD=change-me-postgres
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
S3_BOOTSTRAP_ACCESS_KEY=bootstrap-admin
|
||||
S3_BOOTSTRAP_SECRET_KEY=change-me-bootstrap-secret
|
||||
S3_APP_ACCESS_KEY=messenger-api
|
||||
S3_APP_SECRET_KEY=change-me-s3-api-secret
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
FROM python:3.13-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN python -m pip install --no-cache-dir -r requirements.txt
|
||||
COPY app.py ./
|
||||
COPY migrations ./migrations/
|
||||
COPY scripts ./scripts/
|
||||
|
||||
RUN useradd --create-home --shell /usr/sbin/nologin appuser && \
|
||||
chown -R appuser:appuser /app
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
CMD ["python", "app.py"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test test-grade4 test-attachments-load load test-persistence test-failover
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -18,6 +18,12 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-grade4:
|
||||
$(PYTHON) tests/grade4_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachments-load:
|
||||
$(PYTHON) tests/load_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
|
||||
@@ -1,37 +1,72 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Антоновва Екатерина бпи2304
|
||||
- Целевая оценка: 4
|
||||
- Выбранный трек 5: не заявлен
|
||||
- Commit/PR: заполняется после публикации
|
||||
|
||||
## Использованные источники, библиотеки и помощь ИИ
|
||||
|
||||
Использованы нормативные `contracts/openapi.json`, `contracts/README.md`, `contracts/infrastructure.json`, публичные тесты из `tests/` и документация FastAPI, PostgreSQL, boto3, Pillow, SeaweedFS и Docker Compose. ИИ применялся как вспомогательный инструмент для проверки реализации, поиска ошибок и редактирования отчёта; итоговые решения проверены локальными тестами и запуском стенда.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
```sh
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
# дождаться: docker compose ps показывает healthy у postgres, s3, api1, api2 и load_balancer
|
||||
make check
|
||||
make validate
|
||||
make test
|
||||
make test-grade4
|
||||
make test-attachments-load
|
||||
```
|
||||
|
||||
Для остановки без удаления данных:
|
||||
|
||||
```sh
|
||||
docker compose down
|
||||
```
|
||||
|
||||
Для намеренного удаления учебных данных вместе с volumes:
|
||||
|
||||
```sh
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
Пароли находятся только в локальном `.env`, который исключён из Git. В текущей проверке Windows `make` отсутствовал, поэтому его цели выполнены эквивалентными командами с `.venv\Scripts\python.exe`; в Linux/CI используются команды Makefile.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Клиент обращается к единственному публичному Nginx на `localhost:8080`. Балансировщик направляет запросы на `api1` и `api2`; обе реплики используют одну PostgreSQL и один SeaweedFS с S3 API. Локального состояния сессий, сообщений и файлов в API нет: PostgreSQL хранит метаданные, а приватный bucket `messenger-attachments` — байты.
|
||||
|
||||
Вложение принимается потоково в `SpooledTemporaryFile` с лимитом 10 MiB, проверяются фактический PNG/JPEG, MIME, имя, SHA-256 и максимум 40 млн пикселей. Сначала выполняется S3 PUT, затем транзакционная запись метаданных; при ошибке БД выполняется компенсационный DELETE. `scripts/reconcile_attachments.py` поддерживает dry-run и удаление только orphan-объектов старше grace period.
|
||||
|
||||
S3 bootstrap идемпотентен. SeaweedFS имеет отдельные bootstrap- и API-credentials; API получает только права `Read/Write/List/Tagging` для нужного bucket, административная консоль наружу не публикуется. Миграция `004_allow_attachment_only_messages.sql` сохраняет возможность сообщений, содержащих только вложение.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Критерий | Команда/сценарий | Результат |
|
||||
| --- | --- | --- |
|
||||
| Контракт и OpenAPI | `scripts/check_contract.py`, `scripts/validate_openapi.py` | PASS |
|
||||
| Публичный smoke | `tests/smoke.py` / `make test` | PASS: регистрация, сессии, чат, сообщение, upload/download и доступ участника |
|
||||
| Ошибки уровня 4 | `tests/grade4_attachments.py` / `make test-grade4` | PASS: >10 MiB, повреждённый файл, MIME spoofing, опасное имя, чужой attachment и другой чат |
|
||||
| Идемпотентный bootstrap | повторный `docker compose up -d --build` | PASS, bucket и миграции не дублируются |
|
||||
| Persistence | `scripts/restart.sh` без удаления volumes | PASS: действующая сессия, метаданные и байты вложения сохранились |
|
||||
| Failover | `scripts/stop-one.sh`, остановка обслужившей реплики | PASS: download через вторую реплику вернул исходные байты |
|
||||
| Частный bucket | API credentials и отсутствие публикации S3-порта | S3 доступен API внутри Compose, прямой публичный GET не используется |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Эксперимент потоковой загрузки
|
||||
|
||||
## Эксперименты и отказы
|
||||
Запуск:
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
```sh
|
||||
make test-attachments-load
|
||||
```
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Скрипт одновременно выполняет 10 загрузок, а Docker RSS API-реплик опрашивается во время эксперимента. Фактический результат текущего прогона сохранён в `evidence/attachments-concurrency.txt`: `uploads=10`, `concurrency=10`, `elapsed_seconds=0.480`, `peak_api_rss_mb=84.2`. Полный файл не удерживается в RAM: после 1 MiB `SpooledTemporaryFile` переходит на временный файл.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
## Ограничения
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Проверен single-node Compose; отказ Docker-host, сетевой partition и HA PostgreSQL не заявляются. Компенсация закрывает обычный отказ записи метаданных, а периодическая сверка orphan-объектов запускается оператором, а не отдельным worker-процессом.
|
||||
|
||||
@@ -0,0 +1,859 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import psycopg
|
||||
import boto3
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from fastapi.responses import StreamingResponse
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from psycopg.rows import dict_row
|
||||
from fastapi import FastAPI, Query, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
|
||||
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.getenv("APP_PORT", "8080"))
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "")
|
||||
DATABASE_URL_FILE = os.getenv("DATABASE_URL_FILE", "")
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
INSTANCE_ID = os.getenv("INSTANCE_ID", "api-" + uuid.uuid4().hex[:8])
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "")
|
||||
S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID", "")
|
||||
S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "")
|
||||
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
MIGRATIONS_DIR = Path(__file__).parent / "migrations"
|
||||
PASSWORD_HASHER = PasswordHasher()
|
||||
LOGGER = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status_code: int, code: str, message: str):
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(..., min_length=1, max_length=100)
|
||||
password: str = Field(..., min_length=12, max_length=128)
|
||||
|
||||
@field_validator("display_name", "password")
|
||||
@classmethod
|
||||
def validate_non_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("value must not be blank")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title: str = Field(..., min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(..., min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def validate_title(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("title must not be blank")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def validate_member_ids(cls, value: list[str]) -> list[str]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("member_ids must be unique")
|
||||
for member_id in value:
|
||||
try:
|
||||
uuid.UUID(member_id)
|
||||
except ValueError as exc:
|
||||
raise ValueError("member_ids must contain UUIDs") from exc
|
||||
return value
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
text: str = Field(..., max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def validate_attachment_ids(cls, value: list[str]) -> list[str]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("attachment_ids must be unique")
|
||||
for attachment_id in value:
|
||||
try:
|
||||
uuid.UUID(attachment_id)
|
||||
except ValueError as exc:
|
||||
raise ValueError("attachment_ids must contain UUIDs") from exc
|
||||
return value
|
||||
|
||||
|
||||
class AttachmentError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def iso(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace(
|
||||
"+00:00", "Z"
|
||||
)
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
if DATABASE_URL:
|
||||
return DATABASE_URL
|
||||
if DATABASE_URL_FILE:
|
||||
try:
|
||||
return Path(DATABASE_URL_FILE).read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
raise RuntimeError("database URL file is not readable") from exc
|
||||
raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required")
|
||||
|
||||
|
||||
def db():
|
||||
return psycopg.connect(database_url(), row_factory=dict_row)
|
||||
|
||||
|
||||
def s3():
|
||||
if not S3_ENDPOINT or not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY:
|
||||
raise RuntimeError("S3 configuration is incomplete")
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
region_name=S3_REGION,
|
||||
aws_access_key_id=S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
||||
)
|
||||
|
||||
|
||||
def wait_for_s3() -> None:
|
||||
deadline = time.monotonic() + 60
|
||||
last_error: Exception | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
s3().head_bucket(Bucket=S3_BUCKET)
|
||||
return
|
||||
except (BotoCoreError, ClientError, OSError) as exc:
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
raise RuntimeError("S3 bucket did not become ready") from last_error
|
||||
|
||||
|
||||
def attachment_public(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": [],
|
||||
"error_code": row["error_code"],
|
||||
"created_at": iso(row["created_at"]),
|
||||
"updated_at": iso(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
def attachment_ids_for_message(conn, message_id: uuid.UUID) -> list[str]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT attachment_id FROM message_attachments
|
||||
WHERE message_id = %s ORDER BY attachment_id
|
||||
""",
|
||||
(message_id,),
|
||||
).fetchall()
|
||||
return [str(row["attachment_id"]) for row in rows]
|
||||
|
||||
|
||||
def public_message(conn, row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": iso(row["created_at"]),
|
||||
"attachment_ids": attachment_ids_for_message(conn, row["id"]),
|
||||
}
|
||||
|
||||
|
||||
def apply_migrations() -> None:
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now()
|
||||
)
|
||||
"""
|
||||
)
|
||||
for path in sorted(MIGRATIONS_DIR.glob("*.sql")):
|
||||
version = path.name.split("_", 1)[0]
|
||||
applied = conn.execute(
|
||||
"SELECT 1 FROM schema_migrations WHERE version = %s", (version,)
|
||||
).fetchone()
|
||||
if applied is None:
|
||||
for statement in path.read_text(encoding="utf-8").split(";"):
|
||||
if statement.strip():
|
||||
conn.execute(statement)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations(version) VALUES (%s)", (version,)
|
||||
)
|
||||
|
||||
|
||||
def wait_for_database() -> None:
|
||||
deadline = time.monotonic() + 60
|
||||
last_error: Exception | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
apply_migrations()
|
||||
return
|
||||
except (OSError, psycopg.Error) as exc:
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
raise RuntimeError("database did not become ready") from last_error
|
||||
|
||||
|
||||
def error_response(request: Request, status_code: int, code: str, message: str):
|
||||
request_id = getattr(request.state, "request_id", str(uuid.uuid4()))
|
||||
headers = {"X-Request-Id": request_id}
|
||||
if status_code == 401:
|
||||
headers["WWW-Authenticate"] = (
|
||||
'Bearer realm="messenger"'
|
||||
if request.url.path != "/api/v1/auth/sessions"
|
||||
else 'Basic realm="messenger", charset="UTF-8"'
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
def bearer_user(request: Request) -> dict[str, Any]:
|
||||
header = request.headers.get("Authorization", "")
|
||||
scheme, _, token = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not token or " " in token:
|
||||
raise APIError(401, "unauthorized", "valid bearer session required")
|
||||
hashed_token = token_hash(token)
|
||||
with db() as conn:
|
||||
user = conn.execute(
|
||||
"""
|
||||
SELECT u.id, u.username, u.display_name, u.created_at
|
||||
FROM sessions s JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = %s AND s.revoked_at IS NULL
|
||||
AND s.expires_at > now()
|
||||
""",
|
||||
(hashed_token,),
|
||||
).fetchone()
|
||||
if user is None:
|
||||
raise APIError(401, "unauthorized", "valid bearer session required")
|
||||
return user
|
||||
|
||||
|
||||
def basic_credentials(request: Request) -> tuple[str, str]:
|
||||
header = request.headers.get("Authorization", "")
|
||||
scheme, _, encoded = header.partition(" ")
|
||||
if scheme.lower() != "basic" or not encoded:
|
||||
raise APIError(401, "unauthorized", "basic credentials required")
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, separator, password = decoded.partition(":")
|
||||
except (binascii.Error, UnicodeDecodeError):
|
||||
raise APIError(401, "unauthorized", "invalid credentials") from None
|
||||
if not separator or not username or not password:
|
||||
raise APIError(401, "unauthorized", "invalid credentials")
|
||||
return username, password
|
||||
|
||||
|
||||
def token_hash(token: str) -> str:
|
||||
try:
|
||||
return hashlib.sha256(token.encode("ascii", "strict")).hexdigest()
|
||||
except UnicodeEncodeError:
|
||||
raise APIError(401, "unauthorized", "valid bearer session required") from None
|
||||
|
||||
|
||||
def encode_cursor(created_at: datetime, object_id: uuid.UUID) -> str:
|
||||
payload = json.dumps(
|
||||
{"created_at": iso(created_at), "id": str(object_id)},
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(cursor: str) -> tuple[datetime, uuid.UUID]:
|
||||
try:
|
||||
padding = "=" * ((4 - len(cursor) % 4) % 4)
|
||||
payload = json.loads(
|
||||
base64.urlsafe_b64decode((cursor + padding).encode()).decode()
|
||||
)
|
||||
created_at = datetime.fromisoformat(payload["created_at"].replace("Z", "+00:00"))
|
||||
return created_at, uuid.UUID(payload["id"])
|
||||
except (KeyError, TypeError, ValueError, UnicodeDecodeError, binascii.Error, json.JSONDecodeError):
|
||||
raise APIError(400, "invalid_request", "cursor is invalid") from None
|
||||
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger Lab2")
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def startup() -> None:
|
||||
wait_for_database()
|
||||
wait_for_s3()
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def add_request_id(request: Request, call_next):
|
||||
request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(APIError)
|
||||
async def api_error_handler(request: Request, exc: APIError):
|
||||
return error_response(request, exc.status_code, exc.code, exc.message)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400, "invalid_request", "invalid request")
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error_handler(request: Request, exc: Exception):
|
||||
return error_response(request, 500, "internal_error", "internal server error")
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def health_live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def health_ready():
|
||||
try:
|
||||
with db() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
s3().head_bucket(Bucket=S3_BUCKET)
|
||||
except (OSError, psycopg.Error, BotoCoreError, ClientError):
|
||||
raise APIError(503, "unavailable", "required storage unavailable")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
def create_user(payload: CreateUser):
|
||||
user_id = uuid.uuid4()
|
||||
password_hash = PASSWORD_HASHER.hash(payload.password)
|
||||
try:
|
||||
with db() as conn:
|
||||
user = conn.execute(
|
||||
"""
|
||||
INSERT INTO users(id, username, display_name, password_hash)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING id, username, display_name, created_at
|
||||
""",
|
||||
(user_id, payload.username, payload.display_name, password_hash),
|
||||
).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise APIError(409, "conflict", "user already exists") from None
|
||||
return {**user, "id": str(user["id"]), "created_at": iso(user["created_at"])}
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
def create_session(request: Request):
|
||||
username, password = basic_credentials(request)
|
||||
with db() as conn:
|
||||
user = conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
if user is None:
|
||||
raise APIError(401, "unauthorized", "invalid credentials")
|
||||
try:
|
||||
PASSWORD_HASHER.verify(user["password_hash"], password)
|
||||
except VerificationError:
|
||||
raise APIError(401, "unauthorized", "invalid credentials") from None
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||
(token_hash(token), user["id"], expires_at),
|
||||
)
|
||||
public_user = {
|
||||
"id": str(user["id"]),
|
||||
"username": user["username"],
|
||||
"display_name": user["display_name"],
|
||||
"created_at": iso(user["created_at"]),
|
||||
}
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": iso(expires_at),
|
||||
"user": public_user,
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def revoke_session(request: Request):
|
||||
bearer_user(request)
|
||||
header = request.headers["Authorization"]
|
||||
token = header.split(" ", 1)[1]
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE sessions SET revoked_at = now() WHERE token_hash = %s AND revoked_at IS NULL",
|
||||
(token_hash(token),),
|
||||
)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def current_user(request: Request):
|
||||
user = bearer_user(request)
|
||||
return {**user, "id": str(user["id"]), "created_at": iso(user["created_at"])}
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
def create_chat(payload: CreateChat, request: Request):
|
||||
actor = bearer_user(request)
|
||||
member_ids = [uuid.UUID(member_id) for member_id in payload.member_ids]
|
||||
if actor["id"] not in member_ids:
|
||||
raise APIError(400, "invalid_request", "actor must be included in member_ids")
|
||||
chat_id = uuid.uuid4()
|
||||
with db() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT id FROM users WHERE id = ANY(%s)", (member_ids,)
|
||||
).fetchall()
|
||||
if len(existing) != len(member_ids):
|
||||
raise APIError(400, "invalid_request", "member not found")
|
||||
chat = conn.execute(
|
||||
"INSERT INTO chats(id, title) VALUES (%s, %s) RETURNING id, title, created_at",
|
||||
(chat_id, payload.title),
|
||||
).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members(chat_id, user_id) VALUES (%s, %s)",
|
||||
[(chat_id, member_id) for member_id in member_ids],
|
||||
)
|
||||
return {
|
||||
"id": str(chat["id"]),
|
||||
"title": chat["title"],
|
||||
"member_ids": sorted(str(member_id) for member_id in member_ids),
|
||||
"created_at": iso(chat["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def chat_for_user(conn, chat_id: uuid.UUID, user_id: uuid.UUID):
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE c.id = %s AND m.user_id = %s
|
||||
""",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
|
||||
|
||||
def public_chat(conn, chat: dict[str, Any]) -> dict[str, Any]:
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", (chat["id"],)
|
||||
).fetchall()
|
||||
return {
|
||||
"id": str(chat["id"]),
|
||||
"title": chat["title"],
|
||||
"member_ids": [str(row["user_id"]) for row in members],
|
||||
"created_at": iso(chat["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def get_chat(chat_id: uuid.UUID, request: Request):
|
||||
actor = bearer_user(request)
|
||||
with db() as conn:
|
||||
chat = chat_for_user(conn, chat_id, actor["id"])
|
||||
if chat is None:
|
||||
raise APIError(404, "not_found", "chat not found")
|
||||
return public_chat(conn, chat)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def list_chats(request: Request, limit: int = Query(50, ge=1, le=100), cursor: str | None = None):
|
||||
actor = bearer_user(request)
|
||||
after = decode_cursor(cursor) if cursor else None
|
||||
with db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE m.user_id = %s
|
||||
AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
""",
|
||||
(actor["id"], after[0] if after else None, after[0] if after else None,
|
||||
after[1] if after else None, limit + 1),
|
||||
).fetchall()
|
||||
items = [public_chat(conn, row) for row in rows[:limit]]
|
||||
next_cursor = (
|
||||
encode_cursor(rows[limit - 1]["created_at"], rows[limit - 1]["id"])
|
||||
if len(rows) > limit
|
||||
else None
|
||||
)
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def create_message(
|
||||
chat_id: uuid.UUID,
|
||||
payload: CreateMessage,
|
||||
request: Request,
|
||||
):
|
||||
actor = bearer_user(request)
|
||||
idempotency_key = request.headers.get("Idempotency-Key", "")
|
||||
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key):
|
||||
raise APIError(400, "invalid_request", "valid Idempotency-Key required")
|
||||
attachment_ids = sorted(uuid.UUID(value) for value in payload.attachment_ids)
|
||||
if not payload.text.strip() and not attachment_ids:
|
||||
raise APIError(400, "invalid_request", "text or attachment is required")
|
||||
request_hash = hashlib.sha256(
|
||||
json.dumps(
|
||||
{"text": payload.text, "attachment_ids": [str(value) for value in attachment_ids]},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
).encode()
|
||||
).hexdigest()
|
||||
with db() as conn:
|
||||
if chat_for_user(conn, chat_id, actor["id"]) is None:
|
||||
raise APIError(404, "not_found", "chat not accessible")
|
||||
if attachment_ids:
|
||||
attachments = conn.execute(
|
||||
"""
|
||||
SELECT id, status FROM attachments
|
||||
WHERE id = ANY(%s) AND chat_id = %s
|
||||
""",
|
||||
(attachment_ids, chat_id),
|
||||
).fetchall()
|
||||
if len(attachments) != len(attachment_ids):
|
||||
raise APIError(404, "not_found", "attachment not found")
|
||||
if any(row["status"] != "ready" for row in attachments):
|
||||
raise APIError(409, "conflict", "attachment is not ready")
|
||||
existing = conn.execute(
|
||||
"""
|
||||
INSERT INTO idempotency_keys(user_id, chat_id, idempotency_key, request_hash)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING
|
||||
RETURNING message_id, request_hash
|
||||
""",
|
||||
(actor["id"], chat_id, idempotency_key, request_hash),
|
||||
).fetchone()
|
||||
if existing is None:
|
||||
existing = conn.execute(
|
||||
"""
|
||||
SELECT message_id, request_hash
|
||||
FROM idempotency_keys
|
||||
WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s
|
||||
FOR UPDATE
|
||||
""",
|
||||
(actor["id"], chat_id, idempotency_key),
|
||||
).fetchone()
|
||||
if existing["request_hash"] != request_hash:
|
||||
raise APIError(409, "conflict", "idempotency key has another request")
|
||||
message = conn.execute(
|
||||
"""
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages WHERE id = %s
|
||||
""",
|
||||
(existing["message_id"],),
|
||||
).fetchone()
|
||||
if message is None:
|
||||
raise APIError(503, "unavailable", "idempotent result is not ready")
|
||||
return public_message(conn, message)
|
||||
message_id = uuid.uuid4()
|
||||
message = conn.execute(
|
||||
"""
|
||||
INSERT INTO messages(id, chat_id, sender_id, text)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING id, chat_id, sender_id, text, created_at
|
||||
""",
|
||||
(message_id, chat_id, actor["id"], payload.text),
|
||||
).fetchone()
|
||||
if attachment_ids:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO message_attachments(message_id, attachment_id)
|
||||
VALUES (%s, %s)
|
||||
""",
|
||||
[(message_id, attachment_id) for attachment_id in attachment_ids],
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE idempotency_keys SET message_id = %s
|
||||
WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s
|
||||
""",
|
||||
(message_id, actor["id"], chat_id, idempotency_key),
|
||||
)
|
||||
return public_message(conn, message)
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> None:
|
||||
if not filename or len(filename) > 255:
|
||||
raise APIError(400, "invalid_request", "filename is invalid")
|
||||
if any(ord(char) < 32 for char in filename) or "/" in filename or "\\" in filename:
|
||||
raise APIError(400, "invalid_request", "filename is invalid")
|
||||
|
||||
|
||||
def inspect_image(path: Any, declared_type: str) -> tuple[str, int, int]:
|
||||
try:
|
||||
with Image.open(path) as image:
|
||||
detected_type = {
|
||||
"PNG": "image/png",
|
||||
"JPEG": "image/jpeg",
|
||||
}.get(image.format)
|
||||
if detected_type is None or detected_type != declared_type:
|
||||
raise APIError(415, "unsupported_media_type", "image type is invalid")
|
||||
width, height = image.size
|
||||
if width * height > MAX_IMAGE_PIXELS:
|
||||
raise APIError(413, "payload_too_large", "image has too many pixels")
|
||||
image.verify()
|
||||
path.seek(0)
|
||||
with Image.open(path) as image:
|
||||
width, height = image.size
|
||||
image.verify()
|
||||
except APIError:
|
||||
raise
|
||||
except (UnidentifiedImageError, OSError, Image.DecompressionBombError):
|
||||
raise APIError(415, "unsupported_media_type", "image is invalid") from None
|
||||
return detected_type, width, height
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
||||
async def upload_attachment(chat_id: uuid.UUID, request: Request, filename: str):
|
||||
actor = bearer_user(request)
|
||||
validate_filename(filename)
|
||||
declared_type = request.headers.get("content-type", "").split(";", 1)[0].lower()
|
||||
if declared_type not in {"image/png", "image/jpeg"}:
|
||||
raise APIError(415, "unsupported_media_type", "content type is invalid")
|
||||
content_length = request.headers.get("content-length")
|
||||
if content_length:
|
||||
try:
|
||||
if int(content_length) > MAX_UPLOAD_BYTES:
|
||||
raise APIError(413, "payload_too_large", "file is too large")
|
||||
except ValueError:
|
||||
raise APIError(400, "invalid_request", "content length is invalid") from None
|
||||
with db() as conn:
|
||||
if chat_for_user(conn, chat_id, actor["id"]) is None:
|
||||
raise APIError(404, "not_found", "chat not accessible")
|
||||
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
object_key = "attachments/" + str(uuid.uuid4())
|
||||
uploaded = False
|
||||
try:
|
||||
with tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") as temporary:
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_UPLOAD_BYTES:
|
||||
raise APIError(413, "payload_too_large", "file is too large")
|
||||
digest.update(chunk)
|
||||
temporary.write(chunk)
|
||||
if size == 0:
|
||||
raise APIError(415, "unsupported_media_type", "image is empty")
|
||||
temporary.seek(0)
|
||||
detected_type, _, _ = inspect_image(temporary, declared_type)
|
||||
temporary.seek(0)
|
||||
try:
|
||||
await run_in_threadpool(
|
||||
s3().upload_fileobj,
|
||||
temporary,
|
||||
S3_BUCKET,
|
||||
object_key,
|
||||
ExtraArgs={"ContentType": detected_type},
|
||||
)
|
||||
uploaded = True
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
raise APIError(503, "unavailable", "object storage unavailable") from None
|
||||
except APIError:
|
||||
raise
|
||||
except (OSError, ValueError):
|
||||
raise APIError(503, "unavailable", "upload could not be completed") from None
|
||||
|
||||
attachment_id = uuid.uuid4()
|
||||
now = utc_now()
|
||||
row = {
|
||||
"id": attachment_id,
|
||||
"chat_id": chat_id,
|
||||
"owner_id": actor["id"],
|
||||
"filename": filename,
|
||||
"content_type": detected_type,
|
||||
"size_bytes": size,
|
||||
"sha256": digest.hexdigest(),
|
||||
"status": "ready",
|
||||
"error_code": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
try:
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO attachments(
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, status, object_key, error_code, created_at, updated_at
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, 'ready', %s, NULL, %s, %s)
|
||||
""",
|
||||
(
|
||||
attachment_id,
|
||||
chat_id,
|
||||
actor["id"],
|
||||
filename,
|
||||
detected_type,
|
||||
size,
|
||||
digest.hexdigest(),
|
||||
object_key,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
except psycopg.Error:
|
||||
if uploaded:
|
||||
try:
|
||||
s3().delete_object(Bucket=S3_BUCKET, Key=object_key)
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
LOGGER.error("attachment compensation failed for object key")
|
||||
raise APIError(503, "unavailable", "attachment metadata unavailable") from None
|
||||
return attachment_public(row)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
def get_attachment(attachment_id: uuid.UUID, request: Request):
|
||||
actor = bearer_user(request)
|
||||
with db() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT a.*
|
||||
FROM attachments a
|
||||
JOIN chat_members m ON m.chat_id = a.chat_id
|
||||
WHERE a.id = %s AND m.user_id = %s
|
||||
""",
|
||||
(attachment_id, actor["id"]),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise APIError(404, "not_found", "attachment not found")
|
||||
return attachment_public(row)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download_attachment(
|
||||
attachment_id: uuid.UUID,
|
||||
request: Request,
|
||||
variant: str = "original",
|
||||
):
|
||||
actor = bearer_user(request)
|
||||
if variant != "original":
|
||||
raise APIError(404, "not_found", "attachment variant not found")
|
||||
with db() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT a.*
|
||||
FROM attachments a
|
||||
JOIN chat_members m ON m.chat_id = a.chat_id
|
||||
WHERE a.id = %s AND m.user_id = %s
|
||||
""",
|
||||
(attachment_id, actor["id"]),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise APIError(404, "not_found", "attachment not found")
|
||||
try:
|
||||
object_response = s3().get_object(Bucket=S3_BUCKET, Key=row["object_key"])
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
raise APIError(503, "unavailable", "object storage unavailable") from None
|
||||
|
||||
body = object_response["Body"]
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while True:
|
||||
chunk = body.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=row["content_type"],
|
||||
headers={"Content-Length": str(row["size_bytes"])},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def list_messages(
|
||||
chat_id: uuid.UUID,
|
||||
request: Request,
|
||||
limit: int = Query(50, ge=1, le=100),
|
||||
cursor: str | None = None,
|
||||
):
|
||||
actor = bearer_user(request)
|
||||
after = decode_cursor(cursor) if cursor else None
|
||||
with db() as conn:
|
||||
if chat_for_user(conn, chat_id, actor["id"]) is None:
|
||||
raise APIError(404, "not_found", "chat not accessible")
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = %s
|
||||
AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
""",
|
||||
(chat_id, after[0] if after else None, after[0] if after else None,
|
||||
after[1] if after else None, limit + 1),
|
||||
).fetchall()
|
||||
items = [public_message(conn, row) for row in rows[:limit]]
|
||||
next_cursor = (
|
||||
encode_cursor(rows[limit - 1]["created_at"], rows[limit - 1]["id"])
|
||||
if len(rows) > limit
|
||||
else None
|
||||
)
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run("app:app", host=APP_HOST, port=APP_PORT, reload=False)
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
|
||||
s3-config:
|
||||
image: alpine:3.20.3
|
||||
environment:
|
||||
S3_BOOTSTRAP_ACCESS_KEY: ${S3_BOOTSTRAP_ACCESS_KEY:?set S3_BOOTSTRAP_ACCESS_KEY in .env}
|
||||
S3_BOOTSTRAP_SECRET_KEY: ${S3_BOOTSTRAP_SECRET_KEY:?set S3_BOOTSTRAP_SECRET_KEY in .env}
|
||||
S3_APP_ACCESS_KEY: ${S3_APP_ACCESS_KEY:?set S3_APP_ACCESS_KEY in .env}
|
||||
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY:?set S3_APP_SECRET_KEY in .env}
|
||||
volumes:
|
||||
- s3_config:/config
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
printf '{"identities":[{"name":"bootstrap","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin"]},{"name":"messenger-api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:messenger-attachments","Write:messenger-attachments","List:messenger-attachments","Tagging:messenger-attachments"]}]}' \
|
||||
"$$S3_BOOTSTRAP_ACCESS_KEY" "$$S3_BOOTSTRAP_SECRET_KEY" \
|
||||
"$$S3_APP_ACCESS_KEY" "$$S3_APP_SECRET_KEY" > /config/s3.json
|
||||
restart: "no"
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.80
|
||||
depends_on:
|
||||
s3-config:
|
||||
condition: service_completed_successfully
|
||||
command: ["server", "-dir=/data", "-s3", "-s3.config=/etc/seaweed/s3.json"]
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- s3_config:/etc/seaweed:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
|
||||
s3-bootstrap:
|
||||
build: .
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: ${S3_BOOTSTRAP_ACCESS_KEY}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_BOOTSTRAP_SECRET_KEY}
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- |
|
||||
import time
|
||||
import boto3
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
client = boto3.client("s3", endpoint_url="http://s3:8333", region_name="us-east-1", aws_access_key_id="${S3_BOOTSTRAP_ACCESS_KEY}", aws_secret_access_key="${S3_BOOTSTRAP_SECRET_KEY}")
|
||||
for attempt in range(60):
|
||||
try:
|
||||
client.create_bucket(Bucket="messenger-attachments")
|
||||
break
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
break
|
||||
raise
|
||||
except (BotoCoreError, OSError):
|
||||
if attempt == 59:
|
||||
raise
|
||||
time.sleep(1)
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
environment:
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
command: ["python", "-c", "import app; app.wait_for_database()"]
|
||||
restart: "no"
|
||||
|
||||
api1:
|
||||
build: .
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
INSTANCE_ID: api1
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: ${S3_APP_ACCESS_KEY}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_APP_SECRET_KEY}
|
||||
MAX_UPLOAD_BYTES: 10485760
|
||||
MAX_IMAGE_PIXELS: 40000000
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
|
||||
api2:
|
||||
build: .
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
INSTANCE_ID: api2
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: ${S3_APP_ACCESS_KEY}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_APP_SECRET_KEY}
|
||||
MAX_UPLOAD_BYTES: 10485760
|
||||
MAX_IMAGE_PIXELS: 40000000
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
|
||||
load_balancer:
|
||||
image: nginx:1.27.2-alpine
|
||||
ports:
|
||||
- "8080:80"
|
||||
volumes:
|
||||
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
depends_on:
|
||||
api1:
|
||||
condition: service_healthy
|
||||
api2:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1/health/live"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
s3_data:
|
||||
s3_config:
|
||||
Binary file not shown.
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username varchar(32) NOT NULL UNIQUE,
|
||||
display_name varchar(100) NOT NULL CHECK (btrim(display_name) <> ''),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title varchar(100) NOT NULL CHECK (btrim(title) <> ''),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text varchar(4000) NOT NULL CHECK (btrim(text) <> ''),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash char(64) PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
|
||||
CREATE INDEX messages_chat_order_idx ON messages(chat_id, created_at, id);
|
||||
CREATE INDEX sessions_user_idx ON sessions(user_id, expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
idempotency_key varchar(128) NOT NULL,
|
||||
request_hash char(64) NOT NULL,
|
||||
message_id uuid REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, idempotency_key)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_created_idx ON idempotency_keys(created_at);
|
||||
@@ -0,0 +1,25 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename varchar(255) NOT NULL,
|
||||
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760),
|
||||
sha256 char(64) NOT NULL,
|
||||
status varchar(16) NOT NULL CHECK (status = 'ready'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
error_code varchar(64),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_idx ON attachments(chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx
|
||||
ON message_attachments(attachment_id, message_id);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT IF EXISTS messages_text_check;
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
upstream messenger_api {
|
||||
server api1:8080 max_fails=1 fail_timeout=2s;
|
||||
server api2:8080 max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
client_max_body_size 11m;
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger_api;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Request-Id $http_x_request_id;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 10s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.35.76
|
||||
fastapi==0.115.5
|
||||
Pillow==11.0.0
|
||||
psycopg[binary]==3.2.3
|
||||
uvicorn==0.32.0
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Report or remove old S3 objects that have no PostgreSQL attachment row."""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--grace-seconds", type=int, default=3600)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 0:
|
||||
parser.error("--grace-seconds must be non-negative")
|
||||
|
||||
with app.db() as conn:
|
||||
known = {
|
||||
row["object_key"]
|
||||
for row in conn.execute("SELECT object_key FROM attachments").fetchall()
|
||||
}
|
||||
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
client = app.s3()
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
candidates = []
|
||||
for page in paginator.paginate(Bucket=app.S3_BUCKET):
|
||||
for item in page.get("Contents", []):
|
||||
key = item["Key"]
|
||||
modified = item["LastModified"]
|
||||
if key not in known and (now - modified).total_seconds() >= args.grace_seconds:
|
||||
candidates.append(key)
|
||||
|
||||
for key in candidates:
|
||||
if args.apply:
|
||||
client.delete_object(Bucket=app.S3_BUCKET, Key=key)
|
||||
print(("deleted" if args.apply else "candidates") + f"={len(candidates)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
docker compose up -d --force-recreate postgres s3
|
||||
docker compose run --rm migrate
|
||||
docker compose run --rm s3-bootstrap
|
||||
docker compose up -d --force-recreate api1 api2 load_balancer
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
case "${TARGET_INSTANCE_ID:-}" in
|
||||
api1) service=api1 ;;
|
||||
api2) service=api2 ;;
|
||||
*) echo "unknown TARGET_INSTANCE_ID" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
docker compose stop "$service"
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative attachment checks required for the level-4 assessment."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("lab4 level-4 attachment checks")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
second_chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
sample = Path(__file__).with_name("fixtures").joinpath("sample.png").read_bytes()
|
||||
|
||||
client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=large.png",
|
||||
413,
|
||||
actor=alice,
|
||||
raw=b"x" * (10 * 1024 * 1024 + 1),
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=broken.png",
|
||||
415,
|
||||
actor=alice,
|
||||
raw=b"not an image",
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=spoof.jpg",
|
||||
415,
|
||||
actor=alice,
|
||||
raw=sample,
|
||||
headers={"Content-Type": "image/jpeg"},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=..%2Fsecret.png",
|
||||
400,
|
||||
actor=alice,
|
||||
raw=sample,
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
check(hashlib.sha256(content).hexdigest() == attachment["sha256"], "fixture checksum changed")
|
||||
client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}",
|
||||
404,
|
||||
actor=eve,
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{second_chat['id']}/messages",
|
||||
404,
|
||||
actor=alice,
|
||||
body={"text": "wrong chat", "attachment_ids": [attachment["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
print("PASS: lab4 level-4 attachment validation and access checks")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run ten concurrent bounded uploads and sample API RSS via Docker."""
|
||||
import concurrent.futures
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def rss_mb() -> float:
|
||||
output = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
|
||||
text=True,
|
||||
)
|
||||
values = []
|
||||
for line in output.splitlines():
|
||||
if "api1" not in line and "api2" not in line:
|
||||
continue
|
||||
used = line.split()[1].split("/")[0]
|
||||
factor = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}
|
||||
values.append(float(used[:-3]) * factor[used[-3:]])
|
||||
return max(values, default=0.0)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("lab4 concurrent bounded upload experiment")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
content = Path(__file__).with_name("fixtures").joinpath("sample.png").read_bytes()
|
||||
stop = threading.Event()
|
||||
samples = [rss_mb()]
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
samples.append(rss_mb())
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
|
||||
sampler = threading.Thread(target=sample, daemon=True)
|
||||
sampler.start()
|
||||
started = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
|
||||
futures = [
|
||||
pool.submit(
|
||||
client.call,
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=load-{index}.png",
|
||||
201,
|
||||
None,
|
||||
alice,
|
||||
{"Content-Type": "image/png"},
|
||||
content,
|
||||
"Attachment",
|
||||
)
|
||||
for index in range(10)
|
||||
]
|
||||
[future.result() for future in futures]
|
||||
elapsed = time.monotonic() - started
|
||||
stop.set()
|
||||
sampler.join(timeout=1)
|
||||
samples.append(rss_mb())
|
||||
check(len(samples) > 0, "RSS sampling did not produce data")
|
||||
print(f"uploads=10 concurrency=10 elapsed_seconds={elapsed:.3f} peak_api_rss_mb={max(samples):.1f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user