forked from mtusi2026/mtusi2026lab4
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de2246a888 |
+6
-19
@@ -1,20 +1,7 @@
|
|||||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
# Copy to .env and replace local-only passwords. Never commit .env.
|
||||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
POSTGRES_PASSWORD=change-me-postgres
|
||||||
APP_HOST=0.0.0.0
|
|
||||||
APP_PORT=8080
|
|
||||||
APP_ENV=lab
|
|
||||||
SESSION_TTL_SECONDS=3600
|
SESSION_TTL_SECONDS=3600
|
||||||
# Choose credentials locally. File paths contain no secret values.
|
S3_BOOTSTRAP_ACCESS_KEY=bootstrap-admin
|
||||||
DATABASE_URL_FILE=/run/secrets/database_url
|
S3_BOOTSTRAP_SECRET_KEY=change-me-bootstrap-secret
|
||||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
S3_APP_ACCESS_KEY=messenger-api
|
||||||
SESSION_STORE=postgres
|
S3_APP_SECRET_KEY=change-me-s3-api-secret
|
||||||
CACHE_URL_FILE=/run/secrets/cache_url
|
|
||||||
API_REPLICAS=2
|
|
||||||
IDEMPOTENCY_TTL_SECONDS=86400
|
|
||||||
S3_ENDPOINT=http://s3:9000
|
|
||||||
S3_REGION=us-east-1
|
|
||||||
S3_BUCKET=messenger-attachments
|
|
||||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
|
||||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
|
||||||
MAX_UPLOAD_BYTES=10485760
|
|
||||||
MAX_IMAGE_PIXELS=40000000
|
|
||||||
|
|||||||
+17
@@ -0,0 +1,17 @@
|
|||||||
|
FROM python:3.13-slim-bookworm
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN python -m pip install --no-cache-dir -r requirements.txt
|
||||||
|
COPY app.py ./
|
||||||
|
COPY migrations ./migrations/
|
||||||
|
COPY scripts ./scripts/
|
||||||
|
|
||||||
|
RUN useradd --create-home --shell /usr/sbin/nologin appuser && \
|
||||||
|
chown -R appuser:appuser /app
|
||||||
|
USER appuser
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
CMD ["python", "app.py"]
|
||||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
|||||||
ACTION_SCRIPT ?=
|
ACTION_SCRIPT ?=
|
||||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||||
|
|
||||||
.PHONY: help check validate test load test-persistence test-failover
|
.PHONY: help check validate test test-grade4 test-attachments-load load test-persistence test-failover
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||||
@@ -18,6 +18,12 @@ validate:
|
|||||||
test:
|
test:
|
||||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||||
|
|
||||||
|
test-grade4:
|
||||||
|
$(PYTHON) tests/grade4_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||||
|
|
||||||
|
test-attachments-load:
|
||||||
|
$(PYTHON) tests/load_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||||
|
|
||||||
load:
|
load:
|
||||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||||
|
|
||||||
|
|||||||
@@ -1,37 +1,72 @@
|
|||||||
# Отчёт — лабораторная 4
|
# Отчёт — лабораторная 4
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- Автор, группа: Антоновва Екатерина бпи2304
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- Целевая оценка: 4
|
||||||
- Commit/PR: TODO после публикации
|
- Выбранный трек 5: не заявлен
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
- Commit/PR: заполняется после публикации
|
||||||
|
|
||||||
|
## Использованные источники, библиотеки и помощь ИИ
|
||||||
|
|
||||||
|
Использованы нормативные `contracts/openapi.json`, `contracts/README.md`, `contracts/infrastructure.json`, публичные тесты из `tests/` и документация FastAPI, PostgreSQL, boto3, Pillow, SeaweedFS и Docker Compose. ИИ применялся как вспомогательный инструмент для проверки реализации, поиска ошибок и редактирования отчёта; итоговые решения проверены локальными тестами и запуском стенда.
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Запуск из чистого clone
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
```sh
|
||||||
|
cp .env.example .env
|
||||||
|
docker compose --env-file .env config --quiet
|
||||||
|
docker compose --env-file .env up -d --build
|
||||||
|
# дождаться: docker compose ps показывает healthy у postgres, s3, api1, api2 и load_balancer
|
||||||
|
make check
|
||||||
|
make validate
|
||||||
|
make test
|
||||||
|
make test-grade4
|
||||||
|
make test-attachments-load
|
||||||
|
```
|
||||||
|
|
||||||
|
Для остановки без удаления данных:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose down
|
||||||
|
```
|
||||||
|
|
||||||
|
Для намеренного удаления учебных данных вместе с volumes:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose down -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Пароли находятся только в локальном `.env`, который исключён из Git. В текущей проверке Windows `make` отсутствовал, поэтому его цели выполнены эквивалентными командами с `.venv\Scripts\python.exe`; в Linux/CI используются команды Makefile.
|
||||||
|
|
||||||
## Архитектура и решения
|
## Архитектура и решения
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
Клиент обращается к единственному публичному Nginx на `localhost:8080`. Балансировщик направляет запросы на `api1` и `api2`; обе реплики используют одну PostgreSQL и один SeaweedFS с S3 API. Локального состояния сессий, сообщений и файлов в API нет: PostgreSQL хранит метаданные, а приватный bucket `messenger-attachments` — байты.
|
||||||
|
|
||||||
|
Вложение принимается потоково в `SpooledTemporaryFile` с лимитом 10 MiB, проверяются фактический PNG/JPEG, MIME, имя, SHA-256 и максимум 40 млн пикселей. Сначала выполняется S3 PUT, затем транзакционная запись метаданных; при ошибке БД выполняется компенсационный DELETE. `scripts/reconcile_attachments.py` поддерживает dry-run и удаление только orphan-объектов старше grace period.
|
||||||
|
|
||||||
|
S3 bootstrap идемпотентен. SeaweedFS имеет отдельные bootstrap- и API-credentials; API получает только права `Read/Write/List/Tagging` для нужного bucket, административная консоль наружу не публикуется. Миграция `004_allow_attachment_only_messages.sql` сохраняет возможность сообщений, содержащих только вложение.
|
||||||
|
|
||||||
## Доказательства критериев
|
## Доказательства критериев
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
| Критерий | Команда/сценарий | Результат |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
| Контракт и OpenAPI | `scripts/check_contract.py`, `scripts/validate_openapi.py` | PASS |
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
| Публичный smoke | `tests/smoke.py` / `make test` | PASS: регистрация, сессии, чат, сообщение, upload/download и доступ участника |
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
| Ошибки уровня 4 | `tests/grade4_attachments.py` / `make test-grade4` | PASS: >10 MiB, повреждённый файл, MIME spoofing, опасное имя, чужой attachment и другой чат |
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
| Идемпотентный bootstrap | повторный `docker compose up -d --build` | PASS, bucket и миграции не дублируются |
|
||||||
|
| Persistence | `scripts/restart.sh` без удаления volumes | PASS: действующая сессия, метаданные и байты вложения сохранились |
|
||||||
|
| Failover | `scripts/stop-one.sh`, остановка обслужившей реплики | PASS: download через вторую реплику вернул исходные байты |
|
||||||
|
| Частный bucket | API credentials и отсутствие публикации S3-порта | S3 доступен API внутри Compose, прямой публичный GET не используется |
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
## Эксперимент потоковой загрузки
|
||||||
|
|
||||||
## Эксперименты и отказы
|
Запуск:
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
```sh
|
||||||
|
make test-attachments-load
|
||||||
|
```
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
Скрипт одновременно выполняет 10 загрузок, а Docker RSS API-реплик опрашивается во время эксперимента. Фактический результат текущего прогона сохранён в `evidence/attachments-concurrency.txt`: `uploads=10`, `concurrency=10`, `elapsed_seconds=0.480`, `peak_api_rss_mb=84.2`. Полный файл не удерживается в RAM: после 1 MiB `SpooledTemporaryFile` переходит на временный файл.
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| TODO | TODO | TODO | TODO |
|
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
## Ограничения
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
Проверен single-node Compose; отказ Docker-host, сетевой partition и HA PostgreSQL не заявляются. Компенсация закрывает обычный отказ записи метаданных, а периодическая сверка orphan-объектов запускается оператором, а не отдельным worker-процессом.
|
||||||
|
|||||||
@@ -0,0 +1,859 @@
|
|||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
import boto3
|
||||||
|
from botocore.exceptions import BotoCoreError, ClientError
|
||||||
|
from fastapi.concurrency import run_in_threadpool
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
|
from PIL import Image, UnidentifiedImageError
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
from fastapi import FastAPI, Query, Request, Response
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||||
|
from argon2 import PasswordHasher
|
||||||
|
from argon2.exceptions import VerificationError
|
||||||
|
|
||||||
|
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
|
||||||
|
APP_PORT = int(os.getenv("APP_PORT", "8080"))
|
||||||
|
DATABASE_URL = os.getenv("DATABASE_URL", "")
|
||||||
|
DATABASE_URL_FILE = os.getenv("DATABASE_URL_FILE", "")
|
||||||
|
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||||
|
INSTANCE_ID = os.getenv("INSTANCE_ID", "api-" + uuid.uuid4().hex[:8])
|
||||||
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "")
|
||||||
|
S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
||||||
|
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||||
|
S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID", "")
|
||||||
|
S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "")
|
||||||
|
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||||
|
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||||
|
MIGRATIONS_DIR = Path(__file__).parent / "migrations"
|
||||||
|
PASSWORD_HASHER = PasswordHasher()
|
||||||
|
LOGGER = logging.getLogger("messenger")
|
||||||
|
|
||||||
|
|
||||||
|
class APIError(Exception):
|
||||||
|
def __init__(self, status_code: int, code: str, message: str):
|
||||||
|
self.status_code = status_code
|
||||||
|
self.code = code
|
||||||
|
self.message = message
|
||||||
|
|
||||||
|
|
||||||
|
class CreateUser(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$")
|
||||||
|
display_name: str = Field(..., min_length=1, max_length=100)
|
||||||
|
password: str = Field(..., min_length=12, max_length=128)
|
||||||
|
|
||||||
|
@field_validator("display_name", "password")
|
||||||
|
@classmethod
|
||||||
|
def validate_non_blank(cls, value: str) -> str:
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError("value must not be blank")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class CreateChat(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
title: str = Field(..., min_length=1, max_length=100)
|
||||||
|
member_ids: list[str] = Field(..., min_length=1, max_length=100)
|
||||||
|
|
||||||
|
@field_validator("title")
|
||||||
|
@classmethod
|
||||||
|
def validate_title(cls, value: str) -> str:
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError("title must not be blank")
|
||||||
|
return value
|
||||||
|
|
||||||
|
@field_validator("member_ids")
|
||||||
|
@classmethod
|
||||||
|
def validate_member_ids(cls, value: list[str]) -> list[str]:
|
||||||
|
if len(set(value)) != len(value):
|
||||||
|
raise ValueError("member_ids must be unique")
|
||||||
|
for member_id in value:
|
||||||
|
try:
|
||||||
|
uuid.UUID(member_id)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ValueError("member_ids must contain UUIDs") from exc
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class CreateMessage(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
text: str = Field(..., max_length=4000)
|
||||||
|
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
|
||||||
|
|
||||||
|
@field_validator("attachment_ids")
|
||||||
|
@classmethod
|
||||||
|
def validate_attachment_ids(cls, value: list[str]) -> list[str]:
|
||||||
|
if len(set(value)) != len(value):
|
||||||
|
raise ValueError("attachment_ids must be unique")
|
||||||
|
for attachment_id in value:
|
||||||
|
try:
|
||||||
|
uuid.UUID(attachment_id)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ValueError("attachment_ids must contain UUIDs") from exc
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class AttachmentError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def utc_now() -> datetime:
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def iso(value: datetime) -> str:
|
||||||
|
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace(
|
||||||
|
"+00:00", "Z"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def database_url() -> str:
|
||||||
|
if DATABASE_URL:
|
||||||
|
return DATABASE_URL
|
||||||
|
if DATABASE_URL_FILE:
|
||||||
|
try:
|
||||||
|
return Path(DATABASE_URL_FILE).read_text(encoding="utf-8").strip()
|
||||||
|
except OSError as exc:
|
||||||
|
raise RuntimeError("database URL file is not readable") from exc
|
||||||
|
raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required")
|
||||||
|
|
||||||
|
|
||||||
|
def db():
|
||||||
|
return psycopg.connect(database_url(), row_factory=dict_row)
|
||||||
|
|
||||||
|
|
||||||
|
def s3():
|
||||||
|
if not S3_ENDPOINT or not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY:
|
||||||
|
raise RuntimeError("S3 configuration is incomplete")
|
||||||
|
return boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=S3_ENDPOINT,
|
||||||
|
region_name=S3_REGION,
|
||||||
|
aws_access_key_id=S3_ACCESS_KEY_ID,
|
||||||
|
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_s3() -> None:
|
||||||
|
deadline = time.monotonic() + 60
|
||||||
|
last_error: Exception | None = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
s3().head_bucket(Bucket=S3_BUCKET)
|
||||||
|
return
|
||||||
|
except (BotoCoreError, ClientError, OSError) as exc:
|
||||||
|
last_error = exc
|
||||||
|
time.sleep(1)
|
||||||
|
raise RuntimeError("S3 bucket did not become ready") from last_error
|
||||||
|
|
||||||
|
|
||||||
|
def attachment_public(row: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": str(row["id"]),
|
||||||
|
"chat_id": str(row["chat_id"]),
|
||||||
|
"owner_id": str(row["owner_id"]),
|
||||||
|
"filename": row["filename"],
|
||||||
|
"content_type": row["content_type"],
|
||||||
|
"size_bytes": row["size_bytes"],
|
||||||
|
"sha256": row["sha256"],
|
||||||
|
"status": row["status"],
|
||||||
|
"variants": [],
|
||||||
|
"error_code": row["error_code"],
|
||||||
|
"created_at": iso(row["created_at"]),
|
||||||
|
"updated_at": iso(row["updated_at"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def attachment_ids_for_message(conn, message_id: uuid.UUID) -> list[str]:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT attachment_id FROM message_attachments
|
||||||
|
WHERE message_id = %s ORDER BY attachment_id
|
||||||
|
""",
|
||||||
|
(message_id,),
|
||||||
|
).fetchall()
|
||||||
|
return [str(row["attachment_id"]) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def public_message(conn, row: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": str(row["id"]),
|
||||||
|
"chat_id": str(row["chat_id"]),
|
||||||
|
"sender_id": str(row["sender_id"]),
|
||||||
|
"text": row["text"],
|
||||||
|
"created_at": iso(row["created_at"]),
|
||||||
|
"attachment_ids": attachment_ids_for_message(conn, row["id"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def apply_migrations() -> None:
|
||||||
|
with db() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||||
|
version text PRIMARY KEY,
|
||||||
|
applied_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
for path in sorted(MIGRATIONS_DIR.glob("*.sql")):
|
||||||
|
version = path.name.split("_", 1)[0]
|
||||||
|
applied = conn.execute(
|
||||||
|
"SELECT 1 FROM schema_migrations WHERE version = %s", (version,)
|
||||||
|
).fetchone()
|
||||||
|
if applied is None:
|
||||||
|
for statement in path.read_text(encoding="utf-8").split(";"):
|
||||||
|
if statement.strip():
|
||||||
|
conn.execute(statement)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO schema_migrations(version) VALUES (%s)", (version,)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_database() -> None:
|
||||||
|
deadline = time.monotonic() + 60
|
||||||
|
last_error: Exception | None = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
apply_migrations()
|
||||||
|
return
|
||||||
|
except (OSError, psycopg.Error) as exc:
|
||||||
|
last_error = exc
|
||||||
|
time.sleep(1)
|
||||||
|
raise RuntimeError("database did not become ready") from last_error
|
||||||
|
|
||||||
|
|
||||||
|
def error_response(request: Request, status_code: int, code: str, message: str):
|
||||||
|
request_id = getattr(request.state, "request_id", str(uuid.uuid4()))
|
||||||
|
headers = {"X-Request-Id": request_id}
|
||||||
|
if status_code == 401:
|
||||||
|
headers["WWW-Authenticate"] = (
|
||||||
|
'Bearer realm="messenger"'
|
||||||
|
if request.url.path != "/api/v1/auth/sessions"
|
||||||
|
else 'Basic realm="messenger", charset="UTF-8"'
|
||||||
|
)
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status_code,
|
||||||
|
content={"error": {"code": code, "message": message, "request_id": request_id}},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def bearer_user(request: Request) -> dict[str, Any]:
|
||||||
|
header = request.headers.get("Authorization", "")
|
||||||
|
scheme, _, token = header.partition(" ")
|
||||||
|
if scheme.lower() != "bearer" or not token or " " in token:
|
||||||
|
raise APIError(401, "unauthorized", "valid bearer session required")
|
||||||
|
hashed_token = token_hash(token)
|
||||||
|
with db() as conn:
|
||||||
|
user = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT u.id, u.username, u.display_name, u.created_at
|
||||||
|
FROM sessions s JOIN users u ON u.id = s.user_id
|
||||||
|
WHERE s.token_hash = %s AND s.revoked_at IS NULL
|
||||||
|
AND s.expires_at > now()
|
||||||
|
""",
|
||||||
|
(hashed_token,),
|
||||||
|
).fetchone()
|
||||||
|
if user is None:
|
||||||
|
raise APIError(401, "unauthorized", "valid bearer session required")
|
||||||
|
return user
|
||||||
|
|
||||||
|
|
||||||
|
def basic_credentials(request: Request) -> tuple[str, str]:
|
||||||
|
header = request.headers.get("Authorization", "")
|
||||||
|
scheme, _, encoded = header.partition(" ")
|
||||||
|
if scheme.lower() != "basic" or not encoded:
|
||||||
|
raise APIError(401, "unauthorized", "basic credentials required")
|
||||||
|
try:
|
||||||
|
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||||
|
username, separator, password = decoded.partition(":")
|
||||||
|
except (binascii.Error, UnicodeDecodeError):
|
||||||
|
raise APIError(401, "unauthorized", "invalid credentials") from None
|
||||||
|
if not separator or not username or not password:
|
||||||
|
raise APIError(401, "unauthorized", "invalid credentials")
|
||||||
|
return username, password
|
||||||
|
|
||||||
|
|
||||||
|
def token_hash(token: str) -> str:
|
||||||
|
try:
|
||||||
|
return hashlib.sha256(token.encode("ascii", "strict")).hexdigest()
|
||||||
|
except UnicodeEncodeError:
|
||||||
|
raise APIError(401, "unauthorized", "valid bearer session required") from None
|
||||||
|
|
||||||
|
|
||||||
|
def encode_cursor(created_at: datetime, object_id: uuid.UUID) -> str:
|
||||||
|
payload = json.dumps(
|
||||||
|
{"created_at": iso(created_at), "id": str(object_id)},
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode()
|
||||||
|
return base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||||
|
|
||||||
|
|
||||||
|
def decode_cursor(cursor: str) -> tuple[datetime, uuid.UUID]:
|
||||||
|
try:
|
||||||
|
padding = "=" * ((4 - len(cursor) % 4) % 4)
|
||||||
|
payload = json.loads(
|
||||||
|
base64.urlsafe_b64decode((cursor + padding).encode()).decode()
|
||||||
|
)
|
||||||
|
created_at = datetime.fromisoformat(payload["created_at"].replace("Z", "+00:00"))
|
||||||
|
return created_at, uuid.UUID(payload["id"])
|
||||||
|
except (KeyError, TypeError, ValueError, UnicodeDecodeError, binascii.Error, json.JSONDecodeError):
|
||||||
|
raise APIError(400, "invalid_request", "cursor is invalid") from None
|
||||||
|
|
||||||
|
|
||||||
|
app = FastAPI(title="MTUSI Messenger Lab2")
|
||||||
|
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
def startup() -> None:
|
||||||
|
wait_for_database()
|
||||||
|
wait_for_s3()
|
||||||
|
|
||||||
|
|
||||||
|
@app.middleware("http")
|
||||||
|
async def add_request_id(request: Request, call_next):
|
||||||
|
request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||||
|
request.state.request_id = request_id
|
||||||
|
response = await call_next(request)
|
||||||
|
response.headers["X-Request-Id"] = request_id
|
||||||
|
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(APIError)
|
||||||
|
async def api_error_handler(request: Request, exc: APIError):
|
||||||
|
return error_response(request, exc.status_code, exc.code, exc.message)
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(RequestValidationError)
|
||||||
|
async def validation_error_handler(request: Request, exc: RequestValidationError):
|
||||||
|
return error_response(request, 400, "invalid_request", "invalid request")
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(Exception)
|
||||||
|
async def unexpected_error_handler(request: Request, exc: Exception):
|
||||||
|
return error_response(request, 500, "internal_error", "internal server error")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health/live")
|
||||||
|
def health_live():
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health/ready")
|
||||||
|
def health_ready():
|
||||||
|
try:
|
||||||
|
with db() as conn:
|
||||||
|
conn.execute("SELECT 1")
|
||||||
|
s3().head_bucket(Bucket=S3_BUCKET)
|
||||||
|
except (OSError, psycopg.Error, BotoCoreError, ClientError):
|
||||||
|
raise APIError(503, "unavailable", "required storage unavailable")
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/users", status_code=201)
|
||||||
|
def create_user(payload: CreateUser):
|
||||||
|
user_id = uuid.uuid4()
|
||||||
|
password_hash = PASSWORD_HASHER.hash(payload.password)
|
||||||
|
try:
|
||||||
|
with db() as conn:
|
||||||
|
user = conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO users(id, username, display_name, password_hash)
|
||||||
|
VALUES (%s, %s, %s, %s)
|
||||||
|
RETURNING id, username, display_name, created_at
|
||||||
|
""",
|
||||||
|
(user_id, payload.username, payload.display_name, password_hash),
|
||||||
|
).fetchone()
|
||||||
|
except psycopg.errors.UniqueViolation:
|
||||||
|
raise APIError(409, "conflict", "user already exists") from None
|
||||||
|
return {**user, "id": str(user["id"]), "created_at": iso(user["created_at"])}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||||
|
def create_session(request: Request):
|
||||||
|
username, password = basic_credentials(request)
|
||||||
|
with db() as conn:
|
||||||
|
user = conn.execute(
|
||||||
|
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||||
|
(username,),
|
||||||
|
).fetchone()
|
||||||
|
if user is None:
|
||||||
|
raise APIError(401, "unauthorized", "invalid credentials")
|
||||||
|
try:
|
||||||
|
PASSWORD_HASHER.verify(user["password_hash"], password)
|
||||||
|
except VerificationError:
|
||||||
|
raise APIError(401, "unauthorized", "invalid credentials") from None
|
||||||
|
token = secrets.token_urlsafe(32)
|
||||||
|
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||||
|
with db() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sessions(token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||||
|
(token_hash(token), user["id"], expires_at),
|
||||||
|
)
|
||||||
|
public_user = {
|
||||||
|
"id": str(user["id"]),
|
||||||
|
"username": user["username"],
|
||||||
|
"display_name": user["display_name"],
|
||||||
|
"created_at": iso(user["created_at"]),
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"token": token,
|
||||||
|
"token_type": "Bearer",
|
||||||
|
"expires_at": iso(expires_at),
|
||||||
|
"user": public_user,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||||
|
def revoke_session(request: Request):
|
||||||
|
bearer_user(request)
|
||||||
|
header = request.headers["Authorization"]
|
||||||
|
token = header.split(" ", 1)[1]
|
||||||
|
with db() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE sessions SET revoked_at = now() WHERE token_hash = %s AND revoked_at IS NULL",
|
||||||
|
(token_hash(token),),
|
||||||
|
)
|
||||||
|
return Response(status_code=204)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/users/me")
|
||||||
|
def current_user(request: Request):
|
||||||
|
user = bearer_user(request)
|
||||||
|
return {**user, "id": str(user["id"]), "created_at": iso(user["created_at"])}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/chats", status_code=201)
|
||||||
|
def create_chat(payload: CreateChat, request: Request):
|
||||||
|
actor = bearer_user(request)
|
||||||
|
member_ids = [uuid.UUID(member_id) for member_id in payload.member_ids]
|
||||||
|
if actor["id"] not in member_ids:
|
||||||
|
raise APIError(400, "invalid_request", "actor must be included in member_ids")
|
||||||
|
chat_id = uuid.uuid4()
|
||||||
|
with db() as conn:
|
||||||
|
existing = conn.execute(
|
||||||
|
"SELECT id FROM users WHERE id = ANY(%s)", (member_ids,)
|
||||||
|
).fetchall()
|
||||||
|
if len(existing) != len(member_ids):
|
||||||
|
raise APIError(400, "invalid_request", "member not found")
|
||||||
|
chat = conn.execute(
|
||||||
|
"INSERT INTO chats(id, title) VALUES (%s, %s) RETURNING id, title, created_at",
|
||||||
|
(chat_id, payload.title),
|
||||||
|
).fetchone()
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.executemany(
|
||||||
|
"INSERT INTO chat_members(chat_id, user_id) VALUES (%s, %s)",
|
||||||
|
[(chat_id, member_id) for member_id in member_ids],
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"id": str(chat["id"]),
|
||||||
|
"title": chat["title"],
|
||||||
|
"member_ids": sorted(str(member_id) for member_id in member_ids),
|
||||||
|
"created_at": iso(chat["created_at"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def chat_for_user(conn, chat_id: uuid.UUID, user_id: uuid.UUID):
|
||||||
|
return conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT c.id, c.title, c.created_at
|
||||||
|
FROM chats c JOIN chat_members m ON m.chat_id = c.id
|
||||||
|
WHERE c.id = %s AND m.user_id = %s
|
||||||
|
""",
|
||||||
|
(chat_id, user_id),
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
|
||||||
|
def public_chat(conn, chat: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
members = conn.execute(
|
||||||
|
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", (chat["id"],)
|
||||||
|
).fetchall()
|
||||||
|
return {
|
||||||
|
"id": str(chat["id"]),
|
||||||
|
"title": chat["title"],
|
||||||
|
"member_ids": [str(row["user_id"]) for row in members],
|
||||||
|
"created_at": iso(chat["created_at"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/chats/{chat_id}")
|
||||||
|
def get_chat(chat_id: uuid.UUID, request: Request):
|
||||||
|
actor = bearer_user(request)
|
||||||
|
with db() as conn:
|
||||||
|
chat = chat_for_user(conn, chat_id, actor["id"])
|
||||||
|
if chat is None:
|
||||||
|
raise APIError(404, "not_found", "chat not found")
|
||||||
|
return public_chat(conn, chat)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/chats")
|
||||||
|
def list_chats(request: Request, limit: int = Query(50, ge=1, le=100), cursor: str | None = None):
|
||||||
|
actor = bearer_user(request)
|
||||||
|
after = decode_cursor(cursor) if cursor else None
|
||||||
|
with db() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT c.id, c.title, c.created_at
|
||||||
|
FROM chats c JOIN chat_members m ON m.chat_id = c.id
|
||||||
|
WHERE m.user_id = %s
|
||||||
|
AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))
|
||||||
|
ORDER BY c.created_at, c.id
|
||||||
|
LIMIT %s
|
||||||
|
""",
|
||||||
|
(actor["id"], after[0] if after else None, after[0] if after else None,
|
||||||
|
after[1] if after else None, limit + 1),
|
||||||
|
).fetchall()
|
||||||
|
items = [public_chat(conn, row) for row in rows[:limit]]
|
||||||
|
next_cursor = (
|
||||||
|
encode_cursor(rows[limit - 1]["created_at"], rows[limit - 1]["id"])
|
||||||
|
if len(rows) > limit
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
return {"items": items, "next_cursor": next_cursor}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||||
|
def create_message(
|
||||||
|
chat_id: uuid.UUID,
|
||||||
|
payload: CreateMessage,
|
||||||
|
request: Request,
|
||||||
|
):
|
||||||
|
actor = bearer_user(request)
|
||||||
|
idempotency_key = request.headers.get("Idempotency-Key", "")
|
||||||
|
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key):
|
||||||
|
raise APIError(400, "invalid_request", "valid Idempotency-Key required")
|
||||||
|
attachment_ids = sorted(uuid.UUID(value) for value in payload.attachment_ids)
|
||||||
|
if not payload.text.strip() and not attachment_ids:
|
||||||
|
raise APIError(400, "invalid_request", "text or attachment is required")
|
||||||
|
request_hash = hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
{"text": payload.text, "attachment_ids": [str(value) for value in attachment_ids]},
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
).encode()
|
||||||
|
).hexdigest()
|
||||||
|
with db() as conn:
|
||||||
|
if chat_for_user(conn, chat_id, actor["id"]) is None:
|
||||||
|
raise APIError(404, "not_found", "chat not accessible")
|
||||||
|
if attachment_ids:
|
||||||
|
attachments = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id, status FROM attachments
|
||||||
|
WHERE id = ANY(%s) AND chat_id = %s
|
||||||
|
""",
|
||||||
|
(attachment_ids, chat_id),
|
||||||
|
).fetchall()
|
||||||
|
if len(attachments) != len(attachment_ids):
|
||||||
|
raise APIError(404, "not_found", "attachment not found")
|
||||||
|
if any(row["status"] != "ready" for row in attachments):
|
||||||
|
raise APIError(409, "conflict", "attachment is not ready")
|
||||||
|
existing = conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO idempotency_keys(user_id, chat_id, idempotency_key, request_hash)
|
||||||
|
VALUES (%s, %s, %s, %s)
|
||||||
|
ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING
|
||||||
|
RETURNING message_id, request_hash
|
||||||
|
""",
|
||||||
|
(actor["id"], chat_id, idempotency_key, request_hash),
|
||||||
|
).fetchone()
|
||||||
|
if existing is None:
|
||||||
|
existing = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT message_id, request_hash
|
||||||
|
FROM idempotency_keys
|
||||||
|
WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s
|
||||||
|
FOR UPDATE
|
||||||
|
""",
|
||||||
|
(actor["id"], chat_id, idempotency_key),
|
||||||
|
).fetchone()
|
||||||
|
if existing["request_hash"] != request_hash:
|
||||||
|
raise APIError(409, "conflict", "idempotency key has another request")
|
||||||
|
message = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id, chat_id, sender_id, text, created_at
|
||||||
|
FROM messages WHERE id = %s
|
||||||
|
""",
|
||||||
|
(existing["message_id"],),
|
||||||
|
).fetchone()
|
||||||
|
if message is None:
|
||||||
|
raise APIError(503, "unavailable", "idempotent result is not ready")
|
||||||
|
return public_message(conn, message)
|
||||||
|
message_id = uuid.uuid4()
|
||||||
|
message = conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO messages(id, chat_id, sender_id, text)
|
||||||
|
VALUES (%s, %s, %s, %s)
|
||||||
|
RETURNING id, chat_id, sender_id, text, created_at
|
||||||
|
""",
|
||||||
|
(message_id, chat_id, actor["id"], payload.text),
|
||||||
|
).fetchone()
|
||||||
|
if attachment_ids:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.executemany(
|
||||||
|
"""
|
||||||
|
INSERT INTO message_attachments(message_id, attachment_id)
|
||||||
|
VALUES (%s, %s)
|
||||||
|
""",
|
||||||
|
[(message_id, attachment_id) for attachment_id in attachment_ids],
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE idempotency_keys SET message_id = %s
|
||||||
|
WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s
|
||||||
|
""",
|
||||||
|
(message_id, actor["id"], chat_id, idempotency_key),
|
||||||
|
)
|
||||||
|
return public_message(conn, message)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_filename(filename: str) -> None:
|
||||||
|
if not filename or len(filename) > 255:
|
||||||
|
raise APIError(400, "invalid_request", "filename is invalid")
|
||||||
|
if any(ord(char) < 32 for char in filename) or "/" in filename or "\\" in filename:
|
||||||
|
raise APIError(400, "invalid_request", "filename is invalid")
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_image(path: Any, declared_type: str) -> tuple[str, int, int]:
|
||||||
|
try:
|
||||||
|
with Image.open(path) as image:
|
||||||
|
detected_type = {
|
||||||
|
"PNG": "image/png",
|
||||||
|
"JPEG": "image/jpeg",
|
||||||
|
}.get(image.format)
|
||||||
|
if detected_type is None or detected_type != declared_type:
|
||||||
|
raise APIError(415, "unsupported_media_type", "image type is invalid")
|
||||||
|
width, height = image.size
|
||||||
|
if width * height > MAX_IMAGE_PIXELS:
|
||||||
|
raise APIError(413, "payload_too_large", "image has too many pixels")
|
||||||
|
image.verify()
|
||||||
|
path.seek(0)
|
||||||
|
with Image.open(path) as image:
|
||||||
|
width, height = image.size
|
||||||
|
image.verify()
|
||||||
|
except APIError:
|
||||||
|
raise
|
||||||
|
except (UnidentifiedImageError, OSError, Image.DecompressionBombError):
|
||||||
|
raise APIError(415, "unsupported_media_type", "image is invalid") from None
|
||||||
|
return detected_type, width, height
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
||||||
|
async def upload_attachment(chat_id: uuid.UUID, request: Request, filename: str):
|
||||||
|
actor = bearer_user(request)
|
||||||
|
validate_filename(filename)
|
||||||
|
declared_type = request.headers.get("content-type", "").split(";", 1)[0].lower()
|
||||||
|
if declared_type not in {"image/png", "image/jpeg"}:
|
||||||
|
raise APIError(415, "unsupported_media_type", "content type is invalid")
|
||||||
|
content_length = request.headers.get("content-length")
|
||||||
|
if content_length:
|
||||||
|
try:
|
||||||
|
if int(content_length) > MAX_UPLOAD_BYTES:
|
||||||
|
raise APIError(413, "payload_too_large", "file is too large")
|
||||||
|
except ValueError:
|
||||||
|
raise APIError(400, "invalid_request", "content length is invalid") from None
|
||||||
|
with db() as conn:
|
||||||
|
if chat_for_user(conn, chat_id, actor["id"]) is None:
|
||||||
|
raise APIError(404, "not_found", "chat not accessible")
|
||||||
|
|
||||||
|
size = 0
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
object_key = "attachments/" + str(uuid.uuid4())
|
||||||
|
uploaded = False
|
||||||
|
try:
|
||||||
|
with tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") as temporary:
|
||||||
|
async for chunk in request.stream():
|
||||||
|
size += len(chunk)
|
||||||
|
if size > MAX_UPLOAD_BYTES:
|
||||||
|
raise APIError(413, "payload_too_large", "file is too large")
|
||||||
|
digest.update(chunk)
|
||||||
|
temporary.write(chunk)
|
||||||
|
if size == 0:
|
||||||
|
raise APIError(415, "unsupported_media_type", "image is empty")
|
||||||
|
temporary.seek(0)
|
||||||
|
detected_type, _, _ = inspect_image(temporary, declared_type)
|
||||||
|
temporary.seek(0)
|
||||||
|
try:
|
||||||
|
await run_in_threadpool(
|
||||||
|
s3().upload_fileobj,
|
||||||
|
temporary,
|
||||||
|
S3_BUCKET,
|
||||||
|
object_key,
|
||||||
|
ExtraArgs={"ContentType": detected_type},
|
||||||
|
)
|
||||||
|
uploaded = True
|
||||||
|
except (BotoCoreError, ClientError, OSError):
|
||||||
|
raise APIError(503, "unavailable", "object storage unavailable") from None
|
||||||
|
except APIError:
|
||||||
|
raise
|
||||||
|
except (OSError, ValueError):
|
||||||
|
raise APIError(503, "unavailable", "upload could not be completed") from None
|
||||||
|
|
||||||
|
attachment_id = uuid.uuid4()
|
||||||
|
now = utc_now()
|
||||||
|
row = {
|
||||||
|
"id": attachment_id,
|
||||||
|
"chat_id": chat_id,
|
||||||
|
"owner_id": actor["id"],
|
||||||
|
"filename": filename,
|
||||||
|
"content_type": detected_type,
|
||||||
|
"size_bytes": size,
|
||||||
|
"sha256": digest.hexdigest(),
|
||||||
|
"status": "ready",
|
||||||
|
"error_code": None,
|
||||||
|
"created_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
with db() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO attachments(
|
||||||
|
id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||||
|
sha256, status, object_key, error_code, created_at, updated_at
|
||||||
|
)
|
||||||
|
VALUES (%s, %s, %s, %s, %s, %s, %s, 'ready', %s, NULL, %s, %s)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
attachment_id,
|
||||||
|
chat_id,
|
||||||
|
actor["id"],
|
||||||
|
filename,
|
||||||
|
detected_type,
|
||||||
|
size,
|
||||||
|
digest.hexdigest(),
|
||||||
|
object_key,
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except psycopg.Error:
|
||||||
|
if uploaded:
|
||||||
|
try:
|
||||||
|
s3().delete_object(Bucket=S3_BUCKET, Key=object_key)
|
||||||
|
except (BotoCoreError, ClientError, OSError):
|
||||||
|
LOGGER.error("attachment compensation failed for object key")
|
||||||
|
raise APIError(503, "unavailable", "attachment metadata unavailable") from None
|
||||||
|
return attachment_public(row)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/attachments/{attachment_id}")
|
||||||
|
def get_attachment(attachment_id: uuid.UUID, request: Request):
|
||||||
|
actor = bearer_user(request)
|
||||||
|
with db() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT a.*
|
||||||
|
FROM attachments a
|
||||||
|
JOIN chat_members m ON m.chat_id = a.chat_id
|
||||||
|
WHERE a.id = %s AND m.user_id = %s
|
||||||
|
""",
|
||||||
|
(attachment_id, actor["id"]),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise APIError(404, "not_found", "attachment not found")
|
||||||
|
return attachment_public(row)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||||
|
def download_attachment(
|
||||||
|
attachment_id: uuid.UUID,
|
||||||
|
request: Request,
|
||||||
|
variant: str = "original",
|
||||||
|
):
|
||||||
|
actor = bearer_user(request)
|
||||||
|
if variant != "original":
|
||||||
|
raise APIError(404, "not_found", "attachment variant not found")
|
||||||
|
with db() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT a.*
|
||||||
|
FROM attachments a
|
||||||
|
JOIN chat_members m ON m.chat_id = a.chat_id
|
||||||
|
WHERE a.id = %s AND m.user_id = %s
|
||||||
|
""",
|
||||||
|
(attachment_id, actor["id"]),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise APIError(404, "not_found", "attachment not found")
|
||||||
|
try:
|
||||||
|
object_response = s3().get_object(Bucket=S3_BUCKET, Key=row["object_key"])
|
||||||
|
except (BotoCoreError, ClientError, OSError):
|
||||||
|
raise APIError(503, "unavailable", "object storage unavailable") from None
|
||||||
|
|
||||||
|
body = object_response["Body"]
|
||||||
|
|
||||||
|
def chunks():
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
chunk = body.read(1024 * 1024)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
yield chunk
|
||||||
|
finally:
|
||||||
|
body.close()
|
||||||
|
|
||||||
|
return StreamingResponse(
|
||||||
|
chunks(),
|
||||||
|
media_type=row["content_type"],
|
||||||
|
headers={"Content-Length": str(row["size_bytes"])},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||||
|
def list_messages(
|
||||||
|
chat_id: uuid.UUID,
|
||||||
|
request: Request,
|
||||||
|
limit: int = Query(50, ge=1, le=100),
|
||||||
|
cursor: str | None = None,
|
||||||
|
):
|
||||||
|
actor = bearer_user(request)
|
||||||
|
after = decode_cursor(cursor) if cursor else None
|
||||||
|
with db() as conn:
|
||||||
|
if chat_for_user(conn, chat_id, actor["id"]) is None:
|
||||||
|
raise APIError(404, "not_found", "chat not accessible")
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id, chat_id, sender_id, text, created_at
|
||||||
|
FROM messages
|
||||||
|
WHERE chat_id = %s
|
||||||
|
AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))
|
||||||
|
ORDER BY created_at, id
|
||||||
|
LIMIT %s
|
||||||
|
""",
|
||||||
|
(chat_id, after[0] if after else None, after[0] if after else None,
|
||||||
|
after[1] if after else None, limit + 1),
|
||||||
|
).fetchall()
|
||||||
|
items = [public_message(conn, row) for row in rows[:limit]]
|
||||||
|
next_cursor = (
|
||||||
|
encode_cursor(rows[limit - 1]["created_at"], rows[limit - 1]["id"])
|
||||||
|
if len(rows) > limit
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
return {"items": items, "next_cursor": next_cursor}
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
|
uvicorn.run("app:app", host=APP_HOST, port=APP_PORT, reload=False)
|
||||||
+167
@@ -0,0 +1,167 @@
|
|||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16.4-bookworm
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: messenger
|
||||||
|
POSTGRES_USER: messenger
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
|
||||||
|
volumes:
|
||||||
|
- postgres_data:/var/lib/postgresql/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
s3-config:
|
||||||
|
image: alpine:3.20.3
|
||||||
|
environment:
|
||||||
|
S3_BOOTSTRAP_ACCESS_KEY: ${S3_BOOTSTRAP_ACCESS_KEY:?set S3_BOOTSTRAP_ACCESS_KEY in .env}
|
||||||
|
S3_BOOTSTRAP_SECRET_KEY: ${S3_BOOTSTRAP_SECRET_KEY:?set S3_BOOTSTRAP_SECRET_KEY in .env}
|
||||||
|
S3_APP_ACCESS_KEY: ${S3_APP_ACCESS_KEY:?set S3_APP_ACCESS_KEY in .env}
|
||||||
|
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY:?set S3_APP_SECRET_KEY in .env}
|
||||||
|
volumes:
|
||||||
|
- s3_config:/config
|
||||||
|
entrypoint: ["/bin/sh", "-c"]
|
||||||
|
command:
|
||||||
|
- |
|
||||||
|
set -eu
|
||||||
|
printf '{"identities":[{"name":"bootstrap","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin"]},{"name":"messenger-api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:messenger-attachments","Write:messenger-attachments","List:messenger-attachments","Tagging:messenger-attachments"]}]}' \
|
||||||
|
"$$S3_BOOTSTRAP_ACCESS_KEY" "$$S3_BOOTSTRAP_SECRET_KEY" \
|
||||||
|
"$$S3_APP_ACCESS_KEY" "$$S3_APP_SECRET_KEY" > /config/s3.json
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
s3:
|
||||||
|
image: chrislusf/seaweedfs:3.80
|
||||||
|
depends_on:
|
||||||
|
s3-config:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
command: ["server", "-dir=/data", "-s3", "-s3.config=/etc/seaweed/s3.json"]
|
||||||
|
volumes:
|
||||||
|
- s3_data:/data
|
||||||
|
- s3_config:/etc/seaweed:ro
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
s3-bootstrap:
|
||||||
|
build: .
|
||||||
|
environment:
|
||||||
|
S3_ENDPOINT: http://s3:8333
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_BOOTSTRAP_ACCESS_KEY}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_BOOTSTRAP_SECRET_KEY}
|
||||||
|
depends_on:
|
||||||
|
s3:
|
||||||
|
condition: service_healthy
|
||||||
|
command:
|
||||||
|
- python
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
import time
|
||||||
|
import boto3
|
||||||
|
from botocore.exceptions import BotoCoreError, ClientError
|
||||||
|
client = boto3.client("s3", endpoint_url="http://s3:8333", region_name="us-east-1", aws_access_key_id="${S3_BOOTSTRAP_ACCESS_KEY}", aws_secret_access_key="${S3_BOOTSTRAP_SECRET_KEY}")
|
||||||
|
for attempt in range(60):
|
||||||
|
try:
|
||||||
|
client.create_bucket(Bucket="messenger-attachments")
|
||||||
|
break
|
||||||
|
except ClientError as exc:
|
||||||
|
if exc.response.get("Error", {}).get("Code") in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||||
|
break
|
||||||
|
raise
|
||||||
|
except (BotoCoreError, OSError):
|
||||||
|
if attempt == 59:
|
||||||
|
raise
|
||||||
|
time.sleep(1)
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
migrate:
|
||||||
|
build: .
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
command: ["python", "-c", "import app; app.wait_for_database()"]
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
api1:
|
||||||
|
build: .
|
||||||
|
environment:
|
||||||
|
APP_HOST: 0.0.0.0
|
||||||
|
APP_PORT: 8080
|
||||||
|
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
INSTANCE_ID: api1
|
||||||
|
S3_ENDPOINT: http://s3:8333
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_APP_ACCESS_KEY}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_APP_SECRET_KEY}
|
||||||
|
MAX_UPLOAD_BYTES: 10485760
|
||||||
|
MAX_IMAGE_PIXELS: 40000000
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-bootstrap:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
api2:
|
||||||
|
build: .
|
||||||
|
environment:
|
||||||
|
APP_HOST: 0.0.0.0
|
||||||
|
APP_PORT: 8080
|
||||||
|
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
INSTANCE_ID: api2
|
||||||
|
S3_ENDPOINT: http://s3:8333
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_APP_ACCESS_KEY}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_APP_SECRET_KEY}
|
||||||
|
MAX_UPLOAD_BYTES: 10485760
|
||||||
|
MAX_IMAGE_PIXELS: 40000000
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-bootstrap:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
load_balancer:
|
||||||
|
image: nginx:1.27.2-alpine
|
||||||
|
ports:
|
||||||
|
- "8080:80"
|
||||||
|
volumes:
|
||||||
|
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||||
|
depends_on:
|
||||||
|
api1:
|
||||||
|
condition: service_healthy
|
||||||
|
api2:
|
||||||
|
condition: service_healthy
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1/health/live"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
postgres_data:
|
||||||
|
s3_data:
|
||||||
|
s3_config:
|
||||||
Binary file not shown.
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:ListBucket"],
|
||||||
|
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
CREATE TABLE users (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
username varchar(32) NOT NULL UNIQUE,
|
||||||
|
display_name varchar(100) NOT NULL CHECK (btrim(display_name) <> ''),
|
||||||
|
password_hash text NOT NULL,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chats (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
title varchar(100) NOT NULL CHECK (btrim(title) <> ''),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chat_members (
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
PRIMARY KEY (chat_id, user_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE messages (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
sender_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
text varchar(4000) NOT NULL CHECK (btrim(text) <> ''),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE sessions (
|
||||||
|
token_hash char(64) PRIMARY KEY,
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
expires_at timestamptz NOT NULL,
|
||||||
|
revoked_at timestamptz
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
|
||||||
|
CREATE INDEX messages_chat_order_idx ON messages(chat_id, created_at, id);
|
||||||
|
CREATE INDEX sessions_user_idx ON sessions(user_id, expires_at);
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
CREATE TABLE idempotency_keys (
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
idempotency_key varchar(128) NOT NULL,
|
||||||
|
request_hash char(64) NOT NULL,
|
||||||
|
message_id uuid REFERENCES messages(id) ON DELETE CASCADE,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (user_id, chat_id, idempotency_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idempotency_created_idx ON idempotency_keys(created_at);
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
CREATE TABLE attachments (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
owner_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
filename varchar(255) NOT NULL,
|
||||||
|
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||||
|
size_bytes bigint NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760),
|
||||||
|
sha256 char(64) NOT NULL,
|
||||||
|
status varchar(16) NOT NULL CHECK (status = 'ready'),
|
||||||
|
object_key text NOT NULL UNIQUE,
|
||||||
|
error_code varchar(64),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
updated_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX attachments_chat_idx ON attachments(chat_id, created_at, id);
|
||||||
|
|
||||||
|
CREATE TABLE message_attachments (
|
||||||
|
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||||
|
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
|
||||||
|
PRIMARY KEY (message_id, attachment_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX message_attachments_attachment_idx
|
||||||
|
ON message_attachments(attachment_id, message_id);
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE messages
|
||||||
|
DROP CONSTRAINT IF EXISTS messages_text_check;
|
||||||
+21
@@ -0,0 +1,21 @@
|
|||||||
|
upstream messenger_api {
|
||||||
|
server api1:8080 max_fails=1 fail_timeout=2s;
|
||||||
|
server api2:8080 max_fails=1 fail_timeout=2s;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
client_max_body_size 11m;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://messenger_api;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Request-Id $http_x_request_id;
|
||||||
|
proxy_connect_timeout 1s;
|
||||||
|
proxy_read_timeout 10s;
|
||||||
|
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||||
|
proxy_next_upstream_tries 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
argon2-cffi==23.1.0
|
||||||
|
boto3==1.35.76
|
||||||
|
fastapi==0.115.5
|
||||||
|
Pillow==11.0.0
|
||||||
|
psycopg[binary]==3.2.3
|
||||||
|
uvicorn==0.32.0
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Report or remove old S3 objects that have no PostgreSQL attachment row."""
|
||||||
|
import argparse
|
||||||
|
import datetime as dt
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
|
import app
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--grace-seconds", type=int, default=3600)
|
||||||
|
parser.add_argument("--apply", action="store_true")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.grace_seconds < 0:
|
||||||
|
parser.error("--grace-seconds must be non-negative")
|
||||||
|
|
||||||
|
with app.db() as conn:
|
||||||
|
known = {
|
||||||
|
row["object_key"]
|
||||||
|
for row in conn.execute("SELECT object_key FROM attachments").fetchall()
|
||||||
|
}
|
||||||
|
|
||||||
|
now = dt.datetime.now(dt.timezone.utc)
|
||||||
|
client = app.s3()
|
||||||
|
paginator = client.get_paginator("list_objects_v2")
|
||||||
|
candidates = []
|
||||||
|
for page in paginator.paginate(Bucket=app.S3_BUCKET):
|
||||||
|
for item in page.get("Contents", []):
|
||||||
|
key = item["Key"]
|
||||||
|
modified = item["LastModified"]
|
||||||
|
if key not in known and (now - modified).total_seconds() >= args.grace_seconds:
|
||||||
|
candidates.append(key)
|
||||||
|
|
||||||
|
for key in candidates:
|
||||||
|
if args.apply:
|
||||||
|
client.delete_object(Bucket=app.S3_BUCKET, Key=key)
|
||||||
|
print(("deleted" if args.apply else "candidates") + f"={len(candidates)}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
docker compose up -d --force-recreate postgres s3
|
||||||
|
docker compose run --rm migrate
|
||||||
|
docker compose run --rm s3-bootstrap
|
||||||
|
docker compose up -d --force-recreate api1 api2 load_balancer
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
case "${TARGET_INSTANCE_ID:-}" in
|
||||||
|
api1) service=api1 ;;
|
||||||
|
api2) service=api2 ;;
|
||||||
|
*) echo "unknown TARGET_INSTANCE_ID" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
docker compose stop "$service"
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Negative attachment checks required for the level-4 assessment."""
|
||||||
|
import hashlib
|
||||||
|
from pathlib import Path
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from client import Client, arguments, check, run
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = arguments("lab4 level-4 attachment checks")
|
||||||
|
client = Client(args.base_url, args.ca_file)
|
||||||
|
alice, bob, eve, chat = client.bootstrap()
|
||||||
|
second_chat = client.call(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/chats",
|
||||||
|
201,
|
||||||
|
actor=alice,
|
||||||
|
body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]},
|
||||||
|
schema="Chat",
|
||||||
|
)["json"]
|
||||||
|
sample = Path(__file__).with_name("fixtures").joinpath("sample.png").read_bytes()
|
||||||
|
|
||||||
|
client.call(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/chats/{chat['id']}/attachments?filename=large.png",
|
||||||
|
413,
|
||||||
|
actor=alice,
|
||||||
|
raw=b"x" * (10 * 1024 * 1024 + 1),
|
||||||
|
headers={"Content-Type": "image/png"},
|
||||||
|
)
|
||||||
|
client.call(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/chats/{chat['id']}/attachments?filename=broken.png",
|
||||||
|
415,
|
||||||
|
actor=alice,
|
||||||
|
raw=b"not an image",
|
||||||
|
headers={"Content-Type": "image/png"},
|
||||||
|
)
|
||||||
|
client.call(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/chats/{chat['id']}/attachments?filename=spoof.jpg",
|
||||||
|
415,
|
||||||
|
actor=alice,
|
||||||
|
raw=sample,
|
||||||
|
headers={"Content-Type": "image/jpeg"},
|
||||||
|
)
|
||||||
|
client.call(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/chats/{chat['id']}/attachments?filename=..%2Fsecret.png",
|
||||||
|
400,
|
||||||
|
actor=alice,
|
||||||
|
raw=sample,
|
||||||
|
headers={"Content-Type": "image/png"},
|
||||||
|
)
|
||||||
|
attachment, content = client.attachment(alice, chat)
|
||||||
|
check(hashlib.sha256(content).hexdigest() == attachment["sha256"], "fixture checksum changed")
|
||||||
|
client.call(
|
||||||
|
"GET",
|
||||||
|
f"/api/v1/attachments/{attachment['id']}",
|
||||||
|
404,
|
||||||
|
actor=eve,
|
||||||
|
)
|
||||||
|
client.call(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/chats/{second_chat['id']}/messages",
|
||||||
|
404,
|
||||||
|
actor=alice,
|
||||||
|
body={"text": "wrong chat", "attachment_ids": [attachment["id"]]},
|
||||||
|
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||||
|
)
|
||||||
|
print("PASS: lab4 level-4 attachment validation and access checks")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
run(main)
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Run ten concurrent bounded uploads and sample API RSS via Docker."""
|
||||||
|
import concurrent.futures
|
||||||
|
import subprocess
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from client import Client, arguments, check, run
|
||||||
|
|
||||||
|
|
||||||
|
def rss_mb() -> float:
|
||||||
|
output = subprocess.check_output(
|
||||||
|
["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
values = []
|
||||||
|
for line in output.splitlines():
|
||||||
|
if "api1" not in line and "api2" not in line:
|
||||||
|
continue
|
||||||
|
used = line.split()[1].split("/")[0]
|
||||||
|
factor = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}
|
||||||
|
values.append(float(used[:-3]) * factor[used[-3:]])
|
||||||
|
return max(values, default=0.0)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = arguments("lab4 concurrent bounded upload experiment")
|
||||||
|
client = Client(args.base_url, args.ca_file)
|
||||||
|
alice, _, _, chat = client.bootstrap()
|
||||||
|
content = Path(__file__).with_name("fixtures").joinpath("sample.png").read_bytes()
|
||||||
|
stop = threading.Event()
|
||||||
|
samples = [rss_mb()]
|
||||||
|
|
||||||
|
def sample():
|
||||||
|
while not stop.is_set():
|
||||||
|
try:
|
||||||
|
samples.append(rss_mb())
|
||||||
|
except (OSError, ValueError, subprocess.SubprocessError):
|
||||||
|
pass
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
sampler = threading.Thread(target=sample, daemon=True)
|
||||||
|
sampler.start()
|
||||||
|
started = time.monotonic()
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
|
||||||
|
futures = [
|
||||||
|
pool.submit(
|
||||||
|
client.call,
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/chats/{chat['id']}/attachments?filename=load-{index}.png",
|
||||||
|
201,
|
||||||
|
None,
|
||||||
|
alice,
|
||||||
|
{"Content-Type": "image/png"},
|
||||||
|
content,
|
||||||
|
"Attachment",
|
||||||
|
)
|
||||||
|
for index in range(10)
|
||||||
|
]
|
||||||
|
[future.result() for future in futures]
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
stop.set()
|
||||||
|
sampler.join(timeout=1)
|
||||||
|
samples.append(rss_mb())
|
||||||
|
check(len(samples) > 0, "RSS sampling did not produce data")
|
||||||
|
print(f"uploads=10 concurrency=10 elapsed_seconds={elapsed:.3f} peak_api_rss_mb={max(samples):.1f}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
run(main)
|
||||||
Reference in New Issue
Block a user