1 Commits
Author SHA1 Message Date
katyatheestalion de2246a888 Сделана лаба 4 2026-11-16 22:00:34 +03:00
19 changed files with 1437 additions and 40 deletions
+6 -19
View File
@@ -1,20 +1,7 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Copy to .env and replace local-only passwords. Never commit .env.
POSTGRES_PASSWORD=change-me-postgres
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
S3_BOOTSTRAP_ACCESS_KEY=bootstrap-admin
S3_BOOTSTRAP_SECRET_KEY=change-me-bootstrap-secret
S3_APP_ACCESS_KEY=messenger-api
S3_APP_SECRET_KEY=change-me-s3-api-secret
+17
View File
@@ -0,0 +1,17 @@
FROM python:3.13-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
WORKDIR /app
COPY requirements.txt .
RUN python -m pip install --no-cache-dir -r requirements.txt
COPY app.py ./
COPY migrations ./migrations/
COPY scripts ./scripts/
RUN useradd --create-home --shell /usr/sbin/nologin appuser && \
chown -R appuser:appuser /app
USER appuser
EXPOSE 8080
CMD ["python", "app.py"]
+7 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test test-grade4 test-attachments-load load test-persistence test-failover
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -18,6 +18,12 @@ validate:
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-grade4:
$(PYTHON) tests/grade4_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachments-load:
$(PYTHON) tests/load_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
+55 -20
View File
@@ -1,37 +1,72 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Антоновва Екатерина бпи2304
- Целевая оценка: 4
- Выбранный трек 5: не заявлен
- Commit/PR: заполняется после публикации
## Использованные источники, библиотеки и помощь ИИ
Использованы нормативные `contracts/openapi.json`, `contracts/README.md`, `contracts/infrastructure.json`, публичные тесты из `tests/` и документация FastAPI, PostgreSQL, boto3, Pillow, SeaweedFS и Docker Compose. ИИ применялся как вспомогательный инструмент для проверки реализации, поиска ошибок и редактирования отчёта; итоговые решения проверены локальными тестами и запуском стенда.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
```sh
cp .env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# дождаться: docker compose ps показывает healthy у postgres, s3, api1, api2 и load_balancer
make check
make validate
make test
make test-grade4
make test-attachments-load
```
Для остановки без удаления данных:
```sh
docker compose down
```
Для намеренного удаления учебных данных вместе с volumes:
```sh
docker compose down -v
```
Пароли находятся только в локальном `.env`, который исключён из Git. В текущей проверке Windows `make` отсутствовал, поэтому его цели выполнены эквивалентными командами с `.venv\Scripts\python.exe`; в Linux/CI используются команды Makefile.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Клиент обращается к единственному публичному Nginx на `localhost:8080`. Балансировщик направляет запросы на `api1` и `api2`; обе реплики используют одну PostgreSQL и один SeaweedFS с S3 API. Локального состояния сессий, сообщений и файлов в API нет: PostgreSQL хранит метаданные, а приватный bucket `messenger-attachments` — байты.
Вложение принимается потоково в `SpooledTemporaryFile` с лимитом 10 MiB, проверяются фактический PNG/JPEG, MIME, имя, SHA-256 и максимум 40 млн пикселей. Сначала выполняется S3 PUT, затем транзакционная запись метаданных; при ошибке БД выполняется компенсационный DELETE. `scripts/reconcile_attachments.py` поддерживает dry-run и удаление только orphan-объектов старше grace period.
S3 bootstrap идемпотентен. SeaweedFS имеет отдельные bootstrap- и API-credentials; API получает только права `Read/Write/List/Tagging` для нужного bucket, административная консоль наружу не публикуется. Миграция `004_allow_attachment_only_messages.sql` сохраняет возможность сообщений, содержащих только вложение.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Критерий | Команда/сценарий | Результат |
| --- | --- | --- |
| Контракт и OpenAPI | `scripts/check_contract.py`, `scripts/validate_openapi.py` | PASS |
| Публичный smoke | `tests/smoke.py` / `make test` | PASS: регистрация, сессии, чат, сообщение, upload/download и доступ участника |
| Ошибки уровня 4 | `tests/grade4_attachments.py` / `make test-grade4` | PASS: >10 MiB, повреждённый файл, MIME spoofing, опасное имя, чужой attachment и другой чат |
| Идемпотентный bootstrap | повторный `docker compose up -d --build` | PASS, bucket и миграции не дублируются |
| Persistence | `scripts/restart.sh` без удаления volumes | PASS: действующая сессия, метаданные и байты вложения сохранились |
| Failover | `scripts/stop-one.sh`, остановка обслужившей реплики | PASS: download через вторую реплику вернул исходные байты |
| Частный bucket | API credentials и отсутствие публикации S3-порта | S3 доступен API внутри Compose, прямой публичный GET не используется |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Эксперимент потоковой загрузки
## Эксперименты и отказы
Запуск:
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
```sh
make test-attachments-load
```
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Скрипт одновременно выполняет 10 загрузок, а Docker RSS API-реплик опрашивается во время эксперимента. Фактический результат текущего прогона сохранён в `evidence/attachments-concurrency.txt`: `uploads=10`, `concurrency=10`, `elapsed_seconds=0.480`, `peak_api_rss_mb=84.2`. Полный файл не удерживается в RAM: после 1 MiB `SpooledTemporaryFile` переходит на временный файл.
## Ограничения и следующие шаги
## Ограничения
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Проверен single-node Compose; отказ Docker-host, сетевой partition и HA PostgreSQL не заявляются. Компенсация закрывает обычный отказ записи метаданных, а периодическая сверка orphan-объектов запускается оператором, а не отдельным worker-процессом.
+859
View File
@@ -0,0 +1,859 @@
import base64
import binascii
import hashlib
import json
import logging
import os
import re
import secrets
import tempfile
import time
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
import psycopg
import boto3
from botocore.exceptions import BotoCoreError, ClientError
from fastapi.concurrency import run_in_threadpool
from fastapi.responses import StreamingResponse
from PIL import Image, UnidentifiedImageError
from psycopg.rows import dict_row
from fastapi import FastAPI, Query, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ConfigDict, Field, field_validator
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
APP_PORT = int(os.getenv("APP_PORT", "8080"))
DATABASE_URL = os.getenv("DATABASE_URL", "")
DATABASE_URL_FILE = os.getenv("DATABASE_URL_FILE", "")
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
INSTANCE_ID = os.getenv("INSTANCE_ID", "api-" + uuid.uuid4().hex[:8])
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "")
S3_REGION = os.getenv("S3_REGION", "us-east-1")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID", "")
S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "")
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
MIGRATIONS_DIR = Path(__file__).parent / "migrations"
PASSWORD_HASHER = PasswordHasher()
LOGGER = logging.getLogger("messenger")
class APIError(Exception):
def __init__(self, status_code: int, code: str, message: str):
self.status_code = status_code
self.code = code
self.message = message
class CreateUser(BaseModel):
model_config = ConfigDict(extra="forbid")
username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(..., min_length=1, max_length=100)
password: str = Field(..., min_length=12, max_length=128)
@field_validator("display_name", "password")
@classmethod
def validate_non_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("value must not be blank")
return value
class CreateChat(BaseModel):
model_config = ConfigDict(extra="forbid")
title: str = Field(..., min_length=1, max_length=100)
member_ids: list[str] = Field(..., min_length=1, max_length=100)
@field_validator("title")
@classmethod
def validate_title(cls, value: str) -> str:
if not value.strip():
raise ValueError("title must not be blank")
return value
@field_validator("member_ids")
@classmethod
def validate_member_ids(cls, value: list[str]) -> list[str]:
if len(set(value)) != len(value):
raise ValueError("member_ids must be unique")
for member_id in value:
try:
uuid.UUID(member_id)
except ValueError as exc:
raise ValueError("member_ids must contain UUIDs") from exc
return value
class CreateMessage(BaseModel):
model_config = ConfigDict(extra="forbid")
text: str = Field(..., max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@field_validator("attachment_ids")
@classmethod
def validate_attachment_ids(cls, value: list[str]) -> list[str]:
if len(set(value)) != len(value):
raise ValueError("attachment_ids must be unique")
for attachment_id in value:
try:
uuid.UUID(attachment_id)
except ValueError as exc:
raise ValueError("attachment_ids must contain UUIDs") from exc
return value
class AttachmentError(Exception):
pass
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def iso(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace(
"+00:00", "Z"
)
def database_url() -> str:
if DATABASE_URL:
return DATABASE_URL
if DATABASE_URL_FILE:
try:
return Path(DATABASE_URL_FILE).read_text(encoding="utf-8").strip()
except OSError as exc:
raise RuntimeError("database URL file is not readable") from exc
raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required")
def db():
return psycopg.connect(database_url(), row_factory=dict_row)
def s3():
if not S3_ENDPOINT or not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY:
raise RuntimeError("S3 configuration is incomplete")
return boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
region_name=S3_REGION,
aws_access_key_id=S3_ACCESS_KEY_ID,
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
)
def wait_for_s3() -> None:
deadline = time.monotonic() + 60
last_error: Exception | None = None
while time.monotonic() < deadline:
try:
s3().head_bucket(Bucket=S3_BUCKET)
return
except (BotoCoreError, ClientError, OSError) as exc:
last_error = exc
time.sleep(1)
raise RuntimeError("S3 bucket did not become ready") from last_error
def attachment_public(row: dict[str, Any]) -> dict[str, Any]:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": [],
"error_code": row["error_code"],
"created_at": iso(row["created_at"]),
"updated_at": iso(row["updated_at"]),
}
def attachment_ids_for_message(conn, message_id: uuid.UUID) -> list[str]:
rows = conn.execute(
"""
SELECT attachment_id FROM message_attachments
WHERE message_id = %s ORDER BY attachment_id
""",
(message_id,),
).fetchall()
return [str(row["attachment_id"]) for row in rows]
def public_message(conn, row: dict[str, Any]) -> dict[str, Any]:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": iso(row["created_at"]),
"attachment_ids": attachment_ids_for_message(conn, row["id"]),
}
def apply_migrations() -> None:
with db() as conn:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
applied_at timestamptz NOT NULL DEFAULT now()
)
"""
)
for path in sorted(MIGRATIONS_DIR.glob("*.sql")):
version = path.name.split("_", 1)[0]
applied = conn.execute(
"SELECT 1 FROM schema_migrations WHERE version = %s", (version,)
).fetchone()
if applied is None:
for statement in path.read_text(encoding="utf-8").split(";"):
if statement.strip():
conn.execute(statement)
conn.execute(
"INSERT INTO schema_migrations(version) VALUES (%s)", (version,)
)
def wait_for_database() -> None:
deadline = time.monotonic() + 60
last_error: Exception | None = None
while time.monotonic() < deadline:
try:
apply_migrations()
return
except (OSError, psycopg.Error) as exc:
last_error = exc
time.sleep(1)
raise RuntimeError("database did not become ready") from last_error
def error_response(request: Request, status_code: int, code: str, message: str):
request_id = getattr(request.state, "request_id", str(uuid.uuid4()))
headers = {"X-Request-Id": request_id}
if status_code == 401:
headers["WWW-Authenticate"] = (
'Bearer realm="messenger"'
if request.url.path != "/api/v1/auth/sessions"
else 'Basic realm="messenger", charset="UTF-8"'
)
return JSONResponse(
status_code=status_code,
content={"error": {"code": code, "message": message, "request_id": request_id}},
headers=headers,
)
def bearer_user(request: Request) -> dict[str, Any]:
header = request.headers.get("Authorization", "")
scheme, _, token = header.partition(" ")
if scheme.lower() != "bearer" or not token or " " in token:
raise APIError(401, "unauthorized", "valid bearer session required")
hashed_token = token_hash(token)
with db() as conn:
user = conn.execute(
"""
SELECT u.id, u.username, u.display_name, u.created_at
FROM sessions s JOIN users u ON u.id = s.user_id
WHERE s.token_hash = %s AND s.revoked_at IS NULL
AND s.expires_at > now()
""",
(hashed_token,),
).fetchone()
if user is None:
raise APIError(401, "unauthorized", "valid bearer session required")
return user
def basic_credentials(request: Request) -> tuple[str, str]:
header = request.headers.get("Authorization", "")
scheme, _, encoded = header.partition(" ")
if scheme.lower() != "basic" or not encoded:
raise APIError(401, "unauthorized", "basic credentials required")
try:
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
username, separator, password = decoded.partition(":")
except (binascii.Error, UnicodeDecodeError):
raise APIError(401, "unauthorized", "invalid credentials") from None
if not separator or not username or not password:
raise APIError(401, "unauthorized", "invalid credentials")
return username, password
def token_hash(token: str) -> str:
try:
return hashlib.sha256(token.encode("ascii", "strict")).hexdigest()
except UnicodeEncodeError:
raise APIError(401, "unauthorized", "valid bearer session required") from None
def encode_cursor(created_at: datetime, object_id: uuid.UUID) -> str:
payload = json.dumps(
{"created_at": iso(created_at), "id": str(object_id)},
separators=(",", ":"),
).encode()
return base64.urlsafe_b64encode(payload).decode().rstrip("=")
def decode_cursor(cursor: str) -> tuple[datetime, uuid.UUID]:
try:
padding = "=" * ((4 - len(cursor) % 4) % 4)
payload = json.loads(
base64.urlsafe_b64decode((cursor + padding).encode()).decode()
)
created_at = datetime.fromisoformat(payload["created_at"].replace("Z", "+00:00"))
return created_at, uuid.UUID(payload["id"])
except (KeyError, TypeError, ValueError, UnicodeDecodeError, binascii.Error, json.JSONDecodeError):
raise APIError(400, "invalid_request", "cursor is invalid") from None
app = FastAPI(title="MTUSI Messenger Lab2")
@app.on_event("startup")
def startup() -> None:
wait_for_database()
wait_for_s3()
@app.middleware("http")
async def add_request_id(request: Request, call_next):
request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4())
request.state.request_id = request_id
response = await call_next(request)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
@app.exception_handler(APIError)
async def api_error_handler(request: Request, exc: APIError):
return error_response(request, exc.status_code, exc.code, exc.message)
@app.exception_handler(RequestValidationError)
async def validation_error_handler(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "invalid request")
@app.exception_handler(Exception)
async def unexpected_error_handler(request: Request, exc: Exception):
return error_response(request, 500, "internal_error", "internal server error")
@app.get("/health/live")
def health_live():
return {"status": "ok"}
@app.get("/health/ready")
def health_ready():
try:
with db() as conn:
conn.execute("SELECT 1")
s3().head_bucket(Bucket=S3_BUCKET)
except (OSError, psycopg.Error, BotoCoreError, ClientError):
raise APIError(503, "unavailable", "required storage unavailable")
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
def create_user(payload: CreateUser):
user_id = uuid.uuid4()
password_hash = PASSWORD_HASHER.hash(payload.password)
try:
with db() as conn:
user = conn.execute(
"""
INSERT INTO users(id, username, display_name, password_hash)
VALUES (%s, %s, %s, %s)
RETURNING id, username, display_name, created_at
""",
(user_id, payload.username, payload.display_name, password_hash),
).fetchone()
except psycopg.errors.UniqueViolation:
raise APIError(409, "conflict", "user already exists") from None
return {**user, "id": str(user["id"]), "created_at": iso(user["created_at"])}
@app.post("/api/v1/auth/sessions", status_code=201)
def create_session(request: Request):
username, password = basic_credentials(request)
with db() as conn:
user = conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,),
).fetchone()
if user is None:
raise APIError(401, "unauthorized", "invalid credentials")
try:
PASSWORD_HASHER.verify(user["password_hash"], password)
except VerificationError:
raise APIError(401, "unauthorized", "invalid credentials") from None
token = secrets.token_urlsafe(32)
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
with db() as conn:
conn.execute(
"INSERT INTO sessions(token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash(token), user["id"], expires_at),
)
public_user = {
"id": str(user["id"]),
"username": user["username"],
"display_name": user["display_name"],
"created_at": iso(user["created_at"]),
}
return {
"token": token,
"token_type": "Bearer",
"expires_at": iso(expires_at),
"user": public_user,
}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def revoke_session(request: Request):
bearer_user(request)
header = request.headers["Authorization"]
token = header.split(" ", 1)[1]
with db() as conn:
conn.execute(
"UPDATE sessions SET revoked_at = now() WHERE token_hash = %s AND revoked_at IS NULL",
(token_hash(token),),
)
return Response(status_code=204)
@app.get("/api/v1/users/me")
def current_user(request: Request):
user = bearer_user(request)
return {**user, "id": str(user["id"]), "created_at": iso(user["created_at"])}
@app.post("/api/v1/chats", status_code=201)
def create_chat(payload: CreateChat, request: Request):
actor = bearer_user(request)
member_ids = [uuid.UUID(member_id) for member_id in payload.member_ids]
if actor["id"] not in member_ids:
raise APIError(400, "invalid_request", "actor must be included in member_ids")
chat_id = uuid.uuid4()
with db() as conn:
existing = conn.execute(
"SELECT id FROM users WHERE id = ANY(%s)", (member_ids,)
).fetchall()
if len(existing) != len(member_ids):
raise APIError(400, "invalid_request", "member not found")
chat = conn.execute(
"INSERT INTO chats(id, title) VALUES (%s, %s) RETURNING id, title, created_at",
(chat_id, payload.title),
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members(chat_id, user_id) VALUES (%s, %s)",
[(chat_id, member_id) for member_id in member_ids],
)
return {
"id": str(chat["id"]),
"title": chat["title"],
"member_ids": sorted(str(member_id) for member_id in member_ids),
"created_at": iso(chat["created_at"]),
}
def chat_for_user(conn, chat_id: uuid.UUID, user_id: uuid.UUID):
return conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats c JOIN chat_members m ON m.chat_id = c.id
WHERE c.id = %s AND m.user_id = %s
""",
(chat_id, user_id),
).fetchone()
def public_chat(conn, chat: dict[str, Any]) -> dict[str, Any]:
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", (chat["id"],)
).fetchall()
return {
"id": str(chat["id"]),
"title": chat["title"],
"member_ids": [str(row["user_id"]) for row in members],
"created_at": iso(chat["created_at"]),
}
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: uuid.UUID, request: Request):
actor = bearer_user(request)
with db() as conn:
chat = chat_for_user(conn, chat_id, actor["id"])
if chat is None:
raise APIError(404, "not_found", "chat not found")
return public_chat(conn, chat)
@app.get("/api/v1/chats")
def list_chats(request: Request, limit: int = Query(50, ge=1, le=100), cursor: str | None = None):
actor = bearer_user(request)
after = decode_cursor(cursor) if cursor else None
with db() as conn:
rows = conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats c JOIN chat_members m ON m.chat_id = c.id
WHERE m.user_id = %s
AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))
ORDER BY c.created_at, c.id
LIMIT %s
""",
(actor["id"], after[0] if after else None, after[0] if after else None,
after[1] if after else None, limit + 1),
).fetchall()
items = [public_chat(conn, row) for row in rows[:limit]]
next_cursor = (
encode_cursor(rows[limit - 1]["created_at"], rows[limit - 1]["id"])
if len(rows) > limit
else None
)
return {"items": items, "next_cursor": next_cursor}
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def create_message(
chat_id: uuid.UUID,
payload: CreateMessage,
request: Request,
):
actor = bearer_user(request)
idempotency_key = request.headers.get("Idempotency-Key", "")
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key):
raise APIError(400, "invalid_request", "valid Idempotency-Key required")
attachment_ids = sorted(uuid.UUID(value) for value in payload.attachment_ids)
if not payload.text.strip() and not attachment_ids:
raise APIError(400, "invalid_request", "text or attachment is required")
request_hash = hashlib.sha256(
json.dumps(
{"text": payload.text, "attachment_ids": [str(value) for value in attachment_ids]},
ensure_ascii=False,
sort_keys=True,
).encode()
).hexdigest()
with db() as conn:
if chat_for_user(conn, chat_id, actor["id"]) is None:
raise APIError(404, "not_found", "chat not accessible")
if attachment_ids:
attachments = conn.execute(
"""
SELECT id, status FROM attachments
WHERE id = ANY(%s) AND chat_id = %s
""",
(attachment_ids, chat_id),
).fetchall()
if len(attachments) != len(attachment_ids):
raise APIError(404, "not_found", "attachment not found")
if any(row["status"] != "ready" for row in attachments):
raise APIError(409, "conflict", "attachment is not ready")
existing = conn.execute(
"""
INSERT INTO idempotency_keys(user_id, chat_id, idempotency_key, request_hash)
VALUES (%s, %s, %s, %s)
ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING
RETURNING message_id, request_hash
""",
(actor["id"], chat_id, idempotency_key, request_hash),
).fetchone()
if existing is None:
existing = conn.execute(
"""
SELECT message_id, request_hash
FROM idempotency_keys
WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s
FOR UPDATE
""",
(actor["id"], chat_id, idempotency_key),
).fetchone()
if existing["request_hash"] != request_hash:
raise APIError(409, "conflict", "idempotency key has another request")
message = conn.execute(
"""
SELECT id, chat_id, sender_id, text, created_at
FROM messages WHERE id = %s
""",
(existing["message_id"],),
).fetchone()
if message is None:
raise APIError(503, "unavailable", "idempotent result is not ready")
return public_message(conn, message)
message_id = uuid.uuid4()
message = conn.execute(
"""
INSERT INTO messages(id, chat_id, sender_id, text)
VALUES (%s, %s, %s, %s)
RETURNING id, chat_id, sender_id, text, created_at
""",
(message_id, chat_id, actor["id"], payload.text),
).fetchone()
if attachment_ids:
with conn.cursor() as cur:
cur.executemany(
"""
INSERT INTO message_attachments(message_id, attachment_id)
VALUES (%s, %s)
""",
[(message_id, attachment_id) for attachment_id in attachment_ids],
)
conn.execute(
"""
UPDATE idempotency_keys SET message_id = %s
WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s
""",
(message_id, actor["id"], chat_id, idempotency_key),
)
return public_message(conn, message)
def validate_filename(filename: str) -> None:
if not filename or len(filename) > 255:
raise APIError(400, "invalid_request", "filename is invalid")
if any(ord(char) < 32 for char in filename) or "/" in filename or "\\" in filename:
raise APIError(400, "invalid_request", "filename is invalid")
def inspect_image(path: Any, declared_type: str) -> tuple[str, int, int]:
try:
with Image.open(path) as image:
detected_type = {
"PNG": "image/png",
"JPEG": "image/jpeg",
}.get(image.format)
if detected_type is None or detected_type != declared_type:
raise APIError(415, "unsupported_media_type", "image type is invalid")
width, height = image.size
if width * height > MAX_IMAGE_PIXELS:
raise APIError(413, "payload_too_large", "image has too many pixels")
image.verify()
path.seek(0)
with Image.open(path) as image:
width, height = image.size
image.verify()
except APIError:
raise
except (UnidentifiedImageError, OSError, Image.DecompressionBombError):
raise APIError(415, "unsupported_media_type", "image is invalid") from None
return detected_type, width, height
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
async def upload_attachment(chat_id: uuid.UUID, request: Request, filename: str):
actor = bearer_user(request)
validate_filename(filename)
declared_type = request.headers.get("content-type", "").split(";", 1)[0].lower()
if declared_type not in {"image/png", "image/jpeg"}:
raise APIError(415, "unsupported_media_type", "content type is invalid")
content_length = request.headers.get("content-length")
if content_length:
try:
if int(content_length) > MAX_UPLOAD_BYTES:
raise APIError(413, "payload_too_large", "file is too large")
except ValueError:
raise APIError(400, "invalid_request", "content length is invalid") from None
with db() as conn:
if chat_for_user(conn, chat_id, actor["id"]) is None:
raise APIError(404, "not_found", "chat not accessible")
size = 0
digest = hashlib.sha256()
object_key = "attachments/" + str(uuid.uuid4())
uploaded = False
try:
with tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") as temporary:
async for chunk in request.stream():
size += len(chunk)
if size > MAX_UPLOAD_BYTES:
raise APIError(413, "payload_too_large", "file is too large")
digest.update(chunk)
temporary.write(chunk)
if size == 0:
raise APIError(415, "unsupported_media_type", "image is empty")
temporary.seek(0)
detected_type, _, _ = inspect_image(temporary, declared_type)
temporary.seek(0)
try:
await run_in_threadpool(
s3().upload_fileobj,
temporary,
S3_BUCKET,
object_key,
ExtraArgs={"ContentType": detected_type},
)
uploaded = True
except (BotoCoreError, ClientError, OSError):
raise APIError(503, "unavailable", "object storage unavailable") from None
except APIError:
raise
except (OSError, ValueError):
raise APIError(503, "unavailable", "upload could not be completed") from None
attachment_id = uuid.uuid4()
now = utc_now()
row = {
"id": attachment_id,
"chat_id": chat_id,
"owner_id": actor["id"],
"filename": filename,
"content_type": detected_type,
"size_bytes": size,
"sha256": digest.hexdigest(),
"status": "ready",
"error_code": None,
"created_at": now,
"updated_at": now,
}
try:
with db() as conn:
conn.execute(
"""
INSERT INTO attachments(
id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, status, object_key, error_code, created_at, updated_at
)
VALUES (%s, %s, %s, %s, %s, %s, %s, 'ready', %s, NULL, %s, %s)
""",
(
attachment_id,
chat_id,
actor["id"],
filename,
detected_type,
size,
digest.hexdigest(),
object_key,
now,
now,
),
)
except psycopg.Error:
if uploaded:
try:
s3().delete_object(Bucket=S3_BUCKET, Key=object_key)
except (BotoCoreError, ClientError, OSError):
LOGGER.error("attachment compensation failed for object key")
raise APIError(503, "unavailable", "attachment metadata unavailable") from None
return attachment_public(row)
@app.get("/api/v1/attachments/{attachment_id}")
def get_attachment(attachment_id: uuid.UUID, request: Request):
actor = bearer_user(request)
with db() as conn:
row = conn.execute(
"""
SELECT a.*
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id
WHERE a.id = %s AND m.user_id = %s
""",
(attachment_id, actor["id"]),
).fetchone()
if row is None:
raise APIError(404, "not_found", "attachment not found")
return attachment_public(row)
@app.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(
attachment_id: uuid.UUID,
request: Request,
variant: str = "original",
):
actor = bearer_user(request)
if variant != "original":
raise APIError(404, "not_found", "attachment variant not found")
with db() as conn:
row = conn.execute(
"""
SELECT a.*
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id
WHERE a.id = %s AND m.user_id = %s
""",
(attachment_id, actor["id"]),
).fetchone()
if row is None:
raise APIError(404, "not_found", "attachment not found")
try:
object_response = s3().get_object(Bucket=S3_BUCKET, Key=row["object_key"])
except (BotoCoreError, ClientError, OSError):
raise APIError(503, "unavailable", "object storage unavailable") from None
body = object_response["Body"]
def chunks():
try:
while True:
chunk = body.read(1024 * 1024)
if not chunk:
break
yield chunk
finally:
body.close()
return StreamingResponse(
chunks(),
media_type=row["content_type"],
headers={"Content-Length": str(row["size_bytes"])},
)
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(
chat_id: uuid.UUID,
request: Request,
limit: int = Query(50, ge=1, le=100),
cursor: str | None = None,
):
actor = bearer_user(request)
after = decode_cursor(cursor) if cursor else None
with db() as conn:
if chat_for_user(conn, chat_id, actor["id"]) is None:
raise APIError(404, "not_found", "chat not accessible")
rows = conn.execute(
"""
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = %s
AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))
ORDER BY created_at, id
LIMIT %s
""",
(chat_id, after[0] if after else None, after[0] if after else None,
after[1] if after else None, limit + 1),
).fetchall()
items = [public_message(conn, row) for row in rows[:limit]]
next_cursor = (
encode_cursor(rows[limit - 1]["created_at"], rows[limit - 1]["id"])
if len(rows) > limit
else None
)
return {"items": items, "next_cursor": next_cursor}
if __name__ == "__main__":
import uvicorn
uvicorn.run("app:app", host=APP_HOST, port=APP_PORT, reload=False)
+167
View File
@@ -0,0 +1,167 @@
services:
postgres:
image: postgres:16.4-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 3s
retries: 30
s3-config:
image: alpine:3.20.3
environment:
S3_BOOTSTRAP_ACCESS_KEY: ${S3_BOOTSTRAP_ACCESS_KEY:?set S3_BOOTSTRAP_ACCESS_KEY in .env}
S3_BOOTSTRAP_SECRET_KEY: ${S3_BOOTSTRAP_SECRET_KEY:?set S3_BOOTSTRAP_SECRET_KEY in .env}
S3_APP_ACCESS_KEY: ${S3_APP_ACCESS_KEY:?set S3_APP_ACCESS_KEY in .env}
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY:?set S3_APP_SECRET_KEY in .env}
volumes:
- s3_config:/config
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
printf '{"identities":[{"name":"bootstrap","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin"]},{"name":"messenger-api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:messenger-attachments","Write:messenger-attachments","List:messenger-attachments","Tagging:messenger-attachments"]}]}' \
"$$S3_BOOTSTRAP_ACCESS_KEY" "$$S3_BOOTSTRAP_SECRET_KEY" \
"$$S3_APP_ACCESS_KEY" "$$S3_APP_SECRET_KEY" > /config/s3.json
restart: "no"
s3:
image: chrislusf/seaweedfs:3.80
depends_on:
s3-config:
condition: service_completed_successfully
command: ["server", "-dir=/data", "-s3", "-s3.config=/etc/seaweed/s3.json"]
volumes:
- s3_data:/data
- s3_config:/etc/seaweed:ro
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status"]
interval: 2s
timeout: 3s
retries: 30
s3-bootstrap:
build: .
environment:
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: ${S3_BOOTSTRAP_ACCESS_KEY}
S3_SECRET_ACCESS_KEY: ${S3_BOOTSTRAP_SECRET_KEY}
depends_on:
s3:
condition: service_healthy
command:
- python
- -c
- |
import time
import boto3
from botocore.exceptions import BotoCoreError, ClientError
client = boto3.client("s3", endpoint_url="http://s3:8333", region_name="us-east-1", aws_access_key_id="${S3_BOOTSTRAP_ACCESS_KEY}", aws_secret_access_key="${S3_BOOTSTRAP_SECRET_KEY}")
for attempt in range(60):
try:
client.create_bucket(Bucket="messenger-attachments")
break
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
break
raise
except (BotoCoreError, OSError):
if attempt == 59:
raise
time.sleep(1)
restart: "no"
migrate:
build: .
environment:
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
depends_on:
postgres:
condition: service_healthy
command: ["python", "-c", "import app; app.wait_for_database()"]
restart: "no"
api1:
build: .
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
INSTANCE_ID: api1
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: ${S3_APP_ACCESS_KEY}
S3_SECRET_ACCESS_KEY: ${S3_APP_SECRET_KEY}
MAX_UPLOAD_BYTES: 10485760
MAX_IMAGE_PIXELS: 40000000
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 2s
timeout: 3s
retries: 30
restart: unless-stopped
api2:
build: .
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
INSTANCE_ID: api2
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: ${S3_APP_ACCESS_KEY}
S3_SECRET_ACCESS_KEY: ${S3_APP_SECRET_KEY}
MAX_UPLOAD_BYTES: 10485760
MAX_IMAGE_PIXELS: 40000000
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 2s
timeout: 3s
retries: 30
restart: unless-stopped
load_balancer:
image: nginx:1.27.2-alpine
ports:
- "8080:80"
volumes:
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
depends_on:
api1:
condition: service_healthy
api2:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1/health/live"]
interval: 2s
timeout: 3s
retries: 30
restart: unless-stopped
volumes:
postgres_data:
s3_data:
s3_config:
Binary file not shown.
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"]
}
]
}
+38
View File
@@ -0,0 +1,38 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username varchar(32) NOT NULL UNIQUE,
display_name varchar(100) NOT NULL CHECK (btrim(display_name) <> ''),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title varchar(100) NOT NULL CHECK (btrim(title) <> ''),
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id),
text varchar(4000) NOT NULL CHECK (btrim(text) <> ''),
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE sessions (
token_hash char(64) PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL,
revoked_at timestamptz
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
CREATE INDEX messages_chat_order_idx ON messages(chat_id, created_at, id);
CREATE INDEX sessions_user_idx ON sessions(user_id, expires_at);
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
idempotency_key varchar(128) NOT NULL,
request_hash char(64) NOT NULL,
message_id uuid REFERENCES messages(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, idempotency_key)
);
CREATE INDEX idempotency_created_idx ON idempotency_keys(created_at);
+25
View File
@@ -0,0 +1,25 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id),
filename varchar(255) NOT NULL,
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760),
sha256 char(64) NOT NULL,
status varchar(16) NOT NULL CHECK (status = 'ready'),
object_key text NOT NULL UNIQUE,
error_code varchar(64),
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX attachments_chat_idx ON attachments(chat_id, created_at, id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
PRIMARY KEY (message_id, attachment_id)
);
CREATE INDEX message_attachments_attachment_idx
ON message_attachments(attachment_id, message_id);
@@ -0,0 +1,2 @@
ALTER TABLE messages
DROP CONSTRAINT IF EXISTS messages_text_check;
+21
View File
@@ -0,0 +1,21 @@
upstream messenger_api {
server api1:8080 max_fails=1 fail_timeout=2s;
server api2:8080 max_fails=1 fail_timeout=2s;
}
server {
listen 80;
client_max_body_size 11m;
location / {
proxy_pass http://messenger_api;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Request-Id $http_x_request_id;
proxy_connect_timeout 1s;
proxy_read_timeout 10s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
}
}
+6
View File
@@ -0,0 +1,6 @@
argon2-cffi==23.1.0
boto3==1.35.76
fastapi==0.115.5
Pillow==11.0.0
psycopg[binary]==3.2.3
uvicorn==0.32.0
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env python3
"""Report or remove old S3 objects that have no PostgreSQL attachment row."""
import argparse
import datetime as dt
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import app
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--grace-seconds", type=int, default=3600)
parser.add_argument("--apply", action="store_true")
args = parser.parse_args()
if args.grace_seconds < 0:
parser.error("--grace-seconds must be non-negative")
with app.db() as conn:
known = {
row["object_key"]
for row in conn.execute("SELECT object_key FROM attachments").fetchall()
}
now = dt.datetime.now(dt.timezone.utc)
client = app.s3()
paginator = client.get_paginator("list_objects_v2")
candidates = []
for page in paginator.paginate(Bucket=app.S3_BUCKET):
for item in page.get("Contents", []):
key = item["Key"]
modified = item["LastModified"]
if key not in known and (now - modified).total_seconds() >= args.grace_seconds:
candidates.append(key)
for key in candidates:
if args.apply:
client.delete_object(Bucket=app.S3_BUCKET, Key=key)
print(("deleted" if args.apply else "candidates") + f"={len(candidates)}")
if __name__ == "__main__":
main()
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -eu
docker compose up -d --force-recreate postgres s3
docker compose run --rm migrate
docker compose run --rm s3-bootstrap
docker compose up -d --force-recreate api1 api2 load_balancer
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -eu
case "${TARGET_INSTANCE_ID:-}" in
api1) service=api1 ;;
api2) service=api2 ;;
*) echo "unknown TARGET_INSTANCE_ID" >&2; exit 2 ;;
esac
docker compose stop "$service"
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""Negative attachment checks required for the level-4 assessment."""
import hashlib
from pathlib import Path
import uuid
from client import Client, arguments, check, run
def main():
args = arguments("lab4 level-4 attachment checks")
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
second_chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
sample = Path(__file__).with_name("fixtures").joinpath("sample.png").read_bytes()
client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=large.png",
413,
actor=alice,
raw=b"x" * (10 * 1024 * 1024 + 1),
headers={"Content-Type": "image/png"},
)
client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=broken.png",
415,
actor=alice,
raw=b"not an image",
headers={"Content-Type": "image/png"},
)
client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=spoof.jpg",
415,
actor=alice,
raw=sample,
headers={"Content-Type": "image/jpeg"},
)
client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=..%2Fsecret.png",
400,
actor=alice,
raw=sample,
headers={"Content-Type": "image/png"},
)
attachment, content = client.attachment(alice, chat)
check(hashlib.sha256(content).hexdigest() == attachment["sha256"], "fixture checksum changed")
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}",
404,
actor=eve,
)
client.call(
"POST",
f"/api/v1/chats/{second_chat['id']}/messages",
404,
actor=alice,
body={"text": "wrong chat", "attachment_ids": [attachment["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
print("PASS: lab4 level-4 attachment validation and access checks")
if __name__ == "__main__":
run(main)
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Run ten concurrent bounded uploads and sample API RSS via Docker."""
import concurrent.futures
import subprocess
import threading
import time
from pathlib import Path
from client import Client, arguments, check, run
def rss_mb() -> float:
output = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
text=True,
)
values = []
for line in output.splitlines():
if "api1" not in line and "api2" not in line:
continue
used = line.split()[1].split("/")[0]
factor = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}
values.append(float(used[:-3]) * factor[used[-3:]])
return max(values, default=0.0)
def main():
args = arguments("lab4 concurrent bounded upload experiment")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
content = Path(__file__).with_name("fixtures").joinpath("sample.png").read_bytes()
stop = threading.Event()
samples = [rss_mb()]
def sample():
while not stop.is_set():
try:
samples.append(rss_mb())
except (OSError, ValueError, subprocess.SubprocessError):
pass
time.sleep(0.1)
sampler = threading.Thread(target=sample, daemon=True)
sampler.start()
started = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
futures = [
pool.submit(
client.call,
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=load-{index}.png",
201,
None,
alice,
{"Content-Type": "image/png"},
content,
"Attachment",
)
for index in range(10)
]
[future.result() for future in futures]
elapsed = time.monotonic() - started
stop.set()
sampler.join(timeout=1)
samples.append(rss_mb())
check(len(samples) > 0, "RSS sampling did not produce data")
print(f"uploads=10 concurrency=10 elapsed_seconds={elapsed:.3f} peak_api_rss_mb={max(samples):.1f}")
if __name__ == "__main__":
run(main)