From ae2bd700b51bc23616b8370bea6711a67f298c75 Mon Sep 17 00:00:00 2001 From: Yuliiiaaaa Date: Tue, 10 Nov 2026 13:12:50 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A04=20-=20=D0=91=D0=92=D0=A22302=20-?= =?UTF-8?q?=20=D0=9D=D0=B8=D0=BA=D0=B8=D1=82=D0=B8=D0=BD=D0=B0=20-=20?= =?UTF-8?q?=D0=BD=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + Dockerfile | 10 + Makefile | 50 ++- REPORT.md | 50 ++- compose.yaml | 86 +++++ evidence/track2_reconcile.txt | 27 ++ migrations/001_init.sql | 34 ++ migrations/002_idempotency.sql | 8 + migrations/003_attachments.sql | 15 + migrations/004_fix_text.sql | 3 + nginx/nginx.conf | 25 ++ requirements.txt | 9 + scripts/generate_data.sql | 41 +++ scripts/reconcile.py | 75 +++++ scripts/restart.bat | 2 + scripts/restart.ps1 | 3 + scripts/restart.sh | 4 + scripts/run_track1.ps1 | 79 +++++ scripts/run_track2.ps1 | 43 +++ scripts/stop-one.bat | 2 + scripts/stop-one.ps1 | 8 + src/app.py | 595 +++++++++++++++++++++++++++++++++ src/s3_client.py | 34 ++ src/security.py | 16 + tests/failover.py | 93 ++++++ tests/test_persistence.py | 61 ++++ 26 files changed, 1332 insertions(+), 42 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/track2_reconcile.txt create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_fix_text.sql create mode 100644 nginx/nginx.conf create mode 100644 requirements.txt create mode 100644 scripts/generate_data.sql create mode 100644 scripts/reconcile.py create mode 100644 scripts/restart.bat create mode 100644 scripts/restart.ps1 create mode 100644 scripts/restart.sh create mode 100644 scripts/run_track1.ps1 create mode 100644 scripts/run_track2.ps1 create mode 100644 scripts/stop-one.bat create mode 100644 scripts/stop-one.ps1 create mode 100644 src/app.py create mode 100644 src/s3_client.py create mode 100644 src/security.py create mode 100644 tests/failover.py create mode 100644 tests/test_persistence.py diff --git a/.gitignore b/.gitignore index b5a71896..fd471046 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,6 @@ coverage/ .DS_Store .idea/ .vscode/ +downloaded.png # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..21b96262 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.7-slim-bookworm +RUN useradd -m -u 10001 appuser +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY src/ /app/src/ +COPY migrations/ /app/migrations/ +USER appuser +EXPOSE 8080 +CMD ["uvicorn", "src.app:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/Makefile b/Makefile index 5d86d171..ce30a8e8 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,48 @@ -PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 -CA_FILE ?= -ACTION_SCRIPT ?= -TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) +PYTHON ?= python +BASE_URL ?= http://127.0.0.1:8080 +CA_FILE ?= +IMAGE ?= mtusi-messenger:lab3 +CONTAINER ?= mtusi-messenger-lab3 -.PHONY: help check validate test load test-persistence test-failover +# Флаг -X utf8 заставляет Python всегда использовать кодировку UTF-8 (лечит ошибку на Windows) +PY = $(PYTHON) -X utf8 + +.PHONY: help check validate test build run stop student-tests experiment test-persistence test-failover help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "targets: check validate test build run stop student-tests experiment test-persistence test-failover" check: - $(PYTHON) scripts/check_contract.py + $(PY) scripts/check_contract.py validate: - $(PYTHON) scripts/validate_openapi.py + $(PY) scripts/validate_openapi.py test: - $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PY) tests/smoke.py --base-url $(BASE_URL) -load: - $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +# Для Лабораторной 3 мы используем Docker Compose с масштабированием +build: + docker compose build + +run: + docker compose up -d --scale api=2 + +stop: + docker compose down + +student-tests: + $(PY) student-tests/test_negative.py --base-url $(BASE_URL) + $(PY) student-tests/test_pagination.py --base-url $(BASE_URL) + $(PY) student-tests/test_concurrency.py --base-url $(BASE_URL) + +experiment: + $(PY) student-tests/load_experiment.py --base-url $(BASE_URL) \ + --history 1000 --concurrency 4 --duration 20 --output evidence/run.json test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) - $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PY) tests/test_persistence.py --base-url $(BASE_URL) --action-script $(ACTION_SCRIPT) + test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) - $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PY) tests/failover.py --base-url $(BASE_URL) --action-script "$(ACTION_SCRIPT)" \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..99069000 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,35 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Никитина Юлия Владимировна, БВТ2302 +- Целевая оценка: 5 +- Commit/PR: https://gitea.privcloud.tech/Julia_Nikitina/mtusi2026lab4 +- Использованные источники, библиотеки и помощь ИИ: Официальная документация MinIO (S3), FastAPI, PostgreSQL, Boto3. ИИ использовался для генерации шаблона отчета и ревью кода. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker Desktop с Docker Compose, Python 3.11+ (для тестов). +Все зависимости (PostgreSQL, Redis, MinIO, Nginx, API) поднимаются из `compose.yaml`. -## Архитектура и решения +```powershell +git clone && cd mtusi2026lab4 -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +# 1) Создание локального .env +@" +POSTGRES_PASSWORD=messenger_super_secret_password +S3_ACCESS_KEY=minioadmin +S3_SECRET_KEY=minioadmin +"@ | Out-File -FilePath .env -Encoding utf8 -## Доказательства критериев +# 2) Сборка и запуск +docker compose up -d --build --scale api=1 -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +# 3) Ожидание готовности +Start-Sleep -Seconds 5 +curl.exe -s http://127.0.0.1:8080/health/ready +# Ожидается: {"status":"ok"} -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +# 4) Публичный smoke-тест +make test -## Эксперименты и отказы - -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +# 5) Демонстрация Трека 2 (Согласованность S3 и БД) +docker compose exec api python /app/scripts/reconcile.py \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f7ef3740 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,86 @@ +services: + db: + image: postgres:16.3-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: messenger + volumes: + - pgdata:/var/lib/postgresql/data + - ./migrations:/docker-entrypoint-initdb.d/ + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger"] + interval: 5s + retries: 5 + + redis: + image: redis:7.2.5-bookworm + command: redis-server --appendonly yes + volumes: + - redisdata:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + + # АЛЬТЕРНАТИВА: стабильный S3-сервер, совместимый с MinIO + minio: + image: pgsty/silo:latest # Форк MinIO (сохраняет S3 API и флаги MINIO_*) + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY:-minioadmin} + MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-minioadmin} + volumes: + - miniodata:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + retries: 5 + + # АЛЬТЕРНАТИВА: легкий публичный клиент для создания бакета + minio-init: + image: netvark/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 $$S3_ACCESS_KEY $$S3_SECRET_KEY && + mc mb --ignore-existing local/attachments + " + environment: + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-minioadmin} + S3_SECRET_KEY: ${S3_SECRET_KEY:-minioadmin} + + api: + build: . + environment: + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-minioadmin} + S3_SECRET_KEY: ${S3_SECRET_KEY:-minioadmin} + S3_BUCKET: attachments + volumes: + - ./scripts:/app/scripts + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + + + nginx: + image: nginx:1.25.5-alpine + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - api + +volumes: + pgdata: + redisdata: + miniodata: \ No newline at end of file diff --git a/evidence/track2_reconcile.txt b/evidence/track2_reconcile.txt new file mode 100644 index 00000000..bb5ac877 --- /dev/null +++ b/evidence/track2_reconcile.txt @@ -0,0 +1,27 @@ +=== ТРЕК 2: Согласованность S3 и PostgreSQL === + +[1] Исходное состояние (всё синхронизировано): +Total records in DB: 1 +Orphan objects (in S3, not in DB): 0 +Missing objects (in DB, not in S3): 0 + +[2] Создаем Orphan object (загружаем файл в S3 в обход API): +docker compose exec api python -c "... s3.put_object(..., Key='garbage_file.png', ...)" +(Файл garbage_file.png загружен в бакет) + +[3] Запускаем проверку (Dry-run): +Total objects in S3: 2 +Total records in DB: 1 +Orphan objects (in S3, not in DB): 1 +[ORPHANS] (Safe to delete, no metadata in DB): + - garbage_file.png + +[4] Запускаем исправление (--fix): +[FIX] Deleting orphan objects from S3... + Deleted garbage_file.png + +[5] Финальная проверка: +Total objects in S3: 1 +Total records in DB: 1 +Orphan objects (in S3, not in DB): 0 +Missing objects (in DB, not in S3): 0 \ No newline at end of file diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..9c805d55 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,34 @@ +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT username_format CHECK (username ~ '^[a-z0-9_]{3,32}$') +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE messages ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Критический индекс для keyset-пагинации и сортировки истории +CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id); \ No newline at end of file diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..eefccf97 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,8 @@ +CREATE TABLE idempotency_keys ( + key VARCHAR(128) NOT NULL, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + request_hash VARCHAR(64) NOT NULL, + message_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (key, user_id) +); \ No newline at end of file diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..42065545 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,15 @@ +CREATE TABLE attachments ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(100) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 VARCHAR(64) NOT NULL, + s3_key VARCHAR(512) NOT NULL UNIQUE, + status VARCHAR(20) NOT NULL DEFAULT 'ready', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +ALTER TABLE messages ADD COLUMN attachment_ids UUID[] DEFAULT '{}'; \ No newline at end of file diff --git a/migrations/004_fix_text.sql b/migrations/004_fix_text.sql new file mode 100644 index 00000000..656fe678 --- /dev/null +++ b/migrations/004_fix_text.sql @@ -0,0 +1,3 @@ +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check +CHECK (char_length(text) BETWEEN 1 AND 4000 OR cardinality(attachment_ids) > 0); \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..7e484529 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,25 @@ +worker_processes 1; + +events { worker_connections 1024; } + +http { + # Используем встроенный DNS Docker для динамического разрешения имен + resolver 127.0.0.11 valid=5s; + + server { + listen 8080; + + location / { + # Переменная заставляет Nginx перепроверять IP контейнеров + set $backend "http://api:8080"; + proxy_pass $backend; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + + # Таймауты для быстрого отсечения упавших реплик + proxy_connect_timeout 2s; + proxy_next_upstream error timeout http_502 http_503 http_504; + } + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..efb1308d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +openapi-spec-validator==0.7.2 +fastapi==0.115.0 +uvicorn[standard]==0.30.6 +psycopg[binary,pool]==3.2.1 +redis==5.0.8 +pydantic==2.9.2 +boto3==1.34.162 +Pillow==10.4.0 +python-multipart==0.0.9 \ No newline at end of file diff --git a/scripts/generate_data.sql b/scripts/generate_data.sql new file mode 100644 index 00000000..d2c60ec5 --- /dev/null +++ b/scripts/generate_data.sql @@ -0,0 +1,41 @@ +-- Создаем тестового пользователя +INSERT INTO users (id, username, display_name, password_hash) +VALUES ('00000000-0000-0000-0000-000000000001', 'load_user', 'Load User', 'fake_hash') +ON CONFLICT DO NOTHING; + +-- Создаем большой чат (50 000 сообщений) +INSERT INTO chats (id, title) VALUES ('00000000-0000-0000-0000-000000000002', 'Big Chat'); +INSERT INTO chat_members (chat_id, user_id) VALUES ('00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001'); + +-- Создаем средний чат (30 000 сообщений) +INSERT INTO chats (id, title) VALUES ('00000000-0000-0000-0000-000000000003', 'Medium Chat'); +INSERT INTO chat_members (chat_id, user_id) VALUES ('00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001'); + +-- Создаем маленький чат (20 000 сообщений) +INSERT INTO chats (id, title) VALUES ('00000000-0000-0000-0000-000000000004', 'Small Chat'); +INSERT INTO chat_members (chat_id, user_id) VALUES ('00000000-0000-0000-0000-000000000004', '00000000-0000-0000-0000-000000000001'); + +-- Генерируем сообщения +INSERT INTO messages (chat_id, sender_id, text, created_at) +SELECT + '00000000-0000-0000-0000-000000000002', + '00000000-0000-0000-0000-000000000001', + 'Big chat message ' || i, + now() - (100000 - i) * interval '1 second' +FROM generate_series(1, 50000) AS i; + +INSERT INTO messages (chat_id, sender_id, text, created_at) +SELECT + '00000000-0000-0000-0000-000000000003', + '00000000-0000-0000-0000-000000000001', + 'Medium chat message ' || i, + now() - (100000 - i) * interval '1 second' +FROM generate_series(1, 30000) AS i; + +INSERT INTO messages (chat_id, sender_id, text, created_at) +SELECT + '00000000-0000-0000-0000-000000000004', + '00000000-0000-0000-0000-000000000001', + 'Small chat message ' || i, + now() - (100000 - i) * interval '1 second' +FROM generate_series(1, 20000) AS i; \ No newline at end of file diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 00000000..452e8d50 --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +import os +import argparse +import boto3 +from botocore.client import Config +import psycopg + +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@db:5432/messenger") +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://minio:9000") +S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "minioadmin") +S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "minioadmin") +S3_BUCKET = os.environ.get("S3_BUCKET", "attachments") + +s3 = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + +def main(): + parser = argparse.ArgumentParser(description="Reconcile S3 objects and DB metadata.") + parser.add_argument("--fix", action="store_true", help="Delete orphan objects from S3") + args = parser.parse_args() + + print("Connecting to S3 and PostgreSQL...") + + # 1. Получаем все ключи из S3 + s3_keys = set() + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=S3_BUCKET): + for obj in page.get("Contents", []): + s3_keys.add(obj["Key"]) + + # 2. Получаем все s3_key из БД + db_keys = set() + with psycopg.connect(DATABASE_URL) as conn: + with conn.cursor() as cur: + cur.execute("SELECT s3_key FROM attachments") + for row in cur.fetchall(): + db_keys.add(row[0]) + + # 3. Сравниваем + orphans = s3_keys - db_keys + missing = db_keys - s3_keys + + print(f"\n--- RECONCILIATION REPORT ---") + print(f"Total objects in S3: {len(s3_keys)}") + print(f"Total records in DB: {len(db_keys)}") + print(f"Orphan objects (in S3, not in DB): {len(orphans)}") + print(f"Missing objects (in DB, not in S3): {len(missing)}") + + if orphans: + print("\n[ORPHANS] (Safe to delete, no metadata in DB):") + for key in orphans: + print(f" - {key}") + + if missing: + print("\n[MISSING] (CRITICAL! Metadata exists, but file is lost):") + for key in missing: + print(f" - {key}") + + if args.fix and orphans: + print("\n[FIX] Deleting orphan objects from S3...") + for key in orphans: + s3.delete_object(Bucket=S3_BUCKET, Key=key) + print(f" Deleted {key}") + + if not args.fix and (orphans or missing): + print("\nHint: run with `--fix` to delete orphan objects (missing objects require manual review).") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 00000000..8c776894 --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,2 @@ +@echo off +docker compose restart \ No newline at end of file diff --git a/scripts/restart.ps1 b/scripts/restart.ps1 new file mode 100644 index 00000000..de167b9c --- /dev/null +++ b/scripts/restart.ps1 @@ -0,0 +1,3 @@ +Write-Host "Restarting the whole stack..." +docker compose down +docker compose up -d --scale api=2 \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..8f5a83b7 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +echo "Restarting the whole stack..." +docker compose down +docker compose up -d --scale api=2 \ No newline at end of file diff --git a/scripts/run_track1.ps1 b/scripts/run_track1.ps1 new file mode 100644 index 00000000..30b7b2f3 --- /dev/null +++ b/scripts/run_track1.ps1 @@ -0,0 +1,79 @@ +param( + [string]$Image = "mtusi-messenger:lab1", + [string]$Container = "mtusi-messenger-lab1", + [string]$BaseUrl = "http://127.0.0.1:8080", + [string]$History = "1000,5000,20000", + [string]$Concurrency = "1,4,16", + [int]$Duration = 15, + [int]$Warmup = 2 +) + +$ErrorActionPreference = "Stop" + +function Stop-App { + $running = docker ps -q -f "name=$Container" + if ($running) { + Write-Host " stopping container..." + docker kill --signal SIGTERM $Container | Out-Null + for ($i = 0; $i -lt 60; $i++) { + $still = docker ps -q -f "name=$Container" + if (-not $still) { break } + Start-Sleep -Milliseconds 250 + } + } +} + +function Test-Ready { + $code = (& curl.exe -s -o NUL -m 2 -w "%{http_code}" "$BaseUrl/health/ready" 2>$null) + return ("$code".Trim() -eq "200") +} + +function Start-App([string]$Mode) { + docker run --rm -d --name $Container ` + -p 127.0.0.1:8080:8080 ` + -e HOST=0.0.0.0 -e PORT=8080 ` + -e PAGINATION_MODE=$Mode ` + -e MAX_HISTORY_PER_CHAT=100000 ` + -e MAX_INFLIGHT=256 ` + $Image | Out-Null + + for ($i = 0; $i -lt 80; $i++) { + if (Test-Ready) { + Write-Host " server ready (PAGINATION_MODE=$Mode)" + return + } + Start-Sleep -Milliseconds 300 + } + + Write-Host "---- diagnostics ----" + docker ps -a --filter "name=$Container" + Write-Host "---- last logs ----" + docker logs $Container --tail 30 + throw "Server did not become ready" +} + +# ------------- LINEAR ------------- +Write-Host "=== LINEAR ===" +Stop-App +Start-App "linear" +python student-tests\load_experiment.py ` + --base-url $BaseUrl ` + --mode linear --docker-container $Container ` + --history $History --concurrency $Concurrency ` + --duration $Duration --warmup $Warmup --read-ratio 0.8 ` + --output-dir evidence + +# ------------- INDEX ------------- +Write-Host "=== INDEX ===" +Stop-App +Start-App "index" +python student-tests\load_experiment.py ` + --base-url $BaseUrl ` + --mode index --docker-container $Container ` + --history $History --concurrency $Concurrency ` + --duration $Duration --warmup $Warmup --read-ratio 0.8 ` + --output-dir evidence + +Stop-App +Write-Host "" +Write-Host "DONE. See evidence\linear_summary.json and evidence\index_summary.json" \ No newline at end of file diff --git a/scripts/run_track2.ps1 b/scripts/run_track2.ps1 new file mode 100644 index 00000000..43823a01 --- /dev/null +++ b/scripts/run_track2.ps1 @@ -0,0 +1,43 @@ +$ErrorActionPreference = "Stop" +$EvidDir = "evidence" +if (-not (Test-Path $EvidDir)) { New-Item -ItemType Directory -Path $EvidDir | Out-Null } + +Write-Host "=== TRACK 2: Gathering evidence ===" -ForegroundColor Cyan + +# 1. EXPLAIN WITH INDEX +Write-Host "1. Read query (WITH index)..." -ForegroundColor Yellow +$sql_with_idx = "EXPLAIN (ANALYZE, BUFFERS) SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '00000000-0000-0000-0000-000000000002' AND created_at > now() - interval '50 days' ORDER BY created_at ASC LIMIT 50;" +$sql_with_idx | docker compose exec -T db psql -U messenger -d messenger | Out-File -FilePath "$EvidDir\track2_explain_with_index.txt" -Encoding utf8 + +# 2. EXPLAIN WITHOUT INDEX +Write-Host "2. Read query (WITHOUT index)..." -ForegroundColor Yellow +"DROP INDEX idx_messages_chat_created;" | docker compose exec -T db psql -U messenger -d messenger | Out-Null +$sql_with_idx | docker compose exec -T db psql -U messenger -d messenger | Out-File -FilePath "$EvidDir\track2_explain_no_index.txt" -Encoding utf8 +"CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);" | docker compose exec -T db psql -U messenger -d messenger | Out-Null + +# 3. WRITE COST +Write-Host "3. Write cost analysis..." -ForegroundColor Yellow +$insert_script = @" +DROP INDEX idx_messages_chat_created; +EXPLAIN ANALYZE INSERT INTO messages (chat_id, sender_id, text) SELECT '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', 'test ' || i FROM generate_series(1, 1000) i; +CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id); +EXPLAIN ANALYZE INSERT INTO messages (chat_id, sender_id, text) SELECT '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', 'test2 ' || i FROM generate_series(1, 1000) i; +"@ +$insert_script | docker compose exec -T db psql -U messenger -d messenger | Out-File -FilePath "$EvidDir\track2_write_cost.txt" -Encoding utf8 + +# 4. BACKUP +Write-Host "4. Creating backup..." -ForegroundColor Yellow +docker compose exec -T db pg_dump -U messenger -d messenger -Fc -f /tmp/backup.dump +docker cp mtusi2026lab2-db-1:/tmp/backup.dump ./backup.dump | Out-Null + +# 5. RESTORE +Write-Host "5. Restoring from backup..." -ForegroundColor Yellow +docker compose exec -T db psql -U messenger -d postgres -c "DROP DATABASE IF EXISTS restored_db;" | Out-Null +docker compose exec -T db psql -U messenger -d postgres -c "CREATE DATABASE restored_db;" | Out-Null + +$time = Measure-Command { docker compose exec -T db pg_restore -U messenger -d restored_db -Fc /tmp/backup.dump } +"Restore time: $($time.TotalSeconds) seconds" | Out-File -FilePath "$EvidDir\track2_restore.txt" -Encoding utf8 +docker compose exec -T db psql -U messenger -d restored_db -c "SELECT COUNT(*) AS restored_messages FROM messages;" | Out-File -FilePath "$EvidDir\track2_restore.txt" -Encoding utf8 -Append + +Write-Host "=== DONE! All results saved to evidence/ ===" -ForegroundColor Green +Get-ChildItem $EvidDir \ No newline at end of file diff --git a/scripts/stop-one.bat b/scripts/stop-one.bat new file mode 100644 index 00000000..f4dbc75b --- /dev/null +++ b/scripts/stop-one.bat @@ -0,0 +1,2 @@ +@echo off +powershell -ExecutionPolicy Bypass -File scripts/stop-one.ps1 \ No newline at end of file diff --git a/scripts/stop-one.ps1 b/scripts/stop-one.ps1 new file mode 100644 index 00000000..fd6e6cc5 --- /dev/null +++ b/scripts/stop-one.ps1 @@ -0,0 +1,8 @@ +$containers = docker ps -q -f "name=mtusi2026lab4-api-*" +if (-not $containers) { + Write-Host "No API containers running" + exit 1 +} +$target = ($containers -split "`n")[0].Trim() +Write-Host "Stopping API replica $target..." +docker stop $target \ No newline at end of file diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..1868d188 --- /dev/null +++ b/src/app.py @@ -0,0 +1,595 @@ +#!/usr/bin/env python3 +import base64 +import datetime as dt +import hashlib +import json +import os +import secrets +import uuid +import socket +import io +import asyncio +from contextlib import asynccontextmanager +from typing import List, Optional +from src.s3_client import upload_file, download_file +import boto3 +from botocore.client import Config +import redis.asyncio as redis +from fastapi import FastAPI, Request, Response, HTTPException, Header, Depends, Query +from fastapi.responses import JSONResponse +from fastapi.exceptions import RequestValidationError +from psycopg_pool import AsyncConnectionPool +from pydantic import BaseModel, Field, ConfigDict +from PIL import Image + +# Configuration & S3 Setup +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@db:5432/messenger") +REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379/0") +SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://minio:9000") +S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "minioadmin") +S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "minioadmin") +S3_BUCKET = os.environ.get("S3_BUCKET", "attachments") + +s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + +MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MiB +MAX_PIXELS = 40_000_000 + +# Global state +pool: Optional[AsyncConnectionPool] = None +redis_client: Optional[redis.Redis] = None + + +# Pydantic Models +class UserCreate(BaseModel): + model_config = ConfigDict(extra='forbid') + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + password: str = Field(..., min_length=12, max_length=128) + +class UserOut(BaseModel): + id: str + username: str + display_name: str + created_at: str + +class ChatCreate(BaseModel): + model_config = ConfigDict(extra='forbid') + title: str = Field(..., min_length=1, max_length=100) + member_ids: List[str] = Field(..., min_length=1, max_length=100) + +class ChatOut(BaseModel): + id: str + title: str + member_ids: List[str] + created_at: str + +class MessageCreate(BaseModel): + model_config = ConfigDict(extra='forbid') + text: str = Field(..., max_length=4000) + attachment_ids: List[str] = Field(default_factory=list, max_length=10) + +class MessageOut(BaseModel): + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: List[str] = [] + +class PageResponse(BaseModel): + items: List[dict] + next_cursor: Optional[str] + +class SessionResponse(BaseModel): + token: str + token_type: str = "Bearer" + expires_at: str + user: UserOut + + +# Helpers +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + key = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=64) + return f"{salt.hex()}${key.hex()}" + +def verify_password(password: str, stored: str) -> bool: + try: + salt_hex, key_hex = stored.split("$") + salt = bytes.fromhex(salt_hex) + expected_key = bytes.fromhex(key_hex) + actual_key = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=64) + return secrets.compare_digest(actual_key, expected_key) + except Exception: + return False + +def generate_token() -> str: + return secrets.token_urlsafe(48) + +def encode_cursor(scope: str, created_at: str) -> str: + payload = json.dumps({"s": scope, "t": created_at}, separators=(",", ":")) + return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=") + +def decode_cursor(cursor: str, expected_scope: str) -> Optional[str]: + try: + pad = "=" * (-len(cursor) % 4) + data = json.loads(base64.urlsafe_b64decode(cursor + pad)) + if data.get("s") == expected_scope and isinstance(data.get("t"), str): + return data["t"] + except Exception: + pass + return None + +def validate_image(data: bytes, content_type: str) -> str: + if len(data) > MAX_FILE_SIZE: + raise HTTPException(413, "File too large") + try: + img = Image.open(io.BytesIO(data)) + img.verify() + except Exception: + raise HTTPException(415, "Invalid image file") + if img.format not in ("PNG", "JPEG"): + raise HTTPException(415, "Only PNG and JPEG are supported") + if img.width * img.height > MAX_PIXELS: + raise HTTPException(413, "Image has too many pixels") + return "image/png" if img.format == "PNG" else "image/jpeg" + + +# FastAPI Lifespan +@asynccontextmanager +async def lifespan(app: FastAPI): + global pool, redis_client + pool = AsyncConnectionPool(DATABASE_URL, min_size=5, max_size=50, open=False) + await pool.open() + redis_client = redis.from_url(REDIS_URL, decode_responses=True) + + async with pool.connection() as conn: + await conn.execute("SELECT 1") + await redis_client.ping() + + try: + await asyncio.to_thread(s3_client.head_bucket, Bucket=S3_BUCKET) + except Exception: + try: + await asyncio.to_thread(s3_client.create_bucket, Bucket=S3_BUCKET) + except Exception as e: + print(f"Warning: Could not create S3 bucket: {e}") + + yield + + await pool.close() + await redis_client.aclose() + +app = FastAPI(lifespan=lifespan) + + +# Middleware & Error Handling +@app.middleware("http") +async def add_instance_id(request: Request, call_next): + response = await call_next(request) + response.headers["X-Instance-Id"] = os.environ.get("HOSTNAME", socket.gethostname()) + return response + +@app.middleware("http") +async def add_request_id(request: Request, call_next): + request.state.request_id = uuid.uuid4().hex + response = await call_next(request) + response.headers["X-Request-Id"] = request.state.request_id + return response + +@app.exception_handler(HTTPException) +async def custom_http_exception_handler(request: Request, exc: HTTPException): + code_map = { + 400: "invalid_request", 401: "unauthorized", 404: "not_found", + 409: "conflict", 413: "payload_too_large", 415: "unsupported_media_type", + 503: "unavailable", 500: "internal_error" + } + error_code = code_map.get(exc.status_code, "internal_error") + payload = { + "error": { + "code": error_code, + "message": str(exc.detail), + "request_id": getattr(request.state, "request_id", "unknown") + } + } + headers = {} + if exc.status_code == 401: + if request.url.path == "/api/v1/auth/sessions": + headers["WWW-Authenticate"] = 'Basic realm="messenger", charset="UTF-8"' + else: + headers["WWW-Authenticate"] = 'Bearer realm="messenger"' + return JSONResponse(status_code=exc.status_code, content=payload, headers=headers) + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + payload = { + "error": { + "code": "invalid_request", + "message": "Invalid request parameters", + "request_id": getattr(request.state, "request_id", "unknown") + } + } + return JSONResponse(status_code=400, content=payload) + + +# Dependencies +async def get_current_user(request: Request, authorization: Optional[str] = Header(None)) -> dict: + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(401, "Missing or invalid Authorization header") + token = authorization[7:] + user_id = await redis_client.get(f"session:{token}") + if not user_id: + raise HTTPException(401, "Session expired or invalid") + + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute( + "SELECT id::text, username, display_name, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM users WHERE id = %s", + (user_id,) + ) + row = await cur.fetchone() + if not row: + raise HTTPException(401, "User no longer exists") + return {"id": row[0], "username": row[1], "display_name": row[2], "created_at": row[3]} + +# Health Endpoints +@app.get("/health/live") +async def health_live(): + return {"status": "ok"} + +@app.get("/health/ready") +async def health_ready(): + try: + async with pool.connection() as conn: + await conn.execute("SELECT 1") + await redis_client.ping() + return {"status": "ok"} + except Exception: + raise HTTPException(503, "Dependencies unavailable") + +# Auth & Users +@app.post("/api/v1/users", status_code=201, response_model=UserOut) +async def create_user(body: UserCreate): + pwd_hash = hash_password(body.password) + async with pool.connection() as conn: + try: + async with conn.cursor() as cur: + await cur.execute( + "INSERT INTO users (username, display_name, password_hash) VALUES (%s, %s, %s) RETURNING id::text, username, display_name, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')", + (body.username, body.display_name, pwd_hash) + ) + row = await cur.fetchone() + await conn.commit() + return {"id": row[0], "username": row[1], "display_name": row[2], "created_at": row[3]} + except Exception as e: + await conn.rollback() + if "unique" in str(e).lower(): + raise HTTPException(409, "Username already taken") + raise HTTPException(400, "Invalid user data") + +@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionResponse) +async def login(authorization: Optional[str] = Header(None)): + if not authorization or not authorization.startswith("Basic "): + raise HTTPException(401, "Basic authentication required") + try: + creds = base64.b64decode(authorization[6:]).decode('utf-8').split(":", 1) + username, password = creds + except Exception: + raise HTTPException(401, "Invalid credentials") + + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute("SELECT id::text, password_hash, username, display_name, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM users WHERE username = %s", (username,)) + row = await cur.fetchone() + + if not row or not verify_password(password, row[1]): + raise HTTPException(401, "Invalid credentials") + + token = generate_token() + await redis_client.setex(f"session:{token}", SESSION_TTL, row[0]) + expires = (dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=SESSION_TTL)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + return { + "token": token, "token_type": "Bearer", "expires_at": expires, + "user": {"id": row[0], "username": row[2], "display_name": row[3], "created_at": row[4]} + } + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(authorization: Optional[str] = Header(None)): + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(401, "Missing Authorization header") + token = authorization[7:] + deleted = await redis_client.delete(f"session:{token}") + if not deleted: + raise HTTPException(401, "Session invalid or already logged out") + return Response(status_code=204) + +@app.get("/api/v1/users/me", response_model=UserOut) +async def get_me(user: dict = Depends(get_current_user)): + return user + +# Chats +@app.post("/api/v1/chats", status_code=201, response_model=ChatOut) +async def create_chat(body: ChatCreate, user: dict = Depends(get_current_user)): + if user["id"] not in body.member_ids: + raise HTTPException(400, "Caller must be a member of the chat") + unique_members = list(set(body.member_ids)) + if len(unique_members) != len(body.member_ids): + raise HTTPException(400, "Duplicate member IDs") + + async with pool.connection() as conn: + try: + async with conn.transaction(): + async with conn.cursor() as cur: + await cur.execute("SELECT id::text FROM users WHERE id = ANY(%s::uuid[])", (unique_members,)) + found_users = {r[0] for r in await cur.fetchall()} + if len(found_users) != len(unique_members): + raise HTTPException(400, "One or more members do not exist") + + await cur.execute("INSERT INTO chats (title) VALUES (%s) RETURNING id::text, title, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')", (body.title,)) + chat_row = await cur.fetchone() + chat_id = chat_row[0] + + for uid in unique_members: + await cur.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat_id, uid)) + return {"id": chat_id, "title": chat_row[1], "member_ids": unique_members, "created_at": chat_row[2]} + except HTTPException: + raise + except Exception: + raise HTTPException(500, "Failed to create chat") + +@app.get("/api/v1/chats") +async def list_chats(limit: int = 50, cursor: Optional[str] = None, user: dict = Depends(get_current_user)): + if not (1 <= limit <= 100): + raise HTTPException(400, "Invalid limit") + cursor_ts = decode_cursor(cursor, "chats") if cursor else None + if cursor and not cursor_ts: + raise HTTPException(400, "Invalid cursor") + + async with pool.connection() as conn: + async with conn.cursor() as cur: + if cursor_ts: + await cur.execute("SELECT c.id::text, c.title, to_char(c.created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM chats c JOIN chat_members cm ON c.id = cm.chat_id WHERE cm.user_id = %s AND c.created_at > %s ORDER BY c.created_at ASC LIMIT %s", (user["id"], cursor_ts, limit + 1)) + else: + await cur.execute("SELECT c.id::text, c.title, to_char(c.created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM chats c JOIN chat_members cm ON c.id = cm.chat_id WHERE cm.user_id = %s ORDER BY c.created_at ASC LIMIT %s", (user["id"], limit + 1)) + rows = await cur.fetchall() + + chat_ids = [r[0] for r in rows[:limit]] + members_map = {} + if chat_ids: + await cur.execute("SELECT chat_id::text, user_id::text FROM chat_members WHERE chat_id = ANY(%s::uuid[])", (chat_ids,)) + for cid, uid in await cur.fetchall(): + members_map.setdefault(cid, []).append(uid) + + items = [{"id": r[0], "title": r[1], "created_at": r[2], "member_ids": members_map.get(r[0], [])} for r in rows[:limit]] + next_cursor = encode_cursor("chats", items[-1]["created_at"]) if len(rows) > limit else None + return {"items": items, "next_cursor": next_cursor} + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, user: dict = Depends(get_current_user)): + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user["id"])) + if not await cur.fetchone(): + raise HTTPException(404, "Chat not found") + await cur.execute("SELECT id::text, title, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM chats WHERE id = %s", (chat_id,)) + chat = await cur.fetchone() + await cur.execute("SELECT user_id::text FROM chat_members WHERE chat_id = %s", (chat_id,)) + members = [r[0] for r in await cur.fetchall()] + return {"id": chat[0], "title": chat[1], "created_at": chat[2], "member_ids": members} + + +# Attachments (Lab 4) +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(..., min_length=1, max_length=255), + user: dict = Depends(get_current_user) +): + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user["id"])) + if not await cur.fetchone(): + raise HTTPException(404, "Chat not found") + + data = await request.body() + if len(data) == 0: + raise HTTPException(400, "Empty file") + + content_type_header = request.headers.get("content-type", "") + real_content_type = validate_image(data, content_type_header) + + sha256 = hashlib.sha256(data).hexdigest() + file_id = str(uuid.uuid4()) + ext = "png" if real_content_type == "image/png" else "jpg" + s3_key = f"{chat_id}/{file_id}.{ext}" + + upload_file(s3_key, data, real_content_type) + + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, s3_key) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING id::text, filename, content_type, size_bytes, sha256, status, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"'), to_char(updated_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')", + (file_id, chat_id, user["id"], filename, real_content_type, len(data), sha256, s3_key) + ) + row = await cur.fetchone() + await conn.commit() + + return { + "id": row[0], "chat_id": chat_id, "owner_id": user["id"], "filename": row[1], + "content_type": row[2], "size_bytes": row[3], "sha256": row[4], + "status": row[5], "created_at": row[6], "updated_at": row[7], + "error_code": None, "variants": [] + } + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment_meta(attachment_id: str, user: dict = Depends(get_current_user)): + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute( + "SELECT a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type, a.size_bytes, a.sha256, a.status, to_char(a.created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"'), to_char(a.updated_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM attachments a JOIN chat_members cm ON a.chat_id = cm.chat_id " + "WHERE a.id = %s AND cm.user_id = %s", + (attachment_id, user["id"]) + ) + row = await cur.fetchone() + if not row: + raise HTTPException(404, "Attachment not found") + return { + "id": row[0], "chat_id": row[1], "owner_id": row[2], "filename": row[3], + "content_type": row[4], "size_bytes": row[5], "sha256": row[6], + "status": row[7], "created_at": row[8], "updated_at": row[9], + "error_code": None, "variants": [] + } + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, user: dict = Depends(get_current_user)): + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute( + "SELECT a.s3_key, a.content_type, a.filename FROM attachments a " + "JOIN chat_members cm ON a.chat_id = cm.chat_id " + "WHERE a.id = %s AND cm.user_id = %s", + (attachment_id, user["id"]) + ) + row = await cur.fetchone() + if not row: + raise HTTPException(404, "Attachment not found") + + s3_key, content_type, filename = row + try: + s3_response = await asyncio.to_thread(s3_client.get_object, Bucket=S3_BUCKET, Key=s3_key) + data = await asyncio.to_thread(s3_response['Body'].read) + except Exception: + raise HTTPException(404, "File content missing in S3") + + return Response( + content=data, + media_type=content_type, + headers={"Content-Disposition": f'attachment; filename="{filename}"'} + ) + +# Messages +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201, response_model=MessageOut) +async def create_message( + chat_id: str, + body: MessageCreate, + user: dict = Depends(get_current_user), + idempotency_key: str = Header(None, alias="Idempotency-Key") +): + if not idempotency_key: + raise HTTPException(400, "Idempotency-Key is required") + + # Валидация: либо текст, либо вложения + if not body.text.strip() and not body.attachment_ids: + raise HTTPException(400, "Message must have either text or attachments") + + req_hash = hashlib.sha256(body.text.encode('utf-8')).hexdigest() + new_msg_id = str(uuid.uuid4()) + attachments = body.attachment_ids if hasattr(body, 'attachment_ids') and body.attachment_ids else [] + + async with pool.connection() as conn: + async with conn.transaction(): + async with conn.cursor() as cur: + # 1. Проверяем доступ к чату + await cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user["id"])) + if not await cur.fetchone(): + raise HTTPException(404, "Chat not found") + + # 2. ПРОВЕРКА: Существуют ли эти вложения и принадлежат ли они ЭТОМУ чату? + if attachments: + # Передаем список строк в PostgreSQL как массив UUID + await cur.execute( + "SELECT id::text FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s", + (attachments, chat_id) + ) + found_attachments = {row[0] for row in await cur.fetchall()} + + # Если найдено меньше, чем передано — значит есть чужие или фейковые ID + if len(found_attachments) != len(attachments): + raise HTTPException(404, "One or more attachments not found or belong to another chat") + + # 3. Проверка идемпотентности + await cur.execute( + "INSERT INTO idempotency_keys (key, user_id, request_hash, message_id) " + "VALUES (%s, %s, %s, %s) ON CONFLICT DO NOTHING", + (idempotency_key, user["id"], req_hash, new_msg_id) + ) + + if cur.rowcount == 1: + # 4. Вставляем сообщение + await cur.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids) " + "VALUES (%s, %s, %s, %s, %s) " + "RETURNING id::text, chat_id::text, sender_id::text, text, attachment_ids::text[], to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')", + (new_msg_id, chat_id, user["id"], body.text, attachments) + ) + row = await cur.fetchone() + return {"id": row[0], "chat_id": row[1], "sender_id": row[2], "text": row[3], "attachment_ids": row[4] or [], "created_at": row[5]} + else: + # 5. Возврат кэшированного ответа при дубле + await cur.execute( + "SELECT request_hash, message_id FROM idempotency_keys WHERE key = %s AND user_id = %s", + (idempotency_key, user["id"]) + ) + key_row = await cur.fetchone() + if key_row[0] != req_hash: + raise HTTPException(409, "Idempotency key reused with different body") + + await cur.execute( + "SELECT id::text, chat_id::text, sender_id::text, text, attachment_ids::text[], to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM messages WHERE id = %s", + (key_row[1],) + ) + row = await cur.fetchone() + return {"id": row[0], "chat_id": row[1], "sender_id": row[2], "text": row[3], "attachment_ids": row[4] or [], "created_at": row[5]} + + +@app.get("/api/v1/chats/{chat_id}/messages", response_model=PageResponse) +async def list_messages(chat_id: str, limit: int = 50, cursor: Optional[str] = None, user: dict = Depends(get_current_user)): + if not (1 <= limit <= 100): + raise HTTPException(400, "Invalid limit") + + scope = f"chat:{chat_id}" + cursor_ts = decode_cursor(cursor, scope) if cursor else None + if cursor and not cursor_ts: + raise HTTPException(400, "Invalid cursor") + + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user["id"])) + if not await cur.fetchone(): + raise HTTPException(404, "Chat not found") + + query = "SELECT id::text, chat_id::text, sender_id::text, text, attachment_ids::text[], to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM messages WHERE chat_id = %s " + params = [chat_id] + if cursor_ts: + query += "AND created_at > %s " + params.append(cursor_ts) + query += "ORDER BY created_at ASC LIMIT %s" + params.append(limit + 1) + + await cur.execute(query, tuple(params)) + rows = await cur.fetchall() + + items = [] + for r in rows[:limit]: + items.append({ + "id": r[0], "chat_id": r[1], "sender_id": r[2], "text": r[3], + "attachment_ids": r[4] or [], + "created_at": r[5] + }) + + next_cursor = encode_cursor(scope, items[-1]["created_at"]) if len(rows) > limit else None + return {"items": items, "next_cursor": next_cursor} \ No newline at end of file diff --git a/src/s3_client.py b/src/s3_client.py new file mode 100644 index 00000000..ccb72723 --- /dev/null +++ b/src/s3_client.py @@ -0,0 +1,34 @@ +import boto3 +import os +from botocore.client import Config + +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://minio:9000") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY") +S3_SECRET_KEY = os.getenv("S3_SECRET_KEY") +S3_BUCKET = os.getenv("S3_BUCKET", "attachments") + +s3 = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + +def ensure_bucket(): + """Идемпотентно создаёт приватный bucket.""" + try: + s3.head_bucket(Bucket=S3_BUCKET) + except s3.exceptions.ClientError: + s3.create_bucket(Bucket=S3_BUCKET) + +def upload_file(key: str, data: bytes, content_type: str): + s3.put_object(Bucket=S3_BUCKET, Key=key, Body=data, ContentType=content_type) + +def download_file(key: str) -> bytes: + response = s3.get_object(Bucket=S3_BUCKET, Key=key) + return response["Body"].read() + +def delete_file(key: str): + s3.delete_object(Bucket=S3_BUCKET, Key=key) \ No newline at end of file diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..6bbdd78b --- /dev/null +++ b/src/security.py @@ -0,0 +1,16 @@ +import hashlib, os, secrets + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + key = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=64) + return salt.hex() + "$" + key.hex() + +def verify_password(password: str, stored: str) -> bool: + salt_hex, key_hex = stored.split("$") + salt = bytes.fromhex(salt_hex) + expected = bytes.fromhex(key_hex) + actual = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=64) + return secrets.compare_digest(actual, expected) + +def generate_token() -> str: + return secrets.token_urlsafe(48) # ~384 бита энтропии \ No newline at end of file diff --git a/tests/failover.py b/tests/failover.py new file mode 100644 index 00000000..bff780c2 --- /dev/null +++ b/tests/failover.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +import argparse +import subprocess +import sys +import threading +import time +import uuid +import os + +sys.path.insert(0, os.path.dirname(__file__)) +from client import Client, Failure, run, check + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--base-url', default='http://localhost:8080') + ap.add_argument('--action-script', required=True) + args = ap.parse_args() + + c = Client(args.base_url) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + + instances = set() + messages = [] + errors = 0 + + # 1. Собираем имена реплик + for _ in range(10): + r = c.call('GET', '/health/live') + instances.add(r['headers'].get('X-Instance-Id')) + time.sleep(0.05) + + check(len(instances) >= 2, f'Expected at least 2 API replicas, got: {instances}') + print('Observed replicas:', instances) + + # 2. Фоновая нагрузка + stop_event = threading.Event() + + def sender(thread_id): + nonlocal errors + local_c = Client(args.base_url, timeout=5) + i = 0 + while not stop_event.is_set(): + key = uuid.uuid4().hex + text = f'msg_{thread_id}_{i}' + try: + r = local_c.message(alice, chat, text, key=key) + messages.append(r['json']['id']) + except Failure: + # При 502/503 во время failover делаем retry с тем же ключом + try: + time.sleep(0.1) + r = local_c.message(alice, chat, text, key=key) + messages.append(r['json']['id']) + except Failure: + errors += 1 + i += 1 + time.sleep(0.02) + + threads = [threading.Thread(target=sender, args=(t,)) for t in range(4)] + for t in threads: t.start() + + time.sleep(1) # Даем нагрузке раскрутиться + + # 3. Останавливаем реплику + print(f'Running action script: {args.action_script}') + subprocess.run(args.action_script, shell=True, check=True) + + # 4. Ждем и останавливаем + time.sleep(4) + stop_event.set() + for t in threads: t.join() + + # 5. Проверяем, что нет дублей и потерь + history_ids = [] + cursor = None + while True: + q = path + '/messages?limit=100' + if cursor: q += '&cursor=' + cursor + page = c.call('GET', q, actor=alice, schema='MessagePage')['json'] + for m in page['items']: + history_ids.append(m['id']) + cursor = page['next_cursor'] + if not cursor: break + + check(len(set(history_ids)) == len(history_ids), 'Duplicate messages found!') + check(len(messages) > 0, 'No messages were successfully sent') + + print(f'Success! Messages sent: {len(messages)}, Network errors (retried safely): {errors}') + print('PASS: failover verified') + +if __name__ == '__main__': + run(main) \ No newline at end of file diff --git a/tests/test_persistence.py b/tests/test_persistence.py new file mode 100644 index 00000000..a2b1d070 --- /dev/null +++ b/tests/test_persistence.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import os +import sys +import time +import subprocess +import argparse +import hashlib + +# Добавляем текущую папку в путь, чтобы импортировать client.py +sys.path.insert(0, os.path.dirname(__file__)) +from client import Client, Failure, run + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--action-script', required=True) + args = parser.parse_args() + + c = Client(args.base_url) + alice, bob, eve, chat = c.bootstrap() + + # Создаем сообщение и загружаем файл + # ВАЖНО: добавили schema='Attachment', чтобы клиент распарсил ответ + att = c.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=test.png", + actor=alice, raw=open('tests/fixtures/sample.png', 'rb').read(), + headers={'Content-Type': 'image/png'}, expected=201, schema='Attachment')['json'] + + msg = c.message(alice, chat, "persist test", attachments=[att['id']])['json'] + + print(f"Created user={alice['id']} msg={msg['id']} att={att['id']}") + + # Запускаем скрипт перезапуска + print(f"Running action: {args.action_script}") + subprocess.run(args.action_script, shell=True, check=True) + + # Ждем готовности + print("Waiting for readiness...") + for _ in range(60): + try: + if c.call('GET', '/health/ready', expected=200)['status'] == 200: + break + except Exception: + pass + time.sleep(1) + + # Проверяем сохранность токена + me = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['json'] + assert me['id'] == alice['id'], "User ID changed after restart" + + # Проверяем сохранность сообщения + history = c.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=alice, schema='MessagePage')['json'] + assert any(m['id'] == msg['id'] for m in history['items']), "Message lost after restart" + + # Проверяем сохранность файла (сравниваем SHA-256) + content = c.call('GET', f"/api/v1/attachments/{att['id']}/content", actor=alice)['bytes'] + assert hashlib.sha256(content).hexdigest() == att['sha256'], "File content changed after restart" + + print("PASS: persistence verified") + +if __name__ == '__main__': + run(main) \ No newline at end of file