diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f49c19ef --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +*.py text eol=lf +*.sql text eol=lf +*.yaml text eol=lf +Dockerfile text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..806a906a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.12.8-slim-bookworm +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app +COPY src/ src/ +COPY migrations/ migrations/ +COPY scripts/ scripts/ +COPY tests/ tests/ +COPY contracts/ contracts/ +COPY lab.json . +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 +USER app +EXPOSE 8080 +CMD ["gunicorn", "--bind", "0.0.0.0:8080", "--workers", "1", "--threads", "4", "--timeout", "30", "--graceful-timeout", "5", "src.app:app"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..c0ed4e29 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,81 @@ -# Отчёт — лабораторная 4 +# Лабораторная 4 — отчёт -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Мартинес Анастасия, БПИ2303. Целевая оценка: **4**. Ветка: +`solution/bpi2303-martines`. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +```sh +python scripts/bootstrap.py +docker compose config --quiet +docker compose up -d --build +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +``` + +Стенд содержит HAProxy, две API-реплики, PostgreSQL, Redis, MinIO и cleanup +worker. Внешний API доступен только на 127.0.0.1:8080; S3 API локально +опубликован для отрицательной проверки private bucket, административная +консоль MinIO наружу не публикуется. Secrets генерируются в игнорируемой +`secrets/`; `docker compose down` сохраняет данные, `down -v` удаляет +учебные volumes намеренно. + +Использованы Python 3.12.8, PostgreSQL 16.6, Redis 7.4.1, HAProxy 3.0.6, +MinIO pinned digest, Flask/Gunicorn/Psycopg/Redis-py/Boto3/Pillow с pinned +версиями. Контракт, README и публичные tests не менялись. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Из lab3 сохранены общие сессии, LB, две реплики и idempotency. Метаданные +attachment лежат в PostgreSQL, байты — в приватном S3 bucket `messenger`. +API получает отдельные credentials с правами на object prefixes; root keys +используются только bootstrap-контейнером. Filename используется для +отображения, object key генерирует сервер. -## Доказательства критериев +Upload принимает только сырые PNG/JPEG, ограничивает тело 10 MiB до записи, +проверяет MIME по содержимому и ограничивает 40 млн пикселей Pillow. Файл +сначала пишется в S3, затем metadata — в БД. Таблица `object_intents` +служит durable журналом намерения: cleanup удаляет старые unreferenced +objects после grace period. Поэтому сбой между S3 и PostgreSQL не оставляет +вечные orphan objects. Запись metadata и удаление intent происходят одной +транзакцией. Download доступен только участнику чата и отдаёт байты через API, +без redirect. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Составной message сохраняет `attachment_ids`; attachment другого чата, +чужой attachment и failed attachment отвергаются до записи сообщения. +Ограниченный semaphore и SpooledTemporaryFile не позволяют одной загрузке +занять неограниченную RAM; 10 одновременных upload операций разрешены в +пределах лимита. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Проверки -## Эксперименты и отказы +Публичный smoke: -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +``` +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +``` -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Smoke проверяет upload 201 ready, checksum/size, авторизованный download, +сообщение с attachment и отсутствие доступа у outsider. Дополнительный +`student-tests/attachments.py` покрывает повреждённый файл, MIME spoof, +опасное имя, >10 MiB, чужой ID, private bucket и 10 concurrent upload. +Сценарии lab3 `tests/resilience.py` используются для persistence/failover; +lab3 проверка до переноса дала: две реплики, 24 concurrent idempotent requests +создали одну запись, отказ каждой реплики восстанавливался примерно за 1.1 с, +rolling recreation сохранял подтверждённые сообщения. -## Ограничения и следующие шаги +Память контейнеров во время расширенной проверки сохраняется в +`evidence/attachment-resources.txt`. Cleanup — dry-run по умолчанию и +`--apply --loop` только внутри Compose. При сбое S3 API возвращает +ограниченную ошибку; неавторизованный пользователь не узнаёт существование +attachment. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Ограничения + +Один MinIO/DB/LB и один Docker host остаются SPOF. Reconciliation удаляет +только intents старше часа; аварийный orphan моложе grace period ожидает +следующего прохода. Тест памяти зависит от ресурсов ноутбука и не является +production SLA. HTTPS и mTLS относятся к lab7, исследовательский track 5 +не заявлен. + +Помощь ИИ: Codex подготовил реализацию, тесты, измерения и отчёт. Все fixtures +синтетические; секреты, токены и private keys в Git не попадают. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..b7397948 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,124 @@ +name: vp-lab4 +x-api: &api + build: . + image: vp-lab4-api + environment: &app-env + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + secrets: [database_url, s3_access_key, s3_secret_key] + depends_on: + migrate: + condition: service_completed_successfully + s3init: + condition: service_completed_successfully + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=4)"] + interval: 3s + timeout: 5s + retries: 20 + stop_grace_period: 8s + mem_limit: 384m + cpus: 1 +services: + api1: + <<: *api + environment: + <<: *app-env + INSTANCE_ID: api1 + api2: + <<: *api + environment: + <<: *app-env + INSTANCE_ID: api2 + migrate: + build: . + image: vp-lab4-api + command: [python, -m, src.db] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + db: + condition: service_healthy + mem_limit: 128m + cpus: 0.5 + lb: + image: haproxy:3.0.6-alpine + ports: ["127.0.0.1:8080:8080", "127.0.0.1:9000:9000"] + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + mem_limit: 128m + cpus: 0.5 + minio: + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: [server, /data, --console-address, ':9001'] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + secrets: [s3_root_user, s3_root_password] + volumes: [s3data:/data] + mem_limit: 512m + cpus: 1 + s3init: + image: minio/mc@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727 + entrypoint: [/bin/sh, /bootstrap/s3-init.sh] + volumes: + - ./infra/s3-init.sh:/bootstrap/s3-init.sh:ro + - ./infra/s3-policy.json:/bootstrap/policy.json:ro + secrets: [s3_root_user, s3_root_password, s3_access_key, s3_secret_key] + depends_on: [minio] + mem_limit: 128m + cpus: 0.5 + cleanup: + <<: *api + command: [python, -m, src.cleanup, --apply, --loop] + healthcheck: + test: [CMD, python, -c, "import sys; sys.exit(0)"] + depends_on: + lb: + condition: service_started + mem_limit: 128m + db: + image: postgres:16.6-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 512m + cpus: 1 + redis: + image: redis:7.4.1-bookworm + command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + mem_limit: 128m + cpus: 0.5 +volumes: + pgdata: + s3data: +secrets: + db_password: + file: ./secrets/db_password + database_url: + file: ./secrets/database_url + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..2ed81403 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,34 @@ +global + maxconn 512 +defaults + mode http + timeout connect 1s + timeout client 30s + timeout server 30s + timeout check 1s + retries 0 +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s +frontend ingress + bind :8080 + default_backend api +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 1 resolvers docker init-addr libc,none + server api1 api1:8080 check + server api2 api2:8080 check +frontend object_ingress + bind :9000 + default_backend objects +backend objects + option httpchk GET /minio/health/live + http-check expect status 200 + server storage minio:9000 check inter 1s fall 1 rise 1 resolvers docker init-addr libc,none diff --git a/infra/s3-init.sh b/infra/s3-init.sh new file mode 100755 index 00000000..87fdc40f --- /dev/null +++ b/infra/s3-init.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +root_user=$(cat /run/secrets/s3_root_user) +root_password=$(cat /run/secrets/s3_root_password) +app_user=$(cat /run/secrets/s3_access_key) +app_password=$(cat /run/secrets/s3_secret_key) +attempt=0 +until mc alias set local http://minio:9000 "$root_user" "$root_password" >/dev/null 2>&1; do + attempt=$((attempt+1)); [ "$attempt" -lt 60 ] || exit 1; sleep 1 +done +mc mb --ignore-existing local/messenger >/dev/null +mc anonymous set none local/messenger >/dev/null +mc admin user add local "$app_user" "$app_password" >/dev/null +mc admin policy create local messenger-app /bootstrap/policy.json >/dev/null +mc admin policy attach local messenger-app --user "$app_user" >/dev/null +echo 'Private bucket and scoped application identity ready' diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 00000000..54679964 --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetBucketLocation"],"Resource":["arn:aws:s3:::messenger"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger/original/*","arn:aws:s3:::messenger/staging/*","arn:aws:s3:::messenger/variants/*"]}]} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..093ce411 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (length(trim(display_name)) > 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (length(trim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX members_user_chat ON members(user_id, chat_id); +CREATE INDEX chats_history ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text varchar(4000) NOT NULL CHECK (length(trim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id, user_id) +); +CREATE INDEX messages_history ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..f23ab309 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key varchar(128) NOT NULL, + request jsonb NOT NULL, + response jsonb NOT NULL, + expires_at timestamptz NOT NULL DEFAULT (clock_timestamp() + interval '24 hours'), + PRIMARY KEY (user_id,chat_id,key) +); +CREATE INDEX idempotency_expiry ON idempotency(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..c52b7f20 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,23 @@ +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CHECK (length(trim(text)) > 0 OR cardinality(attachment_ids) > 0); +CREATE TABLE object_intents ( + object_key text PRIMARY KEY, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename varchar(255) NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL, + status text NOT NULL DEFAULT 'ready', + variants jsonb NOT NULL DEFAULT '[]', + error_code text, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachments_chat ON attachments(chat_id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..1dae88df --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +Flask==3.1.0 +gunicorn==23.0.0 +psycopg[binary,pool]==3.2.3 +redis==5.2.1 +argon2-cffi==23.1.0 +# Transitive dependencies pinned to the verified Linux/Python 3.12 environment. +argon2-cffi-bindings==26.2.0 +blinker==1.9.0 +cffi==2.1.1 +click==8.5.1 +itsdangerous==2.2.0 +Jinja2==3.1.6 +MarkupSafe==3.0.4 +packaging==26.3 +psycopg-binary==3.2.3 +psycopg-pool==3.3.3 +pycparser==3.12 +typing_extensions==4.16.0 +Werkzeug==3.1.9 +boto3==1.35.99 +Pillow==11.1.0 diff --git a/scripts/action.py b/scripts/action.py new file mode 100644 index 00000000..8c29f02e --- /dev/null +++ b/scripts/action.py @@ -0,0 +1,21 @@ +"""Explicit local Compose failure hooks. Never deletes volumes.""" +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +os.chdir(ROOT) +mode = sys.argv[1] +if mode == 'restart': + services = subprocess.check_output(['docker', 'compose', 'ps', '--services', '--status', 'running'], text=True).split() + subprocess.run(['docker', 'compose', 'restart', *services], check=True) +elif mode == 'stop-one': + alias = os.environ['TARGET_INSTANCE_ID'] + if alias not in ('api1', 'api2'): + raise SystemExit('Unknown API alias') + Path('evidence/last-stopped.json').write_text(json.dumps({'service': alias}), encoding='utf-8') + subprocess.run(['docker', 'compose', 'stop', '-t', '2', alias], check=True) +else: + raise SystemExit('Unknown action') diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..99908140 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,26 @@ +"""Create local synthetic secrets once; never overwrite existing credentials.""" +import secrets +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +directory = root / 'secrets' +directory.mkdir(exist_ok=True) +password_file = directory / 'db_password' +if not password_file.exists(): + with password_file.open('x', encoding='utf-8') as file: + file.write(secrets.token_urlsafe(32)) +password = password_file.read_text(encoding='utf-8').strip() +url_file = directory / 'database_url' +if not url_file.exists(): + with url_file.open('x', encoding='utf-8') as file: + file.write('postgresql://messenger:' + password + '@db:5432/messenger') +env = root / '.env' +if not env.exists(): + with env.open('x', encoding='utf-8') as file: + file.write('SESSION_TTL_SECONDS=3600\n') +print('Local configuration ready; credentials are not printed.') +for name in ('s3_root_user','s3_root_password','s3_access_key','s3_secret_key'): + path = directory / name + if not path.exists(): + with path.open('x', encoding='utf-8') as file: + file.write(secrets.token_hex(16)) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..82d4fb96 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +python scripts/action.py restart diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..2651a499 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/action.py restart diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..97bf88df --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +python scripts/action.py stop-one diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..2448345c --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/action.py stop-one diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 00000000..a5b1dada --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,79 @@ +"""Run public and student acceptance, preserving sanitized evidence.""" +import json +import os +import platform +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +os.chdir(ROOT) +EVIDENCE = ROOT / 'evidence' +ENV = {**os.environ, 'PYTHONUTF8': '1', 'BASE_URL': os.getenv('BASE_URL', 'http://127.0.0.1:8080')} + + +def run(args, artifact): + result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, + encoding='utf-8', errors='replace', env=ENV) + output = result.stdout + result.stderr + (EVIDENCE / artifact).write_text(output, encoding='utf-8') + print(output, end='', flush=True) + if result.returncode: + raise SystemExit(result.returncode) + return output + + +def containers(): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], text=True).split() + raw = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True)) + return [{'name': x['Name'], 'id': x['Id'], 'started_at': x['State']['StartedAt'], + 'image': x['Config']['Image']} for x in raw] + + +def main(): + run([sys.executable, 'scripts/check_contract.py'], 'contract.txt') + run([sys.executable, 'tests/smoke.py'], 'smoke.txt') + before = containers() + suffix = '.cmd' if os.name == 'nt' else '.sh' + run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', + '--action-script', 'scripts/restart' + suffix], 'persistence.txt') + after = containers() + old = {x['id']: x for x in before} + assert all(x['id'] in old and x['started_at'] != old[x['id']]['started_at'] for x in after) + (EVIDENCE / 'restart.json').write_text(json.dumps({'before':before,'after':after},indent=2),encoding='utf-8') + try: + run([sys.executable, 'tests/resilience.py', '--mode', 'failover', + '--action-script', 'scripts/stop-one' + suffix], 'failover.txt') + finally: + subprocess.run(['docker','compose','up','-d','--no-deps','--wait','api1','api2'], check=True) + time.sleep(2) + run([sys.executable, 'student-tests/replication.py'], 'replication.txt') + if (ROOT / 'student-tests/attachments.py').exists(): + run([sys.executable, 'student-tests/attachments.py'], 'attachments.txt') + if (ROOT / 'student-tests/pipeline.py').exists(): + run([sys.executable, 'student-tests/pipeline.py'], 'pipeline.txt') + if (ROOT / 'student-tests/observability.py').exists(): + run([sys.executable, 'student-tests/observability.py'], 'observability.txt') + process = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4', + '--output','evidence/load.json'], stdout=subprocess.PIPE,stderr=subprocess.STDOUT, + text=True,encoding='utf-8',env=ENV) + ids = [x['id'] for x in containers()] + samples=[] + while process.poll() is None: + stats=subprocess.check_output(['docker','stats','--no-stream','--format', + '{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}',*ids],text=True) + samples.append({'at':datetime.now(timezone.utc).isoformat(),'stats':stats.strip()}) + time.sleep(2) + output=process.communicate()[0] + (EVIDENCE/'load.txt').write_text(output,encoding='utf-8') + (EVIDENCE/'resources.json').write_text(json.dumps(samples,indent=2),encoding='utf-8') + assert process.returncode==0, output + (EVIDENCE/'environment.json').write_text(json.dumps({'platform':platform.platform(), + 'cpu':platform.processor(),'logical_cpus':os.cpu_count(),'observed_at':datetime.now(timezone.utc).isoformat()},indent=2),encoding='utf-8') + print(output,flush=True) + + +if __name__ == '__main__': + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..5218c71a --- /dev/null +++ b/src/app.py @@ -0,0 +1,421 @@ +"""Lab 2 API: PostgreSQL is authoritative; Redis caches immutable public profiles.""" +import base64 +import binascii +import hashlib +import json +import os +import re +import secrets +import uuid +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout +from datetime import datetime +from threading import BoundedSemaphore + +import psycopg +import redis +from botocore.exceptions import BotoCoreError, ClientError +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from flask import Flask, g, jsonify, request +from psycopg_pool import PoolTimeout, TooManyRequests +from psycopg.types.json import Jsonb +from werkzeug.exceptions import HTTPException + +from src.db import make_pool, setting + +app = Flask(__name__) +app.config['MAX_CONTENT_LENGTH'] = 10485760 +pool = make_pool() +cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.2, + socket_timeout=0.2, decode_responses=True) +cache_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix='cache') +cache_slots = BoundedSemaphore(2) +hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +dummy_hash = hasher.hash(secrets.token_urlsafe(32)) +SESSION_TTL = int(os.getenv('SESSION_TTL_SECONDS', '3600')) +if SESSION_TTL < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') +USER_FIELDS = 'id, username, display_name, created_at' +INSTANCE_ID = os.getenv('INSTANCE_ID', uuid.uuid4().hex) + + +class APIError(Exception): + def __init__(self, status, code, message, challenge=None): + self.status, self.code, self.message, self.challenge = status, code, message, challenge + + +def invalid(): + raise APIError(400, 'invalid_request', 'Invalid request') + + +def unauthorized(basic=False): + challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + raise APIError(401, 'unauthorized', 'Authentication required', challenge) + + +def serial(value): + if isinstance(value, dict): + return {key: serial(item) for key, item in value.items()} + if isinstance(value, list): + return [serial(item) for item in value] + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, datetime): + return value.isoformat().replace('+00:00', 'Z') + return value + + +def response(value, status=200): + return jsonify(serial(value)), status + + +@app.before_request +def identify_request(): + g.request_id = uuid.uuid4().hex + + +@app.after_request +def headers(result): + result.headers['X-Request-Id'] = g.request_id + result.headers['X-Instance-Id'] = INSTANCE_ID + result.headers['Cache-Control'] = 'no-store' + return result + + +@app.errorhandler(Exception) +def error(exc): + if isinstance(exc, APIError): + problem = exc + elif isinstance(exc, psycopg.errors.UniqueViolation): + problem = APIError(409, 'conflict', 'Username already exists') + elif isinstance(exc, psycopg.errors.ForeignKeyViolation): + problem = APIError(400, 'invalid_request', 'Invalid member list') + elif isinstance(exc, (BotoCoreError, ClientError, psycopg.OperationalError, PoolTimeout, TooManyRequests, + psycopg.errors.QueryCanceled, psycopg.errors.LockNotAvailable)): + problem = APIError(503, 'unavailable', 'Database temporarily unavailable') + elif isinstance(exc, HTTPException): + status = exc.code if exc.code in (400, 404, 413, 415) else 400 + code = {400: 'invalid_request', 404: 'not_found', 413: 'payload_too_large', + 415: 'unsupported_media_type'}[status] + problem = APIError(status, code, 'Request cannot be processed') + else: + app.logger.error('Unhandled error type=%s request_id=%s', type(exc).__name__, g.request_id) + problem = APIError(500, 'internal_error', 'Internal error') + result = jsonify(error={'code': problem.code, 'message': problem.message, + 'request_id': g.request_id}) + result.status_code = problem.status + if problem.challenge: + result.headers['WWW-Authenticate'] = problem.challenge + if problem.status == 429: + result.headers['Retry-After'] = '1' + return result + + +def body(fields): + if request.mimetype != 'application/json': + raise APIError(415, 'unsupported_media_type', 'Expected application/json') + value = request.get_json() + if not isinstance(value, dict) or set(value) != set(fields): + invalid() + return value + + +def string(value, minimum, maximum): + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + if '\x00' in value: + invalid() + return value + + +def uid(value): + try: + if not isinstance(value, str) or not re.fullmatch( + r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value): + invalid() + return uuid.UUID(value) + except ValueError: + invalid() + + +def cached_operation(operation, *args, **kwargs): + # Socket timeouts do not bound DNS resolution when a Compose service is down. + # At most two background operations per worker, no unbounded queue. + if not cache_slots.acquire(blocking=False): + return None + future = cache_executor.submit(operation, *args, **kwargs) + future.add_done_callback(lambda _: cache_slots.release()) + try: + return future.result(timeout=0.25) + except (redis.RedisError, FutureTimeout): + return None + + +def profile(conn, user_id): + key = 'profile:v1:' + str(user_id) + cached = cached_operation(cache.get, key) + if cached: + try: + return json.loads(cached) + except ValueError: + pass + user = serial(conn.execute('SELECT ' + USER_FIELDS + ' FROM users WHERE id=%s', + (user_id,)).fetchone()) + if user is None: + unauthorized() + cached_operation(cache.set, key, json.dumps(user), ex=60) + return user + + +def actor(conn): + auth = request.headers.get('Authorization', '').split() + if len(auth) != 2 or auth[0].lower() != 'bearer' or len(auth[1]) > 256: + unauthorized() + digest = hashlib.sha256(auth[1].encode()).hexdigest() + session = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s ' + 'AND expires_at > clock_timestamp()', (digest,)).fetchone() + if not session: + unauthorized() + g.token_hash = digest + return session['user_id'] + + +def chat_access(conn, chat_id, user_id): + chat = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id ' + 'WHERE c.id=%s AND m.user_id=%s', (chat_id, user_id)).fetchone() + if not chat: + raise APIError(404, 'not_found', 'Resource not found') + return chat + + +def chat_members(conn, chat): + chat['member_ids'] = [row['user_id'] for row in conn.execute( + 'SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id', (chat['id'],))] + return chat + + +def page_args(scope): + limit = request.args.get('limit', '50') + if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100: + invalid() + cursor = request.args.get('cursor') + after = None + if cursor is not None: + try: + if len(cursor) > 1024: + invalid() + data = json.loads(base64.b64decode(cursor, altchars=b'-_', validate=True)) + if not isinstance(data, list) or len(data) != 3 or data[0] != scope: + invalid() + date = datetime.fromisoformat(data[1]) + if date.tzinfo is None: + invalid() + after = (date, uid(data[2])) + except (ValueError, TypeError, binascii.Error): + invalid() + return int(limit), after + + +def page(rows, limit, scope): + more = len(rows) > limit + items = rows[:limit] + cursor = None + if more: + last = serial(items[-1]) + cursor = base64.urlsafe_b64encode(json.dumps( + [scope, last['created_at'], last['id']]).encode()).decode() + return {'items': items, 'next_cursor': cursor} + + +@app.get('/health/live') +def live(): + return response({'status': 'ok'}) + + +@app.get('/health/ready') +def ready(): + with pool.connection() as conn: + conn.execute('SELECT 1 FROM schema_migrations LIMIT 1') + return response({'status': 'ok'}) + + +@app.post('/api/v1/users') +def register(): + data = body(('username', 'display_name', 'password')) + name = string(data['username'], 3, 32) + if not re.fullmatch('[a-z0-9_]{3,32}', name): + invalid() + display = string(data['display_name'], 1, 100) + password = string(data['password'], 12, 128) + encoded = hasher.hash(password) + with pool.connection() as conn: + user = conn.execute('INSERT INTO users (id,username,display_name,password_hash) ' + 'VALUES (%s,%s,%s,%s) RETURNING ' + USER_FIELDS, + (uuid.uuid4(), name, display, encoded)).fetchone() + return response(user, 201) + + +@app.post('/api/v1/auth/sessions') +def login(): + auth = request.headers.get('Authorization', '').split() + if len(auth) != 2 or auth[0].lower() != 'basic' or len(auth[1]) > 2048: + unauthorized(True) + try: + name, password = base64.b64decode(auth[1], validate=True).decode('utf-8').split(':', 1) + except (ValueError, UnicodeError, binascii.Error): + unauthorized(True) + if not re.fullmatch('[a-z0-9_]{3,32}', name) or not 12 <= len(password) <= 128: + unauthorized(True) + if request.get_data(): + invalid() + with pool.connection() as conn: + user = conn.execute('SELECT * FROM users WHERE username=%s', (name,)).fetchone() + try: + hasher.verify(user['password_hash'] if user else dummy_hash, password) + except VerificationError: + unauthorized(True) + if not user: + unauthorized(True) + token = secrets.token_urlsafe(32) + with pool.connection() as conn: + expiry = conn.execute('INSERT INTO sessions VALUES (%s,%s,clock_timestamp() ' + '+ %s * interval \'1 second\') RETURNING expires_at', + (hashlib.sha256(token.encode()).hexdigest(), user['id'], + SESSION_TTL)).fetchone()['expires_at'] + user.pop('password_hash') + return response({'token': token, 'token_type': 'Bearer', 'expires_at': expiry, 'user': user}, 201) + + +@app.delete('/api/v1/auth/sessions/current') +def logout(): + with pool.connection() as conn: + actor(conn) + conn.execute('DELETE FROM sessions WHERE token_hash=%s', (g.token_hash,)) + return '', 204 + + +@app.get('/api/v1/users/me') +def me(): + with pool.connection() as conn: + user = profile(conn, actor(conn)) + return response(user) + + +@app.post('/api/v1/chats') +def create_chat(): + with pool.connection() as conn: + user = actor(conn) + data = body(('title', 'member_ids')) + title = string(data['title'], 1, 100) + if not isinstance(data['member_ids'], list) or not 1 <= len(data['member_ids']) <= 100: + invalid() + members = [uid(item) for item in data['member_ids']] + if len(set(members)) != len(members) or user not in members: + invalid() + chat = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *', + (uuid.uuid4(), title)).fetchone() + for member in members: + conn.execute('INSERT INTO members VALUES (%s,%s)', (chat['id'], member)) + chat['member_ids'] = sorted(members) + return response(chat, 201) + + +@app.get('/api/v1/chats') +def list_chats(): + with pool.connection() as conn: + user = actor(conn) + scope = 'chats:' + str(user) + limit, after = page_args(scope) + sql = 'SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s' + params = [user] + if after: + sql += ' AND (c.created_at,c.id) > (%s,%s)' + params.extend(after) + rows = conn.execute(sql + ' ORDER BY c.created_at,c.id LIMIT %s', params + [limit+1]).fetchall() + result = page(rows, limit, scope) + for chat in result['items']: + chat_members(conn, chat) + return response(result) + + +@app.get('/api/v1/chats/') +def get_chat(chat_id): + with pool.connection() as conn: + user = actor(conn) + result = chat_members(conn, chat_access(conn, uid(chat_id), user)) + return response(result) + + +@app.route('/api/v1/chats//messages', methods=['GET', 'POST']) +def messages(chat_id): + with pool.connection() as conn: + user = actor(conn) + chat_id = uid(chat_id) + chat_access(conn, chat_id, user) + if request.method == 'POST': + if request.mimetype != 'application/json': + raise APIError(415, 'unsupported_media_type', 'Expected application/json') + data = request.get_json() + if not isinstance(data, dict) or 'text' not in data or set(data) - {'text', 'attachment_ids'}: + invalid() + text = data['text'] + if not isinstance(text, str) or len(text) > 4000 or '\x00' in text: + invalid() + try: + text.encode('utf-8') + except UnicodeError: + invalid() + attachments = data.get('attachment_ids', []) + if not isinstance(attachments, list) or len(attachments) > 10: + invalid() + attachments = [uid(item) for item in attachments] + if len(set(attachments)) != len(attachments) or not (text.strip() or attachments): + invalid() + for attachment_id in attachments: + attachment = conn.execute('SELECT status FROM attachments WHERE id=%s AND chat_id=%s', + (attachment_id, chat_id)).fetchone() + if not attachment: + raise APIError(404, 'not_found', 'Attachment not found') + if attachment['status'] == 'failed': + raise APIError(409, 'conflict', 'Attachment failed') + key = request.headers.get('Idempotency-Key', '') + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + invalid() + canonical = {'text': text, 'attachment_ids': [str(item) for item in attachments]} + lock = int.from_bytes(hashlib.sha256(f'{user}:{chat_id}:{key}'.encode()).digest()[:8], + 'big', signed=True) + conn.execute('SELECT pg_advisory_xact_lock(%s)', (lock,)) + conn.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s ' + 'AND expires_at<=clock_timestamp()', (user, chat_id, key)) + prior = conn.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s', + (user, chat_id, key)).fetchone() + if prior: + if prior['request'] != canonical: + raise APIError(409, 'conflict', 'Idempotency key already used with a different message') + return response(prior['response'], 201) + result = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) ' + 'VALUES (%s,%s,%s,%s,%s) RETURNING *', + (uuid.uuid4(), chat_id, user, text, attachments)).fetchone() + conn.execute('INSERT INTO idempotency(user_id,chat_id,key,request,response) ' + 'VALUES (%s,%s,%s,%s,%s)', + (user, chat_id, key, Jsonb(canonical), Jsonb(serial(result)))) + status = 201 + else: + scope = 'messages:' + str(chat_id) + limit, after = page_args(scope) + sql = 'SELECT * FROM messages WHERE chat_id=%s' + params = [chat_id] + if after: + sql += ' AND (created_at,id) > (%s,%s)' + params.extend(after) + rows = conn.execute(sql + ' ORDER BY created_at,id LIMIT %s', params + [limit+1]).fetchall() + result, status = page(rows, limit, scope), 200 + return response(result, status) + + +from src.attachments import register_routes +register_routes(app, pool, actor, chat_access, uid, response, APIError, invalid) diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..d89d7380 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,117 @@ +import hashlib +import tempfile +import uuid +import warnings +from threading import BoundedSemaphore + +from flask import Response, request, stream_with_context +from PIL import Image, UnidentifiedImageError +from src.storage import BUCKET, s3 + +MAX_BYTES = 10 * 1024 * 1024 +Image.MAX_IMAGE_PIXELS = 40_000_000 +# Each API worker is bounded by Gunicorn threads; this semaphore prevents an +# unbounded number of uploads without serializing the required concurrent case. +slots = BoundedSemaphore(16) + + +def public(row): + return {k: v for k, v in row.items() if k != 'object_key'} + + +def register_routes(app, pool, actor, chat_access, uid, response, APIError, invalid): + @app.post('/api/v1/chats//attachments') + def upload(chat_id): + with pool.connection() as conn: + user = actor(conn) + chat_id = uid(chat_id) + chat_access(conn, chat_id, user) + filename = request.args.get('filename', '') + if not 1 <= len(filename) <= 255 or not filename.strip() or any( + ord(c) < 32 or ord(c) == 127 or c in '/\\' for c in filename): + invalid() + if request.mimetype not in ('image/png', 'image/jpeg'): + raise APIError(415, 'unsupported_media_type', 'Expected PNG or JPEG') + if request.content_length and request.content_length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Image too large') + if not slots.acquire(timeout=5): + raise APIError(429, 'rate_limited', 'Upload capacity reached') + try: + with tempfile.SpooledTemporaryFile(max_size=1024*1024) as file: + length, digest = 0, hashlib.sha256() + while True: + chunk = request.stream.read(65536) + if not chunk: + break + length += len(chunk) + if length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Image too large') + digest.update(chunk) + file.write(chunk) + file.seek(0) + try: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + with Image.open(file) as image: + if image.width * image.height > 40_000_000: + raise APIError(413, 'payload_too_large', 'Too many pixels') + if Image.MIME.get(image.format) != request.mimetype: + raise APIError(415, 'unsupported_media_type', 'Image type mismatch') + image.verify() + file.seek(0) + with Image.open(file) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise APIError(413, 'payload_too_large', 'Too many pixels') from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise APIError(415, 'unsupported_media_type', 'Invalid image') from None + file.seek(0) + identifier = uuid.uuid4() + key = 'original/' + str(identifier) + with pool.connection() as conn: + conn.execute('INSERT INTO object_intents(object_key) VALUES (%s)', (key,)) + s3.put_object(Bucket=BUCKET, Key=key, Body=file, ContentLength=length, ContentType=request.mimetype) + # Intent survives a crash or failed DB commit; the janitor removes old unreferenced objects. + with pool.connection() as conn: + row = conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *', + (identifier,chat_id,user,filename,request.mimetype,length,digest.hexdigest(),key)).fetchone() + conn.execute('DELETE FROM object_intents WHERE object_key=%s', (key,)) + return response(public(row),201) + finally: + slots.release() + + def authorized(attachment_id): + with pool.connection() as conn: + user = actor(conn) + identifier = uid(attachment_id) + row = conn.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s', (identifier,user)).fetchone() + if not row: + raise APIError(404,'not_found','Attachment not found') + return row + + @app.get('/api/v1/attachments/') + def metadata(attachment_id): + return response(public(authorized(attachment_id))) + + @app.get('/api/v1/attachments//content') + def download(attachment_id): + row = authorized(attachment_id) + if row['status'] != 'ready': + raise APIError(409,'conflict','Attachment not ready') + variant = request.args.get('variant','original') + if variant != 'original': + raise APIError(404,'not_found','Variant not found') + obj = s3.get_object(Bucket=BUCKET,Key=row['object_key']) + def chunks(): + try: + while True: + chunk = obj['Body'].read(65536) + if not chunk: + break + yield chunk + finally: + obj['Body'].close() + return Response(stream_with_context(chunks()),content_type=row['content_type'], + headers={'Content-Length':str(row['size_bytes']),'X-Content-Type-Options':'nosniff'}) diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 00000000..2afc7bd7 --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,37 @@ +"""Old upload intent reconciliation; dry run unless --apply, never deletes fresh objects.""" +import argparse +import json +import time +from src.db import make_pool +from src.storage import s3, BUCKET + + +def sweep(apply=False): + pool = make_pool() + with pool.connection() as conn: + rows = conn.execute("SELECT i.object_key FROM object_intents i LEFT JOIN attachments a " + "ON a.object_key=i.object_key WHERE a.id IS NULL " + "AND i.created_at < clock_timestamp()-interval '1 hour' FOR UPDATE OF i SKIP LOCKED").fetchall() + for row in rows: + if apply: + s3.delete_object(Bucket=BUCKET,Key=row['object_key']) + conn.execute('DELETE FROM object_intents WHERE object_key=%s',(row['object_key'],)) + pool.close() + return {'old_unreferenced_intents':len(rows),'applied':apply} + + +if __name__ == '__main__': + parser=argparse.ArgumentParser() + parser.add_argument('--apply',action='store_true') + parser.add_argument('--loop',action='store_true') + args=parser.parse_args() + while True: + try: + print(json.dumps(sweep(args.apply)),flush=True) + except Exception as exc: + print(json.dumps({'cleanup_error':type(exc).__name__}),flush=True) + if not args.loop: + raise SystemExit(1) + if not args.loop: + break + time.sleep(60) diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..19cf69cb --- /dev/null +++ b/src/db.py @@ -0,0 +1,52 @@ +"""Bounded PostgreSQL pool and serialized, versioned migrations.""" +import hashlib +import os +import time +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + + +def setting(name): + path = os.getenv(name + '_FILE') + return Path(path).read_text().strip() if path else os.environ[name] + + +def migrate(): + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn: + conn.execute('SELECT pg_advisory_xact_lock(202602)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations ' + '(version text PRIMARY KEY, checksum text NOT NULL)') + for file in sorted(Path('migrations').glob('*.sql')): + sql = file.read_text(encoding='utf-8') + digest = hashlib.sha256(sql.encode()).hexdigest() + row = conn.execute('SELECT checksum FROM schema_migrations WHERE version=%s', + (file.name,)).fetchone() + if row: + if row[0] != digest: + raise RuntimeError('Applied migration changed') + continue + conn.execute(sql) + conn.execute('INSERT INTO schema_migrations VALUES (%s,%s)', (file.name, digest)) + + +def make_pool(): + return ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=8, + timeout=3, max_waiting=32, + check=ConnectionPool.check_connection, + kwargs={'connect_timeout': 3, 'row_factory': dict_row, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000'}) + + +if __name__ == '__main__': + for attempt in range(30): + try: + migrate() + break + except psycopg.OperationalError: + if attempt == 29: + raise RuntimeError('Database did not become available') from None + time.sleep(2) + print('Migrations applied successfully') diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..240cd6d7 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,10 @@ +import os +import boto3 +from botocore.config import Config +from src.db import setting + +BUCKET = os.getenv('S3_BUCKET', 'messenger') +s3 = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://lb:9000'), + aws_access_key_id=setting('S3_ACCESS_KEY'), aws_secret_access_key=setting('S3_SECRET_KEY'), + region_name='us-east-1', config=Config(connect_timeout=2, read_timeout=5, + retries={'max_attempts': 0}, s3={'addressing_style': 'path'})) diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 00000000..33cd27ff --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,60 @@ +"""Attachment negative checks, private-bucket check, and concurrent upload budget.""" +import base64 +import hashlib +import os +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.request import Request, urlopen +from urllib.error import HTTPError, URLError +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check, Failure + +PNG = (b'\x89PNG\r\n\x1a\n' + b'\x00' * 100) + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=photo.png' + att, content = c.attachment(alice, chat) + check(att['status'] == 'ready' and att['sha256'] == hashlib.sha256(content).hexdigest(), 'metadata mismatch') + downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + check(downloaded['bytes'] == content, 'download mismatch') + c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), 404, actor=alice) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=bad/name.png', 400, + actor=alice, raw=content, headers={'Content-Type': 'image/png'}) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=spoof.png', 415, + actor=alice, raw=b'not-an-image', headers={'Content-Type': 'image/png'}) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=spoof.txt', 415, + actor=alice, raw=content, headers={'Content-Type': 'text/plain'}) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=huge.png', 413, + actor=alice, raw=b'0' * (10 * 1024 * 1024 + 1), headers={'Content-Type': 'image/png'}) + responses = [] + def upload(_): + return c.attachment(alice, chat)[0]['id'] + with ThreadPoolExecutor(max_workers=10) as pool: + responses = list(pool.map(upload, range(10))) + check(len(set(responses)) == 10, 'concurrent uploads shared an id') + try: + urlopen('http://127.0.0.1:9000/messenger/original/' + str(att['id']), timeout=3) + except HTTPError as error: + check(error.code in (400, 403), 'bucket/object is publicly readable') + else: + raise AssertionError('bucket/object is publicly readable') + stats = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{.Name}}={{.MemUsage}}', + 'vp-lab4-api1-1', 'vp-lab4-api2-1'], text=True) + Path(ROOT / 'evidence/attachment-resources.txt').write_text(stats, encoding='utf-8') + print('PASS: attachment access, validation, private bucket and 10 concurrent uploads') + + +if __name__ == '__main__': + main() diff --git a/student-tests/replication.py b/student-tests/replication.py new file mode 100644 index 00000000..99afac0e --- /dev/null +++ b/student-tests/replication.py @@ -0,0 +1,101 @@ +"""Concurrent idempotency, both replica failures, and rolling recreation under writes.""" +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check, Failure + + +def compose(*args): + subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, + stdout=subprocess.DEVNULL, timeout=120) + + +def history(c, actor, chat): + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' + result = [] + while path: + page = c.call('GET', path, actor=actor, schema='MessagePage')['json'] + result.extend(page['items']) + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='') if page['next_cursor'] else None + return result + + +def main(): + c = Client(timeout=5) + alice, bob, eve, chat = c.bootstrap() + key = uuid.uuid4().hex + def send(_): + return Client().message(alice, chat, 'One logical message', key=key) + with ThreadPoolExecutor(max_workers=12) as executor: + responses = list(executor.map(send, range(24))) + ids = {item['json']['id'] for item in responses} + instances = {item['headers']['X-Instance-Id'] for item in responses} + check(len(ids) == 1 and len(instances) >= 2, 'Concurrency did not cover both replicas or duplicated message') + check(len(history(c, alice, chat)) == 1, 'Duplicate row committed') + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + headers={'Idempotency-Key': key}, body={'text': 'Different'}) + results = [{'scenario': 'concurrent_idempotency', 'requests': 24, + 'replicas': sorted(instances), 'unique_messages': 1}] + + def experiment(label, action): + stop = threading.Event() + writes, errors, times = [], [], [] + def traffic(): + client = Client(timeout=3) + retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex + while not stop.is_set(): + try: + response = client.message(alice, chat, text, key=retry_key) + writes.append(response['json']) + times.append(time.monotonic()) + retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex + except Failure: + errors.append(time.monotonic()) + time.sleep(0.02) + thread = threading.Thread(target=traffic) + thread.start() + started = time.monotonic() + try: + action() + time.sleep(5) + finally: + stop.set() + thread.join(timeout=10) + duration = time.monotonic() - started + stored = {item['id']: item for item in history(c, bob, chat)} + check(writes and all(stored.get(item['id']) == item for item in writes), 'Confirmed writes lost') + bodies = [item['text'] for item in stored.values()] + check(len(bodies) == len(set(bodies)), 'Retry duplicated a logical message') + gaps = [b-a for a,b in zip([started]+times, times+[time.monotonic()])] + check(max(gaps) < 10, 'Recovery exceeded 10 seconds') + results.append({'scenario': label, 'duration_seconds': round(duration,3), + 'writes': len(writes), 'errors': len(errors), + 'rps': round(len(writes)/duration,3), 'max_success_gap_seconds': round(max(gaps),3)}) + + for service in ('api1', 'api2'): + try: + experiment('stop_' + service, lambda: compose('stop', '-t', '2', service)) + finally: + compose('up', '-d', '--no-deps', '--wait', service) + time.sleep(2) + def rolling(): + for service in ('api1', 'api2'): + compose('up', '-d', '--no-deps', '--force-recreate', '--wait', service) + time.sleep(2) + experiment('rolling_recreation', rolling) + Path(ROOT / 'evidence/replication.json').write_text(json.dumps(results, indent=2), encoding='utf-8') + print(json.dumps(results, indent=2)) + + +if __name__ == '__main__': + main()