Рестеу Даниил БВТ2302 - Лаба5 - 4 #82

Closed
SaitWors wants to merge 1 commits from SaitWors/mtusi2026lab5:solution/bvt2302-resteu into main
26 changed files with 1680 additions and 48 deletions
+12 -20
View File
@@ -1,25 +1,17 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Local synthetic lab credentials only. Change all passwords before use.
POSTGRES_DB=messenger
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change-this-for-local-demo
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_ROOT_USER=lab4admin
S3_ROOT_PASSWORD=change-minio-admin-password-123
S3_ACCESS_KEY_ID=lab4api
S3_SECRET_ACCESS_KEY=change-minio-api-password-123
# Policy in infra/s3-policy.json is scoped to this bucket; keep unchanged.
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
BROKER_HOST=broker
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
JOB_LEASE_SECONDS=90
+29
View File
@@ -0,0 +1,29 @@
FROM python:3.14.8-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab
WORKDIR /app
COPY requirements.txt ./
RUN python -m pip install --no-cache-dir -r requirements.txt
RUN groupadd --gid 10001 appuser \
&& useradd --uid 10001 --gid appuser --no-create-home --shell /usr/sbin/nologin appuser
COPY src ./src
COPY migrations ./migrations
COPY tests ./tests
COPY student-tests ./student-tests
COPY scripts ./scripts
COPY contracts ./contracts
COPY lab.json ./
USER 10001:10001
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready', timeout=2).close()"]
CMD ["python", "-m", "src.run"]
+15 -28
View File
@@ -1,37 +1,24 @@
# Отчёт — лабораторная 5
# Отчёт ЛР5 — Рестеу Д. А., БВТ2302
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
**Целевая оценка:** 4.
## Запуск из чистого clone
## Архитектура
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Существующие API1/API2 + Nginx + PostgreSQL + Redis сессий + SeaweedFS сохранены из ЛР4. Добавлены отдельный broker (Redis Streams с AOF), dispatcher и worker. HTTP upload принимает полные bytes, проверяет формат/размер, кладёт оригинал в приватный `staging/` и атомарно сохраняет queued и outbox в PostgreSQL. Только после этого возвращает 202. Отдельный dispatcher публикует IDs заданий в durable stream. Worker читает stream в consumer group и подтверждает сообщения после устойчивого эффекта.
## Архитектура и решения
Оригинал: побайтная копия из staging в originals. Thumbnail — JPEG, до 256×256, без увеличения. Crop — JPEG, центрированный квадрат 128×128. Optimized — WebP, до 1280×1280 без увеличения. EXIF orientation применяется, EXIF не сохраняется в производных файлах; прозрачность компонуется на белый. JPEG quality=85; WebP quality=80/method=4.
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Состояния: queued → processing → ready/failed, с повтором queued при временном отказе. Ограничение 5 попыток, backoff с jitter, 90-секундный lease, выделенный worker с 384 MiB и 0.75 CPU, обработки по одному заданию. Повторная доставка уже ready — no-op. Эффекты S3 имеют детерминированные ключи, метаданные вариантов записываются атомарно.
## Доказательства критериев
## Реальные проверки (заполнить после запуска)
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
- `docker compose config --quiet`: НЕ ПРОВЕРЕНО.
- `python tests/smoke.py`: НЕ ПРОВЕРЕНО.
- Остановка worker → upload 202 → restart → ready: НЕ ПРОВЕРЕНО.
- COMMIT до publish → восстановление dispatcher: НЕ ПРОВЕРЕНО.
- Crash после S3 до БД/ack → восстановление lease: НЕ ПРОВЕРЕНО.
- failed → явный replay: НЕ ПРОВЕРЕНО.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Ограничения
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Единственная PostgreSQL, S3 и единый broker остаются точками отказа. Отказ worker не блокирует HTTP upload, если staging и БД исправны. Accepted означает сохранённые bytes и намерение обработки, но не готовое изображение. Отсутствие именно однократной доставки компенсируется идемпотентной записью эффектов. Staging удаляется только после коммита ready, при ошибках может оставаться; для production требуется дополнительная периодическая очистка бесхозных объектов с retention-политикой. Текущий worker использует ограниченный по времени lease, но жёсткого прерывания отдельной долгой декодировки нет; ресурсный лимит Docker ограничивает память.
+171
View File
@@ -0,0 +1,171 @@
services:
postgres:
image: postgres:17.11-bookworm
environment:
POSTGRES_DB: ${POSTGRES_DB:?Set POSTGRES_DB in .env}
POSTGRES_USER: ${POSTGRES_USER:?Set POSTGRES_USER in .env}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
volumes: [postgres_data:/var/lib/postgresql/data]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 5s
timeout: 3s
retries: 12
restart: unless-stopped
redis:
image: redis:8.2.10-bookworm
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
volumes: [redis_data:/data]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 12
restart: unless-stopped
s3_config:
image: python:3.14.8-slim-bookworm
command: ["python", "/bootstrap/generate-s3-config.py"]
environment:
S3_ROOT_USER: ${S3_ROOT_USER:?Set S3_ROOT_USER}
S3_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Set S3_ROOT_PASSWORD}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
volumes:
- ./infra/generate-s3-config.py:/bootstrap/generate-s3-config.py:ro
- s3_config_data:/s3-config
restart: "no"
s3:
image: chrislusf/seaweedfs:4.47
command: ["mini", "-dir=/data", "-s3.config=/s3-config/s3.json", "-bucket=messenger-attachments"]
volumes:
- s3_data:/data
- s3_config_data:/s3-config:ro
depends_on:
s3_config: {condition: service_completed_successfully}
# Nothing is published: API and Admin UI exist only inside Compose network.
restart: unless-stopped
migrate:
build: .
command: ["python", "-m", "src.migrate"]
environment: &db_env
POSTGRES_DB: ${POSTGRES_DB}
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
depends_on:
postgres:
condition: service_healthy
restart: "no"
api1:
build: .
environment: &api_env
<<: *db_env
INSTANCE_ID: api1
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_REGION: us-east-1
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
depends_on: &api_depends
postgres: {condition: service_healthy}
redis: {condition: service_healthy}
migrate: {condition: service_completed_successfully}
s3: {condition: service_started}
healthcheck: &api_health
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).close()"]
interval: 5s
timeout: 5s
retries: 4
start_period: 15s
cpus: 0.50
mem_limit: 256m
stop_grace_period: 15s
restart: unless-stopped
api2:
build: .
environment:
<<: *api_env
INSTANCE_ID: api2
depends_on: *api_depends
healthcheck: *api_health
cpus: 0.50
mem_limit: 256m
stop_grace_period: 15s
restart: unless-stopped
broker:
image: redis:8.2.10-bookworm
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
volumes: [broker_data:/data]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 12
restart: unless-stopped
dispatcher:
build: .
command: ["python", "-m", "src.jobs", "dispatch"]
healthcheck:
disable: true
environment: &job_env
<<: *api_env
BROKER_HOST: broker
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
s3: {condition: service_started}
restart: unless-stopped
mem_limit: 256m
worker:
build: .
command: ["python", "-m", "src.jobs", "worker"]
healthcheck:
disable: true
environment:
<<: *job_env
LAB5_CRASH_AFTER_S3: ${LAB5_CRASH_AFTER_S3:-0}
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
s3: {condition: service_started}
restart: unless-stopped
mem_limit: 384m
cpus: 0.75
stop_grace_period: 15s
lb:
image: nginx:1.28.0-alpine
volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro
ports: ["127.0.0.1:8080:8080"]
depends_on:
api1: {condition: service_healthy}
api2: {condition: service_healthy}
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 4s
retries: 3
restart: unless-stopped
volumes:
broker_data:
postgres_data:
redis_data:
s3_data:
s3_config_data:
+30
View File
@@ -0,0 +1,30 @@
"""Bootstraps a private S3 identity configuration from non-committed .env.
The server reads it from a shared internal-only configuration volume.
"""
import json
import os
from pathlib import Path
bucket = os.environ["S3_BUCKET"]
assert bucket == "messenger-attachments", "Fixed bucket name; change policy and config deliberately"
config = {
"identities": [
{
"name": "lab4-admin",
"credentials": [{"accessKey": os.environ["S3_ROOT_USER"],
"secretKey": os.environ["S3_ROOT_PASSWORD"]}],
"actions": ["Admin", "Read", "Write", "List", "Tagging"],
},
{
"name": "lab4-api",
"credentials": [{"accessKey": os.environ["S3_ACCESS_KEY_ID"],
"secretKey": os.environ["S3_SECRET_ACCESS_KEY"]}],
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"],
},
],
}
target = Path("/s3-config/s3.json")
target.write_text(json.dumps(config, indent=2), encoding="utf-8")
target.chmod(0o644) # Only mounted into S3 service; SeaweedFS runs non-root on some images.
print("PASS: private S3 identity config generated; credentials not logged")
+30
View File
@@ -0,0 +1,30 @@
CREATE TABLE users (
id UUID PRIMARY KEY,
username VARCHAR(32) NOT NULL UNIQUE,
display_name VARCHAR(100) NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE chats (
id UUID PRIMARY KEY,
title VARCHAR(100) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE chat_members (
chat_id UUID NOT NULL REFERENCES chats(id),
user_id UUID NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
sender_id UUID NOT NULL REFERENCES users(id),
text VARCHAR(4000) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
+1
View File
@@ -0,0 +1 @@
CREATE INDEX chats_order_idx ON chats (created_at, id);
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE message_requests (
sender_id UUID NOT NULL REFERENCES users(id),
chat_id UUID NOT NULL REFERENCES chats(id),
request_key VARCHAR(128) NOT NULL,
request_hash CHAR(64) NOT NULL,
message_id UUID REFERENCES messages(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (sender_id, chat_id, request_key)
);
CREATE INDEX message_requests_created_idx ON message_requests (created_at);
+22
View File
@@ -0,0 +1,22 @@
-- Object bytes are deliberately NOT stored in the database.
CREATE TABLE attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
owner_id UUID NOT NULL REFERENCES users(id),
filename VARCHAR(255) NOT NULL,
content_type VARCHAR(10) NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes INTEGER NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 CHAR(64) NOT NULL,
status VARCHAR(12) NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
object_key TEXT NOT NULL UNIQUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
CREATE TABLE message_attachments (
message_id UUID NOT NULL REFERENCES messages(id),
attachment_id UUID NOT NULL REFERENCES attachments(id),
position INTEGER NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id, position)
);
+31
View File
@@ -0,0 +1,31 @@
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status_check;
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code TEXT;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempts INTEGER NOT NULL DEFAULT 0;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS lease_token UUID;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS lease_until TIMESTAMPTZ;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INTEGER NOT NULL DEFAULT 1;
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INTEGER NOT NULL DEFAULT 1,
name TEXT NOT NULL CHECK (name IN ('thumbnail','crop','optimized')),
object_key TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
sha256 CHAR(64) NOT NULL,
PRIMARY KEY (attachment_id,pipeline_version,name)
);
CREATE TABLE IF NOT EXISTS attachment_outbox (
id BIGSERIAL PRIMARY KEY,
attachment_id UUID NOT NULL REFERENCES attachments(id),
pipeline_version INTEGER NOT NULL DEFAULT 1,
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
published_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS attachment_outbox_pending ON attachment_outbox(available_at,id) WHERE published_at IS NULL;
CREATE INDEX IF NOT EXISTS attachment_jobs_lease ON attachments(status,lease_until);
+27
View File
@@ -0,0 +1,27 @@
upstream api_backends {
zone api_backends 64k;
resolver 127.0.0.11 valid=2s ipv6=off;
server api1:8080 resolve max_fails=1 fail_timeout=2s;
server api2:8080 resolve max_fails=1 fail_timeout=2s;
}
server {
listen 8080;
server_name _;
client_max_body_size 0; # API enforces 10 MiB during streamed reads (uniform JSON errors).
location / {
proxy_pass http://api_backends;
proxy_request_buffering off;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Connection "";
proxy_connect_timeout 1s;
proxy_read_timeout 45s;
proxy_send_timeout 45s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
}
}
+7
View File
@@ -0,0 +1,7 @@
worker_processes auto;
events { worker_connections 1024; }
http {
access_log /dev/stdout;
error_log /dev/stderr warn;
include /etc/nginx/conf.d/*.conf;
}
+20
View File
@@ -0,0 +1,20 @@
annotated-doc==0.0.5
annotated-types==0.8.0
anyio==4.15.1
click==8.5.0
fastapi==0.143.0
h11==0.16.0
idna==3.20
opentelemetry-api==1.45.1
pydantic==2.14.1
pydantic_core==2.50.1
starlette==1.7.0
typing-inspection==0.4.4
typing_extensions==4.16.0
uvicorn==0.54.0
psycopg[binary]==3.3.6
psycopg-pool==3.3.3
argon2-cffi==25.1.0
redis==8.1.0
boto3==1.40.52
Pillow==12.0.0
+28
View File
@@ -0,0 +1,28 @@
import argparse
from uuid import UUID
from src.db import pool
def main():
parser = argparse.ArgumentParser()
parser.add_argument('attachment_id')
args = parser.parse_args()
identifier = UUID(args.attachment_id)
pool.open(wait=True)
with pool.connection() as conn:
with conn.transaction():
row = conn.execute('SELECT status,pipeline_version FROM attachments WHERE id=%s FOR UPDATE',
(identifier,)).fetchone()
if row is None or row['status'] != 'failed':
raise SystemExit('Только failed-вложение можно повторно отправить в обработку')
conn.execute('''UPDATE attachments SET status='queued',attempts=0,error_code=NULL,
lease_token=NULL,lease_until=NULL,next_attempt_at=now(),updated_at=now()
WHERE id=%s''', (identifier,))
conn.execute('INSERT INTO attachment_outbox(attachment_id,pipeline_version) VALUES (%s,%s)',
(identifier,row['pipeline_version']))
print('QUEUED:',identifier)
if __name__=='__main__':
main()
View File
+29
View File
@@ -0,0 +1,29 @@
import os
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
def connection_kwargs():
return {
"host": os.getenv("POSTGRES_HOST", "postgres"),
"port": int(os.getenv("POSTGRES_PORT", "5432")),
"dbname": os.environ["POSTGRES_DB"],
"user": os.environ["POSTGRES_USER"],
"password": os.environ["POSTGRES_PASSWORD"],
"connect_timeout": 5,
"options": "-c statement_timeout=5000 -c lock_timeout=3000",
"autocommit": True,
"row_factory": dict_row,
}
pool = ConnectionPool(
kwargs=connection_kwargs(),
min_size=1,
max_size=5,
max_waiting=20,
timeout=3,
check=ConnectionPool.check_connection,
open=False,
)
+14
View File
@@ -0,0 +1,14 @@
class ApiError(Exception):
def __init__(self, status, code, message):
super().__init__(message)
self.status = status
self.code = code
self.message = message
def invalid(message="Неверный запрос"):
raise ApiError(400, "invalid_request", message)
def unauthorized():
raise ApiError(401, "unauthorized", "Неверные данные авторизации")
+268
View File
@@ -0,0 +1,268 @@
import hashlib
import io
import os
import random
import time
from datetime import datetime, timedelta, timezone
from uuid import UUID, uuid4
import redis
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, ImageOps, UnidentifiedImageError
from src.db import pool
from src.storage import bucket, s3, MAX_IMAGE_PIXELS
STREAM = 'lab5:image-jobs'
GROUP = 'lab5-workers'
ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
LEASE_SECONDS = int(os.getenv('JOB_LEASE_SECONDS', '90'))
WORKER_MAX_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000'))
client = redis.Redis(host=os.getenv('BROKER_HOST', 'broker'), port=6379, decode_responses=True,
socket_connect_timeout=3, socket_timeout=3)
Image.MAX_IMAGE_PIXELS = WORKER_MAX_PIXELS
def stage_upload(file, key, size, content_type):
file.seek(0)
s3.put_object(Bucket=bucket, Key=key, Body=file, ContentLength=size, ContentType=content_type)
def queue_attachment(attachment_id, chat_id, owner_id, filename, content_type, size, digest, stage_key):
with pool.connection() as conn:
with conn.transaction():
membership = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s',
(chat_id, owner_id)).fetchone()
if not membership:
return None
item = conn.execute('''INSERT INTO attachments
(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key,staging_key)
VALUES (%s,%s,%s,%s,%s,%s,%s,'queued',%s,%s) RETURNING *''',
(attachment_id, chat_id, owner_id, filename, content_type, size, digest,
'originals/'+attachment_id.hex, stage_key)).fetchone()
conn.execute('INSERT INTO attachment_outbox (attachment_id) VALUES (%s)', (attachment_id,))
return item
def variants_for(attachment_id, pipeline_version=1):
with pool.connection() as conn:
rows = conn.execute('''SELECT name,content_type,size_bytes,width,height,sha256
FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name''',
(attachment_id, pipeline_version)).fetchall()
return [dict(r) for r in rows]
def fetch_variant(attachment_id, name, pipeline_version=1):
with pool.connection() as conn:
return conn.execute('''SELECT * FROM attachment_variants
WHERE attachment_id=%s AND pipeline_version=%s AND name=%s''',
(attachment_id, pipeline_version, name)).fetchone()
def _make_images(data):
if len(data) > 10485760:
raise ValueError('size_limit')
with Image.open(io.BytesIO(data)) as probe:
w,h = probe.size
if w*h > WORKER_MAX_PIXELS:
raise ValueError('pixel_limit')
image = ImageOps.exif_transpose(probe)
image.load()
if image.mode == 'RGB':
image = image.copy()
else:
rgba = image.convert('RGBA')
background = Image.new('RGB', rgba.size, 'white')
background.paste(rgba, mask=rgba.getchannel('A'))
image = background
variants = []
thumb = image.copy()
thumb.thumbnail((256,256), Image.Resampling.LANCZOS)
variants.append(('thumbnail',thumb,'JPEG',{'quality':85,'optimize':True}))
crop = ImageOps.fit(image,(128,128),Image.Resampling.LANCZOS,centering=(0.5,0.5))
variants.append(('crop',crop,'JPEG',{'quality':85,'optimize':True}))
opt = image.copy()
opt.thumbnail((1280,1280),Image.Resampling.LANCZOS)
variants.append(('optimized',opt,'WEBP',{'quality':80,'method':4}))
out = []
for name,im,fmt,kwargs in variants:
buf=io.BytesIO()
im.save(buf, format=fmt, **kwargs)
raw=buf.getvalue()
out.append((name,raw,'image/webp' if fmt=='WEBP' else 'image/jpeg',im.width,im.height,hashlib.sha256(raw).hexdigest()))
return out
def _publish_pending():
with pool.connection() as conn:
with conn.transaction():
rows=conn.execute('''SELECT o.id,o.attachment_id,o.pipeline_version
FROM attachment_outbox o JOIN attachments a ON a.id=o.attachment_id
WHERE o.published_at IS NULL AND o.available_at<=now() AND a.status='queued'
ORDER BY o.id LIMIT 25 FOR UPDATE OF o SKIP LOCKED''').fetchall()
for row in rows:
client.xadd(STREAM,{'attachment_id':str(row['attachment_id']),
'job_id':str(row['attachment_id']),
'pipeline_version':str(row['pipeline_version'])})
conn.execute('UPDATE attachment_outbox SET published_at=now() WHERE id=%s',(row['id'],))
def _recover():
with pool.connection() as conn:
with conn.transaction():
expired=conn.execute('''UPDATE attachments SET status='queued',lease_token=NULL,
lease_until=NULL,next_attempt_at=now(),updated_at=now()
WHERE status='processing' AND lease_until<now() RETURNING id,pipeline_version''').fetchall()
for row in expired:
conn.execute('INSERT INTO attachment_outbox(attachment_id,pipeline_version) VALUES (%s,%s)',
(row['id'],row['pipeline_version']))
retry=conn.execute('''SELECT a.id,a.pipeline_version FROM attachments a WHERE a.status='queued'
AND a.next_attempt_at<=now() AND NOT EXISTS
(SELECT 1 FROM attachment_outbox o WHERE o.attachment_id=a.id AND o.published_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM attachment_outbox o WHERE o.attachment_id=a.id
AND o.created_at>now()-interval '30 seconds') LIMIT 50''').fetchall()
for row in retry:
conn.execute('INSERT INTO attachment_outbox(attachment_id,pipeline_version) VALUES (%s,%s)',
(row['id'],row['pipeline_version']))
def dispatcher():
pool.open(wait=True)
while True:
try:
_recover()
_publish_pending()
except Exception as exc:
print('dispatcher:',type(exc).__name__,flush=True)
time.sleep(2)
def _claim(attachment_id, version):
token=uuid4()
with pool.connection() as conn:
with conn.transaction():
row=conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(attachment_id,)).fetchone()
if not row or row['pipeline_version']!=version or row['status'] in ('ready','failed'):
return None
if row['status']=='processing' or row['next_attempt_at']>datetime.now(timezone.utc):
return None
conn.execute('''UPDATE attachments SET status='processing',attempts=attempts+1,
lease_token=%s,lease_until=now()+(%s * interval '1 second'),updated_at=now()
WHERE id=%s''',(token,LEASE_SECONDS,attachment_id))
return (row,token)
def _finish(row,token,outputs):
with pool.connection() as conn:
with conn.transaction():
locked=conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(row['id'],)).fetchone()
if not locked or locked['status']!='processing' or locked['lease_token']!=token:
return False
for name,data,ct,w,h,sha in outputs:
key=f"variants/{row['id'].hex}/v{row['pipeline_version']}/{name}"
conn.execute('''INSERT INTO attachment_variants
(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,width,height,sha256)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
ON CONFLICT (attachment_id,pipeline_version,name) DO UPDATE SET
object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,
size_bytes=EXCLUDED.size_bytes,width=EXCLUDED.width,height=EXCLUDED.height,
sha256=EXCLUDED.sha256''',
(row['id'],row['pipeline_version'],name,key,ct,len(data),w,h,sha))
conn.execute('''UPDATE attachments SET status='ready',error_code=NULL,
lease_token=NULL,lease_until=NULL,updated_at=now() WHERE id=%s''',(row['id'],))
return True
def _failure(row,token,exc):
permanent = isinstance(exc,(ValueError,UnidentifiedImageError,Image.DecompressionBombError))
with pool.connection() as conn:
with conn.transaction():
current=conn.execute('SELECT attempts,lease_token,status FROM attachments WHERE id=%s FOR UPDATE',
(row['id'],)).fetchone()
if not current or current['status']!='processing' or current['lease_token']!=token:
return
attempts=current['attempts']
if permanent or attempts>=ATTEMPTS:
code='invalid_image' if permanent else 'retry_exhausted'
conn.execute('''UPDATE attachments SET status='failed',error_code=%s,
lease_token=NULL,lease_until=NULL,updated_at=now() WHERE id=%s''',(code,row['id']))
else:
delay=min(60, 2**min(attempts,5))+random.uniform(0,1.5)
conn.execute('''UPDATE attachments SET status='queued',error_code=NULL,
lease_token=NULL,lease_until=NULL,
next_attempt_at=now()+(%s * interval '1 second'),updated_at=now()
WHERE id=%s''',(delay,row['id']))
conn.execute('''INSERT INTO attachment_outbox(attachment_id,pipeline_version,available_at)
VALUES (%s,%s,now()+(%s * interval '1 second'))''',
(row['id'],row['pipeline_version'],delay))
def _process(attachment_id,version):
claim=_claim(attachment_id,version)
if not claim:
return
row,token=claim
try:
response=s3.get_object(Bucket=bucket,Key=row['staging_key'])
data=response['Body'].read(10485761)
response['Body'].close()
if hashlib.sha256(data).hexdigest()!=row['sha256']:
raise ValueError('checksum_mismatch')
outputs=_make_images(data)
s3.copy_object(Bucket=bucket,CopySource={'Bucket':bucket,'Key':row['staging_key']},
Key=row['object_key'],ContentType=row['content_type'],MetadataDirective='REPLACE')
for name,content,ct,w,h,sha in outputs:
s3.put_object(Bucket=bucket,
Key=f"variants/{row['id'].hex}/v{version}/{name}",
Body=content,ContentType=ct)
if os.getenv('LAB5_CRASH_AFTER_S3')=='1':
os._exit(91)
if _finish(row,token,outputs):
try:
s3.delete_object(Bucket=bucket,Key=row['staging_key'])
except Exception:
pass
except Exception as exc:
_failure(row,token,exc)
def worker():
pool.open(wait=True)
consumer=os.getenv('HOSTNAME','worker')+'-'+uuid4().hex[:6]
try:
client.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
except redis.exceptions.ResponseError as e:
if 'BUSYGROUP' not in str(e):
raise
last_scan=0
while True:
try:
if time.monotonic()-last_scan>5:
pending=client.xautoclaim(STREAM,GROUP,consumer,min_idle_time=LEASE_SECONDS*1000,
start_id='0-0',count=10)
for msgid,fields in pending[1]:
try:
_process(UUID(fields['attachment_id']),int(fields.get('pipeline_version','1')))
finally:
client.xack(STREAM,GROUP,msgid)
last_scan=time.monotonic()
entries=client.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=2000)
for _,messages in entries:
for msgid,fields in messages:
try:
_process(UUID(fields['attachment_id']),int(fields.get('pipeline_version','1')))
except Exception as e:
print('worker:',type(e).__name__,flush=True)
else:
client.xack(STREAM,GROUP,msgid)
except Exception as e:
print('broker:',type(e).__name__,flush=True)
time.sleep(1)
if __name__=='__main__':
import sys
if len(sys.argv)>1 and sys.argv[1]=='dispatch':
dispatcher()
else:
worker()
+663
View File
@@ -0,0 +1,663 @@
import base64
import binascii
import hashlib
import json
import os
import logging
import re
import time
import tempfile
import threading
import unicodedata
from io import BytesIO
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from uuid import UUID, uuid4
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError
from fastapi import FastAPI, Header, Query, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response, StreamingResponse
from psycopg import InterfaceError, OperationalError
from psycopg.errors import ForeignKeyViolation, UniqueViolation
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from redis.exceptions import RedisError
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException as StarletteHTTPException
from src.db import pool
from src.jobs import stage_upload, queue_attachment, variants_for, fetch_variant
from src.errors import ApiError, invalid, unauthorized
from src.pagination import encode_cursor, parse_page
from src.sessions import issue_session, redis_client
from src.storage import s3, bucket, check_storage, upload_original, download_original, delete_original, StorageFailure, MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, UnidentifiedImageError
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
log = logging.getLogger("messenger")
password_hasher = PasswordHasher()
INSTANCE_ID = os.getenv("INSTANCE_ID", uuid4().hex)
ready = False
IMAGE_DECODE_SLOT = threading.BoundedSemaphore(1) # prevent concurrent decoded-image RAM spikes
def parse_uuid(value):
try:
return UUID(value)
except (ValueError, TypeError, AttributeError):
invalid("Некорректный UUID")
def row_response(row):
result = {}
for key, value in row.items():
if isinstance(value, UUID):
value = str(value)
elif isinstance(value, datetime):
value = value.astimezone(timezone.utc).isoformat(
timespec="microseconds"
).replace("+00:00", "Z")
elif key in ("member_ids", "attachment_ids"):
value = [str(member) for member in value]
result[key] = value
return result
class InputModel(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
@field_validator("*", check_fields=False)
@classmethod
def reject_blank(cls, value):
if isinstance(value, str) and not value.strip():
raise ValueError("Строка не должна быть пустой")
return value
@field_validator("display_name", "title", "text", check_fields=False)
@classmethod
def reject_nul(cls, value):
if "\u0000" in value:
raise ValueError("Нулевой символ недопустим в текстовом поле")
return value
class CreateUser(InputModel):
username: str = Field(min_length=3, max_length=32)
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def validate_username(cls, value):
if re.fullmatch(r"[a-z0-9_]{3,32}", value) is None:
raise ValueError("Некорректный username")
return value
class CreateChat(InputModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
class CreateMessage(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
text: str = Field(min_length=0, max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@model_validator(mode="after")
def validate_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Нужен текст или вложение")
if "\x00" in self.text:
raise ValueError("Недопустимый текст")
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("Повторные attachment_ids")
for value in self.attachment_ids:
try:
UUID(value)
except (ValueError, TypeError):
raise ValueError("Некорректный attachment_id") from None
return self
def bearer_token(value):
scheme, _, token = (value or "").partition(" ")
if scheme.lower() != "bearer" or not token:
unauthorized()
return token
def current_user(value):
token = bearer_token(value)
user_id = redis_client.get("session:" + token)
if user_id is None:
unauthorized()
with pool.connection() as conn:
user = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(UUID(user_id),),
).fetchone()
if user is None:
unauthorized()
return row_response(user)
@asynccontextmanager
async def lifespan(app):
global ready
try:
await run_in_threadpool(pool.open, wait=True, timeout=10)
await run_in_threadpool(redis_client.ping)
# Container may have started before the S3 service finished bootstrap.
storage_ok = False
for _ in range(30):
try:
await run_in_threadpool(check_storage)
storage_ok = True
break
except (BotoCoreError, ClientError, StorageFailure):
import asyncio
await asyncio.sleep(2)
if not storage_ok:
raise RuntimeError("S3 not ready")
ready = True
log.info("startup ready=true")
yield
finally:
ready = False
redis_client.close()
pool.close()
log.info("shutdown complete")
app = FastAPI(title="Lab 4 Messenger", lifespan=lifespan)
def error_response(request, status, code, message):
request_id = request.state.request_id
headers = {"X-Request-Id": request_id}
if status == 401:
if request.method == "POST" and request.url.path == "/api/v1/auth/sessions":
headers["WWW-Authenticate"] = 'Basic realm="messenger", charset="UTF-8"'
else:
headers["WWW-Authenticate"] = 'Bearer realm="messenger"'
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": request_id}},
headers=headers,
)
@app.exception_handler(ApiError)
async def api_error(request, exc):
return error_response(request, exc.status, exc.code, exc.message)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
@app.exception_handler((StorageFailure))
async def storage_error(request, exc):
return error_response(request, 503, "unavailable", "Хранилище недоступно")
@app.exception_handler(ClientError)
async def boto_error(request, exc):
return error_response(request, 503, "unavailable", "Хранилище недоступно")
@app.exception_handler(StarletteHTTPException)
async def http_error(request, exc):
code = "not_found" if exc.status_code == 404 else "invalid_request"
return error_response(request, exc.status_code, code, "Некорректный запрос")
@app.middleware("http")
async def process_request(request: Request, call_next):
request.state.request_id = str(uuid4())
started = time.perf_counter()
try:
is_login = request.method == "POST" and request.url.path == "/api/v1/auth/sessions"
public = request.method == "POST" and request.url.path in (
"/api/v1/users", "/api/v1/auth/sessions"
)
if request.url.path.startswith("/api/v1/") and not public:
request.state.user = await run_in_threadpool(
current_user, request.headers.get("Authorization")
)
if is_login:
if await request.body():
invalid("Для входа тело запроса должно быть пустым")
elif request.method == "POST" and request.url.path.startswith("/api/v1/"):
media_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
if re.fullmatch(r"/api/v1/chats/[^/]+/attachments", request.url.path):
if media_type not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Только PNG/JPEG")
elif media_type != "application/json":
raise ApiError(415, "unsupported_media_type", "Нужен application/json")
response = await call_next(request)
except ApiError as exc:
response = error_response(request, exc.status, exc.code, exc.message)
except (OperationalError, InterfaceError, RedisError, StorageFailure, BotoCoreError, ClientError):
response = error_response(request, 503, "unavailable", "Хранилище недоступно")
except Exception as exc:
log.error("request_failed type=%s request_id=%s",
type(exc).__name__, request.state.request_id)
response = error_response(request, 500, "internal_error", "Внутренняя ошибка")
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
route = request.scope.get("route")
log.info("request method=%s route=%s status=%s duration_ms=%.2f request_id=%s",
request.method, getattr(route, "path", "unmatched"), response.status_code,
(time.perf_counter() - started) * 1000, request.state.request_id)
return response
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def readiness():
if not ready:
raise ApiError(503, "unavailable", "Приложение не готово")
with pool.connection() as conn:
conn.execute("SELECT 1")
redis_client.ping()
check_storage()
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
def create_user(body: CreateUser):
password_hash = password_hasher.hash(body.password)
try:
with pool.connection() as conn:
user = conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) "
"VALUES (%s, %s, %s, %s) RETURNING id, username, display_name, created_at",
(uuid4(), body.username, body.display_name, password_hash),
).fetchone()
except UniqueViolation as exc:
if exc.diag.constraint_name == "users_username_key":
raise ApiError(409, "conflict", "Username занят") from None
raise
return row_response(user)
@app.post("/api/v1/auth/sessions", status_code=201)
def login(authorization: str | None = Header(default=None)):
scheme, _, encoded = (authorization or "").partition(" ")
if scheme.lower() != "basic" or not encoded:
unauthorized()
try:
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, binascii.Error):
unauthorized()
if re.fullmatch(r"[a-z0-9_]{3,32}", username) is None or not 12 <= len(password) <= 128:
unauthorized()
with pool.connection() as conn:
user = conn.execute(
"SELECT id, username, display_name, password_hash, created_at "
"FROM users WHERE username = %s", (username,),
).fetchone()
if user is None:
unauthorized()
try:
password_hasher.verify(user["password_hash"], password)
except VerifyMismatchError:
unauthorized()
token, expires_at = issue_session(user["id"])
del user["password_hash"]
return {"token": token, "token_type": "Bearer", "expires_at": expires_at,
"user": row_response(user)}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(request: Request):
token = bearer_token(request.headers.get("Authorization"))
redis_client.delete("session:" + token)
return Response(status_code=204)
@app.get("/api/v1/users/me")
def get_me(request: Request):
return request.state.user
def require_member(conn, chat_id, user_id):
found = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, UUID(user_id)),
).fetchone()
if found is None:
raise ApiError(404, "not_found", "Чат не найден")
CHAT_SELECT = """
SELECT c.id, c.title, c.created_at,
ARRAY(SELECT user_id FROM chat_members
WHERE chat_id = c.id ORDER BY user_id) AS member_ids
FROM chats c
JOIN chat_members mine ON mine.chat_id = c.id
WHERE mine.user_id = %s
"""
MESSAGE_SELECT = """
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT ma.attachment_id FROM message_attachments ma
WHERE ma.message_id=m.id ORDER BY ma.position)::uuid[] AS attachment_ids
FROM messages m
"""
HISTORY_FIRST_SQL = MESSAGE_SELECT + """
WHERE m.chat_id = %s ORDER BY m.created_at, m.id LIMIT %s
"""
HISTORY_AFTER_SQL = MESSAGE_SELECT + """
WHERE m.chat_id = %s AND (m.created_at, m.id) > (%s, %s)
ORDER BY m.created_at, m.id LIMIT %s
"""
def page_response(rows, size, scope):
items = [row_response(row) for row in rows[:size]]
cursor = encode_cursor(scope, items[-1]) if len(rows) > size else None
return {"items": items, "next_cursor": cursor}
@app.post("/api/v1/chats", status_code=201)
def create_chat(body: CreateChat, request: Request):
members = sorted(parse_uuid(value) for value in body.member_ids)
if len(members) != len(set(members)):
invalid("Участники повторяются")
if UUID(request.state.user["id"]) not in members:
invalid("Вызывающий должен входить в member_ids")
try:
with pool.connection() as conn:
with conn.transaction():
chat = conn.execute(
"INSERT INTO chats (id, title) VALUES (%s, %s) "
"RETURNING id, title, created_at", (uuid4(), body.title),
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
[(chat["id"], member) for member in members],
)
except ForeignKeyViolation:
invalid("Неизвестный участник")
chat["member_ids"] = members
return row_response(chat)
@app.get("/api/v1/chats")
def list_chats(request: Request, limit: str = Query(default="50"),
cursor: str | None = Query(default=None)):
user_id = request.state.user["id"]
scope = "chats:" + user_id
size, after = parse_page(limit, cursor, scope)
query = CHAT_SELECT
params = [UUID(user_id)]
if after is not None:
query += " AND (c.created_at, c.id) > (%s, %s)"
params.extend(after)
query += " ORDER BY c.created_at, c.id LIMIT %s"
params.append(size + 1)
with pool.connection() as conn:
rows = conn.execute(query, params).fetchall()
return page_response(rows, size, scope)
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: str, request: Request):
with pool.connection() as conn:
chat = conn.execute(
CHAT_SELECT + " AND c.id = %s",
(UUID(request.state.user["id"]), parse_uuid(chat_id)),
).fetchone()
if chat is None:
raise ApiError(404, "not_found", "Чат не найден")
return row_response(chat)
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def create_message(
chat_id: str,
body: CreateMessage,
request: Request,
idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"),
):
if idempotency_key is None or re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) is None:
invalid("Некорректный Idempotency-Key")
chat_uuid = parse_uuid(chat_id)
user_id = UUID(request.state.user["id"])
digest = hashlib.sha256(
json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest()
with pool.connection() as conn:
with conn.transaction():
require_member(conn, chat_uuid, str(user_id))
if body.attachment_ids:
attachment_uuids = [UUID(v) for v in body.attachment_ids]
found = conn.execute(
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)",
(attachment_uuids,),
).fetchall()
if len(found) != len(attachment_uuids) or any(row["chat_id"] != chat_uuid for row in found):
raise ApiError(404, "not_found", "Вложение не найдено")
if any(row["status"] == "failed" for row in found):
raise ApiError(409, "conflict", "Вложение не готово")
conn.execute(
"DELETE FROM message_requests WHERE sender_id = %s AND chat_id = %s "
"AND request_key = %s AND created_at < now() - interval '24 hours'",
(user_id, chat_uuid, idempotency_key),
)
created = conn.execute(
"INSERT INTO message_requests (sender_id, chat_id, request_key, request_hash) "
"VALUES (%s, %s, %s, %s) ON CONFLICT DO NOTHING RETURNING request_key",
(user_id, chat_uuid, idempotency_key, digest),
).fetchone()
if created is None:
existing = conn.execute(
"SELECT request_hash, message_id FROM message_requests "
"WHERE sender_id = %s AND chat_id = %s AND request_key = %s",
(user_id, chat_uuid, idempotency_key),
).fetchone()
if existing is None:
raise ApiError(503, "unavailable", "Повторите запрос")
if existing["request_hash"] != digest:
raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения")
message = conn.execute(
MESSAGE_SELECT + " WHERE m.id = %s",
(existing["message_id"],),
).fetchone()
else:
message = conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text) "
"VALUES (%s, %s, %s, %s) "
"RETURNING id, chat_id, sender_id, text, created_at",
(uuid4(), chat_uuid, user_id, body.text),
).fetchone()
for position, attach in enumerate(body.attachment_ids):
conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
(message["id"], UUID(attach), position),
)
conn.execute(
"UPDATE message_requests SET message_id = %s "
"WHERE sender_id = %s AND chat_id = %s AND request_key = %s",
(message["id"], user_id, chat_uuid, idempotency_key),
)
if "attachment_ids" not in message:
message["attachment_ids"] = [UUID(v) for v in body.attachment_ids]
return row_response(message)
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: str, request: Request, limit: str = Query(default="50"),
cursor: str | None = Query(default=None)):
chat_uuid = parse_uuid(chat_id)
scope = "messages:" + str(chat_uuid)
size, after = parse_page(limit, cursor, scope)
with pool.connection() as conn:
require_member(conn, chat_uuid, request.state.user["id"])
if after is None:
rows = conn.execute(HISTORY_FIRST_SQL, (chat_uuid, size + 1)).fetchall()
else:
rows = conn.execute(
HISTORY_AFTER_SQL, (chat_uuid, after[0], after[1], size + 1)
).fetchall()
return page_response(rows, size, scope)
# Lab 4: binary media are kept in private S3; PostgreSQL stores only metadata.
# Raw request bodies are streamed in bounded chunks to a temporary spool file.
def validate_image(stream, advertised_type):
stream.seek(0)
try:
with IMAGE_DECODE_SLOT, Image.open(stream) as probe:
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(probe.format)
width, height = probe.size
if width * height > MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
if actual != advertised_type:
raise ApiError(415, "unsupported_media_type", "Тип изображения не совпадает с содержимым")
probe.verify()
stream.seek(0)
# Verifying full decode rejects truncated/corrupt image streams.
with IMAGE_DECODE_SLOT, Image.open(stream) as image:
image.load()
except ApiError:
raise
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError):
raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None
stream.seek(0)
def attachment_response(row):
result = row_response(row)
for key in ("object_key", "staging_key", "attempts", "lease_token", "lease_until", "next_attempt_at", "pipeline_version"):
result.pop(key, None)
result["variants"] = variants_for(row["id"]) if row["status"] == "ready" else []
return result
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def create_attachment(chat_id: str, request: Request, filename: str = Query(...)):
chat_uuid = parse_uuid(chat_id)
user_id = UUID(request.state.user["id"])
if not 1 <= len(filename) <= 255 or any(c in filename for c in "/\\\x00") or any(unicodedata.category(c) == "Cc" for c in filename):
invalid("Опасное имя файла")
content_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
length_header = request.headers.get("content-length")
if length_header and length_header.isdigit() and int(length_header) > MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
# ACL before touching storage.
await run_in_threadpool(check_membership, chat_uuid, user_id)
file = tempfile.SpooledTemporaryFile(max_size=256 * 1024)
digest = hashlib.sha256()
size = 0
try:
async for chunk in request.stream():
size += len(chunk)
if size > MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
digest.update(chunk)
file.write(chunk)
if size == 0:
raise ApiError(415, "unsupported_media_type", "Пустой файл")
await run_in_threadpool(validate_image, file, content_type)
attachment_id = uuid4()
object_key = f"staging/{attachment_id.hex}"
file.seek(0)
# One S3 PUT; no multipart uploads left by a client disconnect.
try:
await run_in_threadpool(stage_upload, file, object_key, size, content_type)
except (StorageFailure, BotoCoreError, ClientError):
raise ApiError(503, "unavailable", "Объектное хранилище недоступно") from None
try:
result = await run_in_threadpool(queue_attachment, attachment_id, chat_uuid, user_id, filename,
content_type, size, digest.hexdigest(), object_key)
if result is None:
raise ApiError(404, "not_found", "Чат не найден")
except Exception:
# Best effort compensation; orphan reconciler handles crash between S3 PUT and DB COMMIT.
try:
await run_in_threadpool(delete_original, object_key)
except Exception:
log.warning("upload compensation failed for %s", object_key)
raise
return attachment_response(result)
finally:
file.close()
def check_membership(chat_uuid, user_id):
with pool.connection() as conn:
require_member(conn, chat_uuid, str(user_id))
def save_attachment(attachment_id, chat_uuid, user_id, filename, content_type, size, digest, key):
with pool.connection() as conn:
with conn.transaction():
require_member(conn, chat_uuid, str(user_id))
return conn.execute(
"INSERT INTO attachments (id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',%s) RETURNING *",
(attachment_id, chat_uuid, user_id, filename, content_type, size, digest, key)
).fetchone()
def owned_attachment(attachment_id, user_id):
with pool.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN chat_members cm ON cm.chat_id = a.chat_id "
"WHERE a.id = %s AND cm.user_id = %s",
(attachment_id, UUID(user_id)),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Вложение не найдено")
return row
@app.get("/api/v1/attachments/{attachment_id}")
def attachment_metadata(attachment_id: str, request: Request):
return attachment_response(owned_attachment(parse_uuid(attachment_id), request.state.user["id"]))
@app.get("/api/v1/attachments/{attachment_id}/content")
def attachment_content(attachment_id: str, request: Request, variant: str = Query(default="original")):
row = owned_attachment(parse_uuid(attachment_id), request.state.user["id"])
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение не готово")
item = None
if variant != "original":
if variant not in ("thumbnail", "crop", "optimized"):
raise ApiError(404, "not_found", "Вариант не найден")
item = fetch_variant(row["id"], variant)
if item is None:
raise ApiError(404, "not_found", "Вариант не найден")
key = row["object_key"] if item is None else item["object_key"]
mime = row["content_type"] if item is None else item["content_type"]
size = row["size_bytes"] if item is None else item["size_bytes"]
try:
s3_body = download_original(key)
except (BotoCoreError, ClientError, StorageFailure):
raise ApiError(503, "unavailable", "Объектное хранилище недоступно") from None
def chunks():
try:
for data in s3_body.iter_chunks(chunk_size=64 * 1024):
if data:
yield data
finally:
s3_body.close()
return StreamingResponse(chunks(), media_type=mime,
headers={"Content-Length": str(size), "Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff"})
+36
View File
@@ -0,0 +1,36 @@
from pathlib import Path
import psycopg
from src.db import connection_kwargs
def main():
files = sorted((Path(__file__).resolve().parents[1] / "migrations").glob("*.sql"))
if not files:
raise RuntimeError("Миграции не найдены")
with psycopg.connect(**connection_kwargs()) as conn:
conn.execute("""
CREATE TABLE IF NOT EXISTS schema_migrations (
version TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""")
for file in files:
applied = False
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(202602)")
exists = conn.execute(
"SELECT 1 FROM schema_migrations WHERE version = %s", (file.name,)
).fetchone()
if exists is None:
conn.execute(file.read_text(encoding="utf-8"))
conn.execute(
"INSERT INTO schema_migrations (version) VALUES (%s)", (file.name,)
)
applied = True
print(("APPLIED " if applied else "SKIP ") + file.name)
if __name__ == "__main__":
main()
+47
View File
@@ -0,0 +1,47 @@
import base64
import binascii
import json
import re
from datetime import datetime
from uuid import UUID
from src.errors import invalid
def encode_cursor(scope, item):
data = {"scope": scope, "created_at": item["created_at"], "id": item["id"]}
raw = json.dumps(data, separators=(",", ":")).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def parse_page(limit, cursor, scope):
if len(limit) > 10 or re.fullmatch(r"[0-9]+", limit) is None:
invalid("limit должен быть целым числом от 1 до 100")
size = int(limit)
if not 1 <= size <= 100:
invalid("limit должен быть от 1 до 100")
if cursor is None:
return size, None
if not cursor or len(cursor) > 2048:
invalid("Некорректный cursor")
try:
raw = base64.b64decode(
cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True
)
data = json.loads(raw)
if not isinstance(data, dict) or set(data) != {"scope", "created_at", "id"}:
raise ValueError
if data["scope"] != scope or not isinstance(data["created_at"], str):
raise ValueError
if re.fullmatch(
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z",
data["created_at"],
) is None:
raise ValueError
created_at = datetime.fromisoformat(data["created_at"][:-1] + "+00:00")
if created_at.tzinfo is None:
raise ValueError
item_id = UUID(data["id"])
except (ValueError, TypeError, AttributeError, binascii.Error):
invalid("Некорректный cursor или cursor от другого ресурса")
return size, (created_at, item_id)
+57
View File
@@ -0,0 +1,57 @@
"""Dry-run by default: find unused S3 originals older than safe grace period.
Usage:
docker compose exec -T api1 python -m src.reconcile
docker compose exec -T api1 python -m src.reconcile --apply --min-age-seconds 3600
No database objects or live originals are ever deleted. Only orphans, and only
under the server-owned originals/ prefix, are eligible for cleanup.
"""
import argparse
import datetime as dt
import json
import logging
from src.db import connection_kwargs
from src.storage import s3, bucket
import psycopg
log = logging.getLogger("attachment-reconcile")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true", help="Delete eligible orphan objects")
parser.add_argument("--min-age-seconds", type=int, default=3600)
args = parser.parse_args()
if args.min_age_seconds < 3600:
parser.error("Safety grace period must be at least 3600s")
now = dt.datetime.now(dt.timezone.utc)
checked = 0
orphan = 0
removed = 0
with psycopg.connect(**connection_kwargs()) as conn:
for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix="originals/"):
for item in page.get("Contents", []):
checked += 1
key = item["Key"]
age = (now - item["LastModified"]).total_seconds()
if age < args.min_age_seconds:
continue
exists = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (key,)).fetchone()
if exists:
continue
orphan += 1
if args.apply:
# Recheck immediately before deletion; do not delete newly referenced keys.
exists = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (key,)).fetchone()
if exists:
continue
s3.delete_object(Bucket=bucket, Key=key)
removed += 1
print(json.dumps({"mode": "apply" if args.apply else "dry-run", "scanned": checked,
"eligible_orphans": orphan, "removed": removed,
"min_age_seconds": args.min_age_seconds}, ensure_ascii=False))
if __name__ == "__main__":
main()
+23
View File
@@ -0,0 +1,23 @@
import logging
import os
import uvicorn
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
logging.getLogger("messenger").info(
"config app_env=%s", os.getenv("APP_ENV", "lab")
)
uvicorn.run(
"src.main:app",
host=os.getenv("APP_HOST", "0.0.0.0"),
port=int(os.getenv("APP_PORT", "8080")),
workers=1,
timeout_graceful_shutdown=5,
timeout_keep_alive=1,
access_log=False,
)
+33
View File
@@ -0,0 +1,33 @@
import os
import secrets
import time
from datetime import datetime, timezone
from redis import Redis
from redis.backoff import NoBackoff
from redis.retry import Retry
redis_client = Redis(
host=os.getenv("REDIS_HOST", "redis"),
port=int(os.getenv("REDIS_PORT", "6379")),
decode_responses=True,
socket_connect_timeout=2,
socket_timeout=2,
max_connections=10,
retry=Retry(NoBackoff(), 0),
)
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
if SESSION_TTL_SECONDS <= 0:
raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным")
def issue_session(user_id):
token = secrets.token_urlsafe(32)
expires_ms = int(time.time() * 1000) + SESSION_TTL_SECONDS * 1000
redis_client.set("session:" + token, str(user_id), pxat=expires_ms)
expires_at = datetime.fromtimestamp(
expires_ms / 1000, timezone.utc
).isoformat(timespec="milliseconds").replace("+00:00", "Z")
return token, expires_at
+43
View File
@@ -0,0 +1,43 @@
"""S3-compatible private storage. Only API-scoped credentials are available here."""
import os
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError, BotoCoreError
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
bucket = os.getenv("S3_BUCKET", "messenger-attachments")
s3 = boto3.client(
"s3", endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
region_name=os.getenv("S3_REGION", "us-east-1"),
aws_access_key_id=os.environ.get("S3_ACCESS_KEY_ID", "unset"),
aws_secret_access_key=os.environ.get("S3_SECRET_ACCESS_KEY", "unset"),
config=Config(s3={"addressing_style": "path"}, connect_timeout=2, read_timeout=30,
retries={"mode": "standard", "max_attempts": 2}),
)
class StorageFailure(Exception):
pass
def check_storage():
# Scoped user has ListBucket but no privileges outside this bucket.
s3.list_objects_v2(Bucket=bucket, MaxKeys=1)
def upload_original(file, key, size, content_type):
file.seek(0)
s3.put_object(Bucket=bucket, Key=key, Body=file, ContentLength=size, ContentType=content_type)
def delete_original(key):
s3.delete_object(Bucket=bucket, Key=key)
def download_original(key):
try:
return s3.get_object(Bucket=bucket, Key=key)["Body"]
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"):
raise StorageFailure("Object missing") from None
raise
+33
View File
@@ -0,0 +1,33 @@
"""Unit-level image checks; run in built api1 container."""
import tempfile
import unittest
from pathlib import Path
from src.main import validate_image
from src.errors import ApiError
PNG = (Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes()
class ImageValidationTest(unittest.TestCase):
def check(self, content, content_type):
with tempfile.SpooledTemporaryFile() as stream:
stream.write(content)
validate_image(stream, content_type)
def test_fixture(self):
self.check(PNG,'image/png')
def test_corrupt(self):
with self.assertRaises(ApiError):
self.check(PNG[:70], 'image/png')
def test_spoof_mime(self):
with self.assertRaises(ApiError):
self.check(PNG,'image/jpeg')
def test_random_bytes(self):
with self.assertRaises(ApiError):
self.check(b'unrelated bytes', 'image/png')
if __name__=='__main__':
unittest.main()