This commit is contained in:
2026-10-30 20:08:17 +03:00
parent 4c7a796795
commit 959eee7f9d
25 changed files with 562 additions and 46 deletions
+8 -20
View File
@@ -1,29 +1,17 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
# Copy to .env and choose local credentials. Never commit .env.
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Settings used by the inherited lab3 application and compose.yaml.
POSTGRES_DB=messenger
POSTGRES_USER=messenger
POSTGRES_PASSWORD=replace-with-local-password
SESSION_TTL_SECONDS=3600
CACHE_TTL_SECONDS=30
MAX_REQUEST_BYTES=65536
MAX_INFLIGHT=32
# Compose sets POSTGRES_HOST=postgres and REDIS_URL=redis://redis:6379/0.
# S3 settings below belong to the next implementation stage.
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MINIO_ROOT_USER=local-admin
MINIO_ROOT_PASSWORD=replace-with-admin-password
S3_ACCESS_KEY_ID=messenger-api
S3_SECRET_ACCESS_KEY=replace-with-api-password
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Compose sets internal PostgreSQL, Redis and S3 addresses.
# S3 bucket: messenger-attachments. Root credentials go only to s3/s3-init.
+4 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load upload-load test-persistence test-failover
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -21,6 +21,9 @@ test:
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
upload-load:
$(PYTHON) scripts/upload_load.py --base-url "$(BASE_URL)"
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+63
View File
@@ -16,6 +16,13 @@ x-api: &api
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30}
MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536}
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
networks: [backend]
cpus: 1.0
mem_limit: 512m
@@ -31,6 +38,8 @@ services:
condition: service_completed_successfully
redis:
condition: service_healthy
s3-init:
condition: service_completed_successfully
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"]
interval: 2s
@@ -48,6 +57,59 @@ services:
postgres:
condition: service_healthy
s3:
# RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest.
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
command: [server, /data, --console-address, '127.0.0.1:9001']
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
MINIO_BROWSER: 'off'
volumes:
- s3_data:/data
networks: [backend]
healthcheck:
test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live]
interval: 2s
timeout: 2s
retries: 30
cpus: 1.0
mem_limit: 512m
s3-init:
# The pinned server image also contains mc; no second image is needed.
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
entrypoint: [sh, /bootstrap.sh]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
volumes:
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
- ./infra/s3-policy.json:/policy.json:ro
depends_on:
s3:
condition: service_healthy
networks: [backend]
restart: 'no'
cpus: 0.25
mem_limit: 128m
cleanup:
<<: *api
command: [python, -m, src.cleanup]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
nginx:
image: nginx:1.28.0-alpine
entrypoint: [sh, /etc/nginx/readiness.sh]
@@ -110,3 +172,4 @@ networks:
volumes:
postgres_data:
redis_data:
s3_data:
+9 -4
View File
@@ -11,19 +11,24 @@ http {
access_log /var/log/nginx/access.log upstream;
server {
listen 8080;
client_max_body_size 64k;
# API enforces the byte limit and returns the contracted JSON error.
client_max_body_size 0;
client_body_timeout 5s;
client_header_timeout 5s;
keepalive_timeout 5s;
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_request_buffering off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header Connection close;
proxy_connect_timeout 1s;
proxy_send_timeout 6s;
proxy_read_timeout 6s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_send_timeout 35s;
proxy_read_timeout 120s;
# A 503 is an API response (for example, S3 is unavailable), so
# preserve it for the client instead of marking every replica down.
proxy_next_upstream error timeout http_502 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 8s;
# Never transparently retry a POST already sent to an API.
+9 -3
View File
@@ -5,12 +5,18 @@ update_backends() {
: > "$next"
for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do
if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then
echo "server $address:8080 max_fails=1 fail_timeout=2s;" >> "$next"
echo "server $address:8080;" >> "$next"
fi
done
# An empty upstream is invalid; a down placeholder returns 502 without
# sending business requests to an unready API.
# If every running API temporarily loses a shared dependency (for example
# S3), retain the last addresses. The API can then return the contracted
# 503 instead of Nginx replacing it with a 502. At initial startup there
# is no previous healthy set, so use an inert placeholder.
if [ ! -s "$next" ]; then
if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then
rm -f "$next"
return
fi
echo 'server 127.0.0.1:9 down;' > "$next"
fi
if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID"
mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
mc mb --ignore-existing local/messenger-attachments
mc anonymous set none local/messenger-attachments
mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null
mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json
mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID"
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*"]
}
]
}
+18
View File
@@ -0,0 +1,18 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id),
object_key text NOT NULL UNIQUE,
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0);
ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10);
+2
View File
@@ -15,3 +15,5 @@ psycopg[binary]==3.2.6
SQLAlchemy==2.0.41
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.40.70
Pillow==12.0.0
+2 -2
View File
@@ -15,12 +15,12 @@ def snapshot():
def main():
before = snapshot()
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 'nginx'],
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 's3', 'cleanup', 'nginx'],
cwd=ROOT, check=True, timeout=60)
after = snapshot()
(ROOT / 'evidence').mkdir(exist_ok=True)
(ROOT / 'evidence' / 'restart.txt').write_text(
'Command: docker compose restart api postgres redis nginx\nBefore (UTC):\n' + before
'Command: docker compose restart api postgres redis s3 cleanup nginx\nBefore (UTC):\n' + before
+ 'After (UTC):\n' + after, encoding='utf-8')
+71
View File
@@ -0,0 +1,71 @@
"""Upload one fixture concurrently and record the peak RSS of API containers."""
import argparse
import json
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def api_memory_bytes():
text = subprocess.check_output(
['docker', 'compose', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', 'api'],
cwd=ROOT, text=True,
)
values = []
for line in text.splitlines():
amount = line.split()[1]
number, unit = float(amount[:-3]), amount[-3:].lower()
values.append(int(number * {'b': 1, 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3}[unit]))
return sum(values)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--output', default='evidence/upload-load.json')
args = parser.parse_args()
check(args.concurrency >= 10, 'concurrency must be at least 10')
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
content = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
start = threading.Event()
done = threading.Event()
peak = [0]
def monitor():
while not done.is_set():
peak[0] = max(peak[0], api_memory_bytes())
time.sleep(.2)
def upload(index):
start.wait()
return Client(args.base_url, timeout=60).call(
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load-%d.png' % index,
201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
watcher = threading.Thread(target=monitor, daemon=True)
watcher.start()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
jobs = [pool.submit(upload, item) for item in range(args.concurrency)]
start.set()
results = [job.result() for job in jobs]
done.set()
watcher.join(timeout=1)
report = {'concurrency': args.concurrency, 'fixture_bytes': len(content),
'successful_uploads': len(results), 'peak_api_memory_bytes': peak[0]}
destination = ROOT / args.output
destination.parent.mkdir(exist_ok=True)
destination.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps(report, ensure_ascii=False))
if __name__ == '__main__':
main()
+82
View File
@@ -0,0 +1,82 @@
import os
import unicodedata
from uuid import uuid4
from PIL import Image, UnidentifiedImageError
from sqlalchemy import func, select
from .errors import APIError
from .models import Attachment, ChatMember
from .object_storage import BUCKET, PREFIX, s3_client
from .store import ensure_uuid, serialize, store
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
if MAX_BYTES < 1 or MAX_PIXELS < 1:
raise ValueError('Upload limits must be positive')
def validate_filename(filename):
if (not filename or not filename.strip() or len(filename) > 255
or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)):
raise APIError(400, 'invalid_request', 'Неверное имя файла')
def validate_image(source, content_type):
try:
source.seek(0)
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
if picture.width * picture.height > MAX_PIXELS:
raise APIError(413, 'payload_too_large', 'Слишком много пикселей')
if Image.MIME[picture.format] != content_type:
raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type')
picture.verify()
source.seek(0)
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
picture.load() # verify alone does not decode JPEG or detect truncated pixel data.
except Image.DecompressionBombError as exc:
raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc
source.seek(0)
def lock_object(session, key):
# Shared by upload and cleanup: no deletion while a COMMIT is in flight.
session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0))))
def metadata(attachment):
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
'content_type', 'size_bytes', 'sha256', 'created_at')
return {**result, 'updated_at': result['created_at'], 'status': 'ready',
'variants': [], 'error_code': None}
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
validate_image(source, content_type)
attachment_id = str(uuid4())
key = PREFIX + attachment_id
with store.sessions.begin() as session:
store._chat(session, chat_id, actor_id)
lock_object(session, key)
s3_client().put_object(Bucket=BUCKET, Key=key, Body=source,
ContentLength=size, ContentType=content_type)
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
object_key=key, filename=filename, content_type=content_type,
size_bytes=size, sha256=digest)
session.add(attachment)
session.flush()
result = metadata(attachment)
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
return result
def get_attachment(attachment_id, actor_id):
with store.sessions() as session:
attachment = session.scalar(select(Attachment).join(
ChatMember, ChatMember.chat_id == Attachment.chat_id).where(
Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id))
if attachment is None:
raise APIError(404, 'not_found', 'Объект не найден')
return attachment
+1 -1
View File
@@ -37,4 +37,4 @@ class HistoryCache:
return page
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')))
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4')
+61
View File
@@ -0,0 +1,61 @@
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
import argparse
import logging
import time
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from .attachments import lock_object
from .database import create_db_engine
from .models import Attachment
from .object_storage import BUCKET, PREFIX, s3_client
GRACE = timedelta(hours=1)
def cleanup(sessions, client, now=None):
cutoff = (now or datetime.now(timezone.utc)) - GRACE
deleted = 0
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
for obj in page.get('Contents', []):
if obj['LastModified'] >= cutoff:
continue
key = obj['Key']
with sessions.begin() as session:
lock_object(session, key)
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
continue
# Recheck age after taking the same lock as the uploader.
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
continue
client.delete_object(Bucket=BUCKET, Key=key)
deleted += 1
return deleted
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--once', action='store_true')
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
engine = create_db_engine()
try:
while True:
try:
count = cleanup(sessionmaker(engine), s3_client())
logging.info('Cleanup completed: deleted=%d', count)
except Exception as exc:
logging.error('Cleanup deferred: %s', type(exc).__name__)
if args.once:
raise SystemExit(1) from None
if args.once:
break
time.sleep(3600)
finally:
engine.dispose()
if __name__ == '__main__':
main()
+4
View File
@@ -5,6 +5,7 @@ from dataclasses import dataclass
from starlette.requests import Request
from .errors import error_response
from .middleware import is_upload
@dataclass(frozen=True)
@@ -54,6 +55,9 @@ class ResourceLimitMiddleware:
return await reject(503, 'unavailable', 'Слишком много одновременных запросов')
self.active += 1
try:
# The upload endpoint authenticates and bounds its stream itself.
if is_upload(scope['method'], scope['path']):
return await self.app(scope, receive, send)
length = request.headers.get('content-length')
if length is not None:
try:
+4 -1
View File
@@ -5,6 +5,7 @@ from contextlib import asynccontextmanager
from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout
from sqlalchemy.orm import sessionmaker
from redis.exceptions import RedisError
from botocore.exceptions import BotoCoreError, ClientError
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler
from .database import create_db_engine
@@ -16,6 +17,7 @@ from .routers.health import router as health_router
from .routers.users import router as users_router
from .routers.chats import router as chats_router
from .routers.auth import router as auth_router
from .routers.attachments import router as attachments_router
@asynccontextmanager
@@ -36,10 +38,11 @@ app.add_exception_handler(APIError, api_error_handler)
app.add_exception_handler(RequestValidationError, validation_error_handler)
app.add_exception_handler(HTTPException, http_error_handler)
app.add_exception_handler(Exception, internal_error_handler)
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError):
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError):
app.add_exception_handler(error, dependency_error_handler)
app.include_router(health_router)
app.include_router(users_router)
app.include_router(chats_router)
app.include_router(auth_router)
app.include_router(attachments_router)
+6
View File
@@ -3,6 +3,11 @@ from socket import gethostname
from starlette.responses import JSONResponse
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
import re
def is_upload(method, path):
return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None
INSTANCE_ID = gethostname()
@@ -14,6 +19,7 @@ class RequestIdMiddleware(BaseHTTPMiddleware):
if (request.method in {"POST", "PUT", "PATCH"}
and request.url.path.startswith("/api/")
and not is_upload(request.method, request.url.path)
and request.url.path != '/api/v1/auth/sessions'):
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_type != "application/json":
+17 -1
View File
@@ -3,6 +3,7 @@ from uuid import uuid4
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from sqlalchemy.dialects.postgresql import ARRAY
class Base(DeclarativeBase):
@@ -44,13 +45,15 @@ class ChatMember(Base):
class Message(Base):
__tablename__ = 'messages'
__table_args__ = (
CheckConstraint('char_length(text) BETWEEN 1 AND 4000'),
CheckConstraint('char_length(text) <= 4000'),
CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"),
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
)
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
text: Mapped[str] = mapped_column(Text)
attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}')
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
@@ -60,6 +63,19 @@ class SchemaMigration(Base):
applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True)
class Attachment(Base):
__tablename__ = 'attachments'
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True)
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
object_key: Mapped[str] = mapped_column(Text, unique=True)
filename: Mapped[str] = mapped_column(Text)
content_type: Mapped[str] = mapped_column(Text)
size_bytes: Mapped[int] = mapped_column(BigInteger)
sha256: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
class IdempotencyRecord(Base):
__tablename__ = 'message_idempotency'
actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
+23
View File
@@ -0,0 +1,23 @@
import os
from functools import lru_cache
import boto3
from botocore.config import Config
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
PREFIX = 'originals/'
@lru_cache(maxsize=1)
def s3_client():
return boto3.client(
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'],
config=Config(connect_timeout=2, read_timeout=5,
retries={'total_max_attempts': 1},
s3={'addressing_style': 'path'},
request_checksum_calculation='when_required',
response_checksum_validation='when_required'),
)
+84
View File
@@ -0,0 +1,84 @@
import asyncio
import hashlib
from tempfile import TemporaryFile
from urllib.parse import quote
from fastapi import APIRouter, Depends, Query, Request
from starlette.background import BackgroundTask
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from starlette.responses import StreamingResponse
from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename
from ..dependencies import current_user
from ..errors import APIError
from ..object_storage import BUCKET, s3_client
from ..store import ensure_uuid, store
router = APIRouter(prefix='/api/v1', tags=['attachments'])
# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk.
decode_slot = asyncio.Semaphore(1)
@router.post('/chats/{chat_id}/attachments', status_code=201)
async def upload(chat_id: str, request: Request, filename: str = Query(...),
user: dict = Depends(current_user)):
chat_id = ensure_uuid(chat_id)
await run_in_threadpool(store.get_chat_for_user, chat_id, user['id'])
validate_filename(filename)
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
if content_type not in ('image/png', 'image/jpeg'):
raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG')
length = request.headers.get('content-length')
if length is not None:
try:
length = int(length)
if length < 0:
raise ValueError()
except ValueError as exc:
raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc
if length > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
size = 0
digest = hashlib.sha256()
with TemporaryFile() as source:
try:
async with asyncio.timeout(30):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
digest.update(chunk)
await run_in_threadpool(source.write, chunk)
except (TimeoutError, ClientDisconnect) as exc:
raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc
if length is not None and size != length:
raise APIError(400, 'invalid_request', 'Неполное тело запроса')
async with decode_slot:
return await run_in_threadpool(save_attachment, source, chat_id, user['id'],
filename, content_type, size, digest.hexdigest())
@router.get('/attachments/{attachment_id}')
def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
return metadata(get_attachment(attachment_id, user['id']))
@router.get('/attachments/{attachment_id}/content')
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
attachment = get_attachment(attachment_id, user['id'])
if variant != 'original':
raise APIError(404, 'not_found', 'Вариант не найден')
body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body']
def chunks():
try:
yield from body.iter_chunks(chunk_size=64 * 1024)
finally:
body.close()
return StreamingResponse(chunks(), media_type=attachment.content_type,
headers={'Content-Length': str(attachment.size_bytes),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
'X-Content-Type-Options': 'nosniff'},
background=BackgroundTask(body.close))
+1 -1
View File
@@ -36,7 +36,7 @@ def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(cu
idempotency_key: Optional[str] = Header(default=None)):
if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key):
raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key')
return store.create_message(chat_id, user["id"], payload.text, idempotency_key)
return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids)
@router.get("/{chat_id}/messages", response_model=MessagePage)
+2
View File
@@ -4,6 +4,7 @@ from sqlalchemy import select
from ..schemas.common import Health
from ..auth import sessions
from ..store import store
from ..object_storage import BUCKET, s3_client
router = APIRouter(prefix='/health')
@@ -18,4 +19,5 @@ def health_ready():
with store.sessions() as session:
session.execute(select(1))
sessions.client.ping()
s3_client().head_bucket(Bucket=BUCKET)
return {"status": "ok"}
+17 -7
View File
@@ -1,19 +1,28 @@
from typing import List, Optional
from pydantic import Field, StrictStr, field_validator
from pydantic import Field, StrictStr, field_validator, model_validator
from uuid import UUID
from .common import ContractModel
class CreateMessage(ContractModel):
text: StrictStr = Field(min_length=1, max_length=4000)
text: StrictStr = Field(max_length=4000)
attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10)
@field_validator("text")
def validate_text(cls, value):
if not value.strip():
raise ValueError("blank text")
return value
@field_validator('attachment_ids')
def validate_ids(cls, value):
ids = [str(UUID(item)) for item in value]
if len(ids) != len(set(ids)):
raise ValueError('duplicate attachments')
return ids
@model_validator(mode='after')
def validate_content(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError('empty message')
return self
class Message(ContractModel):
@@ -21,6 +30,7 @@ class Message(ContractModel):
chat_id: str
sender_id: str
text: str
attachment_ids: List[str]
created_at: str
+11 -5
View File
@@ -10,7 +10,7 @@ from sqlalchemy.exc import IntegrityError
from .auth import password_hasher, verify_password, unauthorized
from .errors import APIError
from .cache import history_cache
from .models import User, Chat, ChatMember, Message, IdempotencyRecord
from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment
def ensure_uuid(value):
@@ -36,7 +36,7 @@ def serialize(model, *fields):
USER_FIELDS = ('id', 'username', 'display_name', 'created_at')
CHAT_FIELDS = ('id', 'title', 'created_at')
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at')
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids')
def after_condition(model, after):
@@ -130,10 +130,16 @@ class PostgresStore:
items = [self._members(session, chat) for chat in chats[:limit]]
return self._page(items, len(chats) > limit, 'chats', actor_id)
def create_message(self, chat_id, actor_id, text, idempotency_key=None):
def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None):
chat_id = ensure_uuid(chat_id)
attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])]
with self.sessions.begin() as session:
self._chat(session, chat_id, actor_id)
if attachment_ids:
found = list(session.scalars(select(Attachment.id).where(
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)))
if len(found) != len(attachment_ids):
raise APIError(404, 'not_found', 'Объект не найден')
if idempotency_key is not None:
# Serialize writers to this chat before looking up the key.
# The same row is updated below for history cache invalidation.
@@ -141,12 +147,12 @@ class PostgresStore:
record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key))
if record is not None:
previous = session.get(Message, record.message_id)
if previous.text != text:
if previous.text != text or previous.attachment_ids != attachment_ids:
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
return serialize(previous, *MESSAGE_FIELDS)
session.execute(update(Chat).where(Chat.id == chat_id).values(
history_version=Chat.history_version + 1))
message = Message(chat_id=chat_id, sender_id=actor_id, text=text)
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
session.add(message)
session.flush()
if idempotency_key is not None:
+39
View File
@@ -0,0 +1,39 @@
"""Attachment negative cases. Run against the Compose API at localhost:8080."""
import sys
from pathlib import Path
from uuid import uuid4
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
def main():
client = Client()
alice, bob, eve, chat = client.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png'
image = (Path(__file__).resolve().parents[1] / 'tests' / 'fixtures' / 'sample.png').read_bytes()
client.call('POST', path, 415, actor=alice, raw=b'not an image',
headers={'Content-Type': 'image/png'})
client.call('POST', path, 415, actor=alice, raw=image,
headers={'Content-Type': 'image/jpeg'})
client.call('POST', path, 413, actor=alice, raw=b'x' * (10 * 1024 * 1024 + 1),
headers={'Content-Type': 'image/png'})
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=../sample.png',
400, actor=alice, raw=image, headers={'Content-Type': 'image/png'})
attachment = client.call('POST', path, 201, actor=alice, raw=image,
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
client.call('GET', '/api/v1/attachments/' + attachment['id'], 404, actor=eve)
client.call('GET', '/api/v1/attachments/' + str(uuid4()), 404, actor=alice)
other = client.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Other chat', 'member_ids': [alice['id']]}, schema='Chat')['json']
client.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice,
body={'text': '', 'attachment_ids': [attachment['id']]},
headers={'Idempotency-Key': uuid4().hex})
if __name__ == '__main__':
main()
print('PASS: attachment validation and access checks')