forked from mtusi2026/mtusi2026lab4
Grade 4
This commit is contained in:
+8
-20
@@ -1,29 +1,17 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
# Copy to .env and choose local credentials. Never commit .env.
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Settings used by the inherited lab3 application and compose.yaml.
|
||||
POSTGRES_DB=messenger
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=replace-with-local-password
|
||||
SESSION_TTL_SECONDS=3600
|
||||
CACHE_TTL_SECONDS=30
|
||||
MAX_REQUEST_BYTES=65536
|
||||
MAX_INFLIGHT=32
|
||||
# Compose sets POSTGRES_HOST=postgres and REDIS_URL=redis://redis:6379/0.
|
||||
# S3 settings below belong to the next implementation stage.
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MINIO_ROOT_USER=local-admin
|
||||
MINIO_ROOT_PASSWORD=replace-with-admin-password
|
||||
S3_ACCESS_KEY_ID=messenger-api
|
||||
S3_SECRET_ACCESS_KEY=replace-with-api-password
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Compose sets internal PostgreSQL, Redis and S3 addresses.
|
||||
# S3 bucket: messenger-attachments. Root credentials go only to s3/s3-init.
|
||||
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load upload-load test-persistence test-failover
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -21,6 +21,9 @@ test:
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
upload-load:
|
||||
$(PYTHON) scripts/upload_load.py --base-url "$(BASE_URL)"
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
@@ -16,6 +16,13 @@ x-api: &api
|
||||
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30}
|
||||
MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536}
|
||||
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
networks: [backend]
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
@@ -31,6 +38,8 @@ services:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"]
|
||||
interval: 2s
|
||||
@@ -48,6 +57,59 @@ services:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
s3:
|
||||
# RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest.
|
||||
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||
command: [server, /data, --console-address, '127.0.0.1:9001']
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||
MINIO_BROWSER: 'off'
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
|
||||
s3-init:
|
||||
# The pinned server image also contains mc; no second image is needed.
|
||||
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||
entrypoint: [sh, /bootstrap.sh]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
volumes:
|
||||
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
|
||||
- ./infra/s3-policy.json:/policy.json:ro
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
networks: [backend]
|
||||
restart: 'no'
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
|
||||
cleanup:
|
||||
<<: *api
|
||||
command: [python, -m, src.cleanup]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
|
||||
nginx:
|
||||
image: nginx:1.28.0-alpine
|
||||
entrypoint: [sh, /etc/nginx/readiness.sh]
|
||||
@@ -110,3 +172,4 @@ networks:
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
|
||||
+9
-4
@@ -11,19 +11,24 @@ http {
|
||||
access_log /var/log/nginx/access.log upstream;
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 64k;
|
||||
# API enforces the byte limit and returns the contracted JSON error.
|
||||
client_max_body_size 0;
|
||||
client_body_timeout 5s;
|
||||
client_header_timeout 5s;
|
||||
keepalive_timeout 5s;
|
||||
location / {
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Connection close;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 6s;
|
||||
proxy_read_timeout 6s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_send_timeout 35s;
|
||||
proxy_read_timeout 120s;
|
||||
# A 503 is an API response (for example, S3 is unavailable), so
|
||||
# preserve it for the client instead of marking every replica down.
|
||||
proxy_next_upstream error timeout http_502 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 8s;
|
||||
# Never transparently retry a POST already sent to an API.
|
||||
|
||||
+9
-3
@@ -5,12 +5,18 @@ update_backends() {
|
||||
: > "$next"
|
||||
for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do
|
||||
if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then
|
||||
echo "server $address:8080 max_fails=1 fail_timeout=2s;" >> "$next"
|
||||
echo "server $address:8080;" >> "$next"
|
||||
fi
|
||||
done
|
||||
# An empty upstream is invalid; a down placeholder returns 502 without
|
||||
# sending business requests to an unready API.
|
||||
# If every running API temporarily loses a shared dependency (for example
|
||||
# S3), retain the last addresses. The API can then return the contracted
|
||||
# 503 instead of Nginx replacing it with a 502. At initial startup there
|
||||
# is no previous healthy set, so use an inert placeholder.
|
||||
if [ ! -s "$next" ]; then
|
||||
if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then
|
||||
rm -f "$next"
|
||||
return
|
||||
fi
|
||||
echo 'server 127.0.0.1:9 down;' > "$next"
|
||||
fi
|
||||
if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID"
|
||||
mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||
mc mb --ignore-existing local/messenger-attachments
|
||||
mc anonymous set none local/messenger-attachments
|
||||
mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null
|
||||
mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json
|
||||
mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID"
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
|
||||
|
||||
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
|
||||
ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0);
|
||||
ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10);
|
||||
@@ -15,3 +15,5 @@ psycopg[binary]==3.2.6
|
||||
SQLAlchemy==2.0.41
|
||||
redis==5.2.1
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.40.70
|
||||
Pillow==12.0.0
|
||||
|
||||
+2
-2
@@ -15,12 +15,12 @@ def snapshot():
|
||||
|
||||
def main():
|
||||
before = snapshot()
|
||||
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 'nginx'],
|
||||
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 's3', 'cleanup', 'nginx'],
|
||||
cwd=ROOT, check=True, timeout=60)
|
||||
after = snapshot()
|
||||
(ROOT / 'evidence').mkdir(exist_ok=True)
|
||||
(ROOT / 'evidence' / 'restart.txt').write_text(
|
||||
'Command: docker compose restart api postgres redis nginx\nBefore (UTC):\n' + before
|
||||
'Command: docker compose restart api postgres redis s3 cleanup nginx\nBefore (UTC):\n' + before
|
||||
+ 'After (UTC):\n' + after, encoding='utf-8')
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Upload one fixture concurrently and record the peak RSS of API containers."""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def api_memory_bytes():
|
||||
text = subprocess.check_output(
|
||||
['docker', 'compose', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', 'api'],
|
||||
cwd=ROOT, text=True,
|
||||
)
|
||||
values = []
|
||||
for line in text.splitlines():
|
||||
amount = line.split()[1]
|
||||
number, unit = float(amount[:-3]), amount[-3:].lower()
|
||||
values.append(int(number * {'b': 1, 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3}[unit]))
|
||||
return sum(values)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--output', default='evidence/upload-load.json')
|
||||
args = parser.parse_args()
|
||||
check(args.concurrency >= 10, 'concurrency must be at least 10')
|
||||
client = Client(args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
content = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
|
||||
start = threading.Event()
|
||||
done = threading.Event()
|
||||
peak = [0]
|
||||
|
||||
def monitor():
|
||||
while not done.is_set():
|
||||
peak[0] = max(peak[0], api_memory_bytes())
|
||||
time.sleep(.2)
|
||||
|
||||
def upload(index):
|
||||
start.wait()
|
||||
return Client(args.base_url, timeout=60).call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load-%d.png' % index,
|
||||
201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
|
||||
watcher = threading.Thread(target=monitor, daemon=True)
|
||||
watcher.start()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
jobs = [pool.submit(upload, item) for item in range(args.concurrency)]
|
||||
start.set()
|
||||
results = [job.result() for job in jobs]
|
||||
done.set()
|
||||
watcher.join(timeout=1)
|
||||
report = {'concurrency': args.concurrency, 'fixture_bytes': len(content),
|
||||
'successful_uploads': len(results), 'peak_api_memory_bytes': peak[0]}
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(exist_ok=True)
|
||||
destination.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
||||
print(json.dumps(report, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,82 @@
|
||||
import os
|
||||
import unicodedata
|
||||
from uuid import uuid4
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from .errors import APIError
|
||||
from .models import Attachment, ChatMember
|
||||
from .object_storage import BUCKET, PREFIX, s3_client
|
||||
from .store import ensure_uuid, serialize, store
|
||||
|
||||
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
|
||||
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
|
||||
if MAX_BYTES < 1 or MAX_PIXELS < 1:
|
||||
raise ValueError('Upload limits must be positive')
|
||||
|
||||
|
||||
def validate_filename(filename):
|
||||
if (not filename or not filename.strip() or len(filename) > 255
|
||||
or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)):
|
||||
raise APIError(400, 'invalid_request', 'Неверное имя файла')
|
||||
|
||||
|
||||
def validate_image(source, content_type):
|
||||
try:
|
||||
source.seek(0)
|
||||
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
|
||||
if picture.width * picture.height > MAX_PIXELS:
|
||||
raise APIError(413, 'payload_too_large', 'Слишком много пикселей')
|
||||
if Image.MIME[picture.format] != content_type:
|
||||
raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type')
|
||||
picture.verify()
|
||||
source.seek(0)
|
||||
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
|
||||
picture.load() # verify alone does not decode JPEG or detect truncated pixel data.
|
||||
except Image.DecompressionBombError as exc:
|
||||
raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
|
||||
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc
|
||||
source.seek(0)
|
||||
|
||||
|
||||
def lock_object(session, key):
|
||||
# Shared by upload and cleanup: no deletion while a COMMIT is in flight.
|
||||
session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0))))
|
||||
|
||||
|
||||
def metadata(attachment):
|
||||
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
|
||||
'content_type', 'size_bytes', 'sha256', 'created_at')
|
||||
return {**result, 'updated_at': result['created_at'], 'status': 'ready',
|
||||
'variants': [], 'error_code': None}
|
||||
|
||||
|
||||
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
|
||||
validate_image(source, content_type)
|
||||
attachment_id = str(uuid4())
|
||||
key = PREFIX + attachment_id
|
||||
with store.sessions.begin() as session:
|
||||
store._chat(session, chat_id, actor_id)
|
||||
lock_object(session, key)
|
||||
s3_client().put_object(Bucket=BUCKET, Key=key, Body=source,
|
||||
ContentLength=size, ContentType=content_type)
|
||||
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
|
||||
object_key=key, filename=filename, content_type=content_type,
|
||||
size_bytes=size, sha256=digest)
|
||||
session.add(attachment)
|
||||
session.flush()
|
||||
result = metadata(attachment)
|
||||
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
|
||||
return result
|
||||
|
||||
|
||||
def get_attachment(attachment_id, actor_id):
|
||||
with store.sessions() as session:
|
||||
attachment = session.scalar(select(Attachment).join(
|
||||
ChatMember, ChatMember.chat_id == Attachment.chat_id).where(
|
||||
Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id))
|
||||
if attachment is None:
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
return attachment
|
||||
+1
-1
@@ -37,4 +37,4 @@ class HistoryCache:
|
||||
return page
|
||||
|
||||
|
||||
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')))
|
||||
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4')
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .attachments import lock_object
|
||||
from .database import create_db_engine
|
||||
from .models import Attachment
|
||||
from .object_storage import BUCKET, PREFIX, s3_client
|
||||
|
||||
GRACE = timedelta(hours=1)
|
||||
|
||||
|
||||
def cleanup(sessions, client, now=None):
|
||||
cutoff = (now or datetime.now(timezone.utc)) - GRACE
|
||||
deleted = 0
|
||||
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
|
||||
for obj in page.get('Contents', []):
|
||||
if obj['LastModified'] >= cutoff:
|
||||
continue
|
||||
key = obj['Key']
|
||||
with sessions.begin() as session:
|
||||
lock_object(session, key)
|
||||
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
|
||||
continue
|
||||
# Recheck age after taking the same lock as the uploader.
|
||||
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
|
||||
continue
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--once', action='store_true')
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
count = cleanup(sessionmaker(engine), s3_client())
|
||||
logging.info('Cleanup completed: deleted=%d', count)
|
||||
except Exception as exc:
|
||||
logging.error('Cleanup deferred: %s', type(exc).__name__)
|
||||
if args.once:
|
||||
raise SystemExit(1) from None
|
||||
if args.once:
|
||||
break
|
||||
time.sleep(3600)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -5,6 +5,7 @@ from dataclasses import dataclass
|
||||
from starlette.requests import Request
|
||||
|
||||
from .errors import error_response
|
||||
from .middleware import is_upload
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -54,6 +55,9 @@ class ResourceLimitMiddleware:
|
||||
return await reject(503, 'unavailable', 'Слишком много одновременных запросов')
|
||||
self.active += 1
|
||||
try:
|
||||
# The upload endpoint authenticates and bounds its stream itself.
|
||||
if is_upload(scope['method'], scope['path']):
|
||||
return await self.app(scope, receive, send)
|
||||
length = request.headers.get('content-length')
|
||||
if length is not None:
|
||||
try:
|
||||
|
||||
+4
-1
@@ -5,6 +5,7 @@ from contextlib import asynccontextmanager
|
||||
from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from redis.exceptions import RedisError
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler
|
||||
from .database import create_db_engine
|
||||
@@ -16,6 +17,7 @@ from .routers.health import router as health_router
|
||||
from .routers.users import router as users_router
|
||||
from .routers.chats import router as chats_router
|
||||
from .routers.auth import router as auth_router
|
||||
from .routers.attachments import router as attachments_router
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -36,10 +38,11 @@ app.add_exception_handler(APIError, api_error_handler)
|
||||
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
||||
app.add_exception_handler(HTTPException, http_error_handler)
|
||||
app.add_exception_handler(Exception, internal_error_handler)
|
||||
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError):
|
||||
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError):
|
||||
app.add_exception_handler(error, dependency_error_handler)
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(users_router)
|
||||
app.include_router(chats_router)
|
||||
app.include_router(auth_router)
|
||||
app.include_router(attachments_router)
|
||||
|
||||
@@ -3,6 +3,11 @@ from socket import gethostname
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
import re
|
||||
|
||||
|
||||
def is_upload(method, path):
|
||||
return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None
|
||||
|
||||
INSTANCE_ID = gethostname()
|
||||
|
||||
@@ -14,6 +19,7 @@ class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||
|
||||
if (request.method in {"POST", "PUT", "PATCH"}
|
||||
and request.url.path.startswith("/api/")
|
||||
and not is_upload(request.method, request.url.path)
|
||||
and request.url.path != '/api/v1/auth/sessions'):
|
||||
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if content_type != "application/json":
|
||||
|
||||
+17
-1
@@ -3,6 +3,7 @@ from uuid import uuid4
|
||||
|
||||
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
from sqlalchemy.dialects.postgresql import ARRAY
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
@@ -44,13 +45,15 @@ class ChatMember(Base):
|
||||
class Message(Base):
|
||||
__tablename__ = 'messages'
|
||||
__table_args__ = (
|
||||
CheckConstraint('char_length(text) BETWEEN 1 AND 4000'),
|
||||
CheckConstraint('char_length(text) <= 4000'),
|
||||
CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"),
|
||||
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
|
||||
)
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
|
||||
sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
|
||||
text: Mapped[str] = mapped_column(Text)
|
||||
attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}')
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
|
||||
|
||||
@@ -60,6 +63,19 @@ class SchemaMigration(Base):
|
||||
applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True)
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
__tablename__ = 'attachments'
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True)
|
||||
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
|
||||
owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
|
||||
object_key: Mapped[str] = mapped_column(Text, unique=True)
|
||||
filename: Mapped[str] = mapped_column(Text)
|
||||
content_type: Mapped[str] = mapped_column(Text)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
sha256: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
|
||||
|
||||
class IdempotencyRecord(Base):
|
||||
__tablename__ = 'message_idempotency'
|
||||
actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import os
|
||||
from functools import lru_cache
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||
PREFIX = 'originals/'
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def s3_client():
|
||||
return boto3.client(
|
||||
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
||||
aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'],
|
||||
config=Config(connect_timeout=2, read_timeout=5,
|
||||
retries={'total_max_attempts': 1},
|
||||
s3={'addressing_style': 'path'},
|
||||
request_checksum_calculation='when_required',
|
||||
response_checksum_validation='when_required'),
|
||||
)
|
||||
@@ -0,0 +1,84 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
from tempfile import TemporaryFile
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from starlette.background import BackgroundTask
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename
|
||||
from ..dependencies import current_user
|
||||
from ..errors import APIError
|
||||
from ..object_storage import BUCKET, s3_client
|
||||
from ..store import ensure_uuid, store
|
||||
|
||||
router = APIRouter(prefix='/api/v1', tags=['attachments'])
|
||||
# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk.
|
||||
decode_slot = asyncio.Semaphore(1)
|
||||
|
||||
|
||||
@router.post('/chats/{chat_id}/attachments', status_code=201)
|
||||
async def upload(chat_id: str, request: Request, filename: str = Query(...),
|
||||
user: dict = Depends(current_user)):
|
||||
chat_id = ensure_uuid(chat_id)
|
||||
await run_in_threadpool(store.get_chat_for_user, chat_id, user['id'])
|
||||
validate_filename(filename)
|
||||
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
|
||||
if content_type not in ('image/png', 'image/jpeg'):
|
||||
raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG')
|
||||
length = request.headers.get('content-length')
|
||||
if length is not None:
|
||||
try:
|
||||
length = int(length)
|
||||
if length < 0:
|
||||
raise ValueError()
|
||||
except ValueError as exc:
|
||||
raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc
|
||||
if length > MAX_BYTES:
|
||||
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
with TemporaryFile() as source:
|
||||
try:
|
||||
async with asyncio.timeout(30):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
|
||||
digest.update(chunk)
|
||||
await run_in_threadpool(source.write, chunk)
|
||||
except (TimeoutError, ClientDisconnect) as exc:
|
||||
raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc
|
||||
if length is not None and size != length:
|
||||
raise APIError(400, 'invalid_request', 'Неполное тело запроса')
|
||||
async with decode_slot:
|
||||
return await run_in_threadpool(save_attachment, source, chat_id, user['id'],
|
||||
filename, content_type, size, digest.hexdigest())
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}')
|
||||
def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
|
||||
return metadata(get_attachment(attachment_id, user['id']))
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}/content')
|
||||
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
|
||||
attachment = get_attachment(attachment_id, user['id'])
|
||||
if variant != 'original':
|
||||
raise APIError(404, 'not_found', 'Вариант не найден')
|
||||
body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body']
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
yield from body.iter_chunks(chunk_size=64 * 1024)
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(chunks(), media_type=attachment.content_type,
|
||||
headers={'Content-Length': str(attachment.size_bytes),
|
||||
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
|
||||
'X-Content-Type-Options': 'nosniff'},
|
||||
background=BackgroundTask(body.close))
|
||||
@@ -36,7 +36,7 @@ def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(cu
|
||||
idempotency_key: Optional[str] = Header(default=None)):
|
||||
if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key):
|
||||
raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key')
|
||||
return store.create_message(chat_id, user["id"], payload.text, idempotency_key)
|
||||
return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids)
|
||||
|
||||
|
||||
@router.get("/{chat_id}/messages", response_model=MessagePage)
|
||||
|
||||
@@ -4,6 +4,7 @@ from sqlalchemy import select
|
||||
from ..schemas.common import Health
|
||||
from ..auth import sessions
|
||||
from ..store import store
|
||||
from ..object_storage import BUCKET, s3_client
|
||||
|
||||
router = APIRouter(prefix='/health')
|
||||
|
||||
@@ -18,4 +19,5 @@ def health_ready():
|
||||
with store.sessions() as session:
|
||||
session.execute(select(1))
|
||||
sessions.client.ping()
|
||||
s3_client().head_bucket(Bucket=BUCKET)
|
||||
return {"status": "ok"}
|
||||
|
||||
+17
-7
@@ -1,19 +1,28 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import Field, StrictStr, field_validator
|
||||
from pydantic import Field, StrictStr, field_validator, model_validator
|
||||
from uuid import UUID
|
||||
|
||||
from .common import ContractModel
|
||||
|
||||
|
||||
|
||||
class CreateMessage(ContractModel):
|
||||
text: StrictStr = Field(min_length=1, max_length=4000)
|
||||
text: StrictStr = Field(max_length=4000)
|
||||
attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("text")
|
||||
def validate_text(cls, value):
|
||||
if not value.strip():
|
||||
raise ValueError("blank text")
|
||||
return value
|
||||
@field_validator('attachment_ids')
|
||||
def validate_ids(cls, value):
|
||||
ids = [str(UUID(item)) for item in value]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError('duplicate attachments')
|
||||
return ids
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_content(self):
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError('empty message')
|
||||
return self
|
||||
|
||||
|
||||
class Message(ContractModel):
|
||||
@@ -21,6 +30,7 @@ class Message(ContractModel):
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
attachment_ids: List[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
|
||||
+11
-5
@@ -10,7 +10,7 @@ from sqlalchemy.exc import IntegrityError
|
||||
from .auth import password_hasher, verify_password, unauthorized
|
||||
from .errors import APIError
|
||||
from .cache import history_cache
|
||||
from .models import User, Chat, ChatMember, Message, IdempotencyRecord
|
||||
from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment
|
||||
|
||||
|
||||
def ensure_uuid(value):
|
||||
@@ -36,7 +36,7 @@ def serialize(model, *fields):
|
||||
|
||||
USER_FIELDS = ('id', 'username', 'display_name', 'created_at')
|
||||
CHAT_FIELDS = ('id', 'title', 'created_at')
|
||||
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at')
|
||||
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids')
|
||||
|
||||
|
||||
def after_condition(model, after):
|
||||
@@ -130,10 +130,16 @@ class PostgresStore:
|
||||
items = [self._members(session, chat) for chat in chats[:limit]]
|
||||
return self._page(items, len(chats) > limit, 'chats', actor_id)
|
||||
|
||||
def create_message(self, chat_id, actor_id, text, idempotency_key=None):
|
||||
def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None):
|
||||
chat_id = ensure_uuid(chat_id)
|
||||
attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])]
|
||||
with self.sessions.begin() as session:
|
||||
self._chat(session, chat_id, actor_id)
|
||||
if attachment_ids:
|
||||
found = list(session.scalars(select(Attachment.id).where(
|
||||
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)))
|
||||
if len(found) != len(attachment_ids):
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
if idempotency_key is not None:
|
||||
# Serialize writers to this chat before looking up the key.
|
||||
# The same row is updated below for history cache invalidation.
|
||||
@@ -141,12 +147,12 @@ class PostgresStore:
|
||||
record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key))
|
||||
if record is not None:
|
||||
previous = session.get(Message, record.message_id)
|
||||
if previous.text != text:
|
||||
if previous.text != text or previous.attachment_ids != attachment_ids:
|
||||
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
|
||||
return serialize(previous, *MESSAGE_FIELDS)
|
||||
session.execute(update(Chat).where(Chat.id == chat_id).values(
|
||||
history_version=Chat.history_version + 1))
|
||||
message = Message(chat_id=chat_id, sender_id=actor_id, text=text)
|
||||
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
|
||||
session.add(message)
|
||||
session.flush()
|
||||
if idempotency_key is not None:
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Attachment negative cases. Run against the Compose API at localhost:8080."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def main():
|
||||
client = Client()
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png'
|
||||
image = (Path(__file__).resolve().parents[1] / 'tests' / 'fixtures' / 'sample.png').read_bytes()
|
||||
|
||||
client.call('POST', path, 415, actor=alice, raw=b'not an image',
|
||||
headers={'Content-Type': 'image/png'})
|
||||
client.call('POST', path, 415, actor=alice, raw=image,
|
||||
headers={'Content-Type': 'image/jpeg'})
|
||||
client.call('POST', path, 413, actor=alice, raw=b'x' * (10 * 1024 * 1024 + 1),
|
||||
headers={'Content-Type': 'image/png'})
|
||||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=../sample.png',
|
||||
400, actor=alice, raw=image, headers={'Content-Type': 'image/png'})
|
||||
|
||||
attachment = client.call('POST', path, 201, actor=alice, raw=image,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
|
||||
client.call('GET', '/api/v1/attachments/' + attachment['id'], 404, actor=eve)
|
||||
client.call('GET', '/api/v1/attachments/' + str(uuid4()), 404, actor=alice)
|
||||
|
||||
other = client.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Other chat', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
||||
client.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice,
|
||||
body={'text': '', 'attachment_ids': [attachment['id']]},
|
||||
headers={'Idempotency-Key': uuid4().hex})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
print('PASS: attachment validation and access checks')
|
||||
Reference in New Issue
Block a user