forked from mtusi2026/mtusi2026lab4
Выполнена лабораторная работа 4
This commit is contained in:
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import asyncio
|
||||
from contextlib import suppress
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import uuid
|
||||
import warnings
|
||||
from urllib.parse import quote
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from src.storage import client, BUCKET, PREFIX
|
||||
|
||||
|
||||
def install(app, api):
|
||||
def view(row):
|
||||
return {key: value for key, value in row.items() if key != 'object_key'} | {'variants': []}
|
||||
|
||||
def authorize(request, chat_id):
|
||||
with api.POOL.connection() as db:
|
||||
user, _ = api.authenticate(request, db)
|
||||
api.access(db, api.identifier(chat_id), user['id'])
|
||||
return user['id']
|
||||
|
||||
def inspect_image(file, content_type):
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
if image.width * image.height > api.MAX_PIXELS:
|
||||
raise api.Problem(413, 'payload_too_large')
|
||||
if {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format) != content_type:
|
||||
raise api.Problem(415, 'unsupported_media_type')
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
image.load()
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise api.Problem(413, 'payload_too_large')
|
||||
except (UnidentifiedImageError, OSError, SyntaxError, ValueError):
|
||||
raise api.Problem(415, 'unsupported_media_type')
|
||||
finally:
|
||||
file.seek(0)
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments')
|
||||
async def upload(request: Request, chat_id: str):
|
||||
owner_id = await run_in_threadpool(authorize, request, chat_id)
|
||||
name = request.query_params.get('filename', '')
|
||||
if set(request.query_params) != {'filename'} or not 1 <= len(name) <= 255 or not name.strip() or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in name):
|
||||
api.invalid()
|
||||
mime = request.headers.get('content-type', '').split(';')[0].lower().strip()
|
||||
if mime not in ('image/png', 'image/jpeg'):
|
||||
raise api.Problem(415, 'unsupported_media_type')
|
||||
declared = request.headers.get('content-length')
|
||||
if declared is not None:
|
||||
if not declared.isdigit():
|
||||
api.invalid()
|
||||
if int(declared) > api.MAX_BYTES:
|
||||
raise api.Problem(413, 'payload_too_large')
|
||||
try:
|
||||
await asyncio.wait_for(api.UPLOAD_SLOTS.acquire(), timeout=2)
|
||||
except TimeoutError:
|
||||
raise api.Problem(503, 'unavailable')
|
||||
try:
|
||||
with tempfile.TemporaryFile() as file:
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
async with asyncio.timeout(30):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > api.MAX_BYTES:
|
||||
raise api.Problem(413, 'payload_too_large')
|
||||
digest.update(chunk)
|
||||
await run_in_threadpool(file.write, chunk)
|
||||
await run_in_threadpool(inspect_image, file, mime)
|
||||
attachment_id = uuid.uuid4()
|
||||
key = PREFIX + str(attachment_id)
|
||||
def save():
|
||||
s3 = client()
|
||||
with api.POOL.connection() as db:
|
||||
db.execute('SELECT pg_advisory_xact_lock_shared(202604)')
|
||||
s3.put_object(Bucket=BUCKET, Key=key, Body=file, ContentLength=size, ContentType=mime)
|
||||
try:
|
||||
row = db.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES(%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *', (attachment_id, api.identifier(chat_id), owner_id, name, key, mime, size, digest.hexdigest())).fetchone()
|
||||
except Exception:
|
||||
with suppress(Exception):
|
||||
s3.delete_object(Bucket=BUCKET, Key=key)
|
||||
raise
|
||||
db.commit()
|
||||
return row
|
||||
row = await run_in_threadpool(save)
|
||||
return JSONResponse(api.clean(view(row)), status_code=201)
|
||||
except TimeoutError:
|
||||
raise api.Problem(400, 'invalid_request', 'Upload timeout')
|
||||
finally:
|
||||
api.UPLOAD_SLOTS.release()
|
||||
|
||||
def find(request, attachment_id):
|
||||
with api.POOL.connection() as db:
|
||||
user, _ = api.authenticate(request, db)
|
||||
row = db.execute('SELECT * FROM attachments WHERE id=%s', (api.identifier(attachment_id),)).fetchone()
|
||||
if not row:
|
||||
raise api.Problem(404, 'not_found')
|
||||
api.access(db, row['chat_id'], user['id'])
|
||||
if row['status'] != 'ready':
|
||||
raise api.Problem(503, 'unavailable')
|
||||
return row
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}')
|
||||
async def metadata(request: Request, attachment_id: str):
|
||||
return JSONResponse(api.clean(view(await run_in_threadpool(find, request, attachment_id))))
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}/content')
|
||||
async def content(request: Request, attachment_id: str):
|
||||
row = await run_in_threadpool(find, request, attachment_id)
|
||||
if row['status'] != 'ready':
|
||||
raise api.Problem(409, 'conflict')
|
||||
if set(request.query_params) - {'variant'}:
|
||||
api.invalid()
|
||||
if request.query_params.get('variant', 'original') != 'original':
|
||||
raise api.Problem(404, 'not_found')
|
||||
try:
|
||||
result = await run_in_threadpool(lambda: client().get_object(Bucket=BUCKET, Key=row['object_key']))
|
||||
except ClientError as exc:
|
||||
if exc.response['Error']['Code'] in ('NoSuchKey', '404'):
|
||||
raise api.Problem(503, 'unavailable')
|
||||
raise
|
||||
stream = result['Body']
|
||||
def chunks():
|
||||
try:
|
||||
while chunk := stream.read(65536):
|
||||
yield chunk
|
||||
finally:
|
||||
stream.close()
|
||||
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
|
||||
'Content-Length': str(row['size_bytes']),
|
||||
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(row['filename'], safe=''),
|
||||
'X-Content-Type-Options': 'nosniff'})
|
||||
@@ -0,0 +1,32 @@
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from src.storage import BUCKET
|
||||
|
||||
|
||||
def main():
|
||||
s3 = boto3.client('s3', endpoint_url='http://s3:9000', region_name='us-east-1',
|
||||
aws_access_key_id=Path('/run/secrets/minio_root_user').read_text().strip(),
|
||||
aws_secret_access_key=Path('/run/secrets/minio_root_password').read_text().strip(),
|
||||
config=Config(connect_timeout=2, read_timeout=3, retries={'total_max_attempts': 1}))
|
||||
for _ in range(45):
|
||||
try:
|
||||
try:
|
||||
s3.head_bucket(Bucket=BUCKET)
|
||||
except ClientError as exc:
|
||||
if exc.response['Error']['Code'] not in ('404', 'NoSuchBucket', 'NotFound'):
|
||||
raise
|
||||
s3.create_bucket(Bucket=BUCKET)
|
||||
print('Private bucket prepared; identities loaded from S3 configuration')
|
||||
return
|
||||
except (BotoCoreError, ClientError):
|
||||
time.sleep(1)
|
||||
raise RuntimeError('S3 bootstrap timeout')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,9 @@
|
||||
import socket
|
||||
|
||||
|
||||
with socket.create_connection(('127.0.0.1', 8080), timeout=1) as connection:
|
||||
connection.sendall(b'GET /health/ready HTTP/1.0\r\nHost: localhost\r\nConnection: close\r\n\r\n')
|
||||
with connection.makefile('rb') as stream:
|
||||
status = stream.readline(128)
|
||||
if status.split(b' ', 2)[1:2] != [b'200']:
|
||||
raise SystemExit(1)
|
||||
+367
@@ -0,0 +1,367 @@
|
||||
import base64
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from PIL import Image
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool, ConnectionPool, PoolTimeout
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from src.migrate import database_url
|
||||
from src.storage import client, BUCKET
|
||||
|
||||
|
||||
INSTANCE = os.environ.get('HOSTNAME', str(uuid.uuid4()))
|
||||
PASSWORDS = PasswordHasher()
|
||||
DUMMY_HASH = PASSWORDS.hash(secrets.token_hex(32))
|
||||
POOL = ConnectionPool(database_url(), min_size=1, max_size=int(os.getenv('DB_POOL_SIZE', '1')), open=False, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=3000 -c lock_timeout=2000'})
|
||||
HEALTH_POOL = AsyncConnectionPool(database_url(), min_size=1, max_size=1, open=False, timeout=0.5, kwargs={'connect_timeout': 1, 'options': '-c statement_timeout=500 -c lock_timeout=500'})
|
||||
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
|
||||
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
|
||||
UPLOAD_SLOTS = asyncio.Semaphore(2)
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
POOL.open()
|
||||
POOL.wait(timeout=20)
|
||||
await HEALTH_POOL.open()
|
||||
await HEALTH_POOL.wait(timeout=20)
|
||||
async def record_pool():
|
||||
while True:
|
||||
await asyncio.sleep(5)
|
||||
print(json.dumps({'event': 'pool_metrics', 'instance': INSTANCE, 'at': datetime.now(timezone.utc).isoformat(), 'stats': POOL.get_stats()}), flush=True)
|
||||
recorder = asyncio.create_task(record_pool())
|
||||
yield
|
||||
recorder.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await recorder
|
||||
await HEALTH_POOL.close()
|
||||
POOL.close()
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
class Problem(Exception):
|
||||
def __init__(self, status, code, message='Request cannot be completed'):
|
||||
self.status, self.code, self.message = status, code, message
|
||||
|
||||
|
||||
def invalid():
|
||||
raise Problem(400, 'invalid_request')
|
||||
|
||||
|
||||
def clean(value):
|
||||
if isinstance(value, dict):
|
||||
return {key: clean(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [clean(item) for item in value]
|
||||
if isinstance(value, uuid.UUID):
|
||||
return str(value)
|
||||
if isinstance(value, datetime):
|
||||
return value.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')
|
||||
return value
|
||||
|
||||
|
||||
@app.middleware('http')
|
||||
async def headers(request, call_next):
|
||||
request.state.request_id = str(uuid.uuid4())
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
logging.exception('Unhandled request failure')
|
||||
response = error(request, Problem(500, 'internal_error'))
|
||||
response.headers['X-Request-Id'] = request.state.request_id
|
||||
response.headers['X-Instance-Id'] = INSTANCE
|
||||
return response
|
||||
|
||||
|
||||
def error(request, exc):
|
||||
headers = {}
|
||||
if exc.status == 401:
|
||||
headers['WWW-Authenticate'] = 'Basic realm="messenger", charset="UTF-8"' if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
|
||||
return JSONResponse({'error': {'code': exc.code, 'message': exc.message, 'request_id': request.state.request_id}}, status_code=exc.status, headers=headers)
|
||||
|
||||
|
||||
@app.exception_handler(Problem)
|
||||
async def problem_handler(request, exc):
|
||||
return error(request, exc)
|
||||
|
||||
|
||||
@app.exception_handler(psycopg.Error)
|
||||
@app.exception_handler(PoolTimeout)
|
||||
@app.exception_handler(BotoCoreError)
|
||||
@app.exception_handler(ClientError)
|
||||
async def database_handler(request, exc):
|
||||
return error(request, Problem(503, 'unavailable'))
|
||||
|
||||
|
||||
@app.exception_handler(404)
|
||||
@app.exception_handler(405)
|
||||
async def missing_handler(request, exc):
|
||||
return error(request, Problem(404, 'not_found'))
|
||||
|
||||
|
||||
async def body(request, fields, empty_attachments=False):
|
||||
if request.headers.get('content-type', '').split(';')[0].strip().lower() != 'application/json':
|
||||
raise Problem(415, 'unsupported_media_type')
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
if len(raw) + len(chunk) > 65536:
|
||||
raise Problem(413, 'payload_too_large')
|
||||
raw.extend(chunk)
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except (ValueError, UnicodeError):
|
||||
invalid()
|
||||
if empty_attachments and isinstance(value, dict) and 'attachment_ids' in value:
|
||||
fields = (*fields, 'attachment_ids')
|
||||
if not isinstance(value, dict) or set(value) != set(fields):
|
||||
invalid()
|
||||
if empty_attachments:
|
||||
value.setdefault('attachment_ids', [])
|
||||
return value
|
||||
|
||||
|
||||
def text(value, maximum, minimum=1):
|
||||
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
def identifier(value):
|
||||
try:
|
||||
return uuid.UUID(value)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
invalid()
|
||||
|
||||
|
||||
USER_COLUMNS = 'id, username, display_name, created_at'
|
||||
|
||||
|
||||
def authenticate(request, db):
|
||||
authorization = request.headers.get('authorization', '')
|
||||
if not authorization.startswith('Bearer ') or not authorization[7:]:
|
||||
raise Problem(401, 'unauthorized')
|
||||
digest = hashlib.sha256(authorization[7:].encode()).hexdigest()
|
||||
user = db.execute('SELECT u.* FROM users u JOIN sessions s ON s.user_id=u.id WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()', (digest,)).fetchone()
|
||||
if not user:
|
||||
raise Problem(401, 'unauthorized')
|
||||
return user, digest
|
||||
|
||||
|
||||
def user_view(user):
|
||||
return {key: user[key] for key in ('id', 'username', 'display_name', 'created_at')}
|
||||
|
||||
|
||||
def access(db, chat_id, user_id):
|
||||
if not db.execute('SELECT 1 FROM members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone():
|
||||
raise Problem(404, 'not_found')
|
||||
|
||||
|
||||
def chat_view(db, chat):
|
||||
chat['member_ids'] = [row['user_id'] for row in db.execute('SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id', (chat['id'],))]
|
||||
return chat
|
||||
|
||||
|
||||
def pagination(request, scope):
|
||||
if set(request.query_params) - {'limit', 'cursor'}:
|
||||
invalid()
|
||||
raw_limit = request.query_params.get('limit', '50')
|
||||
if not re.fullmatch('[0-9]+', raw_limit) or not 1 <= int(raw_limit) <= 100:
|
||||
invalid()
|
||||
cursor = request.query_params.get('cursor')
|
||||
position = None
|
||||
if cursor is not None:
|
||||
try:
|
||||
resource, when, item = json.loads(base64.urlsafe_b64decode(cursor + '=' * (-len(cursor) % 4)))
|
||||
if resource != scope:
|
||||
invalid()
|
||||
position = (datetime.fromisoformat(when.replace('Z', '+00:00')), identifier(item))
|
||||
if position[0].tzinfo is None:
|
||||
invalid()
|
||||
except (ValueError, TypeError, UnicodeError, OverflowError):
|
||||
invalid()
|
||||
return int(raw_limit), position
|
||||
|
||||
|
||||
def page(rows, limit, scope):
|
||||
selected = rows[:limit]
|
||||
cursor = None
|
||||
if len(rows) > limit:
|
||||
last = clean(selected[-1])
|
||||
cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=')
|
||||
return {'items': selected, 'next_cursor': cursor}
|
||||
|
||||
|
||||
@app.get('/health/live')
|
||||
def live():
|
||||
return {'status': 'ok'}
|
||||
|
||||
|
||||
@app.get('/health/ready')
|
||||
async def ready():
|
||||
async with HEALTH_POOL.connection() as db:
|
||||
await db.execute('SELECT 1 FROM users LIMIT 1')
|
||||
await run_in_threadpool(lambda: client().head_bucket(Bucket=BUCKET))
|
||||
return {'status': 'ok'}
|
||||
|
||||
|
||||
@app.post('/api/v1/users')
|
||||
async def register(request: Request):
|
||||
data = await body(request, ('username', 'display_name', 'password'))
|
||||
if not isinstance(data['username'], str) or not re.fullmatch('[a-z0-9_]{3,32}', data['username']):
|
||||
invalid()
|
||||
text(data['display_name'], 100)
|
||||
text(data['password'], 128, 12)
|
||||
def create():
|
||||
hashed = PASSWORDS.hash(data['password'])
|
||||
try:
|
||||
with POOL.connection() as db:
|
||||
return db.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES(%s,%s,%s,%s) RETURNING ' + USER_COLUMNS, (uuid.uuid4(), data['username'], data['display_name'], hashed)).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise Problem(409, 'conflict')
|
||||
return JSONResponse(clean(await run_in_threadpool(create)), status_code=201)
|
||||
|
||||
|
||||
@app.post('/api/v1/auth/sessions')
|
||||
async def login(request: Request):
|
||||
authorization = request.headers.get('authorization', '')
|
||||
try:
|
||||
if not authorization.startswith('Basic '):
|
||||
raise ValueError()
|
||||
username, password = base64.b64decode(authorization[6:], validate=True).decode('utf-8').split(':', 1)
|
||||
except (ValueError, UnicodeError):
|
||||
raise Problem(401, 'unauthorized')
|
||||
if await request.body():
|
||||
invalid()
|
||||
def create():
|
||||
with POOL.connection() as db:
|
||||
user = db.execute('SELECT * FROM users WHERE username=%s', (username,)).fetchone()
|
||||
try:
|
||||
PASSWORDS.verify(user['password_hash'] if user else DUMMY_HASH, password)
|
||||
except VerificationError:
|
||||
raise Problem(401, 'unauthorized')
|
||||
if not user:
|
||||
raise Problem(401, 'unauthorized')
|
||||
token = secrets.token_urlsafe(32)
|
||||
expiry = db.execute("INSERT INTO sessions VALUES(%s,%s,clock_timestamp()+(%s * interval '1 second')) RETURNING expires_at", (hashlib.sha256(token.encode()).hexdigest(), user['id'], int(os.getenv('SESSION_TTL_SECONDS', '3600')))).fetchone()['expires_at']
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': expiry, 'user': user_view(user)}
|
||||
return JSONResponse(clean(await run_in_threadpool(create)), status_code=201)
|
||||
|
||||
|
||||
from src.attachments import install
|
||||
import sys
|
||||
|
||||
install(app, sys.modules[__name__])
|
||||
|
||||
|
||||
@app.api_route('/api/v1/{path:path}', methods=['GET', 'POST', 'DELETE'])
|
||||
async def business(request: Request, path: str):
|
||||
data = None
|
||||
if request.method == 'POST':
|
||||
def verify_identity():
|
||||
with POOL.connection() as db:
|
||||
authenticate(request, db)
|
||||
await run_in_threadpool(verify_identity)
|
||||
data = await body(request, ('title', 'member_ids') if path == 'chats' else ('text',), empty_attachments=path.endswith('/messages'))
|
||||
def execute():
|
||||
with POOL.connection() as db:
|
||||
user, digest = authenticate(request, db)
|
||||
if path == 'users/me' and request.method == 'GET':
|
||||
return user_view(user)
|
||||
if path == 'auth/sessions/current' and request.method == 'DELETE':
|
||||
db.execute('DELETE FROM sessions WHERE token_hash=%s', (digest,))
|
||||
return None
|
||||
if path == 'chats' and request.method == 'POST':
|
||||
text(data['title'], 100)
|
||||
ids = data['member_ids']
|
||||
if not isinstance(ids, list) or not 1 <= len(ids) <= 100:
|
||||
invalid()
|
||||
ids = [identifier(value) for value in ids]
|
||||
if len(set(ids)) != len(ids) or user['id'] not in ids:
|
||||
invalid()
|
||||
if db.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s)', (ids,)).fetchone()['n'] != len(ids):
|
||||
invalid()
|
||||
chat = db.execute('INSERT INTO chats(id,title) VALUES(%s,%s) RETURNING *', (uuid.uuid4(), data['title'])).fetchone()
|
||||
for member in ids:
|
||||
db.execute('INSERT INTO members VALUES(%s,%s)', (chat['id'], member))
|
||||
return chat_view(db, chat)
|
||||
if path == 'chats' and request.method == 'GET':
|
||||
scope = 'chats:' + str(user['id'])
|
||||
limit, position = pagination(request, scope)
|
||||
query = 'SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s'
|
||||
values = [user['id']]
|
||||
if position:
|
||||
query += ' AND (c.created_at,c.id)>(%s,%s)'
|
||||
values.extend(position)
|
||||
rows = db.execute(query + ' ORDER BY c.created_at,c.id LIMIT %s', values + [limit + 1]).fetchall()
|
||||
return page([chat_view(db, row) for row in rows], limit, scope)
|
||||
parts = path.split('/')
|
||||
if len(parts) not in (2, 3) or parts[0] != 'chats':
|
||||
raise Problem(404, 'not_found')
|
||||
chat_id = identifier(parts[1])
|
||||
access(db, chat_id, user['id'])
|
||||
if len(parts) == 2 and request.method == 'GET':
|
||||
return chat_view(db, db.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone())
|
||||
if len(parts) != 3 or parts[2] != 'messages':
|
||||
raise Problem(404, 'not_found')
|
||||
if request.method == 'GET':
|
||||
scope = 'messages:' + str(chat_id)
|
||||
limit, position = pagination(request, scope)
|
||||
query = 'SELECT * FROM messages WHERE chat_id=%s'
|
||||
values = [chat_id]
|
||||
if position:
|
||||
query += ' AND (created_at,id)>(%s,%s)'
|
||||
values.extend(position)
|
||||
rows = db.execute(query + ' ORDER BY created_at,id LIMIT %s', values + [limit + 1]).fetchall()
|
||||
return page(rows, limit, scope)
|
||||
if request.method != 'POST':
|
||||
raise Problem(404, 'not_found')
|
||||
attachments = data['attachment_ids']
|
||||
if not isinstance(attachments, list) or len(attachments) > 10:
|
||||
invalid()
|
||||
attachments = [identifier(item) for item in attachments]
|
||||
if len(set(attachments)) != len(attachments):
|
||||
invalid()
|
||||
if not isinstance(data['text'], str) or len(data['text']) > 4000 or (not data['text'].strip() and not attachments):
|
||||
invalid()
|
||||
for item in attachments:
|
||||
row = db.execute('SELECT status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE', (item, chat_id)).fetchone()
|
||||
if not row:
|
||||
raise Problem(404, 'not_found')
|
||||
if row['status'] != 'ready':
|
||||
raise Problem(409, 'conflict')
|
||||
canonical = json.dumps({'text': data['text'], 'attachment_ids': [str(item) for item in attachments]}, sort_keys=True)
|
||||
key = request.headers.get('idempotency-key', '')
|
||||
if not re.fullmatch('[A-Za-z0-9._:-]{1,128}', key):
|
||||
invalid()
|
||||
lock = str(user['id']) + ':' + str(chat_id) + ':' + key
|
||||
db.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))', (lock,))
|
||||
db.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()', (user['id'], chat_id, key))
|
||||
previous = db.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s', (user['id'], chat_id, key)).fetchone()
|
||||
if previous:
|
||||
if previous['body'] != canonical:
|
||||
raise Problem(409, 'conflict')
|
||||
return db.execute('SELECT * FROM messages WHERE id=%s', (previous['message_id'],)).fetchone()
|
||||
message = db.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES(%s,%s,%s,%s,%s) RETURNING *', (uuid.uuid4(), chat_id, user['id'], data['text'], attachments)).fetchone()
|
||||
db.execute("INSERT INTO idempotency VALUES(%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')", (user['id'], chat_id, key, canonical, message['id']))
|
||||
return message
|
||||
result = await run_in_threadpool(execute)
|
||||
return Response(status_code=204) if result is None else JSONResponse(clean(result), status_code=201 if request.method == 'POST' else 200)
|
||||
@@ -0,0 +1,18 @@
|
||||
from pathlib import Path
|
||||
import os
|
||||
import psycopg
|
||||
|
||||
|
||||
def database_url():
|
||||
return Path(os.environ.get('DATABASE_URL_FILE', '/run/secrets/database_url')).read_text().strip()
|
||||
|
||||
|
||||
def main():
|
||||
with psycopg.connect(database_url()) as connection:
|
||||
connection.execute('SELECT pg_advisory_xact_lock(202603)')
|
||||
for migration in sorted(Path('migrations').glob('*.sql')):
|
||||
connection.execute(migration.read_text())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
import argparse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
import time
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from src.migrate import database_url
|
||||
from src.storage import client, BUCKET, PREFIX
|
||||
|
||||
|
||||
def run(apply=False, grace_seconds=3600):
|
||||
if grace_seconds < 60:
|
||||
raise ValueError('Grace period must be at least 60 seconds')
|
||||
started = time.monotonic()
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
report = dict(apply=apply, grace_seconds=grace_seconds, objects=0, metadata=0,
|
||||
list_requests=0, head_requests=0, orphan_candidates=0, missing_candidates=0,
|
||||
deleted=0, marked_failed=0, fresh_preserved=0, referenced_preserved=0)
|
||||
s3 = client()
|
||||
with psycopg.connect(database_url(), row_factory=dict_row) as db:
|
||||
db.execute("SET lock_timeout='5s'")
|
||||
db.execute('SELECT pg_advisory_xact_lock(202604)')
|
||||
for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
|
||||
report['list_requests'] += 1
|
||||
for obj in page.get('Contents', []):
|
||||
report['objects'] += 1
|
||||
if db.execute('SELECT 1 FROM attachments WHERE object_key=%s', (obj['Key'],)).fetchone():
|
||||
report['referenced_preserved'] += 1
|
||||
continue
|
||||
if obj['LastModified'] >= cutoff:
|
||||
report['fresh_preserved'] += 1
|
||||
continue
|
||||
report['head_requests'] += 1
|
||||
current = s3.head_object(Bucket=BUCKET, Key=obj['Key'])
|
||||
if current['LastModified'] >= cutoff:
|
||||
report['fresh_preserved'] += 1
|
||||
continue
|
||||
report['orphan_candidates'] += 1
|
||||
if apply:
|
||||
s3.delete_object(Bucket=BUCKET, Key=obj['Key'])
|
||||
report['deleted'] += 1
|
||||
with db.cursor(name='attachments_scan') as cursor:
|
||||
cursor.execute("SELECT id,object_key,created_at FROM attachments WHERE status='ready' ORDER BY id")
|
||||
while rows := cursor.fetchmany(100):
|
||||
for row in rows:
|
||||
report['metadata'] += 1
|
||||
if row['created_at'] >= cutoff:
|
||||
continue
|
||||
report['head_requests'] += 1
|
||||
try:
|
||||
s3.head_object(Bucket=BUCKET, Key=row['object_key'])
|
||||
except ClientError as exc:
|
||||
if exc.response['Error']['Code'] not in ('NoSuchKey', '404', 'NotFound'):
|
||||
raise
|
||||
report['missing_candidates'] += 1
|
||||
if apply:
|
||||
db.execute("UPDATE attachments SET status='missing',error_code='original_missing',updated_at=clock_timestamp() WHERE id=%s AND status='ready'", (row['id'],))
|
||||
report['marked_failed'] += 1
|
||||
report['elapsed_seconds'] = round(time.monotonic() - started, 6)
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--grace-seconds', type=int, default=3600)
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(run(args.apply, args.grace_seconds)))
|
||||
@@ -0,0 +1,21 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
|
||||
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||
PREFIX = 'originals/'
|
||||
|
||||
|
||||
def secret(name):
|
||||
return Path(os.getenv(name + '_FILE', '/run/secrets/' + name.lower())).read_text().strip()
|
||||
|
||||
|
||||
def client():
|
||||
return boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||
region_name='us-east-1', aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
|
||||
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
|
||||
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required',
|
||||
s3={'addressing_style': 'path'}))
|
||||
Reference in New Issue
Block a user