forked from mtusi2026/mtusi2026lab7
Implement lab 7 TLS trust boundary
This commit is contained in:
+433
@@ -0,0 +1,433 @@
|
||||
"""Lab 2 API: PostgreSQL is authoritative; Redis caches immutable public profiles."""
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout
|
||||
from datetime import datetime
|
||||
from threading import BoundedSemaphore
|
||||
|
||||
import psycopg
|
||||
import redis
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
from flask import Flask, g, jsonify, request
|
||||
from psycopg_pool import PoolTimeout, TooManyRequests
|
||||
from psycopg.types.json import Jsonb
|
||||
from werkzeug.exceptions import HTTPException
|
||||
|
||||
from src.db import make_pool, setting
|
||||
from src.telemetry import HTTP_REQUESTS, HTTP_DURATION, metrics_response, setup
|
||||
|
||||
app = Flask(__name__)
|
||||
setup()
|
||||
app.config['MAX_CONTENT_LENGTH'] = 10485760
|
||||
pool = make_pool()
|
||||
cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2, decode_responses=True)
|
||||
cache_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix='cache')
|
||||
cache_slots = BoundedSemaphore(2)
|
||||
hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
dummy_hash = hasher.hash(secrets.token_urlsafe(32))
|
||||
SESSION_TTL = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
|
||||
if SESSION_TTL < 1:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||
USER_FIELDS = 'id, username, display_name, created_at'
|
||||
INSTANCE_ID = os.getenv('INSTANCE_ID', uuid.uuid4().hex)
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status, code, message, challenge=None):
|
||||
self.status, self.code, self.message, self.challenge = status, code, message, challenge
|
||||
|
||||
|
||||
def invalid():
|
||||
raise APIError(400, 'invalid_request', 'Invalid request')
|
||||
|
||||
|
||||
def unauthorized(basic=False):
|
||||
challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
|
||||
raise APIError(401, 'unauthorized', 'Authentication required', challenge)
|
||||
|
||||
|
||||
def serial(value):
|
||||
if isinstance(value, dict):
|
||||
return {key: serial(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [serial(item) for item in value]
|
||||
if isinstance(value, uuid.UUID):
|
||||
return str(value)
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat().replace('+00:00', 'Z')
|
||||
return value
|
||||
|
||||
|
||||
def response(value, status=200):
|
||||
return jsonify(serial(value)), status
|
||||
|
||||
|
||||
@app.before_request
|
||||
def identify_request():
|
||||
g.request_id = uuid.uuid4().hex
|
||||
g.started_at = time.monotonic()
|
||||
|
||||
|
||||
@app.after_request
|
||||
def headers(result):
|
||||
result.headers['X-Request-Id'] = g.request_id
|
||||
result.headers['X-Instance-Id'] = INSTANCE_ID
|
||||
result.headers['Cache-Control'] = 'no-store'
|
||||
route = request.url_rule.rule if request.url_rule else 'unknown'
|
||||
HTTP_REQUESTS.labels(request.method, route, result.status_code).inc()
|
||||
HTTP_DURATION.labels(request.method, route).observe(time.monotonic() - g.started_at)
|
||||
app.logger.info('http_request method=%s route=%s status=%s request_id=%s', request.method, route, result.status_code, g.request_id)
|
||||
return result
|
||||
|
||||
@app.get('/metrics')
|
||||
def metrics():
|
||||
return metrics_response()
|
||||
|
||||
|
||||
@app.errorhandler(Exception)
|
||||
def error(exc):
|
||||
if isinstance(exc, APIError):
|
||||
problem = exc
|
||||
elif isinstance(exc, psycopg.errors.UniqueViolation):
|
||||
problem = APIError(409, 'conflict', 'Username already exists')
|
||||
elif isinstance(exc, psycopg.errors.ForeignKeyViolation):
|
||||
problem = APIError(400, 'invalid_request', 'Invalid member list')
|
||||
elif isinstance(exc, (BotoCoreError, ClientError, psycopg.OperationalError, PoolTimeout, TooManyRequests,
|
||||
psycopg.errors.QueryCanceled, psycopg.errors.LockNotAvailable)):
|
||||
problem = APIError(503, 'unavailable', 'Database temporarily unavailable')
|
||||
elif isinstance(exc, HTTPException):
|
||||
status = exc.code if exc.code in (400, 404, 413, 415) else 400
|
||||
code = {400: 'invalid_request', 404: 'not_found', 413: 'payload_too_large',
|
||||
415: 'unsupported_media_type'}[status]
|
||||
problem = APIError(status, code, 'Request cannot be processed')
|
||||
else:
|
||||
app.logger.error('Unhandled error type=%s request_id=%s', type(exc).__name__, g.request_id)
|
||||
problem = APIError(500, 'internal_error', 'Internal error')
|
||||
result = jsonify(error={'code': problem.code, 'message': problem.message,
|
||||
'request_id': g.request_id})
|
||||
result.status_code = problem.status
|
||||
if problem.challenge:
|
||||
result.headers['WWW-Authenticate'] = problem.challenge
|
||||
if problem.status == 429:
|
||||
result.headers['Retry-After'] = '1'
|
||||
return result
|
||||
|
||||
|
||||
def body(fields):
|
||||
if request.mimetype != 'application/json':
|
||||
raise APIError(415, 'unsupported_media_type', 'Expected application/json')
|
||||
value = request.get_json()
|
||||
if not isinstance(value, dict) or set(value) != set(fields):
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
def string(value, minimum, maximum):
|
||||
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
|
||||
invalid()
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
if '\x00' in value:
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
def uid(value):
|
||||
try:
|
||||
if not isinstance(value, str) or not re.fullmatch(
|
||||
r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value):
|
||||
invalid()
|
||||
return uuid.UUID(value)
|
||||
except ValueError:
|
||||
invalid()
|
||||
|
||||
|
||||
def cached_operation(operation, *args, **kwargs):
|
||||
# Socket timeouts do not bound DNS resolution when a Compose service is down.
|
||||
# At most two background operations per worker, no unbounded queue.
|
||||
if not cache_slots.acquire(blocking=False):
|
||||
return None
|
||||
future = cache_executor.submit(operation, *args, **kwargs)
|
||||
future.add_done_callback(lambda _: cache_slots.release())
|
||||
try:
|
||||
return future.result(timeout=0.25)
|
||||
except (redis.RedisError, FutureTimeout):
|
||||
return None
|
||||
|
||||
|
||||
def profile(conn, user_id):
|
||||
key = 'profile:v1:' + str(user_id)
|
||||
cached = cached_operation(cache.get, key)
|
||||
if cached:
|
||||
try:
|
||||
return json.loads(cached)
|
||||
except ValueError:
|
||||
pass
|
||||
user = serial(conn.execute('SELECT ' + USER_FIELDS + ' FROM users WHERE id=%s',
|
||||
(user_id,)).fetchone())
|
||||
if user is None:
|
||||
unauthorized()
|
||||
cached_operation(cache.set, key, json.dumps(user), ex=60)
|
||||
return user
|
||||
|
||||
|
||||
def actor(conn):
|
||||
auth = request.headers.get('Authorization', '').split()
|
||||
if len(auth) != 2 or auth[0].lower() != 'bearer' or len(auth[1]) > 256:
|
||||
unauthorized()
|
||||
digest = hashlib.sha256(auth[1].encode()).hexdigest()
|
||||
session = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s '
|
||||
'AND expires_at > clock_timestamp()', (digest,)).fetchone()
|
||||
if not session:
|
||||
unauthorized()
|
||||
g.token_hash = digest
|
||||
return session['user_id']
|
||||
|
||||
|
||||
def chat_access(conn, chat_id, user_id):
|
||||
chat = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id '
|
||||
'WHERE c.id=%s AND m.user_id=%s', (chat_id, user_id)).fetchone()
|
||||
if not chat:
|
||||
raise APIError(404, 'not_found', 'Resource not found')
|
||||
return chat
|
||||
|
||||
|
||||
def chat_members(conn, chat):
|
||||
chat['member_ids'] = [row['user_id'] for row in conn.execute(
|
||||
'SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id', (chat['id'],))]
|
||||
return chat
|
||||
|
||||
|
||||
def page_args(scope):
|
||||
limit = request.args.get('limit', '50')
|
||||
if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100:
|
||||
invalid()
|
||||
cursor = request.args.get('cursor')
|
||||
after = None
|
||||
if cursor is not None:
|
||||
try:
|
||||
if len(cursor) > 1024:
|
||||
invalid()
|
||||
data = json.loads(base64.b64decode(cursor, altchars=b'-_', validate=True))
|
||||
if not isinstance(data, list) or len(data) != 3 or data[0] != scope:
|
||||
invalid()
|
||||
date = datetime.fromisoformat(data[1])
|
||||
if date.tzinfo is None:
|
||||
invalid()
|
||||
after = (date, uid(data[2]))
|
||||
except (ValueError, TypeError, binascii.Error):
|
||||
invalid()
|
||||
return int(limit), after
|
||||
|
||||
|
||||
def page(rows, limit, scope):
|
||||
more = len(rows) > limit
|
||||
items = rows[:limit]
|
||||
cursor = None
|
||||
if more:
|
||||
last = serial(items[-1])
|
||||
cursor = base64.urlsafe_b64encode(json.dumps(
|
||||
[scope, last['created_at'], last['id']]).encode()).decode()
|
||||
return {'items': items, 'next_cursor': cursor}
|
||||
|
||||
|
||||
@app.get('/health/live')
|
||||
def live():
|
||||
return response({'status': 'ok'})
|
||||
|
||||
|
||||
@app.get('/health/ready')
|
||||
def ready():
|
||||
with pool.connection() as conn:
|
||||
conn.execute('SELECT 1 FROM schema_migrations LIMIT 1')
|
||||
return response({'status': 'ok'})
|
||||
|
||||
|
||||
@app.post('/api/v1/users')
|
||||
def register():
|
||||
data = body(('username', 'display_name', 'password'))
|
||||
name = string(data['username'], 3, 32)
|
||||
if not re.fullmatch('[a-z0-9_]{3,32}', name):
|
||||
invalid()
|
||||
display = string(data['display_name'], 1, 100)
|
||||
password = string(data['password'], 12, 128)
|
||||
encoded = hasher.hash(password)
|
||||
with pool.connection() as conn:
|
||||
user = conn.execute('INSERT INTO users (id,username,display_name,password_hash) '
|
||||
'VALUES (%s,%s,%s,%s) RETURNING ' + USER_FIELDS,
|
||||
(uuid.uuid4(), name, display, encoded)).fetchone()
|
||||
return response(user, 201)
|
||||
|
||||
|
||||
@app.post('/api/v1/auth/sessions')
|
||||
def login():
|
||||
auth = request.headers.get('Authorization', '').split()
|
||||
if len(auth) != 2 or auth[0].lower() != 'basic' or len(auth[1]) > 2048:
|
||||
unauthorized(True)
|
||||
try:
|
||||
name, password = base64.b64decode(auth[1], validate=True).decode('utf-8').split(':', 1)
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
unauthorized(True)
|
||||
if not re.fullmatch('[a-z0-9_]{3,32}', name) or not 12 <= len(password) <= 128:
|
||||
unauthorized(True)
|
||||
if request.get_data():
|
||||
invalid()
|
||||
with pool.connection() as conn:
|
||||
user = conn.execute('SELECT * FROM users WHERE username=%s', (name,)).fetchone()
|
||||
try:
|
||||
hasher.verify(user['password_hash'] if user else dummy_hash, password)
|
||||
except VerificationError:
|
||||
unauthorized(True)
|
||||
if not user:
|
||||
unauthorized(True)
|
||||
token = secrets.token_urlsafe(32)
|
||||
with pool.connection() as conn:
|
||||
expiry = conn.execute('INSERT INTO sessions VALUES (%s,%s,clock_timestamp() '
|
||||
'+ %s * interval \'1 second\') RETURNING expires_at',
|
||||
(hashlib.sha256(token.encode()).hexdigest(), user['id'],
|
||||
SESSION_TTL)).fetchone()['expires_at']
|
||||
user.pop('password_hash')
|
||||
return response({'token': token, 'token_type': 'Bearer', 'expires_at': expiry, 'user': user}, 201)
|
||||
|
||||
|
||||
@app.delete('/api/v1/auth/sessions/current')
|
||||
def logout():
|
||||
with pool.connection() as conn:
|
||||
actor(conn)
|
||||
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (g.token_hash,))
|
||||
return '', 204
|
||||
|
||||
|
||||
@app.get('/api/v1/users/me')
|
||||
def me():
|
||||
with pool.connection() as conn:
|
||||
user = profile(conn, actor(conn))
|
||||
return response(user)
|
||||
|
||||
|
||||
@app.post('/api/v1/chats')
|
||||
def create_chat():
|
||||
with pool.connection() as conn:
|
||||
user = actor(conn)
|
||||
data = body(('title', 'member_ids'))
|
||||
title = string(data['title'], 1, 100)
|
||||
if not isinstance(data['member_ids'], list) or not 1 <= len(data['member_ids']) <= 100:
|
||||
invalid()
|
||||
members = [uid(item) for item in data['member_ids']]
|
||||
if len(set(members)) != len(members) or user not in members:
|
||||
invalid()
|
||||
chat = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *',
|
||||
(uuid.uuid4(), title)).fetchone()
|
||||
for member in members:
|
||||
conn.execute('INSERT INTO members VALUES (%s,%s)', (chat['id'], member))
|
||||
chat['member_ids'] = sorted(members)
|
||||
return response(chat, 201)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats')
|
||||
def list_chats():
|
||||
with pool.connection() as conn:
|
||||
user = actor(conn)
|
||||
scope = 'chats:' + str(user)
|
||||
limit, after = page_args(scope)
|
||||
sql = 'SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s'
|
||||
params = [user]
|
||||
if after:
|
||||
sql += ' AND (c.created_at,c.id) > (%s,%s)'
|
||||
params.extend(after)
|
||||
rows = conn.execute(sql + ' ORDER BY c.created_at,c.id LIMIT %s', params + [limit+1]).fetchall()
|
||||
result = page(rows, limit, scope)
|
||||
for chat in result['items']:
|
||||
chat_members(conn, chat)
|
||||
return response(result)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats/<chat_id>')
|
||||
def get_chat(chat_id):
|
||||
with pool.connection() as conn:
|
||||
user = actor(conn)
|
||||
result = chat_members(conn, chat_access(conn, uid(chat_id), user))
|
||||
return response(result)
|
||||
|
||||
|
||||
@app.route('/api/v1/chats/<chat_id>/messages', methods=['GET', 'POST'])
|
||||
def messages(chat_id):
|
||||
with pool.connection() as conn:
|
||||
user = actor(conn)
|
||||
chat_id = uid(chat_id)
|
||||
chat_access(conn, chat_id, user)
|
||||
if request.method == 'POST':
|
||||
if request.mimetype != 'application/json':
|
||||
raise APIError(415, 'unsupported_media_type', 'Expected application/json')
|
||||
data = request.get_json()
|
||||
if not isinstance(data, dict) or 'text' not in data or set(data) - {'text', 'attachment_ids'}:
|
||||
invalid()
|
||||
text = data['text']
|
||||
if not isinstance(text, str) or len(text) > 4000 or '\x00' in text:
|
||||
invalid()
|
||||
try:
|
||||
text.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
attachments = data.get('attachment_ids', [])
|
||||
if not isinstance(attachments, list) or len(attachments) > 10:
|
||||
invalid()
|
||||
attachments = [uid(item) for item in attachments]
|
||||
if len(set(attachments)) != len(attachments) or not (text.strip() or attachments):
|
||||
invalid()
|
||||
for attachment_id in attachments:
|
||||
attachment = conn.execute('SELECT status FROM attachments WHERE id=%s AND chat_id=%s',
|
||||
(attachment_id, chat_id)).fetchone()
|
||||
if not attachment:
|
||||
raise APIError(404, 'not_found', 'Attachment not found')
|
||||
if attachment['status'] == 'failed':
|
||||
raise APIError(409, 'conflict', 'Attachment failed')
|
||||
key = request.headers.get('Idempotency-Key', '')
|
||||
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
|
||||
invalid()
|
||||
canonical = {'text': text, 'attachment_ids': [str(item) for item in attachments]}
|
||||
lock = int.from_bytes(hashlib.sha256(f'{user}:{chat_id}:{key}'.encode()).digest()[:8],
|
||||
'big', signed=True)
|
||||
conn.execute('SELECT pg_advisory_xact_lock(%s)', (lock,))
|
||||
conn.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s '
|
||||
'AND expires_at<=clock_timestamp()', (user, chat_id, key))
|
||||
prior = conn.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s',
|
||||
(user, chat_id, key)).fetchone()
|
||||
if prior:
|
||||
if prior['request'] != canonical:
|
||||
raise APIError(409, 'conflict', 'Idempotency key already used with a different message')
|
||||
return response(prior['response'], 201)
|
||||
result = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) '
|
||||
'VALUES (%s,%s,%s,%s,%s) RETURNING *',
|
||||
(uuid.uuid4(), chat_id, user, text, attachments)).fetchone()
|
||||
conn.execute('INSERT INTO idempotency(user_id,chat_id,key,request,response) '
|
||||
'VALUES (%s,%s,%s,%s,%s)',
|
||||
(user, chat_id, key, Jsonb(canonical), Jsonb(serial(result))))
|
||||
status = 201
|
||||
else:
|
||||
scope = 'messages:' + str(chat_id)
|
||||
limit, after = page_args(scope)
|
||||
sql = 'SELECT * FROM messages WHERE chat_id=%s'
|
||||
params = [chat_id]
|
||||
if after:
|
||||
sql += ' AND (created_at,id) > (%s,%s)'
|
||||
params.extend(after)
|
||||
rows = conn.execute(sql + ' ORDER BY created_at,id LIMIT %s', params + [limit+1]).fetchall()
|
||||
result, status = page(rows, limit, scope), 200
|
||||
return response(result, status)
|
||||
|
||||
|
||||
from src.attachments import register_routes
|
||||
register_routes(app, pool, actor, chat_access, uid, response, APIError, invalid)
|
||||
@@ -0,0 +1,123 @@
|
||||
import hashlib
|
||||
import tempfile
|
||||
import uuid
|
||||
import warnings
|
||||
from threading import BoundedSemaphore
|
||||
|
||||
from flask import Response, request, stream_with_context
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from src.storage import BUCKET, s3
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
# Each API worker is bounded by Gunicorn threads; this semaphore prevents an
|
||||
# unbounded number of uploads without serializing the required concurrent case.
|
||||
slots = BoundedSemaphore(16)
|
||||
|
||||
|
||||
def public(row):
|
||||
return {k: v for k, v in row.items() if k != 'object_key'}
|
||||
|
||||
|
||||
def register_routes(app, pool, actor, chat_access, uid, response, APIError, invalid):
|
||||
@app.post('/api/v1/chats/<chat_id>/attachments')
|
||||
def upload(chat_id):
|
||||
with pool.connection() as conn:
|
||||
user = actor(conn)
|
||||
chat_id = uid(chat_id)
|
||||
chat_access(conn, chat_id, user)
|
||||
filename = request.args.get('filename', '')
|
||||
if not 1 <= len(filename) <= 255 or not filename.strip() or any(
|
||||
ord(c) < 32 or ord(c) == 127 or c in '/\\' for c in filename):
|
||||
invalid()
|
||||
if request.mimetype not in ('image/png', 'image/jpeg'):
|
||||
raise APIError(415, 'unsupported_media_type', 'Expected PNG or JPEG')
|
||||
if request.content_length and request.content_length > MAX_BYTES:
|
||||
raise APIError(413, 'payload_too_large', 'Image too large')
|
||||
if not slots.acquire(timeout=5):
|
||||
raise APIError(429, 'rate_limited', 'Upload capacity reached')
|
||||
try:
|
||||
with tempfile.SpooledTemporaryFile(max_size=1024*1024) as file:
|
||||
length, digest = 0, hashlib.sha256()
|
||||
while True:
|
||||
chunk = request.stream.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
length += len(chunk)
|
||||
if length > MAX_BYTES:
|
||||
raise APIError(413, 'payload_too_large', 'Image too large')
|
||||
digest.update(chunk)
|
||||
file.write(chunk)
|
||||
file.seek(0)
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
with Image.open(file) as image:
|
||||
if image.width * image.height > 40_000_000:
|
||||
raise APIError(413, 'payload_too_large', 'Too many pixels')
|
||||
if Image.MIME.get(image.format) != request.mimetype:
|
||||
raise APIError(415, 'unsupported_media_type', 'Image type mismatch')
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
image.load()
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise APIError(413, 'payload_too_large', 'Too many pixels') from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
raise APIError(415, 'unsupported_media_type', 'Invalid image') from None
|
||||
file.seek(0)
|
||||
identifier = uuid.uuid4()
|
||||
key = 'original/' + str(identifier)
|
||||
with pool.connection() as conn:
|
||||
conn.execute('INSERT INTO object_intents(object_key) VALUES (%s)', (key,))
|
||||
s3.put_object(Bucket=BUCKET, Key=key, Body=file, ContentLength=length, ContentType=request.mimetype)
|
||||
# Intent survives a crash or failed DB commit; the janitor removes old unreferenced objects.
|
||||
with pool.connection() as conn:
|
||||
row = conn.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,status) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
|
||||
(identifier,chat_id,user,filename,request.mimetype,length,digest.hexdigest(),key)).fetchone()
|
||||
conn.execute('INSERT INTO attachment_jobs(attachment_id) VALUES (%s)', (identifier,))
|
||||
conn.execute('DELETE FROM object_intents WHERE object_key=%s', (key,))
|
||||
return response(public(row),202)
|
||||
finally:
|
||||
slots.release()
|
||||
|
||||
def authorized(attachment_id):
|
||||
with pool.connection() as conn:
|
||||
user = actor(conn)
|
||||
identifier = uid(attachment_id)
|
||||
row = conn.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id '
|
||||
'WHERE a.id=%s AND m.user_id=%s', (identifier,user)).fetchone()
|
||||
if not row:
|
||||
raise APIError(404,'not_found','Attachment not found')
|
||||
return row
|
||||
|
||||
@app.get('/api/v1/attachments/<attachment_id>')
|
||||
def metadata(attachment_id):
|
||||
return response(public(authorized(attachment_id)))
|
||||
|
||||
@app.get('/api/v1/attachments/<attachment_id>/content')
|
||||
def download(attachment_id):
|
||||
row = authorized(attachment_id)
|
||||
if row['status'] != 'ready':
|
||||
raise APIError(409,'conflict','Attachment not ready')
|
||||
variant = request.args.get('variant','original')
|
||||
key, content_type, size_bytes = row['object_key'], row['content_type'], row['size_bytes']
|
||||
if variant != 'original':
|
||||
found = next((item for item in row['variants'] if item['name'] == variant), None)
|
||||
if not found:
|
||||
raise APIError(404,'not_found','Variant not found')
|
||||
key = row['object_key'] + '.' + variant
|
||||
content_type, size_bytes = found['content_type'], found['size_bytes']
|
||||
obj = s3.get_object(Bucket=BUCKET,Key=key)
|
||||
def chunks():
|
||||
try:
|
||||
while True:
|
||||
chunk = obj['Body'].read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
obj['Body'].close()
|
||||
return Response(stream_with_context(chunks()),content_type=content_type,
|
||||
headers={'Content-Length':str(size_bytes),'X-Content-Type-Options':'nosniff'})
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Old upload intent reconciliation; dry run unless --apply, never deletes fresh objects."""
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from src.db import make_pool
|
||||
from src.storage import s3, BUCKET
|
||||
|
||||
|
||||
def sweep(apply=False):
|
||||
pool = make_pool()
|
||||
with pool.connection() as conn:
|
||||
rows = conn.execute("SELECT i.object_key FROM object_intents i LEFT JOIN attachments a "
|
||||
"ON a.object_key=i.object_key WHERE a.id IS NULL "
|
||||
"AND i.created_at < clock_timestamp()-interval '1 hour' FOR UPDATE OF i SKIP LOCKED").fetchall()
|
||||
for row in rows:
|
||||
if apply:
|
||||
s3.delete_object(Bucket=BUCKET,Key=row['object_key'])
|
||||
conn.execute('DELETE FROM object_intents WHERE object_key=%s',(row['object_key'],))
|
||||
pool.close()
|
||||
return {'old_unreferenced_intents':len(rows),'applied':apply}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--apply',action='store_true')
|
||||
parser.add_argument('--loop',action='store_true')
|
||||
args=parser.parse_args()
|
||||
while True:
|
||||
try:
|
||||
print(json.dumps(sweep(args.apply)),flush=True)
|
||||
except Exception as exc:
|
||||
print(json.dumps({'cleanup_error':type(exc).__name__}),flush=True)
|
||||
if not args.loop:
|
||||
raise SystemExit(1)
|
||||
if not args.loop:
|
||||
break
|
||||
time.sleep(60)
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Bounded PostgreSQL pool and serialized, versioned migrations."""
|
||||
import hashlib
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
|
||||
def setting(name):
|
||||
path = os.getenv(name + '_FILE')
|
||||
return Path(path).read_text().strip() if path else os.environ[name]
|
||||
|
||||
|
||||
def migrate():
|
||||
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(202602)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
|
||||
'(version text PRIMARY KEY, checksum text NOT NULL)')
|
||||
for file in sorted(Path('migrations').glob('*.sql')):
|
||||
sql = file.read_text(encoding='utf-8')
|
||||
digest = hashlib.sha256(sql.encode()).hexdigest()
|
||||
row = conn.execute('SELECT checksum FROM schema_migrations WHERE version=%s',
|
||||
(file.name,)).fetchone()
|
||||
if row:
|
||||
if row[0] != digest:
|
||||
raise RuntimeError('Applied migration changed')
|
||||
continue
|
||||
conn.execute(sql)
|
||||
conn.execute('INSERT INTO schema_migrations VALUES (%s,%s)', (file.name, digest))
|
||||
|
||||
|
||||
def make_pool():
|
||||
return ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=8,
|
||||
timeout=3, max_waiting=32,
|
||||
check=ConnectionPool.check_connection,
|
||||
kwargs={'connect_timeout': 3, 'row_factory': dict_row,
|
||||
'options': '-c statement_timeout=3000 -c lock_timeout=2000'})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
for attempt in range(30):
|
||||
try:
|
||||
migrate()
|
||||
break
|
||||
except psycopg.OperationalError:
|
||||
if attempt == 29:
|
||||
raise RuntimeError('Database did not become available') from None
|
||||
time.sleep(2)
|
||||
print('Migrations applied successfully')
|
||||
@@ -0,0 +1,10 @@
|
||||
import os
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from src.db import setting
|
||||
|
||||
BUCKET = os.getenv('S3_BUCKET', 'messenger')
|
||||
s3 = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://lb:9000'),
|
||||
aws_access_key_id=setting('S3_ACCESS_KEY'), aws_secret_access_key=setting('S3_SECRET_KEY'),
|
||||
region_name='us-east-1', config=Config(connect_timeout=2, read_timeout=5,
|
||||
retries={'max_attempts': 0}, s3={'addressing_style': 'path'}))
|
||||
@@ -0,0 +1,22 @@
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from prometheus_client import Counter, Histogram, Gauge, generate_latest, CONTENT_TYPE_LATEST
|
||||
|
||||
HTTP_REQUESTS = Counter('messenger_http_requests_total','HTTP requests',['method','route','status'])
|
||||
HTTP_DURATION = Histogram('messenger_http_request_duration_seconds','HTTP duration',['method','route'])
|
||||
JOBS = Counter('messenger_attachment_jobs_total','Attachment jobs',['status'])
|
||||
QUEUE_AGE = Gauge('messenger_attachment_oldest_job_age_seconds','Oldest pending job age')
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
return json.dumps({'ts': self.formatTime(record), 'level': record.levelname,
|
||||
'message': record.getMessage(), 'trace_id': getattr(record,'trace_id',None)})
|
||||
|
||||
def setup():
|
||||
handler=logging.StreamHandler(); handler.setFormatter(JsonFormatter())
|
||||
logging.getLogger().handlers[:] = [handler]; logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
def metrics_response():
|
||||
from flask import Response
|
||||
return Response(generate_latest(), content_type=CONTENT_TYPE_LATEST)
|
||||
@@ -0,0 +1,60 @@
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import random
|
||||
import time
|
||||
from PIL import Image, ImageOps
|
||||
from src.db import make_pool
|
||||
from src.storage import BUCKET, s3
|
||||
|
||||
MAX_ATTEMPTS = 5
|
||||
|
||||
def make_variant(image, name):
|
||||
image = ImageOps.exif_transpose(image)
|
||||
if name == 'thumbnail':
|
||||
image.thumbnail((256, 256), Image.Resampling.LANCZOS); target = image
|
||||
elif name == 'crop':
|
||||
target = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS)
|
||||
else:
|
||||
image.thumbnail((1280, 1280), Image.Resampling.LANCZOS); target = image
|
||||
if target.mode not in ('RGB', 'L'):
|
||||
bg = Image.new('RGB', target.size, 'white')
|
||||
bg.paste(target, mask=target.getchannel('A') if 'A' in target.getbands() else None)
|
||||
target = bg
|
||||
out = io.BytesIO()
|
||||
content_type = 'image/webp' if name == 'optimized' else 'image/jpeg'
|
||||
target.convert('RGB').save(out, format='WEBP' if name == 'optimized' else 'JPEG', quality=82, exif=b'')
|
||||
raw = out.getvalue()
|
||||
return raw, content_type, target.width, target.height
|
||||
|
||||
def process(row, pool):
|
||||
raw = s3.get_object(Bucket=BUCKET, Key=row['object_key'])['Body'].read()
|
||||
with Image.open(io.BytesIO(raw)) as image:
|
||||
variants=[]
|
||||
for name in ('thumbnail','crop','optimized'):
|
||||
data, content_type, width, height = make_variant(image.copy(), name)
|
||||
s3.put_object(Bucket=BUCKET, Key=row['object_key']+'.'+name, Body=data, ContentType=content_type)
|
||||
variants.append({'name':name,'content_type':content_type,'size_bytes':len(data),'width':width,'height':height,'sha256':hashlib.sha256(data).hexdigest()})
|
||||
with pool.connection() as conn:
|
||||
conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND status<>'ready'", (json.dumps(variants),row['id']))
|
||||
conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),locked_until=NULL WHERE attachment_id=%s',(row['id'],))
|
||||
|
||||
def loop():
|
||||
pool=make_pool()
|
||||
while True:
|
||||
with pool.connection() as conn:
|
||||
job=conn.execute("UPDATE attachment_jobs SET attempts=attempts+1,locked_until=clock_timestamp()+interval '60 seconds' WHERE attachment_id=(SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until<clock_timestamp()) ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *").fetchone()
|
||||
row=conn.execute('SELECT * FROM attachments WHERE id=%s',(job['attachment_id'],)).fetchone() if job else None
|
||||
if not row:
|
||||
time.sleep(.25); continue
|
||||
try: process(row,pool)
|
||||
except Exception as error:
|
||||
with pool.connection() as conn:
|
||||
if job['attempts'] >= MAX_ATTEMPTS:
|
||||
conn.execute("UPDATE attachments SET status='failed',error_code='processing_failed',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
|
||||
conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),last_error=%s WHERE attachment_id=%s',(type(error).__name__,row['id']))
|
||||
else:
|
||||
delay=min(60,2**job['attempts'])+random.random()
|
||||
conn.execute("UPDATE attachment_jobs SET available_at=clock_timestamp()+(%s*interval '1 second'),locked_until=NULL,last_error=%s WHERE attachment_id=%s",(delay,type(error).__name__,row['id']))
|
||||
|
||||
if __name__ == '__main__': loop()
|
||||
Reference in New Issue
Block a user