forked from mtusi2026/mtusi2026lab5
300 lines
18 KiB
Python
300 lines
18 KiB
Python
import base64
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import uuid
|
|
from flask import Flask, request, g
|
|
from werkzeug.utils import secure_filename
|
|
from werkzeug.exceptions import HTTPException
|
|
from argon2 import PasswordHasher
|
|
from argon2.exceptions import VerificationError
|
|
from psycopg.rows import dict_row
|
|
from psycopg.errors import UniqueViolation
|
|
from psycopg_pool import ConnectionPool
|
|
import psycopg
|
|
import redis
|
|
import boto3
|
|
from botocore.config import Config
|
|
from PIL import Image, UnidentifiedImageError
|
|
import io
|
|
from urllib.parse import quote
|
|
from app.migrate import setting
|
|
|
|
app = Flask(__name__)
|
|
MAX_UPLOAD=int(os.getenv('MAX_UPLOAD_BYTES','10485760'))
|
|
MAX_PIXELS=int(os.getenv('MAX_IMAGE_PIXELS','40000000'))
|
|
MAX_QUEUED=int(os.getenv('MAX_QUEUED_JOBS','500'))
|
|
app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD
|
|
pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=int(os.getenv('POOL_MAX','8')), timeout=3,
|
|
kwargs={'row_factory':dict_row,'connect_timeout':3,'options':'-c statement_timeout=3000 -c lock_timeout=2000'})
|
|
cache = redis.Redis.from_url(setting('CACHE_URL'),socket_connect_timeout=.2,socket_timeout=.2)
|
|
broker = redis.Redis.from_url(setting('BROKER_URL'),socket_connect_timeout=.3,socket_timeout=.5)
|
|
kdf = PasswordHasher(time_cost=2,memory_cost=19456,parallelism=1)
|
|
dummy = kdf.hash(secrets.token_urlsafe(32))
|
|
ttl = int(os.getenv('SESSION_TTL_SECONDS','3600'))
|
|
def envsetting(name): return setting(name) if os.getenv(name+'_FILE') else os.environ[name]
|
|
s3=boto3.client('s3',endpoint_url=os.getenv('S3_ENDPOINT','http://rustfs:9000'),region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=envsetting('S3_ACCESS_KEY_ID'),aws_secret_access_key=envsetting('S3_SECRET_ACCESS_KEY'),config=Config(connect_timeout=2,read_timeout=5,retries={'max_attempts':1}))
|
|
bucket=os.getenv('S3_BUCKET','messenger-attachments')
|
|
instance = os.getenv('HOSTNAME','api') + '-' + secrets.token_hex(4)
|
|
|
|
class Problem(Exception):
|
|
def __init__(self,status,code): self.status,self.code=status,code
|
|
|
|
def fail(status,code): raise Problem(status,code)
|
|
def invalid(): fail(400,'invalid_request')
|
|
def uid(value):
|
|
try: return str(uuid.UUID(value))
|
|
except (ValueError,TypeError,AttributeError): invalid()
|
|
def clean(row):
|
|
return {k:(v.isoformat().replace('+00:00','Z') if isinstance(v,dt.datetime) else str(v) if isinstance(v,uuid.UUID) else v) for k,v in row.items()}
|
|
def user(row): return clean({k:row[k] for k in ('id','username','display_name','created_at')})
|
|
def body(fields):
|
|
if request.mimetype != 'application/json': fail(415,'unsupported_media_type')
|
|
try: data = request.get_json()
|
|
except HTTPException: invalid()
|
|
if not isinstance(data,dict) or set(data)!=set(fields): invalid()
|
|
return data
|
|
def string(s,maximum,minimum=1):
|
|
if not isinstance(s,str) or not s.strip() or not minimum<=len(s)<=maximum: invalid()
|
|
try: s.encode('utf-8')
|
|
except UnicodeError: invalid()
|
|
return s
|
|
|
|
def actor(c):
|
|
auth = request.headers.get('Authorization','')
|
|
if not auth.startswith('Bearer ') or not 1<=len(auth[7:])<=256: fail(401,'unauthorized')
|
|
digest = hashlib.sha256(auth[7:].encode()).hexdigest()
|
|
row=c.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp()',(digest,)).fetchone()
|
|
if not row: fail(401,'unauthorized')
|
|
g.digest=digest
|
|
return row['user_id']
|
|
|
|
def public_user(c,ident):
|
|
key='user:'+str(ident)
|
|
try:
|
|
value=cache.get(key)
|
|
if value: return json.loads(value)
|
|
except (redis.RedisError,ValueError): pass
|
|
result=user(c.execute('SELECT * FROM users WHERE id=%s',(ident,)).fetchone())
|
|
try: cache.setex(key,60,json.dumps(result))
|
|
except redis.RedisError: pass
|
|
return result
|
|
|
|
def chat(c,ident,who):
|
|
row=c.execute('SELECT c.* FROM chats c JOIN members m ON c.id=m.chat_id WHERE c.id=%s AND m.user_id=%s',(ident,who)).fetchone()
|
|
if not row: fail(404,'not_found')
|
|
row=clean(row)
|
|
row['member_ids']=[str(x['user_id']) for x in c.execute('SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id',(ident,))]
|
|
return row
|
|
|
|
def page_args(resource):
|
|
if set(request.args)-{'limit','cursor'} or any(len(request.args.getlist(k))!=1 for k in request.args): invalid()
|
|
raw=request.args.get('limit','50')
|
|
if len(raw)>3 or not re.fullmatch(r'[0-9]+',raw) or not 1<=int(raw)<=100: invalid()
|
|
cursor=request.args.get('cursor')
|
|
pair=None
|
|
if cursor is not None:
|
|
try:
|
|
data=json.loads(base64.b64decode(cursor,altchars=b'-_',validate=True))
|
|
if not isinstance(data,list) or len(data)!=3 or data[0]!=resource: invalid()
|
|
stamp=dt.datetime.fromisoformat(data[1].replace('Z','+00:00'))
|
|
if stamp.tzinfo is None: invalid()
|
|
pair=(stamp,uid(data[2]))
|
|
except (ValueError,TypeError,KeyError,AttributeError,OverflowError): invalid()
|
|
return int(raw),pair
|
|
|
|
def page(rows,limit,resource):
|
|
more=len(rows)>limit
|
|
rows=rows[:limit]
|
|
cursor=base64.urlsafe_b64encode(json.dumps([resource,rows[-1]['created_at'],rows[-1]['id']]).encode()).decode() if more else None
|
|
return {'items':rows,'next_cursor':cursor}
|
|
|
|
@app.before_request
|
|
def before(): g.request_id=str(uuid.uuid4())
|
|
@app.after_request
|
|
def after(response):
|
|
response.headers['X-Request-Id']=g.request_id
|
|
response.headers['X-Instance-Id']=instance
|
|
return response
|
|
@app.errorhandler(Exception)
|
|
def error(exc):
|
|
if isinstance(exc,Problem): status,code=exc.status,exc.code
|
|
elif isinstance(exc,UniqueViolation): status,code=409,'conflict'
|
|
elif isinstance(exc,(psycopg.Error, __import__('psycopg_pool').PoolTimeout)): status,code=503,'unavailable'
|
|
elif isinstance(exc,HTTPException):
|
|
status=exc.code
|
|
code={404:'not_found',413:'payload_too_large',415:'unsupported_media_type'}.get(status,'invalid_request')
|
|
else:
|
|
app.logger.error('Unhandled exception type: %s',type(exc).__name__)
|
|
status,code=500,'internal_error'
|
|
response=app.json.response({'error':{'code':code,'message':code,'request_id':g.request_id}})
|
|
response.status_code=status
|
|
if status==401:
|
|
response.headers['WWW-Authenticate']='Basic realm="messenger", charset="UTF-8"' if request.path=='/api/v1/auth/sessions' else 'Bearer realm="messenger"'
|
|
return response
|
|
@app.get('/health/live')
|
|
def live(): return {'status':'ok'}
|
|
@app.get('/health/ready')
|
|
def ready():
|
|
with pool.connection() as c: c.execute('SELECT 1')
|
|
try: broker.ping()
|
|
except redis.RedisError: fail(503,'unavailable')
|
|
return {'status':'ok'}
|
|
|
|
@app.route('/api/v1/<path:path>',methods=['GET','POST','DELETE'])
|
|
def api(path):
|
|
public = (path in ('users','auth/sessions') and request.method=='POST')
|
|
if not public and not request.headers.get('Authorization','').startswith('Bearer '): fail(401,'unauthorized')
|
|
with pool.connection() as c:
|
|
if path=='users' and request.method=='POST':
|
|
d=body(['username','display_name','password'])
|
|
if not isinstance(d['username'],str) or not re.fullmatch('[a-z0-9_]{3,32}',d['username']): invalid()
|
|
string(d['display_name'],100); string(d['password'],128,12)
|
|
row=c.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *',(uuid.uuid4(),d['username'],d['display_name'],kdf.hash(d['password']))).fetchone()
|
|
return user(row),201
|
|
if path=='auth/sessions' and request.method=='POST':
|
|
a=request.headers.get('Authorization','')
|
|
try:
|
|
if not a.startswith('Basic ') or request.get_data(): raise ValueError()
|
|
name,password=base64.b64decode(a[6:],validate=True).decode('utf-8').split(':',1)
|
|
except (ValueError,UnicodeError): fail(401,'unauthorized')
|
|
row=c.execute('SELECT * FROM users WHERE username=%s',(name,)).fetchone()
|
|
try: valid=kdf.verify(row['password_hash'] if row else dummy,password)
|
|
except VerificationError: valid=False
|
|
if not valid or not row: fail(401,'unauthorized')
|
|
token=secrets.token_urlsafe(32)
|
|
expires=c.execute("INSERT INTO sessions VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at",(hashlib.sha256(token.encode()).hexdigest(),row['id'],ttl)).fetchone()['expires_at']
|
|
return {'token':token,'token_type':'Bearer','expires_at':expires.isoformat().replace('+00:00','Z'),'user':user(row)},201
|
|
who=actor(c)
|
|
if path=='auth/sessions/current' and request.method=='DELETE':
|
|
c.execute('DELETE FROM sessions WHERE token_hash=%s',(g.digest,))
|
|
return '',204
|
|
if path=='users/me' and request.method=='GET': return public_user(c,who)
|
|
if path=='chats':
|
|
if request.method=='POST':
|
|
d=body(['title','member_ids']); string(d['title'],100)
|
|
ids=d['member_ids']
|
|
if not isinstance(ids,list) or not 1<=len(ids)<=100: invalid()
|
|
ids=[uid(x) for x in ids]
|
|
if len(ids)!=len(set(ids)) or str(who) not in ids: invalid()
|
|
if c.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s::uuid[])',(ids,)).fetchone()['n']!=len(ids): invalid()
|
|
ident=uuid.uuid4()
|
|
c.execute('INSERT INTO chats(id,title) VALUES (%s,%s)',(ident,d['title']))
|
|
for member in ids: c.execute('INSERT INTO members VALUES (%s,%s)',(ident,member))
|
|
return chat(c,ident,who),201
|
|
if request.method=='GET':
|
|
resource='chats:'+str(who); limit,pair=page_args(resource)
|
|
query='SELECT c.id FROM chats c JOIN members m ON c.id=m.chat_id WHERE m.user_id=%s'
|
|
params=[who]
|
|
if pair: query+=' AND (c.created_at,c.id)>(%s,%s)'; params+=list(pair)
|
|
query+=' ORDER BY c.created_at,c.id LIMIT %s'; params+=[limit+1]
|
|
return page([chat(c,x['id'],who) for x in c.execute(query,params).fetchall()],limit,resource)
|
|
parts=path.split('/')
|
|
if len(parts) in (2,3) and parts[0]=='chats':
|
|
ident=uid(parts[1])
|
|
if len(parts)==3 and parts[2]=='messages' and os.getenv('MESSAGE_ACCESS_FAST','1')=='1':
|
|
# Message endpoints only need membership, not the entire Chat representation.
|
|
if not c.execute('SELECT 1 FROM members WHERE chat_id=%s AND user_id=%s',(ident,who)).fetchone(): fail(404,'not_found')
|
|
result=None
|
|
else:
|
|
result=chat(c,ident,who)
|
|
if len(parts)==2 and request.method=='GET': return result
|
|
if len(parts)==3 and parts[2]=='messages':
|
|
if request.method=='POST':
|
|
key=request.headers.get('Idempotency-Key','')
|
|
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}',key): invalid()
|
|
if request.mimetype != 'application/json': fail(415,'unsupported_media_type')
|
|
try: d=request.get_json()
|
|
except HTTPException: invalid()
|
|
if not isinstance(d,dict) or set(d)-{'text','attachment_ids'} or 'text' not in d: invalid()
|
|
if not isinstance(d['text'],str) or len(d['text'])>4000: invalid()
|
|
aids=d.get('attachment_ids',[])
|
|
if not isinstance(aids,list) or len(aids)>10: invalid()
|
|
aids=[uid(x) for x in aids]
|
|
if len(set(aids))!=len(aids): invalid()
|
|
if not d['text'].strip() and not aids: invalid()
|
|
if aids:
|
|
found=c.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])',(ident,aids)).fetchall()
|
|
if len(found)!=len(aids): fail(404,'not_found')
|
|
if any(x['status']=='failed' for x in found): fail(409,'conflict')
|
|
canonical=json.dumps({'text':d['text'],'attachment_ids':aids},sort_keys=True,ensure_ascii=False)
|
|
# One transaction: lock scope, inspect replay, insert effect and result, commit before response.
|
|
scope=str(who)+':POST:'+ident+':'+key
|
|
c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(scope,))
|
|
old=c.execute('SELECT body,response FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()',(who,ident,key)).fetchone()
|
|
if old:
|
|
if old['body']!=canonical: fail(409,'conflict')
|
|
return old['response'],201
|
|
mid=uuid.uuid4()
|
|
result=clean(c.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *',(mid,ident,who,d['text'])).fetchone())
|
|
result['attachment_ids']=aids
|
|
for pos,aid in enumerate(aids): c.execute('INSERT INTO message_attachments VALUES (%s,%s,%s)',(mid,aid,pos))
|
|
c.execute("INSERT INTO idempotency(user_id,chat_id,key,body,response,expires_at) VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT(user_id,chat_id,key) DO UPDATE SET body=excluded.body,response=excluded.response,expires_at=excluded.expires_at",(who,ident,key,canonical,psycopg.types.json.Jsonb(result)))
|
|
return result,201
|
|
if request.method=='GET':
|
|
limit,pair=page_args(ident)
|
|
query='SELECT * FROM messages WHERE chat_id=%s'; params=[ident]
|
|
if pair: query+=' AND (created_at,id)>(%s,%s)'; params+=list(pair)
|
|
query+=' ORDER BY created_at,id LIMIT %s'; params+=[limit+1]
|
|
items=[clean(x) for x in c.execute(query,params)]
|
|
for m in items: m['attachment_ids']=[str(x['attachment_id']) for x in c.execute('SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position',(m['id'],))]
|
|
return page(items,limit,ident)
|
|
if len(parts)==3 and parts[2]=='attachments' and request.method=='POST':
|
|
backlog=c.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('pending','published','processing')").fetchone()['n']
|
|
if backlog>=MAX_QUEUED: fail(503,'unavailable')
|
|
filename=request.args.get('filename','')
|
|
if len(filename)<1 or len(filename)>255 or any(ord(ch)<32 or ch in '/\\' for ch in filename): invalid()
|
|
if request.mimetype not in ('image/png','image/jpeg'): fail(415,'unsupported_media_type')
|
|
raw=request.get_data(cache=False)
|
|
if not raw: invalid()
|
|
try:
|
|
im=Image.open(io.BytesIO(raw)); fmt=im.format; w,h=im.size
|
|
if w*h>MAX_PIXELS: fail(413,'payload_too_large')
|
|
im.verify()
|
|
except Problem: raise
|
|
except (UnidentifiedImageError,OSError,ValueError): fail(415,'unsupported_media_type')
|
|
actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(fmt)
|
|
if actual!=request.mimetype: fail(415,'unsupported_media_type')
|
|
ident_att=uuid.uuid4(); key='original/'+str(ident)+'/'+str(ident_att); staging='staging/'+str(ident)+'/'+str(ident_att)
|
|
digest=hashlib.sha256(raw).hexdigest()
|
|
try:
|
|
s3.put_object(Bucket=bucket,Key=staging,Body=raw,ContentType=actual,Metadata={'sha256':digest})
|
|
row=c.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,staging_key,filename,content_type,size_bytes,sha256,status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",(ident_att,ident,who,key,staging,filename,actual,len(raw),digest)).fetchone()
|
|
job_id=uuid.uuid4()
|
|
c.execute("INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version) VALUES (%s,%s,1)",(job_id,ident_att))
|
|
except Exception:
|
|
try: s3.delete_object(Bucket=bucket,Key=staging)
|
|
except Exception: app.logger.warning('attachment compensation deferred')
|
|
if isinstance(__import__('sys').exc_info()[1],psycopg.Error): raise
|
|
fail(503,'unavailable')
|
|
return attachment_json(row),202
|
|
if path.startswith('attachments/'):
|
|
bits=path.split('/')
|
|
if len(bits) not in (2,3): fail(404,'not_found')
|
|
aid=uid(bits[1])
|
|
row=c.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(aid,who)).fetchone()
|
|
if not row: fail(404,'not_found')
|
|
if len(bits)==2 and request.method=='GET': return attachment_json(row)
|
|
if len(bits)==3 and bits[2]=='content' and request.method=='GET':
|
|
variant=request.args.get('variant','original')
|
|
if variant not in ('original','thumbnail','crop','optimized'): invalid()
|
|
if row['status']!='ready': fail(409,'conflict')
|
|
if variant!='original':
|
|
v=next((v for v in row['variants'] if v['name']==variant),None)
|
|
if v is None: fail(404,'not_found')
|
|
object_key=f"variants/v1/{aid}/{variant}"; content_type=v['content_type']
|
|
else: object_key=row['object_key']; content_type=row['content_type']
|
|
try: obj=s3.get_object(Bucket=bucket,Key=object_key); data=obj['Body'].read(MAX_UPLOAD+1)
|
|
except Exception: fail(503,'unavailable')
|
|
if len(data)>MAX_UPLOAD or (variant=='original' and hashlib.sha256(data).hexdigest()!=row['sha256']): fail(503,'unavailable')
|
|
disposition='attachment; filename="attachment"; filename*=UTF-8\'\''+quote(row['filename'],safe='')
|
|
return app.response_class(data,200,mimetype=content_type,headers={'Content-Disposition':disposition})
|
|
fail(404,'not_found')
|
|
|
|
def attachment_json(row):
|
|
stamp=row['created_at'].isoformat().replace('+00:00','Z')
|
|
return {'id':str(row['id']),'chat_id':str(row['chat_id']),'owner_id':str(row['owner_id']),'filename':row['filename'],'content_type':row['content_type'],'size_bytes':row['size_bytes'],'sha256':row['sha256'].strip(),'status':row['status'],'variants':row.get('variants',[]),'error_code':row.get('error_code'),'created_at':stamp,'updated_at':row.get('updated_at',row['created_at']).isoformat().replace('+00:00','Z')}
|