Files

300 lines
18 KiB
Python

import base64
import datetime as dt
import hashlib
import json
import os
import re
import secrets
import uuid
from flask import Flask, request, g
from werkzeug.utils import secure_filename
from werkzeug.exceptions import HTTPException
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
from psycopg.rows import dict_row
from psycopg.errors import UniqueViolation
from psycopg_pool import ConnectionPool
import psycopg
import redis
import boto3
from botocore.config import Config
from PIL import Image, UnidentifiedImageError
import io
from urllib.parse import quote
from app.migrate import setting
app = Flask(__name__)
MAX_UPLOAD=int(os.getenv('MAX_UPLOAD_BYTES','10485760'))
MAX_PIXELS=int(os.getenv('MAX_IMAGE_PIXELS','40000000'))
MAX_QUEUED=int(os.getenv('MAX_QUEUED_JOBS','500'))
app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD
pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=int(os.getenv('POOL_MAX','8')), timeout=3,
kwargs={'row_factory':dict_row,'connect_timeout':3,'options':'-c statement_timeout=3000 -c lock_timeout=2000'})
cache = redis.Redis.from_url(setting('CACHE_URL'),socket_connect_timeout=.2,socket_timeout=.2)
broker = redis.Redis.from_url(setting('BROKER_URL'),socket_connect_timeout=.3,socket_timeout=.5)
kdf = PasswordHasher(time_cost=2,memory_cost=19456,parallelism=1)
dummy = kdf.hash(secrets.token_urlsafe(32))
ttl = int(os.getenv('SESSION_TTL_SECONDS','3600'))
def envsetting(name): return setting(name) if os.getenv(name+'_FILE') else os.environ[name]
s3=boto3.client('s3',endpoint_url=os.getenv('S3_ENDPOINT','http://rustfs:9000'),region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=envsetting('S3_ACCESS_KEY_ID'),aws_secret_access_key=envsetting('S3_SECRET_ACCESS_KEY'),config=Config(connect_timeout=2,read_timeout=5,retries={'max_attempts':1}))
bucket=os.getenv('S3_BUCKET','messenger-attachments')
instance = os.getenv('HOSTNAME','api') + '-' + secrets.token_hex(4)
class Problem(Exception):
def __init__(self,status,code): self.status,self.code=status,code
def fail(status,code): raise Problem(status,code)
def invalid(): fail(400,'invalid_request')
def uid(value):
try: return str(uuid.UUID(value))
except (ValueError,TypeError,AttributeError): invalid()
def clean(row):
return {k:(v.isoformat().replace('+00:00','Z') if isinstance(v,dt.datetime) else str(v) if isinstance(v,uuid.UUID) else v) for k,v in row.items()}
def user(row): return clean({k:row[k] for k in ('id','username','display_name','created_at')})
def body(fields):
if request.mimetype != 'application/json': fail(415,'unsupported_media_type')
try: data = request.get_json()
except HTTPException: invalid()
if not isinstance(data,dict) or set(data)!=set(fields): invalid()
return data
def string(s,maximum,minimum=1):
if not isinstance(s,str) or not s.strip() or not minimum<=len(s)<=maximum: invalid()
try: s.encode('utf-8')
except UnicodeError: invalid()
return s
def actor(c):
auth = request.headers.get('Authorization','')
if not auth.startswith('Bearer ') or not 1<=len(auth[7:])<=256: fail(401,'unauthorized')
digest = hashlib.sha256(auth[7:].encode()).hexdigest()
row=c.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp()',(digest,)).fetchone()
if not row: fail(401,'unauthorized')
g.digest=digest
return row['user_id']
def public_user(c,ident):
key='user:'+str(ident)
try:
value=cache.get(key)
if value: return json.loads(value)
except (redis.RedisError,ValueError): pass
result=user(c.execute('SELECT * FROM users WHERE id=%s',(ident,)).fetchone())
try: cache.setex(key,60,json.dumps(result))
except redis.RedisError: pass
return result
def chat(c,ident,who):
row=c.execute('SELECT c.* FROM chats c JOIN members m ON c.id=m.chat_id WHERE c.id=%s AND m.user_id=%s',(ident,who)).fetchone()
if not row: fail(404,'not_found')
row=clean(row)
row['member_ids']=[str(x['user_id']) for x in c.execute('SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id',(ident,))]
return row
def page_args(resource):
if set(request.args)-{'limit','cursor'} or any(len(request.args.getlist(k))!=1 for k in request.args): invalid()
raw=request.args.get('limit','50')
if len(raw)>3 or not re.fullmatch(r'[0-9]+',raw) or not 1<=int(raw)<=100: invalid()
cursor=request.args.get('cursor')
pair=None
if cursor is not None:
try:
data=json.loads(base64.b64decode(cursor,altchars=b'-_',validate=True))
if not isinstance(data,list) or len(data)!=3 or data[0]!=resource: invalid()
stamp=dt.datetime.fromisoformat(data[1].replace('Z','+00:00'))
if stamp.tzinfo is None: invalid()
pair=(stamp,uid(data[2]))
except (ValueError,TypeError,KeyError,AttributeError,OverflowError): invalid()
return int(raw),pair
def page(rows,limit,resource):
more=len(rows)>limit
rows=rows[:limit]
cursor=base64.urlsafe_b64encode(json.dumps([resource,rows[-1]['created_at'],rows[-1]['id']]).encode()).decode() if more else None
return {'items':rows,'next_cursor':cursor}
@app.before_request
def before(): g.request_id=str(uuid.uuid4())
@app.after_request
def after(response):
response.headers['X-Request-Id']=g.request_id
response.headers['X-Instance-Id']=instance
return response
@app.errorhandler(Exception)
def error(exc):
if isinstance(exc,Problem): status,code=exc.status,exc.code
elif isinstance(exc,UniqueViolation): status,code=409,'conflict'
elif isinstance(exc,(psycopg.Error, __import__('psycopg_pool').PoolTimeout)): status,code=503,'unavailable'
elif isinstance(exc,HTTPException):
status=exc.code
code={404:'not_found',413:'payload_too_large',415:'unsupported_media_type'}.get(status,'invalid_request')
else:
app.logger.error('Unhandled exception type: %s',type(exc).__name__)
status,code=500,'internal_error'
response=app.json.response({'error':{'code':code,'message':code,'request_id':g.request_id}})
response.status_code=status
if status==401:
response.headers['WWW-Authenticate']='Basic realm="messenger", charset="UTF-8"' if request.path=='/api/v1/auth/sessions' else 'Bearer realm="messenger"'
return response
@app.get('/health/live')
def live(): return {'status':'ok'}
@app.get('/health/ready')
def ready():
with pool.connection() as c: c.execute('SELECT 1')
try: broker.ping()
except redis.RedisError: fail(503,'unavailable')
return {'status':'ok'}
@app.route('/api/v1/<path:path>',methods=['GET','POST','DELETE'])
def api(path):
public = (path in ('users','auth/sessions') and request.method=='POST')
if not public and not request.headers.get('Authorization','').startswith('Bearer '): fail(401,'unauthorized')
with pool.connection() as c:
if path=='users' and request.method=='POST':
d=body(['username','display_name','password'])
if not isinstance(d['username'],str) or not re.fullmatch('[a-z0-9_]{3,32}',d['username']): invalid()
string(d['display_name'],100); string(d['password'],128,12)
row=c.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *',(uuid.uuid4(),d['username'],d['display_name'],kdf.hash(d['password']))).fetchone()
return user(row),201
if path=='auth/sessions' and request.method=='POST':
a=request.headers.get('Authorization','')
try:
if not a.startswith('Basic ') or request.get_data(): raise ValueError()
name,password=base64.b64decode(a[6:],validate=True).decode('utf-8').split(':',1)
except (ValueError,UnicodeError): fail(401,'unauthorized')
row=c.execute('SELECT * FROM users WHERE username=%s',(name,)).fetchone()
try: valid=kdf.verify(row['password_hash'] if row else dummy,password)
except VerificationError: valid=False
if not valid or not row: fail(401,'unauthorized')
token=secrets.token_urlsafe(32)
expires=c.execute("INSERT INTO sessions VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at",(hashlib.sha256(token.encode()).hexdigest(),row['id'],ttl)).fetchone()['expires_at']
return {'token':token,'token_type':'Bearer','expires_at':expires.isoformat().replace('+00:00','Z'),'user':user(row)},201
who=actor(c)
if path=='auth/sessions/current' and request.method=='DELETE':
c.execute('DELETE FROM sessions WHERE token_hash=%s',(g.digest,))
return '',204
if path=='users/me' and request.method=='GET': return public_user(c,who)
if path=='chats':
if request.method=='POST':
d=body(['title','member_ids']); string(d['title'],100)
ids=d['member_ids']
if not isinstance(ids,list) or not 1<=len(ids)<=100: invalid()
ids=[uid(x) for x in ids]
if len(ids)!=len(set(ids)) or str(who) not in ids: invalid()
if c.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s::uuid[])',(ids,)).fetchone()['n']!=len(ids): invalid()
ident=uuid.uuid4()
c.execute('INSERT INTO chats(id,title) VALUES (%s,%s)',(ident,d['title']))
for member in ids: c.execute('INSERT INTO members VALUES (%s,%s)',(ident,member))
return chat(c,ident,who),201
if request.method=='GET':
resource='chats:'+str(who); limit,pair=page_args(resource)
query='SELECT c.id FROM chats c JOIN members m ON c.id=m.chat_id WHERE m.user_id=%s'
params=[who]
if pair: query+=' AND (c.created_at,c.id)>(%s,%s)'; params+=list(pair)
query+=' ORDER BY c.created_at,c.id LIMIT %s'; params+=[limit+1]
return page([chat(c,x['id'],who) for x in c.execute(query,params).fetchall()],limit,resource)
parts=path.split('/')
if len(parts) in (2,3) and parts[0]=='chats':
ident=uid(parts[1])
if len(parts)==3 and parts[2]=='messages' and os.getenv('MESSAGE_ACCESS_FAST','1')=='1':
# Message endpoints only need membership, not the entire Chat representation.
if not c.execute('SELECT 1 FROM members WHERE chat_id=%s AND user_id=%s',(ident,who)).fetchone(): fail(404,'not_found')
result=None
else:
result=chat(c,ident,who)
if len(parts)==2 and request.method=='GET': return result
if len(parts)==3 and parts[2]=='messages':
if request.method=='POST':
key=request.headers.get('Idempotency-Key','')
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}',key): invalid()
if request.mimetype != 'application/json': fail(415,'unsupported_media_type')
try: d=request.get_json()
except HTTPException: invalid()
if not isinstance(d,dict) or set(d)-{'text','attachment_ids'} or 'text' not in d: invalid()
if not isinstance(d['text'],str) or len(d['text'])>4000: invalid()
aids=d.get('attachment_ids',[])
if not isinstance(aids,list) or len(aids)>10: invalid()
aids=[uid(x) for x in aids]
if len(set(aids))!=len(aids): invalid()
if not d['text'].strip() and not aids: invalid()
if aids:
found=c.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])',(ident,aids)).fetchall()
if len(found)!=len(aids): fail(404,'not_found')
if any(x['status']=='failed' for x in found): fail(409,'conflict')
canonical=json.dumps({'text':d['text'],'attachment_ids':aids},sort_keys=True,ensure_ascii=False)
# One transaction: lock scope, inspect replay, insert effect and result, commit before response.
scope=str(who)+':POST:'+ident+':'+key
c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(scope,))
old=c.execute('SELECT body,response FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()',(who,ident,key)).fetchone()
if old:
if old['body']!=canonical: fail(409,'conflict')
return old['response'],201
mid=uuid.uuid4()
result=clean(c.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *',(mid,ident,who,d['text'])).fetchone())
result['attachment_ids']=aids
for pos,aid in enumerate(aids): c.execute('INSERT INTO message_attachments VALUES (%s,%s,%s)',(mid,aid,pos))
c.execute("INSERT INTO idempotency(user_id,chat_id,key,body,response,expires_at) VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT(user_id,chat_id,key) DO UPDATE SET body=excluded.body,response=excluded.response,expires_at=excluded.expires_at",(who,ident,key,canonical,psycopg.types.json.Jsonb(result)))
return result,201
if request.method=='GET':
limit,pair=page_args(ident)
query='SELECT * FROM messages WHERE chat_id=%s'; params=[ident]
if pair: query+=' AND (created_at,id)>(%s,%s)'; params+=list(pair)
query+=' ORDER BY created_at,id LIMIT %s'; params+=[limit+1]
items=[clean(x) for x in c.execute(query,params)]
for m in items: m['attachment_ids']=[str(x['attachment_id']) for x in c.execute('SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position',(m['id'],))]
return page(items,limit,ident)
if len(parts)==3 and parts[2]=='attachments' and request.method=='POST':
backlog=c.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('pending','published','processing')").fetchone()['n']
if backlog>=MAX_QUEUED: fail(503,'unavailable')
filename=request.args.get('filename','')
if len(filename)<1 or len(filename)>255 or any(ord(ch)<32 or ch in '/\\' for ch in filename): invalid()
if request.mimetype not in ('image/png','image/jpeg'): fail(415,'unsupported_media_type')
raw=request.get_data(cache=False)
if not raw: invalid()
try:
im=Image.open(io.BytesIO(raw)); fmt=im.format; w,h=im.size
if w*h>MAX_PIXELS: fail(413,'payload_too_large')
im.verify()
except Problem: raise
except (UnidentifiedImageError,OSError,ValueError): fail(415,'unsupported_media_type')
actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(fmt)
if actual!=request.mimetype: fail(415,'unsupported_media_type')
ident_att=uuid.uuid4(); key='original/'+str(ident)+'/'+str(ident_att); staging='staging/'+str(ident)+'/'+str(ident_att)
digest=hashlib.sha256(raw).hexdigest()
try:
s3.put_object(Bucket=bucket,Key=staging,Body=raw,ContentType=actual,Metadata={'sha256':digest})
row=c.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,staging_key,filename,content_type,size_bytes,sha256,status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",(ident_att,ident,who,key,staging,filename,actual,len(raw),digest)).fetchone()
job_id=uuid.uuid4()
c.execute("INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version) VALUES (%s,%s,1)",(job_id,ident_att))
except Exception:
try: s3.delete_object(Bucket=bucket,Key=staging)
except Exception: app.logger.warning('attachment compensation deferred')
if isinstance(__import__('sys').exc_info()[1],psycopg.Error): raise
fail(503,'unavailable')
return attachment_json(row),202
if path.startswith('attachments/'):
bits=path.split('/')
if len(bits) not in (2,3): fail(404,'not_found')
aid=uid(bits[1])
row=c.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(aid,who)).fetchone()
if not row: fail(404,'not_found')
if len(bits)==2 and request.method=='GET': return attachment_json(row)
if len(bits)==3 and bits[2]=='content' and request.method=='GET':
variant=request.args.get('variant','original')
if variant not in ('original','thumbnail','crop','optimized'): invalid()
if row['status']!='ready': fail(409,'conflict')
if variant!='original':
v=next((v for v in row['variants'] if v['name']==variant),None)
if v is None: fail(404,'not_found')
object_key=f"variants/v1/{aid}/{variant}"; content_type=v['content_type']
else: object_key=row['object_key']; content_type=row['content_type']
try: obj=s3.get_object(Bucket=bucket,Key=object_key); data=obj['Body'].read(MAX_UPLOAD+1)
except Exception: fail(503,'unavailable')
if len(data)>MAX_UPLOAD or (variant=='original' and hashlib.sha256(data).hexdigest()!=row['sha256']): fail(503,'unavailable')
disposition='attachment; filename="attachment"; filename*=UTF-8\'\''+quote(row['filename'],safe='')
return app.response_class(data,200,mimetype=content_type,headers={'Content-Disposition':disposition})
fail(404,'not_found')
def attachment_json(row):
stamp=row['created_at'].isoformat().replace('+00:00','Z')
return {'id':str(row['id']),'chat_id':str(row['chat_id']),'owner_id':str(row['owner_id']),'filename':row['filename'],'content_type':row['content_type'],'size_bytes':row['size_bytes'],'sha256':row['sha256'].strip(),'status':row['status'],'variants':row.get('variants',[]),'error_code':row.get('error_code'),'created_at':stamp,'updated_at':row.get('updated_at',row['created_at']).isoformat().replace('+00:00','Z')}