import base64 import datetime as dt import hashlib import json import os import re import secrets import uuid from flask import Flask, request, g from werkzeug.utils import secure_filename from werkzeug.exceptions import HTTPException from argon2 import PasswordHasher from argon2.exceptions import VerificationError from psycopg.rows import dict_row from psycopg.errors import UniqueViolation from psycopg_pool import ConnectionPool import psycopg import redis import boto3 from botocore.config import Config from PIL import Image, UnidentifiedImageError import io from urllib.parse import quote from app.migrate import setting app = Flask(__name__) MAX_UPLOAD=int(os.getenv('MAX_UPLOAD_BYTES','10485760')) MAX_PIXELS=int(os.getenv('MAX_IMAGE_PIXELS','40000000')) MAX_QUEUED=int(os.getenv('MAX_QUEUED_JOBS','500')) app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=int(os.getenv('POOL_MAX','8')), timeout=3, kwargs={'row_factory':dict_row,'connect_timeout':3,'options':'-c statement_timeout=3000 -c lock_timeout=2000'}) cache = redis.Redis.from_url(setting('CACHE_URL'),socket_connect_timeout=.2,socket_timeout=.2) broker = redis.Redis.from_url(setting('BROKER_URL'),socket_connect_timeout=.3,socket_timeout=.5) kdf = PasswordHasher(time_cost=2,memory_cost=19456,parallelism=1) dummy = kdf.hash(secrets.token_urlsafe(32)) ttl = int(os.getenv('SESSION_TTL_SECONDS','3600')) def envsetting(name): return setting(name) if os.getenv(name+'_FILE') else os.environ[name] s3=boto3.client('s3',endpoint_url=os.getenv('S3_ENDPOINT','http://rustfs:9000'),region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=envsetting('S3_ACCESS_KEY_ID'),aws_secret_access_key=envsetting('S3_SECRET_ACCESS_KEY'),config=Config(connect_timeout=2,read_timeout=5,retries={'max_attempts':1})) bucket=os.getenv('S3_BUCKET','messenger-attachments') instance = os.getenv('HOSTNAME','api') + '-' + secrets.token_hex(4) class Problem(Exception): def __init__(self,status,code): self.status,self.code=status,code def fail(status,code): raise Problem(status,code) def invalid(): fail(400,'invalid_request') def uid(value): try: return str(uuid.UUID(value)) except (ValueError,TypeError,AttributeError): invalid() def clean(row): return {k:(v.isoformat().replace('+00:00','Z') if isinstance(v,dt.datetime) else str(v) if isinstance(v,uuid.UUID) else v) for k,v in row.items()} def user(row): return clean({k:row[k] for k in ('id','username','display_name','created_at')}) def body(fields): if request.mimetype != 'application/json': fail(415,'unsupported_media_type') try: data = request.get_json() except HTTPException: invalid() if not isinstance(data,dict) or set(data)!=set(fields): invalid() return data def string(s,maximum,minimum=1): if not isinstance(s,str) or not s.strip() or not minimum<=len(s)<=maximum: invalid() try: s.encode('utf-8') except UnicodeError: invalid() return s def actor(c): auth = request.headers.get('Authorization','') if not auth.startswith('Bearer ') or not 1<=len(auth[7:])<=256: fail(401,'unauthorized') digest = hashlib.sha256(auth[7:].encode()).hexdigest() row=c.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp()',(digest,)).fetchone() if not row: fail(401,'unauthorized') g.digest=digest return row['user_id'] def public_user(c,ident): key='user:'+str(ident) try: value=cache.get(key) if value: return json.loads(value) except (redis.RedisError,ValueError): pass result=user(c.execute('SELECT * FROM users WHERE id=%s',(ident,)).fetchone()) try: cache.setex(key,60,json.dumps(result)) except redis.RedisError: pass return result def chat(c,ident,who): row=c.execute('SELECT c.* FROM chats c JOIN members m ON c.id=m.chat_id WHERE c.id=%s AND m.user_id=%s',(ident,who)).fetchone() if not row: fail(404,'not_found') row=clean(row) row['member_ids']=[str(x['user_id']) for x in c.execute('SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id',(ident,))] return row def page_args(resource): if set(request.args)-{'limit','cursor'} or any(len(request.args.getlist(k))!=1 for k in request.args): invalid() raw=request.args.get('limit','50') if len(raw)>3 or not re.fullmatch(r'[0-9]+',raw) or not 1<=int(raw)<=100: invalid() cursor=request.args.get('cursor') pair=None if cursor is not None: try: data=json.loads(base64.b64decode(cursor,altchars=b'-_',validate=True)) if not isinstance(data,list) or len(data)!=3 or data[0]!=resource: invalid() stamp=dt.datetime.fromisoformat(data[1].replace('Z','+00:00')) if stamp.tzinfo is None: invalid() pair=(stamp,uid(data[2])) except (ValueError,TypeError,KeyError,AttributeError,OverflowError): invalid() return int(raw),pair def page(rows,limit,resource): more=len(rows)>limit rows=rows[:limit] cursor=base64.urlsafe_b64encode(json.dumps([resource,rows[-1]['created_at'],rows[-1]['id']]).encode()).decode() if more else None return {'items':rows,'next_cursor':cursor} @app.before_request def before(): g.request_id=str(uuid.uuid4()) @app.after_request def after(response): response.headers['X-Request-Id']=g.request_id response.headers['X-Instance-Id']=instance return response @app.errorhandler(Exception) def error(exc): if isinstance(exc,Problem): status,code=exc.status,exc.code elif isinstance(exc,UniqueViolation): status,code=409,'conflict' elif isinstance(exc,(psycopg.Error, __import__('psycopg_pool').PoolTimeout)): status,code=503,'unavailable' elif isinstance(exc,HTTPException): status=exc.code code={404:'not_found',413:'payload_too_large',415:'unsupported_media_type'}.get(status,'invalid_request') else: app.logger.error('Unhandled exception type: %s',type(exc).__name__) status,code=500,'internal_error' response=app.json.response({'error':{'code':code,'message':code,'request_id':g.request_id}}) response.status_code=status if status==401: response.headers['WWW-Authenticate']='Basic realm="messenger", charset="UTF-8"' if request.path=='/api/v1/auth/sessions' else 'Bearer realm="messenger"' return response @app.get('/health/live') def live(): return {'status':'ok'} @app.get('/health/ready') def ready(): with pool.connection() as c: c.execute('SELECT 1') try: broker.ping() except redis.RedisError: fail(503,'unavailable') return {'status':'ok'} @app.route('/api/v1/',methods=['GET','POST','DELETE']) def api(path): public = (path in ('users','auth/sessions') and request.method=='POST') if not public and not request.headers.get('Authorization','').startswith('Bearer '): fail(401,'unauthorized') with pool.connection() as c: if path=='users' and request.method=='POST': d=body(['username','display_name','password']) if not isinstance(d['username'],str) or not re.fullmatch('[a-z0-9_]{3,32}',d['username']): invalid() string(d['display_name'],100); string(d['password'],128,12) row=c.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *',(uuid.uuid4(),d['username'],d['display_name'],kdf.hash(d['password']))).fetchone() return user(row),201 if path=='auth/sessions' and request.method=='POST': a=request.headers.get('Authorization','') try: if not a.startswith('Basic ') or request.get_data(): raise ValueError() name,password=base64.b64decode(a[6:],validate=True).decode('utf-8').split(':',1) except (ValueError,UnicodeError): fail(401,'unauthorized') row=c.execute('SELECT * FROM users WHERE username=%s',(name,)).fetchone() try: valid=kdf.verify(row['password_hash'] if row else dummy,password) except VerificationError: valid=False if not valid or not row: fail(401,'unauthorized') token=secrets.token_urlsafe(32) expires=c.execute("INSERT INTO sessions VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at",(hashlib.sha256(token.encode()).hexdigest(),row['id'],ttl)).fetchone()['expires_at'] return {'token':token,'token_type':'Bearer','expires_at':expires.isoformat().replace('+00:00','Z'),'user':user(row)},201 who=actor(c) if path=='auth/sessions/current' and request.method=='DELETE': c.execute('DELETE FROM sessions WHERE token_hash=%s',(g.digest,)) return '',204 if path=='users/me' and request.method=='GET': return public_user(c,who) if path=='chats': if request.method=='POST': d=body(['title','member_ids']); string(d['title'],100) ids=d['member_ids'] if not isinstance(ids,list) or not 1<=len(ids)<=100: invalid() ids=[uid(x) for x in ids] if len(ids)!=len(set(ids)) or str(who) not in ids: invalid() if c.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s::uuid[])',(ids,)).fetchone()['n']!=len(ids): invalid() ident=uuid.uuid4() c.execute('INSERT INTO chats(id,title) VALUES (%s,%s)',(ident,d['title'])) for member in ids: c.execute('INSERT INTO members VALUES (%s,%s)',(ident,member)) return chat(c,ident,who),201 if request.method=='GET': resource='chats:'+str(who); limit,pair=page_args(resource) query='SELECT c.id FROM chats c JOIN members m ON c.id=m.chat_id WHERE m.user_id=%s' params=[who] if pair: query+=' AND (c.created_at,c.id)>(%s,%s)'; params+=list(pair) query+=' ORDER BY c.created_at,c.id LIMIT %s'; params+=[limit+1] return page([chat(c,x['id'],who) for x in c.execute(query,params).fetchall()],limit,resource) parts=path.split('/') if len(parts) in (2,3) and parts[0]=='chats': ident=uid(parts[1]) if len(parts)==3 and parts[2]=='messages' and os.getenv('MESSAGE_ACCESS_FAST','1')=='1': # Message endpoints only need membership, not the entire Chat representation. if not c.execute('SELECT 1 FROM members WHERE chat_id=%s AND user_id=%s',(ident,who)).fetchone(): fail(404,'not_found') result=None else: result=chat(c,ident,who) if len(parts)==2 and request.method=='GET': return result if len(parts)==3 and parts[2]=='messages': if request.method=='POST': key=request.headers.get('Idempotency-Key','') if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}',key): invalid() if request.mimetype != 'application/json': fail(415,'unsupported_media_type') try: d=request.get_json() except HTTPException: invalid() if not isinstance(d,dict) or set(d)-{'text','attachment_ids'} or 'text' not in d: invalid() if not isinstance(d['text'],str) or len(d['text'])>4000: invalid() aids=d.get('attachment_ids',[]) if not isinstance(aids,list) or len(aids)>10: invalid() aids=[uid(x) for x in aids] if len(set(aids))!=len(aids): invalid() if not d['text'].strip() and not aids: invalid() if aids: found=c.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])',(ident,aids)).fetchall() if len(found)!=len(aids): fail(404,'not_found') if any(x['status']=='failed' for x in found): fail(409,'conflict') canonical=json.dumps({'text':d['text'],'attachment_ids':aids},sort_keys=True,ensure_ascii=False) # One transaction: lock scope, inspect replay, insert effect and result, commit before response. scope=str(who)+':POST:'+ident+':'+key c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(scope,)) old=c.execute('SELECT body,response FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()',(who,ident,key)).fetchone() if old: if old['body']!=canonical: fail(409,'conflict') return old['response'],201 mid=uuid.uuid4() result=clean(c.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *',(mid,ident,who,d['text'])).fetchone()) result['attachment_ids']=aids for pos,aid in enumerate(aids): c.execute('INSERT INTO message_attachments VALUES (%s,%s,%s)',(mid,aid,pos)) c.execute("INSERT INTO idempotency(user_id,chat_id,key,body,response,expires_at) VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT(user_id,chat_id,key) DO UPDATE SET body=excluded.body,response=excluded.response,expires_at=excluded.expires_at",(who,ident,key,canonical,psycopg.types.json.Jsonb(result))) return result,201 if request.method=='GET': limit,pair=page_args(ident) query='SELECT * FROM messages WHERE chat_id=%s'; params=[ident] if pair: query+=' AND (created_at,id)>(%s,%s)'; params+=list(pair) query+=' ORDER BY created_at,id LIMIT %s'; params+=[limit+1] items=[clean(x) for x in c.execute(query,params)] for m in items: m['attachment_ids']=[str(x['attachment_id']) for x in c.execute('SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position',(m['id'],))] return page(items,limit,ident) if len(parts)==3 and parts[2]=='attachments' and request.method=='POST': backlog=c.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('pending','published','processing')").fetchone()['n'] if backlog>=MAX_QUEUED: fail(503,'unavailable') filename=request.args.get('filename','') if len(filename)<1 or len(filename)>255 or any(ord(ch)<32 or ch in '/\\' for ch in filename): invalid() if request.mimetype not in ('image/png','image/jpeg'): fail(415,'unsupported_media_type') raw=request.get_data(cache=False) if not raw: invalid() try: im=Image.open(io.BytesIO(raw)); fmt=im.format; w,h=im.size if w*h>MAX_PIXELS: fail(413,'payload_too_large') im.verify() except Problem: raise except (UnidentifiedImageError,OSError,ValueError): fail(415,'unsupported_media_type') actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(fmt) if actual!=request.mimetype: fail(415,'unsupported_media_type') ident_att=uuid.uuid4(); key='original/'+str(ident)+'/'+str(ident_att); staging='staging/'+str(ident)+'/'+str(ident_att) digest=hashlib.sha256(raw).hexdigest() try: s3.put_object(Bucket=bucket,Key=staging,Body=raw,ContentType=actual,Metadata={'sha256':digest}) row=c.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,staging_key,filename,content_type,size_bytes,sha256,status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",(ident_att,ident,who,key,staging,filename,actual,len(raw),digest)).fetchone() job_id=uuid.uuid4() c.execute("INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version) VALUES (%s,%s,1)",(job_id,ident_att)) except Exception: try: s3.delete_object(Bucket=bucket,Key=staging) except Exception: app.logger.warning('attachment compensation deferred') if isinstance(__import__('sys').exc_info()[1],psycopg.Error): raise fail(503,'unavailable') return attachment_json(row),202 if path.startswith('attachments/'): bits=path.split('/') if len(bits) not in (2,3): fail(404,'not_found') aid=uid(bits[1]) row=c.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(aid,who)).fetchone() if not row: fail(404,'not_found') if len(bits)==2 and request.method=='GET': return attachment_json(row) if len(bits)==3 and bits[2]=='content' and request.method=='GET': variant=request.args.get('variant','original') if variant not in ('original','thumbnail','crop','optimized'): invalid() if row['status']!='ready': fail(409,'conflict') if variant!='original': v=next((v for v in row['variants'] if v['name']==variant),None) if v is None: fail(404,'not_found') object_key=f"variants/v1/{aid}/{variant}"; content_type=v['content_type'] else: object_key=row['object_key']; content_type=row['content_type'] try: obj=s3.get_object(Bucket=bucket,Key=object_key); data=obj['Body'].read(MAX_UPLOAD+1) except Exception: fail(503,'unavailable') if len(data)>MAX_UPLOAD or (variant=='original' and hashlib.sha256(data).hexdigest()!=row['sha256']): fail(503,'unavailable') disposition='attachment; filename="attachment"; filename*=UTF-8\'\''+quote(row['filename'],safe='') return app.response_class(data,200,mimetype=content_type,headers={'Content-Disposition':disposition}) fail(404,'not_found') def attachment_json(row): stamp=row['created_at'].isoformat().replace('+00:00','Z') return {'id':str(row['id']),'chat_id':str(row['chat_id']),'owner_id':str(row['owner_id']),'filename':row['filename'],'content_type':row['content_type'],'size_bytes':row['size_bytes'],'sha256':row['sha256'].strip(),'status':row['status'],'variants':row.get('variants',[]),'error_code':row.get('error_code'),'created_at':stamp,'updated_at':row.get('updated_at',row['created_at']).isoformat().replace('+00:00','Z')}