177 lines
11 KiB
Python
177 lines
11 KiB
Python
import hashlib
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
from uuid import uuid4
|
|
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
from psycopg_pool import AsyncConnectionPool
|
|
from redis.asyncio import Redis
|
|
from redis.exceptions import RedisError
|
|
|
|
from .common import ApiError, encode_cursor, invalid
|
|
from . import telemetry as tel
|
|
|
|
|
|
def clean(row):
|
|
if isinstance(row,dict): return {k:clean(v) for k,v in row.items()}
|
|
if isinstance(row,list): return [clean(v) for v in row]
|
|
if hasattr(row,'isoformat'): return row.isoformat(timespec='microseconds').replace('+00:00','Z')
|
|
if type(row).__name__=='UUID': return str(row)
|
|
return row
|
|
|
|
|
|
class Store:
|
|
def __init__(self,cfg):
|
|
self.cfg=cfg
|
|
self.secret=cfg.cursor_secret.encode()
|
|
self.pool=AsyncConnectionPool(cfg.database_url,min_size=1,max_size=cfg.db_pool_size,
|
|
timeout=2,max_waiting=32,open=False,
|
|
kwargs={'row_factory':dict_row,'connect_timeout':3,
|
|
'options':'-c statement_timeout=2000 -c lock_timeout=1000 -c timezone=UTC'})
|
|
self.cache=Redis.from_url(cfg.redis_url,socket_timeout=.2,socket_connect_timeout=.2,
|
|
max_connections=8,decode_responses=True)
|
|
|
|
async def open(self): await self.pool.open(wait=True,timeout=20)
|
|
async def close(self):
|
|
await self.cache.aclose()
|
|
await self.pool.close()
|
|
|
|
async def ready(self):
|
|
async with self.pool.connection() as c: await c.execute('SELECT 1')
|
|
|
|
async def register(self,username,display_name,password_hash):
|
|
try:
|
|
async with self.pool.connection() as c:
|
|
row=await (await c.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at',
|
|
(uuid4(),username,display_name,password_hash))).fetchone()
|
|
return clean(row)
|
|
except psycopg.errors.UniqueViolation:
|
|
raise ApiError(409,'conflict','Username already exists') from None
|
|
|
|
async def credentials(self,username):
|
|
async with self.pool.connection() as c:
|
|
return await (await c.execute('SELECT * FROM users WHERE username=%s',(username,))).fetchone()
|
|
|
|
async def create_session(self,user_id,token):
|
|
async with self.pool.connection() as c:
|
|
row=await (await c.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,clock_timestamp()+%s*interval '1 second') RETURNING expires_at",
|
|
(hashlib.sha256(token.encode()).hexdigest(),user_id,self.cfg.session_ttl_seconds))).fetchone()
|
|
return clean(row)['expires_at']
|
|
|
|
async def authenticate(self,token):
|
|
async with self.pool.connection() as c:
|
|
session=await (await c.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp()',
|
|
(hashlib.sha256(token.encode()).hexdigest(),))).fetchone()
|
|
if session is None: raise ApiError(401,'unauthorized','Invalid session',{'WWW-Authenticate':'Bearer realm="messenger"'})
|
|
key='user:'+str(session['user_id'])
|
|
try:
|
|
cached=await self.cache.get(key)
|
|
if cached: return json.loads(cached)
|
|
except (RedisError,ValueError):
|
|
logging.getLogger('messenger').warning('{"event":"cache_bypass"}')
|
|
row=await (await c.execute('SELECT id,username,display_name,created_at FROM users WHERE id=%s',(session['user_id'],))).fetchone()
|
|
user=clean(row)
|
|
try: await self.cache.set(key,json.dumps(user),ex=60)
|
|
except RedisError: pass
|
|
return user
|
|
|
|
async def logout(self,token):
|
|
async with self.pool.connection() as c:
|
|
await c.execute('DELETE FROM sessions WHERE token_hash=%s',(hashlib.sha256(token.encode()).hexdigest(),))
|
|
|
|
async def _chat(self,c,actor,chat_id):
|
|
row=await (await c.execute('SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY user_id) AS member_ids FROM chats c JOIN members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s',(chat_id,actor))).fetchone()
|
|
if row is None: raise ApiError(404,'not_found','Object not found')
|
|
return clean(row)
|
|
|
|
async def chat(self,actor,chat_id):
|
|
async with self.pool.connection() as c: return await self._chat(c,actor,chat_id)
|
|
|
|
async def create_chat(self,actor,title,members):
|
|
if actor not in members: invalid('Members must include the current user')
|
|
try:
|
|
async with self.pool.connection() as c:
|
|
cid=uuid4()
|
|
await c.execute('INSERT INTO chats(id,title) VALUES (%s,%s)',(cid,title))
|
|
async with c.cursor() as cur:
|
|
await cur.executemany('INSERT INTO members(chat_id,user_id) VALUES (%s,%s)',[(cid,m) for m in members])
|
|
return await self._chat(c,actor,cid)
|
|
except psycopg.errors.ForeignKeyViolation:
|
|
invalid('All members must exist')
|
|
|
|
async def create_message(self,actor,chat_id,text,key,attachment_ids):
|
|
digest=hashlib.sha256(json.dumps({'text':text,'attachment_ids':attachment_ids},sort_keys=True,ensure_ascii=False,separators=(',',':')).encode()).hexdigest()
|
|
async with self.pool.connection() as c:
|
|
await self._chat(c,actor,chat_id)
|
|
# Fixed operation POST /chats/{id}/messages; lock and both rows share one transaction.
|
|
scope='POST:'+chat_id+':'+actor+':'+key
|
|
await c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(scope,))
|
|
previous=await (await c.execute("SELECT request_hash,message_id FROM idempotency WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()",(actor,chat_id,key))).fetchone()
|
|
if previous:
|
|
if previous['request_hash']!=digest: raise ApiError(409,'conflict','Key was used for another message')
|
|
return clean(await (await c.execute('SELECT id,chat_id,sender_id,text,created_at,attachment_ids FROM messages WHERE id=%s',(previous['message_id'],))).fetchone())
|
|
for aid in attachment_ids:
|
|
attachment=await (await c.execute('SELECT status,storage_state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE',(aid,chat_id))).fetchone()
|
|
if attachment is None: raise ApiError(404,'not_found','Object not found')
|
|
if attachment['status']=='failed': raise ApiError(409,'conflict','Attachment processing failed')
|
|
if attachment['status']=='ready' and attachment['storage_state']!='ready': raise ApiError(503,'unavailable','Object temporarily unavailable')
|
|
await c.execute('DELETE FROM idempotency WHERE actor_id=%s AND chat_id=%s AND key=%s',(actor,chat_id,key))
|
|
row=await (await c.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES (%s,%s,%s,%s,%s::uuid[]) RETURNING id,chat_id,sender_id,text,created_at,attachment_ids',
|
|
(uuid4(),chat_id,actor,text,attachment_ids))).fetchone()
|
|
for aid in attachment_ids:
|
|
await c.execute('INSERT INTO message_attachments(message_id,attachment_id) VALUES (%s,%s)',(row['id'],aid))
|
|
await c.execute("INSERT INTO idempotency(actor_id,chat_id,key,request_hash,message_id,expires_at) VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')",(actor,chat_id,key,digest,row['id']))
|
|
return clean(row)
|
|
|
|
async def page(self,actor,chat_id,limit,cursor):
|
|
async with self.pool.connection() as c:
|
|
if chat_id:
|
|
await self._chat(c,actor,chat_id)
|
|
query='SELECT id,chat_id,sender_id,text,created_at,attachment_ids FROM messages WHERE chat_id=%s'
|
|
args=[chat_id]; resource='messages:'+chat_id
|
|
if cursor:
|
|
query+=' AND (created_at,id)>(%s::timestamptz,%s::uuid)'; args.extend(cursor)
|
|
query+=' ORDER BY created_at,id LIMIT %s'; args.append(limit+1)
|
|
else:
|
|
query='SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY user_id) AS member_ids FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s'
|
|
args=[actor]; resource='chats:'+actor
|
|
if cursor:
|
|
query+=' AND (c.created_at,c.id)>(%s::timestamptz,%s::uuid)'; args.extend(cursor)
|
|
query+=' ORDER BY c.created_at,c.id LIMIT %s'; args.append(limit+1)
|
|
rows=[clean(row) for row in await (await c.execute(query,args)).fetchall()]
|
|
selected=rows[:limit]
|
|
next_cursor=encode_cursor(resource,(selected[-1]['created_at'],selected[-1]['id']),self.secret) if len(rows)>limit else None
|
|
return {'items':selected,'next_cursor':next_cursor}
|
|
|
|
async def attachment(self,actor,identifier):
|
|
async with self.pool.connection() as c:
|
|
row=await (await c.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(identifier,actor))).fetchone()
|
|
if row is None: raise ApiError(404,'not_found','Object not found')
|
|
if row['status']=='ready' and row['storage_state']!='ready': raise ApiError(503,'unavailable','Object temporarily unavailable')
|
|
return clean(row)
|
|
|
|
async def save_attachment(self,actor,chat_id,filename,mime,size,digest,file,storage):
|
|
identifier=uuid4()
|
|
key='staging/'+str(identifier)+'/v1'
|
|
# The same lock is taken by reconciliation before it can delete an orphan.
|
|
# DB COMMIT uncertainty must NEVER trigger an unconditional S3 delete: the row
|
|
# may have committed. Leave a possible orphan for the grace-period reconciler.
|
|
async with self.pool.connection() as c:
|
|
await self._chat(c,actor,chat_id)
|
|
# Admission is global and serialized: no unbounded durable backlog.
|
|
await c.execute("SELECT pg_advisory_xact_lock(hashtextextended('job-admission',0))")
|
|
pending=await (await c.execute("SELECT count(*) AS n FROM jobs WHERE status IN ('queued','processing')")).fetchone()
|
|
if pending['n']>=1000: raise ApiError(503,'unavailable','Processing backlog is full')
|
|
await c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(key,))
|
|
from .offload import complete_thread
|
|
await complete_thread(storage.put,key,file,size,mime,digest)
|
|
row=await (await c.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,status,staging_key) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s) RETURNING *",
|
|
(identifier,chat_id,actor,filename,key,mime,size,digest,key))).fetchone()
|
|
job_id=uuid4()
|
|
context=tel.carrier()
|
|
await c.execute('INSERT INTO jobs(id,attachment_id,pipeline_version,traceparent,tracestate) VALUES (%s,%s,1,%s,%s)',(job_id,identifier,context.get('traceparent'),context.get('tracestate')))
|
|
await c.execute('INSERT INTO outbox(job_id) VALUES (%s)',(job_id,))
|
|
return clean(row)
|