feat: add private attachments and safe reconciliation experiment
This commit is contained in:
@@ -0,0 +1,324 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from threading import BoundedSemaphore
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
import redis
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
||||
|
||||
from .config import secret
|
||||
from .domain import Failure, timestamp
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
KDF_SLOTS = BoundedSemaphore(2)
|
||||
KDF = PasswordHasher(time_cost=2, memory_cost=65536, parallelism=1)
|
||||
DUMMY_HASH = KDF.hash("invalid-user-dummy-password")
|
||||
|
||||
|
||||
class DeadlineConnection(psycopg.Connection):
|
||||
def wait(self, gen, interval=0.1, timeout=None):
|
||||
try:
|
||||
return super().wait(gen, interval=interval, timeout=timeout or 3.0)
|
||||
except psycopg.errors._WaitTimeout:
|
||||
# A paused server cannot complete cancellation. Discard an uncertain
|
||||
# connection instead of returning it to the pool or waiting forever.
|
||||
self.pgconn.finish()
|
||||
raise
|
||||
|
||||
|
||||
def public(row):
|
||||
return {
|
||||
k: (
|
||||
str(v)
|
||||
if isinstance(v, UUID)
|
||||
else v.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
if isinstance(v, datetime)
|
||||
else bytes(v).decode("utf-8")
|
||||
if isinstance(v, (bytes, memoryview))
|
||||
else v
|
||||
)
|
||||
for k, v in row.items()
|
||||
if k not in {"password_hash", "token_digest"}
|
||||
}
|
||||
|
||||
|
||||
class PostgresStore:
|
||||
def __init__(self):
|
||||
self.pool = ConnectionPool(
|
||||
secret("DATABASE_URL"),
|
||||
connection_class=DeadlineConnection,
|
||||
min_size=1,
|
||||
max_size=int(os.getenv("POOL_MAX_SIZE", "8")),
|
||||
timeout=2,
|
||||
max_waiting=32,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=2500 -c lock_timeout=1500",
|
||||
},
|
||||
check=ConnectionPool.check_connection,
|
||||
)
|
||||
self.pool.wait(timeout=15)
|
||||
self.cache = redis.Redis.from_url(
|
||||
secret("CACHE_URL"),
|
||||
socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2,
|
||||
decode_responses=True,
|
||||
)
|
||||
self.ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
if self.ttl < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным")
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
yield conn
|
||||
except (
|
||||
psycopg.OperationalError,
|
||||
psycopg.InterfaceError,
|
||||
PoolTimeout,
|
||||
TooManyRequests,
|
||||
psycopg.errors.QueryCanceled,
|
||||
psycopg.errors.LockNotAvailable,
|
||||
):
|
||||
raise Failure(503, "Хранилище временно недоступно") from None
|
||||
|
||||
def close(self):
|
||||
self.cache.close()
|
||||
self.pool.close()
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
return True
|
||||
except Failure:
|
||||
return False
|
||||
|
||||
def _cached_user(self, uid):
|
||||
key = "user:v1:" + uid
|
||||
try:
|
||||
value = self.cache.get(key)
|
||||
if value:
|
||||
cached = json.loads(value)
|
||||
if (
|
||||
isinstance(cached, dict)
|
||||
and cached.get("id") == uid
|
||||
and set(cached) == {"id", "username", "display_name", "created_at"}
|
||||
):
|
||||
return cached
|
||||
except (redis.RedisError, ValueError, TypeError):
|
||||
logger.warning(json.dumps({"event": "cache_bypass"}))
|
||||
user = self.user(uid)
|
||||
if user:
|
||||
try:
|
||||
self.cache.setex(key, 60, json.dumps(user))
|
||||
except redis.RedisError:
|
||||
logger.warning(json.dumps({"event": "cache_bypass"}))
|
||||
return user
|
||||
|
||||
def user(self, uid):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id,username,display_name,created_at FROM users WHERE id=%s", (uid,)
|
||||
).fetchone()
|
||||
return public(row) if row else None
|
||||
|
||||
def register(self, data):
|
||||
with KDF_SLOTS:
|
||||
hashed = KDF.hash(data.pop("password"))
|
||||
uid = str(uuid4())
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users VALUES (%s,%s,%s,%s,%s) "
|
||||
"RETURNING id,username,display_name,created_at",
|
||||
(
|
||||
uid,
|
||||
data["username"],
|
||||
data["display_name"].encode("utf-8"),
|
||||
hashed,
|
||||
timestamp(),
|
||||
),
|
||||
).fetchone()
|
||||
return public(row)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise Failure(409, "Имя пользователя занято") from None
|
||||
|
||||
def login(self, username, password):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
try:
|
||||
with KDF_SLOTS:
|
||||
KDF.verify(row["password_hash"] if row else DUMMY_HASH, password)
|
||||
except VerificationError:
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode()).hexdigest()
|
||||
expires = datetime.now(UTC) + timedelta(seconds=self.ttl)
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT INTO sessions VALUES (%s,%s,%s)", (digest, row["id"], expires))
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": public({"v": expires})["v"],
|
||||
"user": public(row),
|
||||
}
|
||||
|
||||
def authenticate(self, token):
|
||||
if not token or len(token) > 512:
|
||||
return None
|
||||
digest = hashlib.sha256(token.encode()).hexdigest()
|
||||
# Session validity always comes from PostgreSQL, including on a cache hit.
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT s.user_id FROM sessions s WHERE s.token_digest=%s AND s.expires_at>now()",
|
||||
(digest,),
|
||||
).fetchone()
|
||||
return self._cached_user(str(row["user_id"])) if row else None
|
||||
|
||||
def logout(self, token):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"DELETE FROM sessions WHERE token_digest=%s AND expires_at>now() RETURNING user_id",
|
||||
(hashlib.sha256(token.encode()).hexdigest(),),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def create_chat(self, actor, data):
|
||||
if actor not in data["member_ids"]:
|
||||
raise Failure(400, "Вызывающий должен входить в чат")
|
||||
cid = str(uuid4())
|
||||
try:
|
||||
with self.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats VALUES (%s,%s,%s) RETURNING *",
|
||||
(cid, data["title"].encode("utf-8"), timestamp()),
|
||||
).fetchone()
|
||||
for index, uid in enumerate(data["member_ids"]):
|
||||
conn.execute("INSERT INTO members VALUES (%s,%s,%s)", (cid, uid, index))
|
||||
return {**public(row), "member_ids": data["member_ids"]}
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise Failure(400, "Неизвестный участник") from None
|
||||
|
||||
@staticmethod
|
||||
def _chat(conn, actor, cid):
|
||||
row = conn.execute(
|
||||
"SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id "
|
||||
"WHERE c.id=%s AND m.user_id=%s",
|
||||
(cid, actor),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise Failure(404, "Объект не найден")
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM members WHERE chat_id=%s ORDER BY position", (cid,)
|
||||
).fetchall()
|
||||
return {**public(row), "member_ids": [str(m["user_id"]) for m in members]}
|
||||
|
||||
def chat(self, actor, cid):
|
||||
with self.connection() as conn:
|
||||
return self._chat(conn, actor, cid)
|
||||
|
||||
def list_chats(self, actor, after, limit):
|
||||
with self.connection() as conn:
|
||||
sql = "SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s"
|
||||
params = [actor]
|
||||
if after:
|
||||
sql += " AND (c.created_at,c.id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
sql += " ORDER BY c.created_at,c.id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
return [self._chat(conn, actor, str(row["id"])) for row in rows]
|
||||
|
||||
def send(self, actor, cid, text, key, attachment_ids=None):
|
||||
attachment_ids = attachment_ids or []
|
||||
body_digest = hashlib.sha256(
|
||||
json.dumps(
|
||||
{"text": text, "attachment_ids": attachment_ids},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).hexdigest()
|
||||
with self.connection() as conn, conn.transaction():
|
||||
self._chat(conn, actor, cid)
|
||||
# Serializes exactly this semantic key across all API processes. Hash
|
||||
# collisions can only serialize unrelated requests, never mix results.
|
||||
scope = json.dumps([actor, "POST", cid, key], separators=(",", ":"))
|
||||
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
old = conn.execute(
|
||||
"SELECT body_digest,response FROM message_requests "
|
||||
"WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>now()",
|
||||
(actor, cid, key),
|
||||
).fetchone()
|
||||
if old:
|
||||
if old["body_digest"] != body_digest:
|
||||
raise Failure(409, "Ключ уже использован для другого сообщения")
|
||||
return json.loads(bytes(old["response"]))
|
||||
conn.execute(
|
||||
"DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s",
|
||||
(actor, cid, key),
|
||||
)
|
||||
for aid in attachment_ids:
|
||||
attachment = conn.execute(
|
||||
"SELECT id,status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE",
|
||||
(aid, cid),
|
||||
).fetchone()
|
||||
if not attachment:
|
||||
raise Failure(404, "Объект не найден")
|
||||
if attachment["status"] == "failed":
|
||||
raise Failure(409, "Вложение недоступно")
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages VALUES (%s,%s,%s,%s,%s) RETURNING *",
|
||||
(str(uuid4()), cid, actor, text.encode("utf-8"), timestamp()),
|
||||
).fetchone()
|
||||
for position, aid in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments VALUES (%s,%s,%s)", (row["id"], aid, position)
|
||||
)
|
||||
response = {**public(row), "attachment_ids": attachment_ids}
|
||||
conn.execute(
|
||||
"INSERT INTO message_requests(actor_id,chat_id,key,body_digest,response,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,%s,now()+interval '24 hours')",
|
||||
(actor, cid, key, body_digest, json.dumps(response).encode("utf-8")),
|
||||
)
|
||||
return response
|
||||
|
||||
def history(self, actor, cid, after, limit):
|
||||
with self.connection() as conn:
|
||||
self._chat(conn, actor, cid)
|
||||
sql = "SELECT * FROM messages WHERE chat_id=%s"
|
||||
params = [cid]
|
||||
if after:
|
||||
sql += " AND (created_at,id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
sql += " ORDER BY created_at,id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
result = []
|
||||
for row in conn.execute(sql, params).fetchall():
|
||||
attachments = conn.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
result.append(
|
||||
{
|
||||
**public(row),
|
||||
"attachment_ids": [str(a["attachment_id"]) for a in attachments],
|
||||
}
|
||||
)
|
||||
return result
|
||||
Reference in New Issue
Block a user