import logging import time from contextlib import contextmanager from datetime import datetime from uuid import UUID, uuid4 import psycopg from psycopg import errors as pg_errors from psycopg.rows import dict_row from psycopg_pool import ConnectionPool, PoolTimeout import dbconn import pipeline import settings import telemetry from errors import bad_request, conflict, not_found, unavailable log = logging.getLogger("messenger.db") def check_connection(conn) -> None: """Проверка соединения при выдаче из пула (пустой запрос) — без спана: это не работа запроса.""" from opentelemetry.instrumentation.utils import suppress_instrumentation with suppress_instrumentation(): ConnectionPool.check_connection(conn) def pool_kwargs() -> dict: """lab7: вызывается на КАЖДОЕ новое соединение пула — пароль роли берётся из OpenBao (текущая версия).""" return dbconn.connect_kwargs( row_factory=dict_row, options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", ) pool = ConnectionPool( conninfo="", min_size=settings.DB_POOL_MIN, max_size=settings.DB_POOL_MAX, timeout=settings.DB_POOL_WAIT_SECONDS, kwargs=pool_kwargs, check=check_connection, open=False, ) # lab6: насыщение пула соединений API (contracts/observability.md, «DB pool used/waiting»). POOL_NAME = {"db.client.connection.pool.name": "messenger-api"} POOL_WAIT = telemetry.meter.create_histogram( "db.client.connection.wait_time", unit="s", description="Time to obtain a connection from the pool") def _pool_stats(): try: return pool.get_stats() except Exception: # noqa: BLE001 — пул ещё не открыт/закрыт: метрик нет return None def _observe_connections(_options): from opentelemetry.metrics import Observation stats = _pool_stats() if not stats: return [] size, available = stats.get("pool_size", 0), stats.get("pool_available", 0) return [Observation(size - available, POOL_NAME | {"db.client.connection.state": "used"}), Observation(available, POOL_NAME | {"db.client.connection.state": "idle"})] def _observe_stat(key): def callback(_options): from opentelemetry.metrics import Observation stats = _pool_stats() return [Observation(stats.get(key, 0), POOL_NAME)] if stats else [] return callback telemetry.meter.create_observable_up_down_counter( "db.client.connection.count", [_observe_connections], unit="{connection}", description="Connections in the pool by state (used/idle)") telemetry.meter.create_observable_up_down_counter( "db.client.connection.max", [_observe_stat("pool_max")], unit="{connection}", description="Maximum pool size (DB_POOL_MAX)") telemetry.meter.create_observable_up_down_counter( "db.client.connection.pending_requests", [_observe_stat("requests_waiting")], unit="{request}", description="Requests waiting for a free connection now") telemetry.meter.create_observable_counter( "db.client.connection.timeouts", [_observe_stat("requests_errors")], unit="{timeout}", description="Requests that did not get a connection within DB_POOL_WAIT_SECONDS") @contextmanager def connection(wait_seconds: float | None = None): try: started = time.perf_counter() try: with pool.connection(timeout=wait_seconds) as conn: POOL_WAIT.record(time.perf_counter() - started, POOL_NAME) yield conn except PoolTimeout: POOL_WAIT.record(time.perf_counter() - started, POOL_NAME) raise except (psycopg.OperationalError, PoolTimeout) as exc: log.warning("PostgreSQL is unavailable: %s", type(exc).__name__, extra={"event": "db.unavailable", "error.type": type(exc).__name__}) raise unavailable() from None except psycopg.DataError as exc: log.warning("PostgreSQL rejected a value: %s", type(exc).__name__, extra={"event": "db.rejected_value", "error.type": type(exc).__name__}) raise bad_request("Invalid request data") from None TEXT_FIELDS = ("display_name", "title", "text", "filename") def to_db(value: str) -> bytes: return value.encode("utf-8") def from_db(row: dict | None) -> dict | None: """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" if row is None: return None return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} def ping() -> None: with connection(wait_seconds=1) as conn: conn.execute("SELECT 1") MIGRATIONS_LOCK = 20260202 def migration_files() -> list: return sorted(settings.MIGRATIONS_DIR.glob("*.sql")) def run_migrations() -> list[str]: files = migration_files() applied = [] with dbconn.connect(autocommit=True) as conn: conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,)) try: conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())""") done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} for path in files: if path.name in done: continue with conn.transaction(): conn.execute(path.read_text(encoding="utf-8")) conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,)) applied.append(path.name) finally: conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,)) return applied def wait_and_migrate(max_wait_seconds: int = 60) -> None: deadline = time.monotonic() + max_wait_seconds while True: try: applied = run_migrations() log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)") return except psycopg.OperationalError: if time.monotonic() > deadline: raise log.warning("PostgreSQL is not ready yet, retry in 1 s") time.sleep(1) def missing_migrations() -> list[str]: expected = [path.name for path in migration_files()] with dbconn.connect(autocommit=True) as conn: try: done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} except (pg_errors.UndefinedTable, pg_errors.InsufficientPrivilege): # lab7: права ролей выдаёт миграция 006 — до неё схема для приложения «не готова». return expected return [name for name in expected if name not in done] def wait_for_schema(max_wait_seconds: int) -> None: deadline = time.monotonic() + max_wait_seconds while True: try: missing = missing_migrations() except psycopg.OperationalError: missing = None if missing == []: log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files())) return if time.monotonic() > deadline: raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}") log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None else "missing " + ", ".join(missing)) time.sleep(1) USER_COLUMNS = "id, username, display_name, created_at" def create_user(username: str, display_name: str, password_hash: str) -> dict: try: with connection() as conn: return from_db(conn.execute( f"""INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", (uuid4(), username, to_db(display_name), password_hash), ).fetchone()) except pg_errors.UniqueViolation: raise conflict("Username already exists") from None def get_user(user_id: UUID) -> dict | None: with connection() as conn: return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) def get_user_with_password(username: str) -> dict | None: with connection() as conn: return from_db(conn.execute( f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) ).fetchone()) def update_password_hash(user_id: UUID, old_hash: str, new_hash: str) -> bool: """lab7: rehash при входе (параметры Argon2id выросли). Условие на старый hash — без гонки с другим входом.""" with connection() as conn: cursor = conn.execute("UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s", (new_hash, user_id, old_hash)) return cursor.rowcount == 1 CHAT_COLUMNS = """c.id, c.title, c.created_at, ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids""" def create_chat(title: str, member_ids: list[UUID]) -> dict: chat_id = uuid4() members = sorted(member_ids) try: with connection() as conn: with conn.transaction(): row = conn.execute( "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", (chat_id, to_db(title)), ).fetchone() with conn.cursor() as cur: cur.executemany( "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", [(chat_id, member_id) for member_id in members], ) except pg_errors.ForeignKeyViolation: raise bad_request("Unknown member") from None return {**from_db(row), "member_ids": members} def get_chat(chat_id: UUID, user_id: UUID) -> dict: with connection() as conn: chat = conn.execute( f"""SELECT {CHAT_COLUMNS} FROM chats c JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s WHERE c.id = %s""", (user_id, chat_id), ).fetchone() if chat is None: raise not_found("Chat not found") return from_db(chat) def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: """Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё».""" where, params = "", [user_id] if after is not None: where = "WHERE (c.created_at, c.id) > (%s, %s)" params += list(after) with connection() as conn: rows = conn.execute( f"""SELECT {CHAT_COLUMNS} FROM chats c JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s {where} ORDER BY c.created_at, c.id LIMIT %s""", (*params, limit + 1), ).fetchall() return [from_db(row) for row in rows] MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, ARRAY(SELECT a.attachment_id FROM message_attachments a WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids""" def is_member(conn, chat_id: UUID, user_id: UUID) -> bool: return conn.execute( "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) ).fetchone() is not None def check_member(chat_id: UUID, user_id: UUID) -> None: with connection() as conn: if not is_member(conn, chat_id, user_id): raise not_found("Chat not found") def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID], idempotency_key: str, request_hash: bytes) -> dict: try: with connection() as conn: with conn.transaction(): if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404 raise not_found("Chat not found") if attachment_ids: # Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409). # FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние. found = conn.execute( """SELECT id, state FROM attachments WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""", (attachment_ids, chat_id), ).fetchall() if len(found) != len(attachment_ids): raise not_found("Attachment not found") failed = any(row["state"] == "failed" for row in found) else: failed = False saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s""" if failed: # Повтор уже созданного сообщения возвращает прежний 201, даже если вложение # с тех пор стало failed. Новое сообщение с failed-вложением — 409. saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() if saved is None: raise conflict("Failed attachment cannot be attached") created = None else: created = conn.execute( """INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash) VALUES (%s, %s, %s, %s, %s, %s) ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING RETURNING id, chat_id, sender_id, text, created_at""", (uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash), ).fetchone() if created is None: saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() if created is not None: if attachment_ids: with conn.cursor() as cur: cur.executemany( """INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) VALUES (%s, %s, %s, %s)""", [(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)], ) return {**from_db(created), "attachment_ids": list(attachment_ids)} except pg_errors.ForeignKeyViolation: raise not_found("Chat not found") from None if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен raise unavailable() if bytes(saved.pop("request_hash")) != request_hash: raise conflict("Idempotency-Key was already used with a different request") return from_db(saved) def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: """История чата после курсора (keyset-пагинация), до limit+1 строк.""" where, params = "m.chat_id = %s", [chat_id] if after is not None: where += " AND (m.created_at, m.id) > (%s, %s)" params += list(after) with connection() as conn: if not is_member(conn, chat_id, user_id): raise not_found("Chat not found") rows = conn.execute( f"""SELECT {MESSAGE_COLUMNS} FROM messages m WHERE {where} ORDER BY m.created_at, m.id LIMIT %s""", (*params, limit + 1), ).fetchall() return [from_db(row) for row in rows] # ---------- вложения ---------- ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, " "pipeline_version, created_at, updated_at, object_key") VISIBLE_STATES = ("queued", "processing", "ready", "failed") def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str, size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None: """Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging. Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД). """ try: with connection() as conn: conn.execute( """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, source_key, state) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""", (attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256, width, height, source_key), ) except pg_errors.ForeignKeyViolation: raise not_found("Chat not found") from None def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None: """Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией. После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL, сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace). """ with connection() as conn: with conn.transaction(): row = conn.execute( f"""UPDATE attachments SET state = 'queued', updated_at = now() WHERE id = %s AND state = 'uploading' RETURNING {ATTACHMENT_COLUMNS}""", (attachment_id,), ).fetchone() if row is None: return None pipeline.enqueue_new(conn, attachment_id, actor) return from_db(row) def delete_upload_intent(attachment_id: UUID) -> None: with connection(wait_seconds=1) as conn: conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,)) def get_attachment(attachment_id: UUID, user_id: UUID) -> dict: """Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404. Возвращает строку и варианты текущей версии pipeline (они есть только у ready). """ with connection() as conn: row = conn.execute( f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))} FROM attachments a JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s WHERE a.id = %s AND a.state = ANY(%s)""", (user_id, attachment_id, list(VISIBLE_STATES)), ).fetchone() if row is None: raise not_found("Attachment not found") variants = [] if row["state"] == "ready": variants = conn.execute( """SELECT name, content_type, size_bytes, width, height, sha256, object_key FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""", (attachment_id, row["pipeline_version"]), ).fetchall() return {**from_db(row), "variants": variants}