forked from mtusi2026/mtusi2026lab6
595 lines
33 KiB
Python
595 lines
33 KiB
Python
from datetime import datetime, timedelta, timezone
|
|
from contextlib import asynccontextmanager
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
from secrets import token_urlsafe
|
|
from uuid import UUID, uuid4
|
|
|
|
import asyncpg
|
|
from argon2 import PasswordHasher
|
|
from argon2.exceptions import InvalidHashError, VerificationError
|
|
|
|
from ..errors import ApiError
|
|
from .. import telemetry as tel
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
MIGRATIONS = ROOT / "migrations"
|
|
PASSWORD_HASHER = PasswordHasher()
|
|
DB_ERRORS = (asyncpg.PostgresError, asyncpg.InterfaceError, OSError, TimeoutError)
|
|
|
|
|
|
class PostgresStore:
|
|
def __init__(self, database_url: str) -> None:
|
|
self._database_url = database_url
|
|
self._pool: asyncpg.Pool | None = None
|
|
|
|
async def connect(self) -> None:
|
|
try:
|
|
self._pool = await asyncpg.create_pool(
|
|
self._database_url, min_size=1, max_size=10, command_timeout=5
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise RuntimeError("PostgreSQL is unavailable") from error
|
|
|
|
async def close(self) -> None:
|
|
if self._pool is not None:
|
|
await self._pool.close()
|
|
|
|
@property
|
|
def pool(self) -> asyncpg.Pool:
|
|
if self._pool is None:
|
|
raise RuntimeError("PostgreSQL pool has not been initialized")
|
|
return self._pool
|
|
|
|
async def migrate(self) -> None:
|
|
async with self.pool.acquire() as connection:
|
|
await connection.execute(
|
|
"CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
|
|
)
|
|
for migration in sorted(MIGRATIONS.glob("*.sql")):
|
|
applied = await connection.fetchval(
|
|
"SELECT 1 FROM schema_migrations WHERE version = $1", migration.name
|
|
)
|
|
if applied:
|
|
continue
|
|
async with connection.transaction():
|
|
await connection.execute(migration.read_text(encoding="utf-8"))
|
|
await connection.execute(
|
|
"INSERT INTO schema_migrations (version) VALUES ($1)", migration.name
|
|
)
|
|
|
|
@staticmethod
|
|
def _user(row: asyncpg.Record) -> dict:
|
|
return {
|
|
"id": row["id"],
|
|
"username": row["username"],
|
|
"display_name": row["display_name"],
|
|
"created_at": row["created_at"],
|
|
}
|
|
|
|
@staticmethod
|
|
def _chat(row: asyncpg.Record) -> dict:
|
|
return {
|
|
"id": row["id"],
|
|
"title": row["title"],
|
|
"member_ids": row["member_ids"],
|
|
"created_at": row["created_at"],
|
|
}
|
|
|
|
@staticmethod
|
|
def _message(row: asyncpg.Record) -> dict:
|
|
return {
|
|
"id": row["id"],
|
|
"chat_id": row["chat_id"],
|
|
"sender_id": row["sender_id"],
|
|
"text": row["text"],
|
|
"attachment_ids": row["attachment_ids"],
|
|
"created_at": row["created_at"],
|
|
}
|
|
|
|
async def ready(self) -> None:
|
|
try:
|
|
await self.pool.fetchval("SELECT 1")
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
|
|
async def create_user(self, username: str, display_name: str, password: str) -> dict:
|
|
try:
|
|
row = await self.pool.fetchrow(
|
|
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) RETURNING id, username, display_name, created_at",
|
|
uuid4(),
|
|
username,
|
|
display_name,
|
|
PASSWORD_HASHER.hash(password),
|
|
)
|
|
except asyncpg.UniqueViolationError as error:
|
|
raise ApiError(409, "conflict", "Username уже занят") from error
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
return self._user(row)
|
|
|
|
async def authenticate(self, username: str, password: str) -> dict | None:
|
|
try:
|
|
row = await self.pool.fetchrow(
|
|
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1",
|
|
username,
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
if row is None:
|
|
return None
|
|
try:
|
|
valid = PASSWORD_HASHER.verify(row["password_hash"], password)
|
|
except (InvalidHashError, VerificationError):
|
|
valid = False
|
|
return self._user(row) if valid else None
|
|
|
|
async def create_session(self, user: dict, ttl_seconds: int) -> dict:
|
|
token = token_urlsafe(32)
|
|
token_hash = sha256(token.encode("ascii")).hexdigest()
|
|
try:
|
|
row = await self.pool.fetchrow(
|
|
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES ($1, $2, now() + $3 * interval '1 second') RETURNING expires_at",
|
|
token_hash,
|
|
user["id"],
|
|
ttl_seconds,
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
return {"token": token, "token_type": "Bearer", "expires_at": row["expires_at"], "user": user}
|
|
|
|
async def user_for_token(self, token: str) -> dict | None:
|
|
if not token.isascii():
|
|
return None
|
|
token_hash = sha256(token.encode("ascii")).hexdigest()
|
|
try:
|
|
row = await self.pool.fetchrow(
|
|
"SELECT u.id, u.username, u.display_name, u.created_at FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token_hash = $1 AND s.expires_at > now()",
|
|
token_hash,
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
return self._user(row) if row else None
|
|
|
|
async def revoke_session(self, token: str) -> None:
|
|
try:
|
|
await self.pool.execute(
|
|
"DELETE FROM sessions WHERE token_hash = $1", sha256(token.encode("ascii")).hexdigest()
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
|
|
async def create_chat(self, creator_id: UUID, title: str, member_ids: list[UUID]) -> dict:
|
|
if len(member_ids) != len(set(member_ids)) or creator_id not in member_ids:
|
|
raise ApiError(400, "invalid_request", "Некорректный список участников")
|
|
try:
|
|
async with self.pool.acquire() as connection, connection.transaction():
|
|
existing = await connection.fetchval(
|
|
"SELECT count(*) FROM users WHERE id = ANY($1::uuid[])", member_ids
|
|
)
|
|
if existing != len(member_ids):
|
|
raise ApiError(400, "invalid_request", "Один или несколько участников не существуют")
|
|
row = await connection.fetchrow(
|
|
"INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING id, title, created_at",
|
|
uuid4(),
|
|
title,
|
|
)
|
|
await connection.executemany(
|
|
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
|
|
[(row["id"], member_id) for member_id in member_ids],
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
return {
|
|
"id": row["id"],
|
|
"title": row["title"],
|
|
"member_ids": member_ids,
|
|
"created_at": row["created_at"],
|
|
}
|
|
|
|
async def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> dict:
|
|
try:
|
|
row = await self.pool.fetchrow(
|
|
"SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids FROM chats c JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $2 JOIN chat_members cm_all ON cm_all.chat_id = c.id WHERE c.id = $1 GROUP BY c.id",
|
|
chat_id,
|
|
user_id,
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Чат не найден")
|
|
return self._chat(row)
|
|
|
|
async def list_chats_for_member(
|
|
self, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int
|
|
) -> list[dict]:
|
|
query = "SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids FROM chats c JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1 JOIN chat_members cm_all ON cm_all.chat_id = c.id"
|
|
args: list = [user_id]
|
|
if after is not None:
|
|
query += " WHERE (c.created_at, c.id) > ($2, $3)"
|
|
args.extend(after)
|
|
query += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT $" + str(len(args) + 1)
|
|
args.append(limit + 1)
|
|
try:
|
|
rows = await self.pool.fetch(query, *args)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
return [self._chat(row) for row in rows]
|
|
|
|
async def create_message_idempotent(
|
|
self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int, attachment_ids: list[UUID] | None = None
|
|
) -> dict:
|
|
"""Создаёт сообщение и фиксирует результат retry в одной транзакции."""
|
|
path = f"/api/v1/chats/{chat_id}/messages"
|
|
attachment_ids = attachment_ids or []
|
|
# Preserve hashes of text-only requests already accepted by lab3.
|
|
canonical = {"text": text}
|
|
if attachment_ids:
|
|
canonical["attachment_ids"] = [str(value) for value in attachment_ids]
|
|
request_hash = sha256(
|
|
json.dumps(canonical, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
|
).hexdigest()
|
|
try:
|
|
async with self.pool.acquire() as connection, connection.transaction():
|
|
member = await connection.fetchval(
|
|
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, sender_id
|
|
)
|
|
if member is None:
|
|
raise ApiError(404, "not_found", "Чат не найден")
|
|
await connection.execute(
|
|
"DELETE FROM idempotency_records WHERE created_at <= now() - $1 * interval '1 second'", ttl_seconds
|
|
)
|
|
if attachment_ids:
|
|
count = await connection.fetchval(
|
|
"SELECT count(*) FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[])",
|
|
chat_id, attachment_ids,
|
|
)
|
|
if count != len(attachment_ids):
|
|
raise ApiError(404, "not_found", "Вложение не найдено")
|
|
failed = await connection.fetchval(
|
|
"SELECT 1 FROM attachments WHERE id=ANY($1::uuid[]) AND status='failed' LIMIT 1", attachment_ids
|
|
)
|
|
if failed:
|
|
raise ApiError(409, 'conflict', 'Вложение завершилось ошибкой')
|
|
claimed = await connection.fetchrow(
|
|
"INSERT INTO idempotency_records (user_id, method, path, idempotency_key, request_hash) "
|
|
"VALUES ($1, 'POST', $2, $3, $4) ON CONFLICT DO NOTHING RETURNING message_id",
|
|
sender_id, path, idempotency_key, request_hash,
|
|
)
|
|
if claimed is None:
|
|
recorded = await connection.fetchrow(
|
|
"SELECT request_hash, message_id FROM idempotency_records "
|
|
"WHERE user_id = $1 AND method = 'POST' AND path = $2 AND idempotency_key = $3 FOR UPDATE",
|
|
sender_id, path, idempotency_key,
|
|
)
|
|
if recorded is None:
|
|
raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса")
|
|
if recorded["request_hash"] != request_hash:
|
|
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом")
|
|
if recorded["message_id"] is None:
|
|
raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса")
|
|
row = await connection.fetchrow(
|
|
"SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE id = $1", recorded["message_id"]
|
|
)
|
|
if row is None:
|
|
raise ApiError(503, "unavailable", "Не удалось получить сохранённое сообщение")
|
|
return self._message(row)
|
|
row = await connection.fetchrow(
|
|
"INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids) VALUES ($1, $2, $3, $4, $5) "
|
|
"RETURNING id, chat_id, sender_id, text, attachment_ids, created_at",
|
|
uuid4(), chat_id, sender_id, text, attachment_ids,
|
|
)
|
|
await connection.execute(
|
|
"UPDATE idempotency_records SET message_id = $1 WHERE user_id = $2 AND method = 'POST' "
|
|
"AND path = $3 AND idempotency_key = $4",
|
|
row["id"], sender_id, path, idempotency_key,
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
return self._message(row)
|
|
|
|
async def list_messages_for_member(
|
|
self, chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int
|
|
) -> list[dict]:
|
|
await self.get_chat_for_member(chat_id, user_id)
|
|
query = "SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE chat_id = $1"
|
|
args: list = [chat_id]
|
|
if after is not None:
|
|
query += " AND (created_at, id) > ($2, $3)"
|
|
args.extend(after)
|
|
query += " ORDER BY created_at, id LIMIT $" + str(len(args) + 1)
|
|
args.append(limit + 1)
|
|
try:
|
|
rows = await self.pool.fetch(query, *args)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
return [self._message(row) for row in rows]
|
|
|
|
|
|
@asynccontextmanager
|
|
async def attachment_transaction(self, *, maintenance=False):
|
|
"""Shared writer / exclusive reconciler lock across all API replicas.
|
|
|
|
Writers hold the transaction from before PUT through metadata COMMIT.
|
|
Maintenance holds the exclusive lock through inventory and deletion.
|
|
All attachment writers must participate in this protocol.
|
|
"""
|
|
try:
|
|
with tel.tracer.start_as_current_span("postgres.attachment_transaction", kind=tel.SpanKind.CLIENT, record_exception=False, set_status_on_exception=False) as span:
|
|
try:
|
|
async with self.pool.acquire() as connection, connection.transaction():
|
|
await connection.execute("SET LOCAL lock_timeout = '5s'")
|
|
lock = "pg_advisory_xact_lock" if maintenance else "pg_advisory_xact_lock_shared"
|
|
await connection.execute(f"SELECT {lock}(41004, 1)")
|
|
yield connection
|
|
except Exception:
|
|
span.set_status(tel.Status(tel.StatusCode.ERROR))
|
|
raise
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
|
|
async def create_attachment(self, attachment_id, chat_id, owner_id, filename,
|
|
content_type, size_bytes, digest, staging_key, job_id, *, connection=None) -> dict:
|
|
try:
|
|
row = await (connection or self.pool).fetchrow(
|
|
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key) "
|
|
"VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING *",
|
|
attachment_id, chat_id, owner_id, filename, content_type, size_bytes, digest, staging_key,
|
|
)
|
|
await (connection or self.pool).execute(
|
|
"INSERT INTO attachment_jobs (job_id, attachment_id, trace_context) VALUES ($1, $2, $3::jsonb)", job_id, attachment_id, json.dumps(tel.carrier())
|
|
)
|
|
await (connection or self.pool).execute(
|
|
"INSERT INTO attachment_slo (attachment_id) VALUES ($1)", attachment_id
|
|
)
|
|
payload = {
|
|
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
|
|
"attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1,
|
|
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
}
|
|
await (connection or self.pool).execute(
|
|
"INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)",
|
|
uuid4(), job_id, json.dumps(payload, separators=(",", ":")),
|
|
)
|
|
await (connection or self.pool).execute(
|
|
"UPDATE attachment_jobs SET next_attempt_at = 'infinity' WHERE job_id = $1", job_id
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
return dict(row)
|
|
|
|
async def ensure_attachment_queue_capacity(self, connection: asyncpg.Connection, maximum: int) -> None:
|
|
"""Globally serialize admission so replicas cannot over-accept queued work."""
|
|
try:
|
|
await connection.execute("SELECT pg_advisory_xact_lock(41005, 1)")
|
|
pending = await connection.fetchval(
|
|
"SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')"
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
if pending >= maximum:
|
|
raise ApiError(429, "rate_limited", "Очередь обработки заполнена")
|
|
|
|
async def attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> dict:
|
|
try:
|
|
row = await self.pool.fetchrow(
|
|
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
|
|
"WHERE a.id = $1 AND m.user_id = $2", attachment_id, user_id,
|
|
)
|
|
except DB_ERRORS as error:
|
|
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Вложение не найдено")
|
|
return await self._attachment_with_variants(row)
|
|
|
|
async def _attachment_with_variants(self, row: asyncpg.Record | dict) -> dict:
|
|
result = dict(row)
|
|
variants = await self.pool.fetch(
|
|
"SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants "
|
|
"WHERE attachment_id = $1 AND pipeline_version = $2 ORDER BY name",
|
|
result["id"], result["pipeline_version"],
|
|
)
|
|
result["variants"] = [dict(item) for item in variants]
|
|
return result
|
|
|
|
async def claim_attachment_job(self, attachment_id: UUID, lease_seconds: int) -> dict | None:
|
|
try:
|
|
async with self.pool.acquire() as connection, connection.transaction():
|
|
row = await connection.fetchrow(
|
|
"SELECT a.*, j.job_id, j.attempt, j.lease_until, "
|
|
"(j.next_attempt_at <> 'infinity'::timestamptz AND j.next_attempt_at > now()) AS retry_waiting "
|
|
"FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id "
|
|
"WHERE a.id = $1 FOR UPDATE", attachment_id
|
|
)
|
|
# Only the dispatcher recovers an expired lease and increments
|
|
# its attempt. Redelivery must not renew it or bypass backoff.
|
|
if row is None or row["status"] != "queued" or row["retry_waiting"]:
|
|
return None
|
|
await connection.execute(
|
|
"UPDATE attachments SET status = 'processing', processing_started_at = now(), updated_at = now() WHERE id = $1",
|
|
attachment_id,
|
|
)
|
|
await connection.execute(
|
|
"UPDATE attachment_jobs SET status = 'processing', first_started_at=COALESCE(first_started_at, now()), lease_until = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1",
|
|
attachment_id, lease_seconds,
|
|
)
|
|
result = dict(row)
|
|
result['claim_time'] = await connection.fetchval('SELECT processing_started_at FROM attachments WHERE id=$1', attachment_id)
|
|
return result
|
|
except DB_ERRORS as error:
|
|
raise RuntimeError("PostgreSQL is unavailable") from error
|
|
|
|
async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variants: list[dict], claim_time) -> bool:
|
|
try:
|
|
async with self.pool.acquire() as connection, connection.transaction():
|
|
valid = await connection.fetchval(
|
|
"SELECT 1 FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id=$1 AND a.status='processing' AND a.processing_started_at=$2 AND j.lease_until > now() FOR UPDATE", attachment_id, claim_time
|
|
)
|
|
if not valid:
|
|
return False
|
|
for variant in variants:
|
|
await connection.execute(
|
|
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) "
|
|
"VALUES ($1, 1, $2, $3, $4, $5, $6, $7, $8) "
|
|
"ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256",
|
|
attachment_id, variant["name"], variant["object_key"], variant["content_type"], variant["size_bytes"], variant["width"], variant["height"], variant["sha256"],
|
|
)
|
|
await connection.execute(
|
|
"UPDATE attachments SET status = 'ready', original_key = $2, error_code = NULL, updated_at = now() WHERE id = $1",
|
|
attachment_id, original_key,
|
|
)
|
|
await connection.execute("UPDATE attachment_jobs SET status = 'ready', lease_until = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id)
|
|
await connection.execute(
|
|
"UPDATE attachment_slo SET ready_xid=pg_current_xact_id()::xid "
|
|
"WHERE attachment_id=$1 AND ready_xid IS NULL", attachment_id
|
|
)
|
|
return True
|
|
except DB_ERRORS as error:
|
|
raise RuntimeError("PostgreSQL is unavailable") from error
|
|
|
|
async def ready_seconds(self, attachment_id: UUID):
|
|
# Both ends are DB commit timestamps, not transaction start or a worker's clock.
|
|
return await self.pool.fetchval("""
|
|
SELECT EXTRACT(EPOCH FROM (
|
|
COALESCE(ready_at, pg_xact_commit_timestamp(ready_xid)) -
|
|
COALESCE(accepted_at, pg_xact_commit_timestamp(accepted_xid))))::float8
|
|
FROM attachment_slo WHERE attachment_id=$1
|
|
""", attachment_id)
|
|
|
|
async def sample_ready_slo(self):
|
|
# Materialize promptly: PostgreSQL does not retain commit timestamps forever.
|
|
async with self.pool.acquire() as connection, connection.transaction():
|
|
await connection.execute("""
|
|
UPDATE attachment_slo SET
|
|
accepted_at=COALESCE(accepted_at, pg_xact_commit_timestamp(accepted_xid)),
|
|
ready_at=COALESCE(ready_at, pg_xact_commit_timestamp(ready_xid))
|
|
WHERE accepted_at IS NULL OR (ready_xid IS NOT NULL AND ready_at IS NULL)
|
|
""")
|
|
await connection.execute("""
|
|
UPDATE attachment_slo SET bad=(ready_at IS NULL OR ready_at>accepted_at+interval '30 seconds')
|
|
WHERE bad IS NULL AND accepted_at<=now()-interval '30 seconds'
|
|
AND (ready_xid IS NULL OR ready_at IS NOT NULL)
|
|
""")
|
|
return await connection.fetchrow("""
|
|
SELECT count(bad) AS eligible, count(*) FILTER (WHERE bad) AS bad,
|
|
count(*) FILTER (WHERE accepted_at IS NULL OR
|
|
(ready_xid IS NOT NULL AND ready_at IS NULL)) AS unknown
|
|
FROM attachment_slo
|
|
""")
|
|
|
|
async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float, claim_time, permanent_code=None) -> bool:
|
|
"""Return true only when a bounded at-least-once retry was recorded."""
|
|
try:
|
|
async with self.pool.acquire() as connection, connection.transaction():
|
|
job = await connection.fetchrow(
|
|
"SELECT j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.id=$1 AND a.status='processing' AND a.processing_started_at=$2 FOR UPDATE", attachment_id, claim_time
|
|
)
|
|
if job is None:
|
|
return False
|
|
if permanent_code is None and job["attempt"] < max_attempts:
|
|
await connection.execute(
|
|
"UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', lease_until = NULL, next_attempt_at = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1",
|
|
attachment_id, delay_seconds,
|
|
)
|
|
await connection.execute(
|
|
"UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", attachment_id
|
|
)
|
|
return True
|
|
code = permanent_code or 'retry_exhausted'
|
|
await connection.execute("UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1", attachment_id, code)
|
|
await connection.execute("UPDATE attachment_jobs SET status='failed', last_error=$2, lease_until=NULL, updated_at=now() WHERE attachment_id=$1", attachment_id, code)
|
|
return False
|
|
except DB_ERRORS as error:
|
|
raise RuntimeError("PostgreSQL is unavailable") from error
|
|
|
|
async def due_outbox_events(self, limit: int = 20) -> list[dict]:
|
|
try:
|
|
return [{**dict(row), 'payload': json.loads(row['payload']) if isinstance(row['payload'], str) else row['payload']} for row in await self.pool.fetch(
|
|
"SELECT o.id, o.payload, j.trace_context FROM attachment_outbox o JOIN attachment_jobs j ON j.job_id=o.job_id WHERE o.published_at IS NULL ORDER BY o.created_at LIMIT $1", limit
|
|
)]
|
|
except DB_ERRORS as error:
|
|
raise RuntimeError("PostgreSQL is unavailable") from error
|
|
|
|
async def mark_outbox_published(self, event_id: UUID) -> None:
|
|
await self.pool.execute("UPDATE attachment_outbox SET published_at = now(), attempts = attempts + 1 WHERE id = $1", event_id)
|
|
|
|
async def schedule_due_jobs(self, max_attempts: int = 5) -> None:
|
|
"""Outbox dispatcher also restores expired leases and scheduled retries."""
|
|
try:
|
|
async with self.pool.acquire() as connection, connection.transaction():
|
|
rows = await connection.fetch(
|
|
"SELECT job_id, attachment_id, pipeline_version, attempt, status FROM attachment_jobs "
|
|
"WHERE (status = 'queued' AND (next_attempt_at <= now() OR "
|
|
"(next_attempt_at = 'infinity'::timestamptz AND updated_at < now() - interval '60 seconds'))) "
|
|
"OR (status = 'processing' AND lease_until < now()) "
|
|
"FOR UPDATE SKIP LOCKED"
|
|
)
|
|
for row in rows:
|
|
row = dict(row)
|
|
if row["status"] == "processing":
|
|
if row['attempt'] >= max_attempts:
|
|
await connection.execute("UPDATE attachment_jobs SET status='failed', lease_until=NULL, last_error='retry_exhausted', updated_at=now() WHERE job_id=$1", row['job_id'])
|
|
await connection.execute("UPDATE attachments SET status='failed', error_code='retry_exhausted', updated_at=now() WHERE id=$1", row['attachment_id'])
|
|
continue
|
|
row['attempt'] += 1
|
|
await connection.execute(
|
|
"UPDATE attachment_jobs SET status = 'queued', attempt=$2, lease_until = NULL, next_attempt_at = now(), last_error = 'lease_expired', updated_at = now() WHERE job_id = $1", row["job_id"], row['attempt']
|
|
)
|
|
await connection.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", row["attachment_id"])
|
|
payload = {"schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": row["pipeline_version"], "attempt": row["attempt"], "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")}
|
|
await connection.execute("INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)", uuid4(), row["job_id"], json.dumps(payload, separators=(",", ":")))
|
|
# Refresh the watchdog on publication intent, not on claim.
|
|
# Unclaimed jobs are republished after 60s. At-least-once
|
|
# duplicates are safe, and a published-but-lost delivery
|
|
# can no longer strand durable accepted work forever.
|
|
await connection.execute("UPDATE attachment_jobs SET next_attempt_at = 'infinity', updated_at=now() WHERE job_id = $1", row["job_id"])
|
|
except DB_ERRORS as error:
|
|
raise RuntimeError("PostgreSQL is unavailable") from error
|
|
|
|
async def replay_failed_attachment(self, attachment_id: UUID) -> bool:
|
|
"""Explicit operator-only replay; never requeues an already successful job."""
|
|
try:
|
|
async with self.pool.acquire() as connection, connection.transaction():
|
|
job = await connection.fetchrow(
|
|
"SELECT job_id, status FROM attachment_jobs WHERE attachment_id = $1 FOR UPDATE", attachment_id
|
|
)
|
|
if job is None or job["status"] != "failed":
|
|
return False
|
|
await connection.execute(
|
|
"UPDATE attachment_jobs SET status = 'queued', attempt = 1, next_attempt_at = now(), lease_until = NULL, replayed_at = now(), last_error = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id
|
|
)
|
|
await connection.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() WHERE id = $1", attachment_id)
|
|
return True
|
|
except DB_ERRORS as error:
|
|
raise RuntimeError("PostgreSQL is unavailable") from error
|
|
|
|
async def attachment_keys(self) -> dict[str, UUID]:
|
|
"""Inventory used only by the maintenance reconciler."""
|
|
try:
|
|
rows = await self.pool.fetch("SELECT staging_key AS object_key, id FROM attachments WHERE status <> 'ready' UNION ALL SELECT original_key, id FROM attachments WHERE original_key IS NOT NULL UNION ALL SELECT object_key, attachment_id AS id FROM attachment_variants")
|
|
except DB_ERRORS as error:
|
|
raise RuntimeError("PostgreSQL is unavailable") from error
|
|
return {row["object_key"]: row["id"] for row in rows}
|
|
|
|
# Span names describe operations only; never export SQL, parameters, credentials or IDs.
|
|
def _trace_db(method):
|
|
from functools import wraps
|
|
@wraps(method)
|
|
async def wrapped(*args, **kwargs):
|
|
with tel.tracer.start_as_current_span('postgres.' + method.__name__, kind=tel.SpanKind.CLIENT,
|
|
record_exception=False, set_status_on_exception=False) as span:
|
|
try:
|
|
return await method(*args, **kwargs)
|
|
except Exception:
|
|
span.set_status(tel.Status(tel.StatusCode.ERROR))
|
|
raise
|
|
return wrapped
|
|
|
|
for _name in ('ready', 'create_user', 'authenticate', 'create_session', 'user_for_token', 'revoke_session',
|
|
'create_chat', 'get_chat_for_member', 'list_chats_for_member', 'create_message_idempotent',
|
|
'list_messages_for_member', 'create_attachment', 'attachment_for_member', 'claim_attachment_job',
|
|
'complete_attachment_job', 'retry_attachment_job'):
|
|
setattr(PostgresStore, _name, _trace_db(getattr(PostgresStore, _name)))
|