import base64 import hashlib import hmac import json import logging import math import os import secrets import uuid from datetime import datetime, timedelta from pathlib import Path from typing import Any, BinaryIO, Dict, List, Optional, Tuple import redis from psycopg import sql from psycopg.rows import dict_row from psycopg_pool import ConnectionPool from .config import ( DB_LOCK_TIMEOUT_MS, DB_POOL_MAX_SIZE, DB_POOL_TIMEOUT_SECONDS, DB_STATEMENT_TIMEOUT_MS, IDEMPOTENCY_TTL_SECONDS, SESSION_TTL_SECONDS, secret_value, ) from .errors import ( AttachmentNotReadyError, CacheUnavailableError, IdempotencyConflictError, ObjectMissingError, ) from .object_store import ObjectStore from .pipeline_versions import PIPELINE_QUALITY from .security import ( hash_password, isoformat, password_hash_needs_upgrade, utc_now, valid_uuid, verify_password, ) LOGGER = logging.getLogger("messenger.store") _DUMMY_PASSWORD_HASH = hash_password(secrets.token_urlsafe(32)) def database_url() -> str: return secret_value("DATABASE_URL") class Store: """PostgreSQL metadata, S3 objects, and optional Redis session cache.""" def __init__(self): self.pool = ConnectionPool( conninfo=database_url(), min_size=1, max_size=DB_POOL_MAX_SIZE, timeout=DB_POOL_TIMEOUT_SECONDS, kwargs={ "row_factory": dict_row, "connect_timeout": 5, "options": ( f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} " f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} " "-c idle_in_transaction_session_timeout=10000" ), }, open=False, ) cache_url = os.getenv("CACHE_URL") self.cache = ( redis.Redis.from_url( cache_url, decode_responses=True, socket_connect_timeout=0.25, socket_timeout=0.25, retry_on_timeout=False, max_connections=16, ) if cache_url else None ) self.objects = ObjectStore() def open(self): self.pool.open(wait=True, timeout=10) def close(self): if self.cache is not None: self.cache.close() self.pool.close() def ready(self) -> bool: with self.pool.connection(timeout=2) as conn: conn.execute("SELECT 1") self.objects.ready() return True def migrate(self): migrations = Path(__file__).resolve().parents[1] / "migrations" with self.pool.connection() as conn: with conn.transaction(): conn.execute( "CREATE TABLE IF NOT EXISTS schema_migrations " "(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" ) conn.execute("SELECT pg_advisory_xact_lock(2741662026)") applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")} for migration in sorted(migrations.glob("*.sql")): if migration.name in applied: continue conn.execute(migration.read_text(encoding="utf-8")) conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,)) def configure_worker_role(self, password: str) -> None: with self.pool.connection() as conn: with conn.transaction(): exists = conn.execute( "SELECT 1 FROM pg_roles WHERE rolname = 'messenger_worker'" ).fetchone() if exists: statement = sql.SQL("ALTER ROLE messenger_worker LOGIN PASSWORD {}").format( sql.Literal(password) ) else: statement = sql.SQL("CREATE ROLE messenger_worker LOGIN PASSWORD {}").format( sql.Literal(password) ) conn.execute(statement) conn.execute("GRANT CONNECT ON DATABASE messenger TO messenger_worker") conn.execute("GRANT USAGE ON SCHEMA public TO messenger_worker") conn.execute( "GRANT SELECT (id, job_id, pipeline_version, attempt, status, lease_until, " "staging_key, object_key, content_type, size_bytes, sha256) ON attachments " "TO messenger_worker" ) conn.execute( "GRANT UPDATE (status, lease_until, updated_at, variants, error_code, attempt) " "ON attachments TO messenger_worker" ) conn.execute( "GRANT INSERT (id, job_id, attachment_id, pipeline_version, attempt, available_at, " "traceparent, tracestate) ON attachment_outbox TO messenger_worker" ) @staticmethod def user(row: Dict[str, Any]) -> Dict[str, Any]: return { "id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": isoformat(row["created_at"]), } @staticmethod def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]: return { "id": str(row["id"]), "title": row["title"], "member_ids": member_ids, "created_at": isoformat(row["created_at"]), } @staticmethod def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]: return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": isoformat(row["created_at"]), "attachment_ids": attachment_ids or [], } @staticmethod def attachment(row: Dict[str, Any]) -> Dict[str, Any]: variants = row.get("variants") or [] public_variants = [ { name: variant[name] for name in ("name", "content_type", "size_bytes", "width", "height", "sha256") } for variant in variants ] return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"], "content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"].strip(), "status": row.get("status", "queued"), "variants": public_variants, "error_code": row.get("error_code"), "created_at": isoformat(row["created_at"]), "updated_at": isoformat(row["updated_at"]), } def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]: with self.pool.connection() as conn: with conn.transaction(): row = conn.execute( "INSERT INTO users (username, display_name, password_hash) " "VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at", (username, display_name, password_hash), ).fetchone() return self.user(row) def get_user(self, user_id: str) -> Optional[Dict[str, Any]]: with self.pool.connection() as conn: row = conn.execute( "SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,), ).fetchone() return self.user(row) if row else None def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]: with self.pool.connection() as conn: row = conn.execute( "SELECT id, username, display_name, created_at, password_hash " "FROM users WHERE username = %s", (username,), ).fetchone() stored_hash = row["password_hash"] if row else _DUMMY_PASSWORD_HASH if not verify_password(password, stored_hash) or not row: return None if password_hash_needs_upgrade(stored_hash): upgraded_hash = hash_password(password) with self.pool.connection() as conn: conn.execute( "UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s", (upgraded_hash, row["id"], stored_hash), ) return self.user(row) def consume_login_attempt(self, username: str, source_ip: str) -> Optional[int]: user_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest() ip_key = hashlib.sha256(b"ip\0" + source_ip.encode("utf-8")).digest() with self.pool.connection() as conn: with conn.transaction(): def bump(key: bytes) -> Dict[str, Any]: return conn.execute( """ INSERT INTO login_rate_limits (username_key, window_started_at, attempts) VALUES (%s, clock_timestamp(), 1) ON CONFLICT (username_key) DO UPDATE SET attempts = CASE WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute' THEN 1 ELSE login_rate_limits.attempts + 1 END, window_started_at = CASE WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute' THEN clock_timestamp() ELSE login_rate_limits.window_started_at END RETURNING attempts, GREATEST(1, ceil(extract(epoch FROM (window_started_at + interval '1 minute' - clock_timestamp()))))::integer AS retry_after """, (key,), ).fetchone() user_row = bump(user_key) ip_row = bump(ip_key) conn.execute( "DELETE FROM login_rate_limits WHERE window_started_at < clock_timestamp() - interval '1 hour'" ) retries = [] if user_row["attempts"] > 5: retries.append(user_row["retry_after"]) if ip_row["attempts"] > 60: retries.append(ip_row["retry_after"]) return max(retries) if retries else None def clear_login_attempts(self, username: str) -> None: username_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest() with self.pool.connection() as conn: conn.execute("DELETE FROM login_rate_limits WHERE username_key = %s", (username_key,)) def create_session(self, user_id: str) -> Tuple[str, datetime]: token = secrets.token_urlsafe(32) expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) token_hash = hashlib.sha256(token.encode("ascii")).digest() with self.pool.connection() as conn: conn.execute( "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", (token_hash, user_id, expires_at), ) return token, expires_at @staticmethod def _session_lock_key(token_hash: bytes) -> int: return int.from_bytes(token_hash[:8], byteorder="big", signed=True) def session_user(self, token: str) -> Optional[Dict[str, Any]]: token_hash = hashlib.sha256(token.encode("utf-8")).digest() cache_key = "messenger:session:" + token_hash.hex() if self.cache is not None: try: cached = self.cache.get(cache_key) if cached: return json.loads(cached) except (redis.RedisError, ValueError, TypeError): LOGGER.warning("session cache read failed; falling back to PostgreSQL") with self.pool.connection() as conn: with conn.transaction(): conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) row = conn.execute( "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at " "FROM sessions s JOIN users u ON u.id = s.user_id " "WHERE s.token_hash = %s AND s.expires_at > now()", (token_hash,), ).fetchone() if not row: return None user = self.user(row) if self.cache is not None: ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000) if ttl_ms > 0: try: self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms) except redis.RedisError: LOGGER.warning("session cache write failed; PostgreSQL remains authoritative") return user def revoke_session(self, token: str) -> bool: token_hash = hashlib.sha256(token.encode("utf-8")).digest() with self.pool.connection() as conn: with conn.transaction(): conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) if self.cache is not None: try: self.cache.delete("messenger:session:" + token_hash.hex()) except redis.RedisError as exc: raise CacheUnavailableError from exc result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,)) return result.rowcount == 1 def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]: if actor_id not in member_ids or len(member_ids) != len(set(member_ids)): raise ValueError("invalid members") with self.pool.connection() as conn: with conn.transaction(): row = conn.execute( "INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", (title,), ).fetchone() for member_id in member_ids: conn.execute( "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (row["id"], member_id), ) return self.chat(row, member_ids) def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]: with self.pool.connection() as conn: row = conn.execute( "SELECT c.id, c.title, c.created_at FROM chats c " "JOIN chat_members m ON m.chat_id = c.id " "WHERE c.id = %s AND m.user_id = %s", (chat_id, user_id), ).fetchone() if not row: return None members = conn.execute( "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", (chat_id,), ).fetchall() return self.chat(row, [str(item["user_id"]) for item in members]) def list_chats( self, user_id: str, limit: int, cursor: Optional[str] ) -> Tuple[List[Dict[str, Any]], Optional[str]]: where = "own.user_id = %s" params: List[Any] = [user_id] if cursor is not None: try: raw = base64.b64decode( cursor.encode("ascii") + b"=" * (-len(cursor) % 4), altchars=b"-_", validate=True, ) cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2) if cursor_user != user_id or not valid_uuid(chat_id): raise ValueError("cursor scope") created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) if created_at.tzinfo is None or created_at.utcoffset() is None: raise ValueError("cursor timestamp must include a timezone") where += " AND (c.created_at, c.id) > (%s, %s)" params.extend((created_at, chat_id)) except (ValueError, UnicodeError, TypeError) as exc: raise ValueError("invalid cursor") from exc params.append(limit + 1) with self.pool.connection() as conn: rows = conn.execute( "SELECT c.id, c.title, c.created_at FROM chats c " "JOIN chat_members own ON own.chat_id = c.id " f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s", params, ).fetchall() has_more = len(rows) > limit page = rows[:limit] result = [] for row in page: members = conn.execute( "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", (row["id"],), ).fetchall() result.append(self.chat(row, [str(item["user_id"]) for item in members])) next_cursor = None if has_more and page: last = page[-1] cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") return result, next_cursor def create_idempotent_message( self, chat_id: str, sender_id: str, text: str, attachment_ids: List[str], idempotency_key: str, canonical_body: str, ) -> Optional[Dict[str, Any]]: request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest() lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8") lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True) with self.pool.connection() as conn: with conn.transaction(): member = conn.execute( "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, sender_id), ).fetchone() if not member: return None if attachment_ids: attachments = conn.execute( "SELECT id, status FROM attachments " "WHERE chat_id = %s AND id = ANY(%s) FOR SHARE", (chat_id, attachment_ids), ).fetchall() if len(attachments) != len(attachment_ids): return None if any(item["status"] == "failed" for item in attachments): raise AttachmentNotReadyError conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) existing = conn.execute( "SELECT request_hash, response, expires_at " "FROM idempotency_records " "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s " "FOR UPDATE", (sender_id, chat_id, idempotency_key), ).fetchone() if existing and existing["expires_at"] > utc_now(): if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash): raise IdempotencyConflictError return existing["response"] if existing: conn.execute( "DELETE FROM idempotency_records " "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s", (sender_id, chat_id, idempotency_key), ) row = conn.execute( "INSERT INTO messages (chat_id, sender_id, text) " "VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at", (chat_id, sender_id, text), ).fetchone() for ordinal, attachment_id in enumerate(attachment_ids): conn.execute( "INSERT INTO message_attachments (message_id, attachment_id, ordinal) " "VALUES (%s, %s, %s)", (row["id"], attachment_id, ordinal), ) message = self.message(row, attachment_ids) conn.execute( "INSERT INTO idempotency_records " "(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) " "VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)", ( sender_id, chat_id, idempotency_key, request_hash, json.dumps(message, ensure_ascii=False, separators=(",", ":")), utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS), ), ) return message def list_messages( self, chat_id: str, user_id: str, limit: int, cursor: Optional[str] ) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]: with self.pool.connection() as conn: is_member = conn.execute( "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id), ).fetchone() if not is_member: return None where = "chat_id = %s" params: List[Any] = [chat_id] if cursor: try: raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4)) decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2) if decoded_chat != chat_id or not valid_uuid(message_id): raise ValueError("cursor scope") created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) where += " AND (created_at, id) > (%s, %s)" params.extend((created_at, message_id)) except (ValueError, UnicodeError, TypeError) as exc: raise ValueError("invalid cursor") from exc params.append(limit + 1) rows = conn.execute( "SELECT id, chat_id, sender_id, text, created_at FROM messages " f"WHERE {where} ORDER BY created_at, id LIMIT %s", params, ).fetchall() ids_by_message = {} if rows: links = conn.execute( "SELECT message_id, attachment_id FROM message_attachments " "WHERE message_id = ANY(%s) ORDER BY message_id, ordinal", ([row["id"] for row in rows],), ).fetchall() for link in links: ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"])) has_more = len(rows) > limit page = rows[:limit] next_cursor = None if has_more and page: last = page[-1] cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor def can_access_chat(self, chat_id: str, user_id: str) -> bool: with self.pool.connection() as conn: return conn.execute( "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id), ).fetchone() is not None def create_attachment( self, chat_id: str, owner_id: str, filename: str, content_type: str, content: BinaryIO, size_bytes: int, digest: str, traceparent: Optional[str] = None, tracestate: Optional[str] = None, ) -> Optional[Dict[str, Any]]: if not self.can_access_chat(chat_id, owner_id): return None attachment_id = uuid.uuid4() job_id = uuid.uuid4() staging_key = f"staging/{attachment_id}" object_key = f"original/{attachment_id}" content.seek(0) self.objects.put(staging_key, content, content_type, size_bytes) try: with self.pool.connection() as conn: with conn.transaction(): row = conn.execute( "INSERT INTO attachments " "(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " "object_key, staging_key, job_id) " "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) " "RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " "status, variants, error_code, created_at, updated_at", ( attachment_id, chat_id, owner_id, filename, content_type, size_bytes, digest, object_key, staging_key, job_id, ), ).fetchone() conn.execute( "INSERT INTO attachment_outbox " "(id, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) " "VALUES (%s, %s, %s, 1, 1, %s, %s)", (uuid.uuid4(), job_id, attachment_id, traceparent, tracestate), ) return self.attachment(row) except Exception: try: self.objects.delete(staging_key) except Exception as cleanup_error: LOGGER.error( "attachment staging compensation failed for %s (%s)", attachment_id, type(cleanup_error).__name__, ) raise def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]: with self.pool.connection() as conn: row = conn.execute( "SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, " "a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at " "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " "WHERE a.id = %s AND m.user_id = %s", (attachment_id, user_id), ).fetchone() return self.attachment(row) if row else None def download_attachment( self, attachment_id: str, user_id: str, variant: str = "original" ) -> Optional[Tuple[str, bytes]]: with self.pool.connection() as conn: row = conn.execute( "SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants " "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " "WHERE a.id = %s AND m.user_id = %s", (attachment_id, user_id), ).fetchone() if not row: return None if row["status"] != "ready": raise AttachmentNotReadyError if variant == "original": key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"] else: match = next( (item for item in (row["variants"] or []) if item["name"] == variant), None, ) if match is None: return None key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"] return content_type, self.objects.get(key, size_bytes) def claim_attachment_job( self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int ) -> Optional[Dict[str, Any]]: with self.pool.connection() as conn: with conn.transaction(): row = conn.execute( "SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, " "object_key, content_type, size_bytes, sha256 " "FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,), ).fetchone() if ( not row or str(row["job_id"]) != job_id or row["pipeline_version"] != pipeline_version or row["status"] in ("ready", "failed") or row["attempt"] != attempt ): return None if row["status"] == "processing" and row["lease_until"] is not None: if row["lease_until"] > utc_now(): return None return conn.execute( "UPDATE attachments SET status = 'processing', " "lease_until = clock_timestamp() + (%s * interval '1 second'), " "updated_at = clock_timestamp() " "WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, " "size_bytes, sha256, attempt, pipeline_version", (lease_seconds, attachment_id), ).fetchone() def complete_attachment_job( self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, variants: List[Dict[str, Any]], ) -> bool: with self.pool.connection() as conn: with conn.transaction(): updated = conn.execute( "UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, " "lease_until = NULL, updated_at = clock_timestamp() " "WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s " "AND status = 'processing' AND lease_until > clock_timestamp()", ( json.dumps(variants, separators=(",", ":")), attachment_id, job_id, pipeline_version, attempt, ), ) return updated.rowcount == 1 def fail_attachment_job( self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, max_attempts: int, error_code: str, retry_delay_seconds: Optional[float], traceparent: Optional[str] = None, tracestate: Optional[str] = None, ) -> None: with self.pool.connection() as conn: with conn.transaction(): row = conn.execute( "SELECT status, attempt FROM attachments " "WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE", (attachment_id, job_id, pipeline_version), ).fetchone() if not row or row["status"] != "processing" or row["attempt"] != attempt: return if retry_delay_seconds is not None and attempt < max_attempts: next_attempt = attempt + 1 conn.execute( "UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, " "error_code = %s, updated_at = clock_timestamp() WHERE id = %s", (next_attempt, error_code, attachment_id), ) conn.execute( "INSERT INTO attachment_outbox " "(id, job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) " "VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second'), %s, %s) " "ON CONFLICT (job_id, attempt) DO NOTHING", ( uuid.uuid4(), job_id, attachment_id, pipeline_version, next_attempt, retry_delay_seconds, traceparent, tracestate, ), ) else: final_code = "retry_exhausted" if attempt >= max_attempts else error_code conn.execute( "UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, " "updated_at = clock_timestamp() WHERE id = %s", (final_code, attachment_id), ) def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]: if pipeline_version not in PIPELINE_QUALITY: raise ValueError("pipeline version is not implemented by this deployment") with self.pool.connection() as conn: current = conn.execute( "SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, " "content_type, size_bytes, sha256 FROM attachments WHERE id = %s", (attachment_id,), ).fetchone() if current is None: raise ValueError("attachment not found") if pipeline_version < current["pipeline_version"]: raise ValueError("pipeline version cannot move backwards") if pipeline_version == current["pipeline_version"] and current["status"] != "failed": raise ValueError("same-version replay is allowed only for failed attachments") if current["status"] not in ("queued", "processing", "ready", "failed"): raise ValueError("attachment cannot be replayed in its current state") try: content = self.objects.get(current["object_key"], current["size_bytes"]) except ObjectMissingError as object_error: try: content = self.objects.get(current["staging_key"], current["size_bytes"]) except ObjectMissingError: raise object_error if ( len(content) != current["size_bytes"] or hashlib.sha256(content).hexdigest() != current["sha256"].strip() ): raise ValueError("stored original does not match attachment metadata") prior_job_id = str(current["job_id"]) next_job_id = uuid.uuid4() staging_key = f"staging/{attachment_id}/{next_job_id}" self.objects.put(staging_key, content, current["content_type"], len(content)) try: with self.pool.connection() as conn: with conn.transaction(): locked = conn.execute( "SELECT job_id, pipeline_version, attempt, status FROM attachments " "WHERE id = %s FOR UPDATE", (attachment_id,), ).fetchone() if ( locked is None or str(locked["job_id"]) != prior_job_id or locked["pipeline_version"] != current["pipeline_version"] or locked["attempt"] != current["attempt"] or locked["status"] != current["status"] ): raise RuntimeError("attachment changed while preparing replay") conn.execute( "UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, " "status = 'queued', staging_key = %s, variants = '[]'::jsonb, " "error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() " "WHERE id = %s", (next_job_id, pipeline_version, staging_key, attachment_id), ) conn.execute( "INSERT INTO attachment_outbox " "(id, job_id, attachment_id, pipeline_version, attempt) " "VALUES (%s, %s, %s, %s, 1)", (uuid.uuid4(), next_job_id, attachment_id, pipeline_version), ) conn.execute( "INSERT INTO attachment_pipeline_events " "(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) " "VALUES (%s, %s, %s, %s, %s, 'replay')", ( attachment_id, prior_job_id, next_job_id, current["pipeline_version"], pipeline_version, ), ) except Exception: try: self.objects.delete(staging_key) except Exception as cleanup_error: LOGGER.error( "replay staging cleanup failed for %s (%s)", attachment_id, type(cleanup_error).__name__, ) raise return { "attachment_id": str(attachment_id), "previous_job_id": prior_job_id, "job_id": str(next_job_id), "pipeline_version": pipeline_version, "attempt": 1, "status": "queued", } def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int: with self.pool.connection() as conn: with conn.transaction(): rows = conn.execute( "SELECT id, job_id, pipeline_version, attempt FROM attachments " "WHERE status = 'processing' AND lease_until < clock_timestamp() " "ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100" ).fetchall() for row in rows: if row["attempt"] >= max_attempts: conn.execute( "UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', " "lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s", (row["id"],), ) continue next_attempt = row["attempt"] + 1 conn.execute( "UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, " "error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s", (next_attempt, row["id"]), ) conn.execute( "INSERT INTO attachment_outbox " "(id, job_id, attachment_id, pipeline_version, attempt) " "VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING", (uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt), ) return len(rows) def claim_outbox(self, conn): return conn.execute( "SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, " "o.traceparent, o.tracestate, " "a.created_at FROM attachment_outbox o " "JOIN attachments a ON a.id = o.attachment_id " "WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() " "ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1" ).fetchone() def queue_snapshot(self) -> Tuple[int, float]: with self.pool.connection() as conn: row = conn.execute( "SELECT count(*) AS depth, " "COALESCE(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS oldest_age " "FROM attachments WHERE status = 'queued'" ).fetchone() return int(row["depth"]), float(row["oldest_age"]) def reconciliation_snapshot(self): with self.pool.connection() as conn: rows = conn.execute( "SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, " "size_bytes, sha256, content_type FROM attachments" ).fetchall() expected = {} for row in rows: if row["status"] == "ready": expected[row["object_key"]] = {**row, "reconcile_type": "original"} for variant in row["variants"] or []: expected[variant["object_key"]] = { **row, "object_key": variant["object_key"], "size_bytes": variant["size_bytes"], "sha256": variant["sha256"], "content_type": variant["content_type"], "reconcile_type": "variant", } elif row["status"] in ("queued", "processing"): expected[row["staging_key"]] = {**row, "reconcile_type": "staging"} expected[row["object_key"]] = {**row, "reconcile_type": "inflight"} for name, extension in ( ("thumbnail", "jpg"), ("crop", "jpg"), ("optimized", "webp"), ): key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}" expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"} return expected def record_missing_object(self, attachment_id: str) -> None: with self.pool.connection() as conn: conn.execute( "INSERT INTO reconciliation_incidents (attachment_id, incident_type) " "VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) " "DO UPDATE SET last_seen_at = clock_timestamp()", (attachment_id,), ) def clear_missing_object(self, attachment_id: str) -> None: with self.pool.connection() as conn: conn.execute( "DELETE FROM reconciliation_incidents " "WHERE attachment_id = %s AND incident_type = 'missing_object'", (attachment_id,), )