Files
mtusi2026lab7/src/store.py
T
kaktuz20000 5a6fcff3ef
Skeleton contracts / contracts (pull_request) Canceled after 0s
выполение на 5
2026-11-04 19:22:35 +03:00

935 lines
42 KiB
Python

import base64
import hashlib
import hmac
import json
import logging
import math
import os
import secrets
import uuid
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
import redis
from psycopg import sql
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from .config import (
DB_LOCK_TIMEOUT_MS,
DB_POOL_MAX_SIZE,
DB_POOL_TIMEOUT_SECONDS,
DB_STATEMENT_TIMEOUT_MS,
IDEMPOTENCY_TTL_SECONDS,
SESSION_TTL_SECONDS,
secret_value,
)
from .errors import (
AttachmentNotReadyError,
CacheUnavailableError,
IdempotencyConflictError,
ObjectMissingError,
)
from .object_store import ObjectStore
from .pipeline_versions import PIPELINE_QUALITY
from .security import (
hash_password,
isoformat,
password_hash_needs_upgrade,
utc_now,
valid_uuid,
verify_password,
)
LOGGER = logging.getLogger("messenger.store")
_DUMMY_PASSWORD_HASH = hash_password(secrets.token_urlsafe(32))
def database_url() -> str:
return secret_value("DATABASE_URL")
class Store:
"""PostgreSQL metadata, S3 objects, and optional Redis session cache."""
def __init__(self):
self.pool = ConnectionPool(
conninfo=database_url(),
min_size=1,
max_size=DB_POOL_MAX_SIZE,
timeout=DB_POOL_TIMEOUT_SECONDS,
kwargs={
"row_factory": dict_row,
"connect_timeout": 5,
"options": (
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
"-c idle_in_transaction_session_timeout=10000"
),
},
open=False,
)
cache_url = os.getenv("CACHE_URL")
self.cache = (
redis.Redis.from_url(
cache_url,
decode_responses=True,
socket_connect_timeout=0.25,
socket_timeout=0.25,
retry_on_timeout=False,
max_connections=16,
)
if cache_url
else None
)
self.objects = ObjectStore()
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
if self.cache is not None:
self.cache.close()
self.pool.close()
def ready(self) -> bool:
with self.pool.connection(timeout=2) as conn:
conn.execute("SELECT 1")
self.objects.ready()
return True
def migrate(self):
migrations = Path(__file__).resolve().parents[1] / "migrations"
with self.pool.connection() as conn:
with conn.transaction():
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
for migration in sorted(migrations.glob("*.sql")):
if migration.name in applied:
continue
conn.execute(migration.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
def configure_worker_role(self, password: str) -> None:
with self.pool.connection() as conn:
with conn.transaction():
exists = conn.execute(
"SELECT 1 FROM pg_roles WHERE rolname = 'messenger_worker'"
).fetchone()
if exists:
statement = sql.SQL("ALTER ROLE messenger_worker LOGIN PASSWORD {}").format(
sql.Literal(password)
)
else:
statement = sql.SQL("CREATE ROLE messenger_worker LOGIN PASSWORD {}").format(
sql.Literal(password)
)
conn.execute(statement)
conn.execute("GRANT CONNECT ON DATABASE messenger TO messenger_worker")
conn.execute("GRANT USAGE ON SCHEMA public TO messenger_worker")
conn.execute(
"GRANT SELECT (id, job_id, pipeline_version, attempt, status, lease_until, "
"staging_key, object_key, content_type, size_bytes, sha256) ON attachments "
"TO messenger_worker"
)
conn.execute(
"GRANT UPDATE (status, lease_until, updated_at, variants, error_code, attempt) "
"ON attachments TO messenger_worker"
)
conn.execute(
"GRANT INSERT (id, job_id, attachment_id, pipeline_version, attempt, available_at, "
"traceparent, tracestate) ON attachment_outbox TO messenger_worker"
)
@staticmethod
def user(row: Dict[str, Any]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": member_ids,
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": isoformat(row["created_at"]),
"attachment_ids": attachment_ids or [],
}
@staticmethod
def attachment(row: Dict[str, Any]) -> Dict[str, Any]:
variants = row.get("variants") or []
public_variants = [
{
name: variant[name]
for name in ("name", "content_type", "size_bytes", "width", "height", "sha256")
}
for variant in variants
]
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"].strip(),
"status": row.get("status", "queued"),
"variants": public_variants,
"error_code": row.get("error_code"),
"created_at": isoformat(row["created_at"]),
"updated_at": isoformat(row["updated_at"]),
}
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO users (username, display_name, password_hash) "
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
(username, display_name, password_hash),
).fetchone()
return self.user(row)
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
).fetchone()
return self.user(row) if row else None
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at, password_hash "
"FROM users WHERE username = %s",
(username,),
).fetchone()
stored_hash = row["password_hash"] if row else _DUMMY_PASSWORD_HASH
if not verify_password(password, stored_hash) or not row:
return None
if password_hash_needs_upgrade(stored_hash):
upgraded_hash = hash_password(password)
with self.pool.connection() as conn:
conn.execute(
"UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s",
(upgraded_hash, row["id"], stored_hash),
)
return self.user(row)
def consume_login_attempt(self, username: str, source_ip: str) -> Optional[int]:
user_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest()
ip_key = hashlib.sha256(b"ip\0" + source_ip.encode("utf-8")).digest()
with self.pool.connection() as conn:
with conn.transaction():
def bump(key: bytes) -> Dict[str, Any]:
return conn.execute(
"""
INSERT INTO login_rate_limits (username_key, window_started_at, attempts)
VALUES (%s, clock_timestamp(), 1)
ON CONFLICT (username_key) DO UPDATE SET
attempts = CASE
WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute'
THEN 1 ELSE login_rate_limits.attempts + 1
END,
window_started_at = CASE
WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute'
THEN clock_timestamp() ELSE login_rate_limits.window_started_at
END
RETURNING attempts,
GREATEST(1, ceil(extract(epoch FROM
(window_started_at + interval '1 minute' - clock_timestamp()))))::integer AS retry_after
""",
(key,),
).fetchone()
user_row = bump(user_key)
ip_row = bump(ip_key)
conn.execute(
"DELETE FROM login_rate_limits WHERE window_started_at < clock_timestamp() - interval '1 hour'"
)
retries = []
if user_row["attempts"] > 5:
retries.append(user_row["retry_after"])
if ip_row["attempts"] > 60:
retries.append(ip_row["retry_after"])
return max(retries) if retries else None
def clear_login_attempts(self, username: str) -> None:
username_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest()
with self.pool.connection() as conn:
conn.execute("DELETE FROM login_rate_limits WHERE username_key = %s", (username_key,))
def create_session(self, user_id: str) -> Tuple[str, datetime]:
token = secrets.token_urlsafe(32)
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
token_hash = hashlib.sha256(token.encode("ascii")).digest()
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash, user_id, expires_at),
)
return token, expires_at
@staticmethod
def _session_lock_key(token_hash: bytes) -> int:
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
cache_key = "messenger:session:" + token_hash.hex()
if self.cache is not None:
try:
cached = self.cache.get(cache_key)
if cached:
return json.loads(cached)
except (redis.RedisError, ValueError, TypeError):
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
row = conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
"FROM sessions s JOIN users u ON u.id = s.user_id "
"WHERE s.token_hash = %s AND s.expires_at > now()",
(token_hash,),
).fetchone()
if not row:
return None
user = self.user(row)
if self.cache is not None:
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
if ttl_ms > 0:
try:
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
except redis.RedisError:
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
return user
def revoke_session(self, token: str) -> bool:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
if self.cache is not None:
try:
self.cache.delete("messenger:session:" + token_hash.hex())
except redis.RedisError as exc:
raise CacheUnavailableError from exc
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
return result.rowcount == 1
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
raise ValueError("invalid members")
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
(title,),
).fetchone()
for member_id in member_ids:
conn.execute(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
(row["id"], member_id),
)
return self.chat(row, member_ids)
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id "
"WHERE c.id = %s AND m.user_id = %s",
(chat_id, user_id),
).fetchone()
if not row:
return None
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(chat_id,),
).fetchall()
return self.chat(row, [str(item["user_id"]) for item in members])
def list_chats(
self, user_id: str, limit: int, cursor: Optional[str]
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
where = "own.user_id = %s"
params: List[Any] = [user_id]
if cursor is not None:
try:
raw = base64.b64decode(
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
altchars=b"-_",
validate=True,
)
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
if cursor_user != user_id or not valid_uuid(chat_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
if created_at.tzinfo is None or created_at.utcoffset() is None:
raise ValueError("cursor timestamp must include a timezone")
where += " AND (c.created_at, c.id) > (%s, %s)"
params.extend((created_at, chat_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members own ON own.chat_id = c.id "
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
params,
).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
result = []
for row in page:
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(row["id"],),
).fetchall()
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return result, next_cursor
def create_idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
attachment_ids: List[str],
idempotency_key: str,
canonical_body: str,
) -> Optional[Dict[str, Any]]:
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest()
lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8")
lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True)
with self.pool.connection() as conn:
with conn.transaction():
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
).fetchone()
if not member:
return None
if attachment_ids:
attachments = conn.execute(
"SELECT id, status FROM attachments "
"WHERE chat_id = %s AND id = ANY(%s) FOR SHARE",
(chat_id, attachment_ids),
).fetchall()
if len(attachments) != len(attachment_ids):
return None
if any(item["status"] == "failed" for item in attachments):
raise AttachmentNotReadyError
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
existing = conn.execute(
"SELECT request_hash, response, expires_at "
"FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s "
"FOR UPDATE",
(sender_id, chat_id, idempotency_key),
).fetchone()
if existing and existing["expires_at"] > utc_now():
if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash):
raise IdempotencyConflictError
return existing["response"]
if existing:
conn.execute(
"DELETE FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s",
(sender_id, chat_id, idempotency_key),
)
row = conn.execute(
"INSERT INTO messages (chat_id, sender_id, text) "
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
(chat_id, sender_id, text),
).fetchone()
for ordinal, attachment_id in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, ordinal) "
"VALUES (%s, %s, %s)",
(row["id"], attachment_id, ordinal),
)
message = self.message(row, attachment_ids)
conn.execute(
"INSERT INTO idempotency_records "
"(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) "
"VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)",
(
sender_id,
chat_id,
idempotency_key,
request_hash,
json.dumps(message, ensure_ascii=False, separators=(",", ":")),
utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS),
),
)
return message
def list_messages(
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
with self.pool.connection() as conn:
is_member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
if not is_member:
return None
where = "chat_id = %s"
params: List[Any] = [chat_id]
if cursor:
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
if decoded_chat != chat_id or not valid_uuid(message_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
where += " AND (created_at, id) > (%s, %s)"
params.extend((created_at, message_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
rows = conn.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
params,
).fetchall()
ids_by_message = {}
if rows:
links = conn.execute(
"SELECT message_id, attachment_id FROM message_attachments "
"WHERE message_id = ANY(%s) ORDER BY message_id, ordinal",
([row["id"] for row in rows],),
).fetchall()
for link in links:
ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"]))
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor
def can_access_chat(self, chat_id: str, user_id: str) -> bool:
with self.pool.connection() as conn:
return conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone() is not None
def create_attachment(
self,
chat_id: str,
owner_id: str,
filename: str,
content_type: str,
content: BinaryIO,
size_bytes: int,
digest: str,
traceparent: Optional[str] = None,
tracestate: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
if not self.can_access_chat(chat_id, owner_id):
return None
attachment_id = uuid.uuid4()
job_id = uuid.uuid4()
staging_key = f"staging/{attachment_id}"
object_key = f"original/{attachment_id}"
content.seek(0)
self.objects.put(staging_key, content, content_type, size_bytes)
try:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO attachments "
"(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"object_key, staging_key, job_id) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, variants, error_code, created_at, updated_at",
(
attachment_id,
chat_id,
owner_id,
filename,
content_type,
size_bytes,
digest,
object_key,
staging_key,
job_id,
),
).fetchone()
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
"VALUES (%s, %s, %s, 1, 1, %s, %s)",
(uuid.uuid4(), job_id, attachment_id, traceparent, tracestate),
)
return self.attachment(row)
except Exception:
try:
self.objects.delete(staging_key)
except Exception as cleanup_error:
LOGGER.error(
"attachment staging compensation failed for %s (%s)",
attachment_id,
type(cleanup_error).__name__,
)
raise
def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, "
"a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at "
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
"WHERE a.id = %s AND m.user_id = %s",
(attachment_id, user_id),
).fetchone()
return self.attachment(row) if row else None
def download_attachment(
self, attachment_id: str, user_id: str, variant: str = "original"
) -> Optional[Tuple[str, bytes]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants "
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
"WHERE a.id = %s AND m.user_id = %s",
(attachment_id, user_id),
).fetchone()
if not row:
return None
if row["status"] != "ready":
raise AttachmentNotReadyError
if variant == "original":
key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"]
else:
match = next(
(item for item in (row["variants"] or []) if item["name"] == variant),
None,
)
if match is None:
return None
key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"]
return content_type, self.objects.get(key, size_bytes)
def claim_attachment_job(
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int
) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, "
"object_key, content_type, size_bytes, sha256 "
"FROM attachments WHERE id = %s FOR UPDATE",
(attachment_id,),
).fetchone()
if (
not row
or str(row["job_id"]) != job_id
or row["pipeline_version"] != pipeline_version
or row["status"] in ("ready", "failed")
or row["attempt"] != attempt
):
return None
if row["status"] == "processing" and row["lease_until"] is not None:
if row["lease_until"] > utc_now():
return None
return conn.execute(
"UPDATE attachments SET status = 'processing', "
"lease_until = clock_timestamp() + (%s * interval '1 second'), "
"updated_at = clock_timestamp() "
"WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, "
"size_bytes, sha256, attempt, pipeline_version",
(lease_seconds, attachment_id),
).fetchone()
def complete_attachment_job(
self,
attachment_id: str,
job_id: str,
pipeline_version: int,
attempt: int,
variants: List[Dict[str, Any]],
) -> bool:
with self.pool.connection() as conn:
with conn.transaction():
updated = conn.execute(
"UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, "
"lease_until = NULL, updated_at = clock_timestamp() "
"WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s "
"AND status = 'processing' AND lease_until > clock_timestamp()",
(
json.dumps(variants, separators=(",", ":")),
attachment_id,
job_id,
pipeline_version,
attempt,
),
)
return updated.rowcount == 1
def fail_attachment_job(
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int,
max_attempts: int, error_code: str, retry_delay_seconds: Optional[float],
traceparent: Optional[str] = None, tracestate: Optional[str] = None,
) -> None:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"SELECT status, attempt FROM attachments "
"WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE",
(attachment_id, job_id, pipeline_version),
).fetchone()
if not row or row["status"] != "processing" or row["attempt"] != attempt:
return
if retry_delay_seconds is not None and attempt < max_attempts:
next_attempt = attempt + 1
conn.execute(
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
"error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
(next_attempt, error_code, attachment_id),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) "
"VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second'), %s, %s) "
"ON CONFLICT (job_id, attempt) DO NOTHING",
(
uuid.uuid4(), job_id, attachment_id, pipeline_version,
next_attempt, retry_delay_seconds, traceparent, tracestate,
),
)
else:
final_code = "retry_exhausted" if attempt >= max_attempts else error_code
conn.execute(
"UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, "
"updated_at = clock_timestamp() WHERE id = %s",
(final_code, attachment_id),
)
def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]:
if pipeline_version not in PIPELINE_QUALITY:
raise ValueError("pipeline version is not implemented by this deployment")
with self.pool.connection() as conn:
current = conn.execute(
"SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, "
"content_type, size_bytes, sha256 FROM attachments WHERE id = %s",
(attachment_id,),
).fetchone()
if current is None:
raise ValueError("attachment not found")
if pipeline_version < current["pipeline_version"]:
raise ValueError("pipeline version cannot move backwards")
if pipeline_version == current["pipeline_version"] and current["status"] != "failed":
raise ValueError("same-version replay is allowed only for failed attachments")
if current["status"] not in ("queued", "processing", "ready", "failed"):
raise ValueError("attachment cannot be replayed in its current state")
try:
content = self.objects.get(current["object_key"], current["size_bytes"])
except ObjectMissingError as object_error:
try:
content = self.objects.get(current["staging_key"], current["size_bytes"])
except ObjectMissingError:
raise object_error
if (
len(content) != current["size_bytes"]
or hashlib.sha256(content).hexdigest() != current["sha256"].strip()
):
raise ValueError("stored original does not match attachment metadata")
prior_job_id = str(current["job_id"])
next_job_id = uuid.uuid4()
staging_key = f"staging/{attachment_id}/{next_job_id}"
self.objects.put(staging_key, content, current["content_type"], len(content))
try:
with self.pool.connection() as conn:
with conn.transaction():
locked = conn.execute(
"SELECT job_id, pipeline_version, attempt, status FROM attachments "
"WHERE id = %s FOR UPDATE",
(attachment_id,),
).fetchone()
if (
locked is None
or str(locked["job_id"]) != prior_job_id
or locked["pipeline_version"] != current["pipeline_version"]
or locked["attempt"] != current["attempt"]
or locked["status"] != current["status"]
):
raise RuntimeError("attachment changed while preparing replay")
conn.execute(
"UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, "
"status = 'queued', staging_key = %s, variants = '[]'::jsonb, "
"error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() "
"WHERE id = %s",
(next_job_id, pipeline_version, staging_key, attachment_id),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt) "
"VALUES (%s, %s, %s, %s, 1)",
(uuid.uuid4(), next_job_id, attachment_id, pipeline_version),
)
conn.execute(
"INSERT INTO attachment_pipeline_events "
"(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) "
"VALUES (%s, %s, %s, %s, %s, 'replay')",
(
attachment_id,
prior_job_id,
next_job_id,
current["pipeline_version"],
pipeline_version,
),
)
except Exception:
try:
self.objects.delete(staging_key)
except Exception as cleanup_error:
LOGGER.error(
"replay staging cleanup failed for %s (%s)",
attachment_id,
type(cleanup_error).__name__,
)
raise
return {
"attachment_id": str(attachment_id),
"previous_job_id": prior_job_id,
"job_id": str(next_job_id),
"pipeline_version": pipeline_version,
"attempt": 1,
"status": "queued",
}
def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int:
with self.pool.connection() as conn:
with conn.transaction():
rows = conn.execute(
"SELECT id, job_id, pipeline_version, attempt FROM attachments "
"WHERE status = 'processing' AND lease_until < clock_timestamp() "
"ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100"
).fetchall()
for row in rows:
if row["attempt"] >= max_attempts:
conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', "
"lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s",
(row["id"],),
)
continue
next_attempt = row["attempt"] + 1
conn.execute(
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
"error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s",
(next_attempt, row["id"]),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt) "
"VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING",
(uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt),
)
return len(rows)
def claim_outbox(self, conn):
return conn.execute(
"SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, "
"o.traceparent, o.tracestate, "
"a.created_at FROM attachment_outbox o "
"JOIN attachments a ON a.id = o.attachment_id "
"WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() "
"ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1"
).fetchone()
def queue_snapshot(self) -> Tuple[int, float]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT count(*) AS depth, "
"COALESCE(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS oldest_age "
"FROM attachments WHERE status = 'queued'"
).fetchone()
return int(row["depth"]), float(row["oldest_age"])
def reconciliation_snapshot(self):
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, "
"size_bytes, sha256, content_type FROM attachments"
).fetchall()
expected = {}
for row in rows:
if row["status"] == "ready":
expected[row["object_key"]] = {**row, "reconcile_type": "original"}
for variant in row["variants"] or []:
expected[variant["object_key"]] = {
**row,
"object_key": variant["object_key"],
"size_bytes": variant["size_bytes"],
"sha256": variant["sha256"],
"content_type": variant["content_type"],
"reconcile_type": "variant",
}
elif row["status"] in ("queued", "processing"):
expected[row["staging_key"]] = {**row, "reconcile_type": "staging"}
expected[row["object_key"]] = {**row, "reconcile_type": "inflight"}
for name, extension in (
("thumbnail", "jpg"),
("crop", "jpg"),
("optimized", "webp"),
):
key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}"
expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"}
return expected
def record_missing_object(self, attachment_id: str) -> None:
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO reconciliation_incidents (attachment_id, incident_type) "
"VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) "
"DO UPDATE SET last_seen_at = clock_timestamp()",
(attachment_id,),
)
def clear_missing_object(self, attachment_id: str) -> None:
with self.pool.connection() as conn:
conn.execute(
"DELETE FROM reconciliation_incidents "
"WHERE attachment_id = %s AND incident_type = 'missing_object'",
(attachment_id,),
)