Files
mtusi2026lab7/src/messenger/pipeline.py
T

267 lines
12 KiB
Python

"""Job state machine. PostgreSQL fences publication; the stream is at least once."""
import os
import random
from datetime import UTC, datetime, timedelta
from psycopg.types.json import Jsonb
from messenger import telemetry
from messenger.domain import DomainError, new_id
def event(conn, job_id, name, **details):
conn.execute(
"INSERT INTO pipeline_audit(job_id,event,details) VALUES (%s,%s,%s)",
(job_id, name, Jsonb(details)),
)
def intent(conn, job_id, when=None):
conn.execute(
"INSERT INTO job_outbox(job_id,due_at) VALUES (%s,coalesce(%s,clock_timestamp())) "
"ON CONFLICT(job_id) DO UPDATE SET due_at=EXCLUDED.due_at,published_at=NULL,stream_id=NULL",
(job_id, when),
)
class Pipeline:
def __init__(self, repository, settings):
self.db, self.settings = repository, settings
def reserve(self, id, key):
with self.db.connection(write=True) as conn:
conn.execute("INSERT INTO upload_reservations(id,object_key) VALUES (%s,%s)", (id, key))
def accept(self, id, chat, owner, name, media, size, digest, key):
with self.db.connection(write=True) as conn:
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (id,)
).fetchone()
if (
not reservation
or reservation["state"] != "uploading"
or reservation["object_key"] != key
or reservation["expires_at"] <= datetime.now(UTC)
):
raise DomainError(503, "unavailable", "Срок приёма файла истёк")
conn.execute("UPDATE upload_reservations SET state='accepted' WHERE id=%s", (id,))
row = conn.execute(
"INSERT INTO attachments "
"(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
"object_key,storage_state) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
(id, chat, owner, name, media, size, digest, key),
).fetchone()
job = new_id()
trace = telemetry.carrier()
conn.execute(
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key,"
"traceparent,tracestate) VALUES (%s,%s,1,%s,%s,%s)",
(job, id, key, trace.get("traceparent"), trace.get("tracestate")),
)
intent(conn, job)
event(conn, job, "accepted")
return row
def claim(self, envelope):
with self.db.connection(write=True) as conn:
# All writers lock attachment before job, including publication and operator replay.
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (envelope.attachment_id,)
).fetchone()
if not a:
return "unknown", None
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE",
(envelope.job_id,),
).fetchone()
if (
not j
or j["attachment_id"] != a["id"]
or j["pipeline_version"] != envelope.pipeline_version
):
return "unknown", None
if a["pipeline_version"] != j["pipeline_version"] or j["state"] == "superseded":
event(conn, j["job_id"], "stale_delivery")
return "stale", None
if j["state"] == "ready":
event(conn, j["job_id"], "duplicate_ready")
return "ready", a
if j["state"] == "failed":
return "failed", None
now = datetime.now(UTC)
if j["state"] == "processing" or j["next_attempt_at"] > now:
return "busy", None
if envelope.attempt != j["attempt"] + 1:
# An older envelope cannot steal a scheduled retry or an explicit replay.
return "obsolete_attempt", None
fence = new_id()
conn.execute(
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,fence=%s,"
"lease_until=clock_timestamp()+(%s*interval '1 "
"second'),updated_at=clock_timestamp() "
"WHERE job_id=%s",
(fence, self.settings.lease_seconds, j["job_id"]),
)
conn.execute(
"UPDATE attachments SET storage_state='processing',"
"updated_at=clock_timestamp() WHERE id=%s",
(a["id"],),
)
event(
conn, j["job_id"], "claimed", attempt=j["attempt"] + 1, fence=fence, pid=os.getpid()
)
return "claimed", {
**a,
"job_id": str(j["job_id"]),
"source_key": j["source_key"],
"fence": fence,
"attempt": j["attempt"] + 1,
"accepted_at": j["created_at"],
}
def publish(self, row, original_key, variants):
with self.db.connection(write=True) as conn:
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
).fetchone()
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
).fetchone()
if not self.owns(a, j, row):
event(conn, row["job_id"], "publication_fenced")
return False
conn.execute(
"UPDATE attachments SET storage_state='ready',object_key=%s,variants=%s,"
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(original_key, Jsonb(variants), row["id"]),
)
conn.execute(
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,"
"updated_at=clock_timestamp() WHERE job_id=%s",
(row["job_id"],),
)
event(conn, row["job_id"], "result_committed", version=row["pipeline_version"])
return True
@staticmethod
def owns(a, j, row):
return (
a
and j
and a["pipeline_version"] == row["pipeline_version"]
and j["state"] == "processing"
and str(j["fence"]) == row["fence"]
and j["lease_until"] > datetime.now(UTC)
)
def failure(self, row, code, permanent=False):
with self.db.connection(write=True) as conn:
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
).fetchone()
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
).fetchone()
if self.owns(a, j, row):
return self._retry(conn, j, code, permanent)
return "fenced"
def _retry(self, conn, j, code, permanent=False):
failed = permanent or j["attempt"] >= self.settings.job_max_attempts
state = "failed" if failed else "queued"
if failed and not permanent:
code = "retry_exhausted"
# Full jitter; durable absolute time survives worker/relay restarts.
delay = random.uniform(0.25, min(30, 2 ** min(j["attempt"], 5)))
due = datetime.now(UTC) + timedelta(seconds=delay)
conn.execute(
"UPDATE attachment_jobs SET state=%s,error_code=%s,next_attempt_at=%s,lease_until=NULL,"
"updated_at=clock_timestamp() WHERE job_id=%s",
(state, code, due, j["job_id"]),
)
conn.execute(
"UPDATE attachments SET storage_state=%s,error_code=%s,updated_at=clock_timestamp() "
"WHERE id=%s",
(state, code if failed else None, j["attachment_id"]),
)
event(conn, j["job_id"], state, reason=code, attempt=j["attempt"], delay_seconds=delay)
if not failed:
intent(conn, j["job_id"], due)
return "failed" if failed else "retry"
def recover(self):
# Bounded SKIP LOCKED batch; multiple relays. No transaction held during Pillow.
with self.db.connection(write=True) as conn:
rows = conn.execute(
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
"AND j.pipeline_version=a.pipeline_version WHERE j.state='processing' "
"AND j.lease_until<clock_timestamp() ORDER BY a.id LIMIT 32 "
"FOR UPDATE OF a SKIP LOCKED"
).fetchall()
for row in rows:
j = conn.execute(
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"AND a.pipeline_version=j.pipeline_version WHERE a.id=%s FOR NO KEY "
"UPDATE OF j",
(row["id"],),
).fetchone()
if j["state"] == "processing" and j["lease_until"] < datetime.now(UTC):
event(conn, j["job_id"], "lease_expired")
self._retry(conn, j, "worker_timeout")
def replay(self, attachment, *, new_version=False):
with self.db.connection(write=True) as conn:
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment,)
).fetchone()
if not a:
raise ValueError("Unknown attachment")
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s "
"AND pipeline_version=%s FOR UPDATE",
(attachment, a["pipeline_version"]),
).fetchone()
if new_version:
if a["pipeline_version"] != 1:
raise ValueError("Only pipeline v1 -> v2 is implemented")
conn.execute(
"UPDATE attachment_jobs SET state='superseded' WHERE job_id=%s "
"AND state IN ('queued','processing','failed')",
(j["job_id"],),
)
key = a["object_key"] if a["storage_state"] == "ready" else j["source_key"]
job = new_id()
conn.execute(
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key) "
"VALUES (%s,%s,2,%s)",
(job, attachment, key),
)
conn.execute(
"UPDATE attachments SET "
"pipeline_version=2,storage_state='queued',variants='[]',"
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(attachment,),
)
event(conn, job, "new_version", previous_job=str(j["job_id"]))
else:
if j["state"] != "failed" or j["source_expired"]:
raise ValueError("Replay requires failed job with retained source")
job = str(j["job_id"])
conn.execute(
"UPDATE attachment_jobs SET "
"state='queued',attempt=0,fence=NULL,error_code=NULL,"
"next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE "
"job_id=%s",
(job,),
)
conn.execute(
"UPDATE attachments SET storage_state='queued',error_code=NULL,"
"updated_at=clock_timestamp() WHERE id=%s",
(attachment,),
)
event(conn, job, "operator_replay")
intent(conn, job)
return job