forked from mtusi2026/mtusi2026lab7
267 lines
12 KiB
Python
267 lines
12 KiB
Python
"""Job state machine. PostgreSQL fences publication; the stream is at least once."""
|
|
|
|
import os
|
|
import random
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from psycopg.types.json import Jsonb
|
|
|
|
from messenger import telemetry
|
|
from messenger.domain import DomainError, new_id
|
|
|
|
|
|
def event(conn, job_id, name, **details):
|
|
conn.execute(
|
|
"INSERT INTO pipeline_audit(job_id,event,details) VALUES (%s,%s,%s)",
|
|
(job_id, name, Jsonb(details)),
|
|
)
|
|
|
|
|
|
def intent(conn, job_id, when=None):
|
|
conn.execute(
|
|
"INSERT INTO job_outbox(job_id,due_at) VALUES (%s,coalesce(%s,clock_timestamp())) "
|
|
"ON CONFLICT(job_id) DO UPDATE SET due_at=EXCLUDED.due_at,published_at=NULL,stream_id=NULL",
|
|
(job_id, when),
|
|
)
|
|
|
|
|
|
class Pipeline:
|
|
def __init__(self, repository, settings):
|
|
self.db, self.settings = repository, settings
|
|
|
|
def reserve(self, id, key):
|
|
with self.db.connection(write=True) as conn:
|
|
conn.execute("INSERT INTO upload_reservations(id,object_key) VALUES (%s,%s)", (id, key))
|
|
|
|
def accept(self, id, chat, owner, name, media, size, digest, key):
|
|
with self.db.connection(write=True) as conn:
|
|
reservation = conn.execute(
|
|
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (id,)
|
|
).fetchone()
|
|
if (
|
|
not reservation
|
|
or reservation["state"] != "uploading"
|
|
or reservation["object_key"] != key
|
|
or reservation["expires_at"] <= datetime.now(UTC)
|
|
):
|
|
raise DomainError(503, "unavailable", "Срок приёма файла истёк")
|
|
conn.execute("UPDATE upload_reservations SET state='accepted' WHERE id=%s", (id,))
|
|
row = conn.execute(
|
|
"INSERT INTO attachments "
|
|
"(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
|
|
"object_key,storage_state) "
|
|
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
|
|
(id, chat, owner, name, media, size, digest, key),
|
|
).fetchone()
|
|
job = new_id()
|
|
trace = telemetry.carrier()
|
|
conn.execute(
|
|
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key,"
|
|
"traceparent,tracestate) VALUES (%s,%s,1,%s,%s,%s)",
|
|
(job, id, key, trace.get("traceparent"), trace.get("tracestate")),
|
|
)
|
|
intent(conn, job)
|
|
event(conn, job, "accepted")
|
|
return row
|
|
|
|
def claim(self, envelope):
|
|
with self.db.connection(write=True) as conn:
|
|
# All writers lock attachment before job, including publication and operator replay.
|
|
a = conn.execute(
|
|
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (envelope.attachment_id,)
|
|
).fetchone()
|
|
if not a:
|
|
return "unknown", None
|
|
j = conn.execute(
|
|
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE",
|
|
(envelope.job_id,),
|
|
).fetchone()
|
|
if (
|
|
not j
|
|
or j["attachment_id"] != a["id"]
|
|
or j["pipeline_version"] != envelope.pipeline_version
|
|
):
|
|
return "unknown", None
|
|
if a["pipeline_version"] != j["pipeline_version"] or j["state"] == "superseded":
|
|
event(conn, j["job_id"], "stale_delivery")
|
|
return "stale", None
|
|
if j["state"] == "ready":
|
|
event(conn, j["job_id"], "duplicate_ready")
|
|
return "ready", a
|
|
if j["state"] == "failed":
|
|
return "failed", None
|
|
now = datetime.now(UTC)
|
|
if j["state"] == "processing" or j["next_attempt_at"] > now:
|
|
return "busy", None
|
|
if envelope.attempt != j["attempt"] + 1:
|
|
# An older envelope cannot steal a scheduled retry or an explicit replay.
|
|
return "obsolete_attempt", None
|
|
fence = new_id()
|
|
conn.execute(
|
|
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,fence=%s,"
|
|
"lease_until=clock_timestamp()+(%s*interval '1 "
|
|
"second'),updated_at=clock_timestamp() "
|
|
"WHERE job_id=%s",
|
|
(fence, self.settings.lease_seconds, j["job_id"]),
|
|
)
|
|
conn.execute(
|
|
"UPDATE attachments SET storage_state='processing',"
|
|
"updated_at=clock_timestamp() WHERE id=%s",
|
|
(a["id"],),
|
|
)
|
|
event(
|
|
conn, j["job_id"], "claimed", attempt=j["attempt"] + 1, fence=fence, pid=os.getpid()
|
|
)
|
|
return "claimed", {
|
|
**a,
|
|
"job_id": str(j["job_id"]),
|
|
"source_key": j["source_key"],
|
|
"fence": fence,
|
|
"attempt": j["attempt"] + 1,
|
|
"accepted_at": j["created_at"],
|
|
}
|
|
|
|
def publish(self, row, original_key, variants):
|
|
with self.db.connection(write=True) as conn:
|
|
a = conn.execute(
|
|
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
|
).fetchone()
|
|
j = conn.execute(
|
|
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
|
|
).fetchone()
|
|
if not self.owns(a, j, row):
|
|
event(conn, row["job_id"], "publication_fenced")
|
|
return False
|
|
conn.execute(
|
|
"UPDATE attachments SET storage_state='ready',object_key=%s,variants=%s,"
|
|
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
|
(original_key, Jsonb(variants), row["id"]),
|
|
)
|
|
conn.execute(
|
|
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,"
|
|
"updated_at=clock_timestamp() WHERE job_id=%s",
|
|
(row["job_id"],),
|
|
)
|
|
event(conn, row["job_id"], "result_committed", version=row["pipeline_version"])
|
|
return True
|
|
|
|
@staticmethod
|
|
def owns(a, j, row):
|
|
return (
|
|
a
|
|
and j
|
|
and a["pipeline_version"] == row["pipeline_version"]
|
|
and j["state"] == "processing"
|
|
and str(j["fence"]) == row["fence"]
|
|
and j["lease_until"] > datetime.now(UTC)
|
|
)
|
|
|
|
def failure(self, row, code, permanent=False):
|
|
with self.db.connection(write=True) as conn:
|
|
a = conn.execute(
|
|
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
|
).fetchone()
|
|
j = conn.execute(
|
|
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
|
|
).fetchone()
|
|
if self.owns(a, j, row):
|
|
return self._retry(conn, j, code, permanent)
|
|
return "fenced"
|
|
|
|
def _retry(self, conn, j, code, permanent=False):
|
|
failed = permanent or j["attempt"] >= self.settings.job_max_attempts
|
|
state = "failed" if failed else "queued"
|
|
if failed and not permanent:
|
|
code = "retry_exhausted"
|
|
# Full jitter; durable absolute time survives worker/relay restarts.
|
|
delay = random.uniform(0.25, min(30, 2 ** min(j["attempt"], 5)))
|
|
due = datetime.now(UTC) + timedelta(seconds=delay)
|
|
conn.execute(
|
|
"UPDATE attachment_jobs SET state=%s,error_code=%s,next_attempt_at=%s,lease_until=NULL,"
|
|
"updated_at=clock_timestamp() WHERE job_id=%s",
|
|
(state, code, due, j["job_id"]),
|
|
)
|
|
conn.execute(
|
|
"UPDATE attachments SET storage_state=%s,error_code=%s,updated_at=clock_timestamp() "
|
|
"WHERE id=%s",
|
|
(state, code if failed else None, j["attachment_id"]),
|
|
)
|
|
event(conn, j["job_id"], state, reason=code, attempt=j["attempt"], delay_seconds=delay)
|
|
if not failed:
|
|
intent(conn, j["job_id"], due)
|
|
return "failed" if failed else "retry"
|
|
|
|
def recover(self):
|
|
# Bounded SKIP LOCKED batch; multiple relays. No transaction held during Pillow.
|
|
with self.db.connection(write=True) as conn:
|
|
rows = conn.execute(
|
|
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
|
|
"AND j.pipeline_version=a.pipeline_version WHERE j.state='processing' "
|
|
"AND j.lease_until<clock_timestamp() ORDER BY a.id LIMIT 32 "
|
|
"FOR UPDATE OF a SKIP LOCKED"
|
|
).fetchall()
|
|
for row in rows:
|
|
j = conn.execute(
|
|
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
|
"AND a.pipeline_version=j.pipeline_version WHERE a.id=%s FOR NO KEY "
|
|
"UPDATE OF j",
|
|
(row["id"],),
|
|
).fetchone()
|
|
if j["state"] == "processing" and j["lease_until"] < datetime.now(UTC):
|
|
event(conn, j["job_id"], "lease_expired")
|
|
self._retry(conn, j, "worker_timeout")
|
|
|
|
def replay(self, attachment, *, new_version=False):
|
|
with self.db.connection(write=True) as conn:
|
|
a = conn.execute(
|
|
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment,)
|
|
).fetchone()
|
|
if not a:
|
|
raise ValueError("Unknown attachment")
|
|
j = conn.execute(
|
|
"SELECT * FROM attachment_jobs WHERE attachment_id=%s "
|
|
"AND pipeline_version=%s FOR UPDATE",
|
|
(attachment, a["pipeline_version"]),
|
|
).fetchone()
|
|
if new_version:
|
|
if a["pipeline_version"] != 1:
|
|
raise ValueError("Only pipeline v1 -> v2 is implemented")
|
|
conn.execute(
|
|
"UPDATE attachment_jobs SET state='superseded' WHERE job_id=%s "
|
|
"AND state IN ('queued','processing','failed')",
|
|
(j["job_id"],),
|
|
)
|
|
key = a["object_key"] if a["storage_state"] == "ready" else j["source_key"]
|
|
job = new_id()
|
|
conn.execute(
|
|
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key) "
|
|
"VALUES (%s,%s,2,%s)",
|
|
(job, attachment, key),
|
|
)
|
|
conn.execute(
|
|
"UPDATE attachments SET "
|
|
"pipeline_version=2,storage_state='queued',variants='[]',"
|
|
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
|
(attachment,),
|
|
)
|
|
event(conn, job, "new_version", previous_job=str(j["job_id"]))
|
|
else:
|
|
if j["state"] != "failed" or j["source_expired"]:
|
|
raise ValueError("Replay requires failed job with retained source")
|
|
job = str(j["job_id"])
|
|
conn.execute(
|
|
"UPDATE attachment_jobs SET "
|
|
"state='queued',attempt=0,fence=NULL,error_code=NULL,"
|
|
"next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE "
|
|
"job_id=%s",
|
|
(job,),
|
|
)
|
|
conn.execute(
|
|
"UPDATE attachments SET storage_state='queued',error_code=NULL,"
|
|
"updated_at=clock_timestamp() WHERE id=%s",
|
|
(attachment,),
|
|
)
|
|
event(conn, job, "operator_replay")
|
|
intent(conn, job)
|
|
return job
|