feat(pipeline): add durable outbox and versioned image workers
This commit is contained in:
+123
@@ -0,0 +1,123 @@
|
||||
"""Explicit local operator actions. There is deliberately no HTTP replay endpoint."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from src.object_store import ObjectStore
|
||||
from src.pipeline import database, enqueue, event
|
||||
|
||||
|
||||
def change(db, store, *, job_id, operator, reason, version=None):
|
||||
if (
|
||||
not operator.strip()
|
||||
or not reason.strip()
|
||||
or len(operator) > 100
|
||||
or len(reason) > 500
|
||||
):
|
||||
raise ValueError("Operator and reason are required")
|
||||
with db.connection() as conn, conn.transaction():
|
||||
job = conn.execute(
|
||||
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
|
||||
).fetchone()
|
||||
if not job:
|
||||
raise ValueError("Unknown job")
|
||||
attachment = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE",
|
||||
(job["attachment_id"],),
|
||||
).fetchone()
|
||||
if job["pipeline_version"] != attachment["pipeline_version"]:
|
||||
raise ValueError("Superseded job cannot be replayed")
|
||||
source = (
|
||||
job["source_key"]
|
||||
if attachment["published_version"] is None
|
||||
else attachment["object_key"]
|
||||
)
|
||||
if store.head(key=source) is None:
|
||||
raise ValueError("Source expired or missing; replay refused")
|
||||
if version is None:
|
||||
if job["state"] != "failed":
|
||||
raise ValueError("Only failed jobs can be replayed")
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state='queued',attempt=0,cycle=cycle+1,fence=fence+1,error_code=NULL,next_at=clock_timestamp(),lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(job_id,),
|
||||
)
|
||||
action = "replay"
|
||||
target = job_id
|
||||
else:
|
||||
if (
|
||||
version not in {1, 2}
|
||||
or version <= attachment["pipeline_version"]
|
||||
):
|
||||
raise ValueError("Supported version must increase (1 or 2)")
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state='obsolete',fence=fence+1,lease_owner=NULL,lease_until=NULL WHERE id=%s",
|
||||
(job_id,),
|
||||
)
|
||||
target = uuid4()
|
||||
conn.execute(
|
||||
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,%s,%s,'queued')",
|
||||
(target, attachment["id"], version, source),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET pipeline_version=%s WHERE id=%s",
|
||||
(version, attachment["id"]),
|
||||
)
|
||||
action = "new_version"
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(attachment["id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_audit(job_id,action,operator,reason) VALUES(%s,%s,%s,%s)",
|
||||
(target, action, operator, reason),
|
||||
)
|
||||
event(conn, target, action, {"previous_job": str(job_id)})
|
||||
enqueue(conn, target)
|
||||
return str(target)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("action", choices=["failed", "replay", "version"])
|
||||
p.add_argument("--job-id", type=UUID)
|
||||
p.add_argument("--operator")
|
||||
p.add_argument("--reason")
|
||||
p.add_argument("--version", type=int)
|
||||
a = p.parse_args()
|
||||
db, store = database(), ObjectStore()
|
||||
try:
|
||||
if a.action == "failed":
|
||||
with db.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id,attachment_id,pipeline_version,attempt,cycle,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at,id"
|
||||
).fetchall()
|
||||
print(json.dumps(rows, default=str))
|
||||
else:
|
||||
if not all([a.job_id, a.operator, a.reason]):
|
||||
p.error("job-id/operator/reason required")
|
||||
if a.action == "version" and a.version is None:
|
||||
p.error("version required")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"job_id": change(
|
||||
db,
|
||||
store,
|
||||
job_id=a.job_id,
|
||||
operator=a.operator,
|
||||
reason=a.reason,
|
||||
version=a.version
|
||||
if a.action == "version"
|
||||
else None,
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user