124 lines
4.5 KiB
Python
124 lines
4.5 KiB
Python
"""Explicit local operator actions. There is deliberately no HTTP replay endpoint."""
|
|
|
|
import argparse
|
|
import json
|
|
from uuid import UUID, uuid4
|
|
|
|
from src.object_store import ObjectStore
|
|
from src.pipeline import database, enqueue, event
|
|
|
|
|
|
def change(db, store, *, job_id, operator, reason, version=None):
|
|
if (
|
|
not operator.strip()
|
|
or not reason.strip()
|
|
or len(operator) > 100
|
|
or len(reason) > 500
|
|
):
|
|
raise ValueError("Operator and reason are required")
|
|
with db.connection() as conn, conn.transaction():
|
|
job = conn.execute(
|
|
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
|
|
).fetchone()
|
|
if not job:
|
|
raise ValueError("Unknown job")
|
|
attachment = conn.execute(
|
|
"SELECT * FROM attachments WHERE id=%s FOR UPDATE",
|
|
(job["attachment_id"],),
|
|
).fetchone()
|
|
if job["pipeline_version"] != attachment["pipeline_version"]:
|
|
raise ValueError("Superseded job cannot be replayed")
|
|
source = (
|
|
job["source_key"]
|
|
if attachment["published_version"] is None
|
|
else attachment["object_key"]
|
|
)
|
|
if store.head(key=source) is None:
|
|
raise ValueError("Source expired or missing; replay refused")
|
|
if version is None:
|
|
if job["state"] != "failed":
|
|
raise ValueError("Only failed jobs can be replayed")
|
|
conn.execute(
|
|
"UPDATE jobs SET state='queued',attempt=0,cycle=cycle+1,fence=fence+1,error_code=NULL,next_at=clock_timestamp(),lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
|
(job_id,),
|
|
)
|
|
action = "replay"
|
|
target = job_id
|
|
else:
|
|
if (
|
|
version not in {1, 2}
|
|
or version <= attachment["pipeline_version"]
|
|
):
|
|
raise ValueError("Supported version must increase (1 or 2)")
|
|
conn.execute(
|
|
"UPDATE jobs SET state='obsolete',fence=fence+1,lease_owner=NULL,lease_until=NULL WHERE id=%s",
|
|
(job_id,),
|
|
)
|
|
target = uuid4()
|
|
conn.execute(
|
|
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,%s,%s,'queued')",
|
|
(target, attachment["id"], version, source),
|
|
)
|
|
conn.execute(
|
|
"UPDATE attachments SET pipeline_version=%s WHERE id=%s",
|
|
(version, attachment["id"]),
|
|
)
|
|
action = "new_version"
|
|
conn.execute(
|
|
"UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
|
(attachment["id"],),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO pipeline_audit(job_id,action,operator,reason) VALUES(%s,%s,%s,%s)",
|
|
(target, action, operator, reason),
|
|
)
|
|
event(conn, target, action, {"previous_job": str(job_id)})
|
|
enqueue(conn, target)
|
|
return str(target)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("action", choices=["failed", "replay", "version"])
|
|
p.add_argument("--job-id", type=UUID)
|
|
p.add_argument("--operator")
|
|
p.add_argument("--reason")
|
|
p.add_argument("--version", type=int)
|
|
a = p.parse_args()
|
|
db, store = database(), ObjectStore()
|
|
try:
|
|
if a.action == "failed":
|
|
with db.connection() as conn:
|
|
rows = conn.execute(
|
|
"SELECT id,attachment_id,pipeline_version,attempt,cycle,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at,id"
|
|
).fetchall()
|
|
print(json.dumps(rows, default=str))
|
|
else:
|
|
if not all([a.job_id, a.operator, a.reason]):
|
|
p.error("job-id/operator/reason required")
|
|
if a.action == "version" and a.version is None:
|
|
p.error("version required")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"job_id": change(
|
|
db,
|
|
store,
|
|
job_id=a.job_id,
|
|
operator=a.operator,
|
|
reason=a.reason,
|
|
version=a.version
|
|
if a.action == "version"
|
|
else None,
|
|
)
|
|
}
|
|
)
|
|
)
|
|
finally:
|
|
db.close()
|
|
store.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|