93 lines
4.0 KiB
Python
93 lines
4.0 KiB
Python
"""Explicit local operator commands; no new public HTTP endpoints."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
from uuid import UUID
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.db import SessionLocal, engine
|
|
from app.pipeline import event
|
|
|
|
POINTS = ("crash_after_commit_before_publish", "crash_after_objects_before_commit",
|
|
"crash_after_commit_before_ack")
|
|
|
|
|
|
async def run(args) -> None:
|
|
try:
|
|
async with SessionLocal() as db:
|
|
if args.command == "fault":
|
|
await db.execute(text("""
|
|
INSERT INTO fault_injections(point,remaining) VALUES (:point,1)
|
|
ON CONFLICT(point) DO UPDATE SET remaining=1,updated_at=now()
|
|
"""), {"point": args.point})
|
|
await db.commit()
|
|
print(f"ARMED one-shot fault: {args.point}")
|
|
return
|
|
if args.command == "failed":
|
|
result = await db.execute(text("""
|
|
SELECT id,job_id,attempt,attempts_run,error_code,staging_deleted,updated_at
|
|
FROM attachments WHERE status='failed' ORDER BY updated_at
|
|
"""))
|
|
print(json.dumps([dict(row) for row in result.mappings()], default=str, indent=2))
|
|
return
|
|
result = await db.execute(text("SELECT * FROM attachments WHERE id=:id FOR UPDATE"),
|
|
{"id": args.attachment_id})
|
|
row = result.mappings().first()
|
|
if row is None or row["job_id"] is None:
|
|
raise ValueError("Unknown asynchronous attachment")
|
|
if args.command == "replay":
|
|
if row["status"] == "ready":
|
|
print("NO-OP: attachment is already ready")
|
|
return
|
|
if row["status"] != "failed":
|
|
raise ValueError("Replay is allowed only for failed attachments")
|
|
if row["staging_deleted"]:
|
|
raise ValueError("Replay source expired; upload the original again")
|
|
if not args.reason.strip() or len(args.reason) > 200:
|
|
raise ValueError("Replay reason must have 1-200 characters")
|
|
result = await db.execute(text("""
|
|
UPDATE attachments SET status='queued',attempt=attempt+1,attempts_run=0,
|
|
error_code=NULL,lease_token=NULL,lease_until=NULL,
|
|
next_attempt_at=now(),updated_at=now() WHERE id=:id RETURNING *
|
|
"""), {"id": row["id"]})
|
|
row = result.mappings().one()
|
|
await event(db, row, "operator_replay", args.reason)
|
|
else:
|
|
# Real duplicate envelope: same job ID and current attempt.
|
|
# Ready effects are checked in PG by the worker and acked.
|
|
await event(db, row, "operator_redeliver")
|
|
await db.execute(text("""
|
|
INSERT INTO outbox(job_id,attachment_id,attempt,available_at,traceparent,tracestate)
|
|
SELECT job_id,id,attempt,GREATEST(now(),next_attempt_at),traceparent,tracestate
|
|
FROM attachments WHERE id=:id
|
|
"""), {"id": row["id"]})
|
|
await db.commit()
|
|
print(f"QUEUED: {args.command} attachment={row['id']} job={row['job_id']}")
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
commands.add_parser("failed")
|
|
fault = commands.add_parser("fault")
|
|
fault.add_argument("point", choices=POINTS)
|
|
replay = commands.add_parser("replay")
|
|
replay.add_argument("attachment_id", type=UUID)
|
|
replay.add_argument("--reason", required=True)
|
|
redeliver = commands.add_parser("redeliver")
|
|
redeliver.add_argument("attachment_id", type=UUID)
|
|
args = parser.parse_args()
|
|
try:
|
|
asyncio.run(run(args))
|
|
except ValueError as exc:
|
|
parser.exit(2, f"REFUSED: {exc}\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|