Files
mtusi2026lab5/app/worker.py
T

342 lines
10 KiB
Python

from __future__ import annotations
import json
import logging
import os
import random
import subprocess
import sys
import time
from uuid import UUID, uuid4
from app.broker import QUEUE_NAME, connect_broker
from app.job_db import connect_db
from app.faults import trip_fault
logger = logging.getLogger("messenger.worker")
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1"))
if WORKER_CONCURRENCY != 1:
raise ValueError("This worker supports WORKER_CONCURRENCY=1 only")
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "120"))
if JOB_TIMEOUT < 1 or LEASE_SECONDS < JOB_TIMEOUT + 15:
raise ValueError("Lease must exceed job timeout by at least 15 seconds")
def claim_job(payload: dict):
job_id = UUID(payload["job_id"])
attachment_id = UUID(payload["attachment_id"])
attempt = payload["attempt"]
version = payload["pipeline_version"]
if (
payload.get("schema_version") != 1
or payload.get("type") != "attachment.process.v1"
or type(attempt) is not int
or attempt < 1
or type(version) is not int
or version < 1
):
raise ValueError("Invalid job message")
token = uuid4()
with connect_db() as db:
with db.transaction():
row = db.execute(
"""
SELECT j.id, j.status, j.attempt,
j.attachment_id, j.pipeline_version,
a.status AS attachment_status,
a.pipeline_version AS current_version
FROM attachment_jobs AS j
JOIN attachments AS a ON a.id = j.attachment_id
WHERE j.id = %s
FOR UPDATE OF j, a
""",
(job_id,),
).fetchone()
if (
row is None
or row["attachment_id"] != attachment_id
or row["pipeline_version"] != version
or row["current_version"] != version
or row["attempt"] != attempt
or row["status"] != "queued"
or row["attachment_status"] != "queued"
):
return None
db.execute(
"""
UPDATE attachment_jobs
SET status = 'processing',
lease_token = %s,
lease_until = clock_timestamp()
+ make_interval(secs => %s),
updated_at = clock_timestamp()
WHERE id = %s
""",
(token, LEASE_SECONDS, job_id),
)
db.execute(
"""
UPDATE attachments
SET status = 'processing',
updated_at = clock_timestamp()
WHERE id = %s
""",
(attachment_id,),
)
return job_id, token
def finish_unsuccessful(job_id, token, *, permanent, error_code):
with connect_db() as db:
with db.transaction():
row = db.execute(
"""
SELECT *
FROM attachment_jobs
WHERE id = %s
FOR UPDATE
""",
(job_id,),
).fetchone()
if (
row is None
or row["status"] != "processing"
or row["lease_token"] != token
):
return
exhausted = row["attempt"] >= row["max_attempts"]
terminal = permanent or exhausted
status = "failed" if terminal else "queued"
code = (
error_code if permanent else "retry_exhausted"
) if terminal else None
delay = random.uniform(
0.5,
min(30.0, 2.0 ** min(row["attempt"], 5)),
)
next_attempt = row["attempt"] if terminal else row["attempt"] + 1
db.execute(
"""
UPDATE attachment_jobs
SET status = %s,
attempt = %s,
error_code = %s,
lease_token = NULL,
lease_until = NULL,
available_at = clock_timestamp()
+ make_interval(secs => %s),
updated_at = clock_timestamp()
WHERE id = %s
""",
(status, next_attempt, code, delay, job_id),
)
db.execute(
"""
UPDATE attachments
SET status = %s,
error_code = %s,
updated_at = clock_timestamp()
WHERE id = %s
AND pipeline_version = %s
AND status <> 'ready'
""",
(
status, code,
row["attachment_id"], row["pipeline_version"],
),
)
if not terminal:
db.execute(
"""
INSERT INTO attachment_outbox (
id, job_id, attempt, available_at
)
VALUES (
%s, %s, %s,
clock_timestamp() + make_interval(secs => %s)
)
""",
(uuid4(), job_id, next_attempt, delay),
)
logger.info(
"job transition job_id=%s status=%s reason=%s",
job_id, status, error_code,
)
def recover_expired():
with connect_db() as db:
rows = db.execute(
"""
SELECT id, lease_token
FROM attachment_jobs
WHERE status = 'processing'
AND lease_until < clock_timestamp()
ORDER BY lease_until
LIMIT 10
"""
).fetchall()
for row in rows:
finish_unsuccessful(
row["id"],
row["lease_token"],
permanent=False,
error_code="lease_expired",
)
def run_child(broker, job_id, token):
process = subprocess.Popen(
[
sys.executable,
"-m",
"app.job_runner",
str(job_id),
str(token),
],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
deadline = time.monotonic() + JOB_TIMEOUT
try:
while process.poll() is None:
if time.monotonic() >= deadline:
process.kill()
process.communicate()
return {"outcome": "retry", "error_code": "processing_timeout"}
broker.process_data_events(time_limit=0.2)
output, _ = process.communicate()
if process.returncode == 75:
logger.warning(
"child_fault job_id=%s stage=after_objects exit_code=75",
job_id,
)
os._exit(75)
if process.returncode != 0:
return {"outcome": "retry", "error_code": "processing_failed"}
result = json.loads(output)
if result.get("outcome") not in ("ready", "stale", "failed", "retry"):
raise ValueError("Invalid child outcome")
return result
finally:
if process.poll() is None:
process.kill()
process.communicate()
def main():
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
)
while True:
broker = None
try:
broker, channel = connect_broker()
channel.basic_qos(prefetch_count=1)
logger.info("worker connected")
next_recovery = 0.0
while True:
if time.monotonic() >= next_recovery:
recover_expired()
next_recovery = time.monotonic() + 5
method, properties, body = channel.basic_get(
queue=QUEUE_NAME,
auto_ack=False,
)
if method is None:
broker.process_data_events(time_limit=0.5)
continue
try:
if len(body) > 16_384:
raise ValueError("Oversized job message")
payload = json.loads(body)
if not isinstance(payload, dict):
raise ValueError("Job must be an object")
claimed = claim_job(payload)
logger.info(
"delivery job_id=%s redelivered=%s",
payload["job_id"],
method.redelivered,
)
except (ValueError, KeyError, TypeError):
logger.warning("invalid broker message rejected")
channel.basic_reject(
delivery_tag=method.delivery_tag,
requeue=False,
)
continue
if claimed is not None:
job_id, token = claimed
result = run_child(broker, job_id, token)
if result["outcome"] in ("failed", "retry"):
finish_unsuccessful(
job_id,
token,
permanent=result["outcome"] == "failed",
error_code=result["error_code"],
)
if claimed is not None and result["outcome"] == "ready":
trip_fault(job_id, "before_ack")
channel.basic_ack(delivery_tag=method.delivery_tag)
logger.info("ack job_id=%s", payload["job_id"])
except KeyboardInterrupt:
return
except Exception as exc:
logger.warning(
"worker reconnect error_type=%s",
type(exc).__name__,
)
time.sleep(2)
finally:
if broker is not None and broker.is_open:
try:
broker.close()
except Exception:
pass
if __name__ == "__main__":
main()