342 lines
10 KiB
Python
342 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from uuid import UUID, uuid4
|
|
|
|
from app.broker import QUEUE_NAME, connect_broker
|
|
from app.job_db import connect_db
|
|
from app.faults import trip_fault
|
|
|
|
|
|
logger = logging.getLogger("messenger.worker")
|
|
|
|
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1"))
|
|
if WORKER_CONCURRENCY != 1:
|
|
raise ValueError("This worker supports WORKER_CONCURRENCY=1 only")
|
|
|
|
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
|
|
LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "120"))
|
|
|
|
if JOB_TIMEOUT < 1 or LEASE_SECONDS < JOB_TIMEOUT + 15:
|
|
raise ValueError("Lease must exceed job timeout by at least 15 seconds")
|
|
|
|
|
|
def claim_job(payload: dict):
|
|
job_id = UUID(payload["job_id"])
|
|
attachment_id = UUID(payload["attachment_id"])
|
|
attempt = payload["attempt"]
|
|
version = payload["pipeline_version"]
|
|
|
|
if (
|
|
payload.get("schema_version") != 1
|
|
or payload.get("type") != "attachment.process.v1"
|
|
or type(attempt) is not int
|
|
or attempt < 1
|
|
or type(version) is not int
|
|
or version < 1
|
|
):
|
|
raise ValueError("Invalid job message")
|
|
|
|
token = uuid4()
|
|
|
|
with connect_db() as db:
|
|
with db.transaction():
|
|
row = db.execute(
|
|
"""
|
|
SELECT j.id, j.status, j.attempt,
|
|
j.attachment_id, j.pipeline_version,
|
|
a.status AS attachment_status,
|
|
a.pipeline_version AS current_version
|
|
FROM attachment_jobs AS j
|
|
JOIN attachments AS a ON a.id = j.attachment_id
|
|
WHERE j.id = %s
|
|
FOR UPDATE OF j, a
|
|
""",
|
|
(job_id,),
|
|
).fetchone()
|
|
|
|
if (
|
|
row is None
|
|
or row["attachment_id"] != attachment_id
|
|
or row["pipeline_version"] != version
|
|
or row["current_version"] != version
|
|
or row["attempt"] != attempt
|
|
or row["status"] != "queued"
|
|
or row["attachment_status"] != "queued"
|
|
):
|
|
return None
|
|
|
|
db.execute(
|
|
"""
|
|
UPDATE attachment_jobs
|
|
SET status = 'processing',
|
|
lease_token = %s,
|
|
lease_until = clock_timestamp()
|
|
+ make_interval(secs => %s),
|
|
updated_at = clock_timestamp()
|
|
WHERE id = %s
|
|
""",
|
|
(token, LEASE_SECONDS, job_id),
|
|
)
|
|
|
|
db.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'processing',
|
|
updated_at = clock_timestamp()
|
|
WHERE id = %s
|
|
""",
|
|
(attachment_id,),
|
|
)
|
|
|
|
return job_id, token
|
|
|
|
|
|
def finish_unsuccessful(job_id, token, *, permanent, error_code):
|
|
with connect_db() as db:
|
|
with db.transaction():
|
|
row = db.execute(
|
|
"""
|
|
SELECT *
|
|
FROM attachment_jobs
|
|
WHERE id = %s
|
|
FOR UPDATE
|
|
""",
|
|
(job_id,),
|
|
).fetchone()
|
|
|
|
if (
|
|
row is None
|
|
or row["status"] != "processing"
|
|
or row["lease_token"] != token
|
|
):
|
|
return
|
|
|
|
exhausted = row["attempt"] >= row["max_attempts"]
|
|
terminal = permanent or exhausted
|
|
status = "failed" if terminal else "queued"
|
|
code = (
|
|
error_code if permanent else "retry_exhausted"
|
|
) if terminal else None
|
|
|
|
delay = random.uniform(
|
|
0.5,
|
|
min(30.0, 2.0 ** min(row["attempt"], 5)),
|
|
)
|
|
next_attempt = row["attempt"] if terminal else row["attempt"] + 1
|
|
|
|
db.execute(
|
|
"""
|
|
UPDATE attachment_jobs
|
|
SET status = %s,
|
|
attempt = %s,
|
|
error_code = %s,
|
|
lease_token = NULL,
|
|
lease_until = NULL,
|
|
available_at = clock_timestamp()
|
|
+ make_interval(secs => %s),
|
|
updated_at = clock_timestamp()
|
|
WHERE id = %s
|
|
""",
|
|
(status, next_attempt, code, delay, job_id),
|
|
)
|
|
|
|
db.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = %s,
|
|
error_code = %s,
|
|
updated_at = clock_timestamp()
|
|
WHERE id = %s
|
|
AND pipeline_version = %s
|
|
AND status <> 'ready'
|
|
""",
|
|
(
|
|
status, code,
|
|
row["attachment_id"], row["pipeline_version"],
|
|
),
|
|
)
|
|
|
|
if not terminal:
|
|
db.execute(
|
|
"""
|
|
INSERT INTO attachment_outbox (
|
|
id, job_id, attempt, available_at
|
|
)
|
|
VALUES (
|
|
%s, %s, %s,
|
|
clock_timestamp() + make_interval(secs => %s)
|
|
)
|
|
""",
|
|
(uuid4(), job_id, next_attempt, delay),
|
|
)
|
|
|
|
logger.info(
|
|
"job transition job_id=%s status=%s reason=%s",
|
|
job_id, status, error_code,
|
|
)
|
|
|
|
|
|
def recover_expired():
|
|
with connect_db() as db:
|
|
rows = db.execute(
|
|
"""
|
|
SELECT id, lease_token
|
|
FROM attachment_jobs
|
|
WHERE status = 'processing'
|
|
AND lease_until < clock_timestamp()
|
|
ORDER BY lease_until
|
|
LIMIT 10
|
|
"""
|
|
).fetchall()
|
|
|
|
for row in rows:
|
|
finish_unsuccessful(
|
|
row["id"],
|
|
row["lease_token"],
|
|
permanent=False,
|
|
error_code="lease_expired",
|
|
)
|
|
|
|
|
|
def run_child(broker, job_id, token):
|
|
process = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"app.job_runner",
|
|
str(job_id),
|
|
str(token),
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
text=True,
|
|
)
|
|
deadline = time.monotonic() + JOB_TIMEOUT
|
|
|
|
try:
|
|
while process.poll() is None:
|
|
if time.monotonic() >= deadline:
|
|
process.kill()
|
|
process.communicate()
|
|
return {"outcome": "retry", "error_code": "processing_timeout"}
|
|
|
|
broker.process_data_events(time_limit=0.2)
|
|
|
|
output, _ = process.communicate()
|
|
|
|
if process.returncode == 75:
|
|
logger.warning(
|
|
"child_fault job_id=%s stage=after_objects exit_code=75",
|
|
job_id,
|
|
)
|
|
os._exit(75)
|
|
|
|
if process.returncode != 0:
|
|
return {"outcome": "retry", "error_code": "processing_failed"}
|
|
|
|
result = json.loads(output)
|
|
if result.get("outcome") not in ("ready", "stale", "failed", "retry"):
|
|
raise ValueError("Invalid child outcome")
|
|
return result
|
|
|
|
finally:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.communicate()
|
|
|
|
|
|
def main():
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(message)s",
|
|
)
|
|
|
|
while True:
|
|
broker = None
|
|
try:
|
|
broker, channel = connect_broker()
|
|
channel.basic_qos(prefetch_count=1)
|
|
logger.info("worker connected")
|
|
|
|
next_recovery = 0.0
|
|
|
|
while True:
|
|
if time.monotonic() >= next_recovery:
|
|
recover_expired()
|
|
next_recovery = time.monotonic() + 5
|
|
|
|
method, properties, body = channel.basic_get(
|
|
queue=QUEUE_NAME,
|
|
auto_ack=False,
|
|
)
|
|
|
|
if method is None:
|
|
broker.process_data_events(time_limit=0.5)
|
|
continue
|
|
|
|
try:
|
|
if len(body) > 16_384:
|
|
raise ValueError("Oversized job message")
|
|
payload = json.loads(body)
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("Job must be an object")
|
|
claimed = claim_job(payload)
|
|
logger.info(
|
|
"delivery job_id=%s redelivered=%s",
|
|
payload["job_id"],
|
|
method.redelivered,
|
|
)
|
|
except (ValueError, KeyError, TypeError):
|
|
logger.warning("invalid broker message rejected")
|
|
channel.basic_reject(
|
|
delivery_tag=method.delivery_tag,
|
|
requeue=False,
|
|
)
|
|
continue
|
|
|
|
if claimed is not None:
|
|
job_id, token = claimed
|
|
result = run_child(broker, job_id, token)
|
|
|
|
if result["outcome"] in ("failed", "retry"):
|
|
finish_unsuccessful(
|
|
job_id,
|
|
token,
|
|
permanent=result["outcome"] == "failed",
|
|
error_code=result["error_code"],
|
|
)
|
|
|
|
if claimed is not None and result["outcome"] == "ready":
|
|
trip_fault(job_id, "before_ack")
|
|
|
|
channel.basic_ack(delivery_tag=method.delivery_tag)
|
|
logger.info("ack job_id=%s", payload["job_id"])
|
|
|
|
except KeyboardInterrupt:
|
|
return
|
|
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"worker reconnect error_type=%s",
|
|
type(exc).__name__,
|
|
)
|
|
time.sleep(2)
|
|
|
|
finally:
|
|
if broker is not None and broker.is_open:
|
|
try:
|
|
broker.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|