287 lines
11 KiB
Python
287 lines
11 KiB
Python
"""Separate serial worker (prefetch=1) and transactional-outbox dispatcher."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from uuid import UUID, uuid4
|
|
|
|
import redis
|
|
from opentelemetry import propagate, trace
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
|
|
from . import telemetry
|
|
from .domain import Failure
|
|
from .jobs import GROUP, STREAM, Jobs, broker_client, fault
|
|
from .postgres import PostgresStore
|
|
from .storage import ObjectStorage
|
|
|
|
logger = logging.getLogger("messenger.worker")
|
|
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
stopping = False
|
|
MAX_JOB_BYTES = 8192
|
|
|
|
|
|
def stop(signum, frame):
|
|
global stopping
|
|
stopping = True
|
|
|
|
|
|
def process(job, jobs, storage):
|
|
fault(jobs.store, "before_effect", job["attachment_id"])
|
|
with tempfile.TemporaryDirectory(prefix="image-job-") as directory:
|
|
root = Path(directory)
|
|
source = root / "source"
|
|
response = storage.get(job["source_key"])
|
|
try:
|
|
if response["ContentLength"] > 10_485_760:
|
|
raise ValueError("invalid_image")
|
|
with source.open("wb") as target:
|
|
while block := response["Body"].read(64 * 1024):
|
|
target.write(block)
|
|
if target.tell() > 10_485_760:
|
|
raise ValueError("invalid_image")
|
|
finally:
|
|
response["Body"].close()
|
|
with source.open("rb") as file:
|
|
digest = hashlib.file_digest(file, "sha256").hexdigest()
|
|
if digest != job["source_sha256"] or source.stat().st_size != job["source_size"]:
|
|
raise ValueError("invalid_image")
|
|
# Resize executes outside the API and is bounded in wall time, CPU and address space.
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"messenger.image_pipeline",
|
|
str(source),
|
|
directory,
|
|
str(job["pipeline_version"]),
|
|
],
|
|
timeout=20,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if result.returncode:
|
|
raise ValueError("invalid_image")
|
|
variants = json.loads((root / "variants.json").read_text())
|
|
prefix = f"attachments/{job['attachment_id']}/v{job['pipeline_version']}"
|
|
with source.open("rb") as file:
|
|
storage.put(
|
|
prefix + "/original", file, source.stat().st_size, job["source_content_type"]
|
|
)
|
|
for variant in variants:
|
|
path = root / variant["name"]
|
|
with path.open("rb") as file:
|
|
storage.put(
|
|
prefix + "/" + variant["name"],
|
|
file,
|
|
variant["size_bytes"],
|
|
variant["content_type"],
|
|
)
|
|
fault(jobs.store, "after_effect", job["attachment_id"])
|
|
published = jobs.complete(job, prefix + "/original", variants)
|
|
if published:
|
|
fault(jobs.store, "after_result_commit", job["attachment_id"])
|
|
cleanup_ready(jobs.store, storage, job["attachment_id"])
|
|
return published
|
|
|
|
|
|
def cleanup_ready(store, storage, attachment_id):
|
|
# Same row lock as reprocess/replay: cannot delete a source chosen by a new job.
|
|
with store.connection() as conn, conn.transaction():
|
|
a = conn.execute(
|
|
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
|
).fetchone()
|
|
if (
|
|
a
|
|
and a["status"] == "ready"
|
|
and a["stage_key"]
|
|
and a["stage_key"].startswith("staging/")
|
|
):
|
|
storage.delete(a["stage_key"])
|
|
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (attachment_id,))
|
|
|
|
|
|
def validate_delivery(body):
|
|
if not isinstance(body, (str, bytes, bytearray)):
|
|
raise ValueError("invalid_job")
|
|
encoded = body.encode("utf-8") if isinstance(body, str) else body
|
|
if len(encoded) > MAX_JOB_BYTES:
|
|
raise ValueError("invalid_job")
|
|
try:
|
|
data = json.loads(encoded)
|
|
except RecursionError:
|
|
raise ValueError("invalid_job") from None
|
|
expected = {
|
|
"schema_version",
|
|
"job_id",
|
|
"type",
|
|
"attachment_id",
|
|
"pipeline_version",
|
|
"attempt",
|
|
"created_at",
|
|
}
|
|
optional = {"traceparent", "tracestate"}
|
|
if not isinstance(data, dict) or not expected <= set(data) or set(data) - expected - optional:
|
|
raise ValueError("invalid_job")
|
|
if (
|
|
type(data["schema_version"]) is not int
|
|
or data["schema_version"] != 1
|
|
or data["type"] != "attachment.process.v1"
|
|
):
|
|
raise ValueError("invalid_job")
|
|
if not isinstance(data["job_id"], str) or not isinstance(data["attachment_id"], str):
|
|
raise ValueError("invalid_job")
|
|
for field in ("job_id", "attachment_id"):
|
|
canonical = str(UUID(data[field]))
|
|
if data[field].lower() != canonical:
|
|
raise ValueError("invalid_job")
|
|
# RFC UUID hex is case-insensitive; PostgreSQL and state comparisons use
|
|
# the same canonical spelling, so a valid uppercase delivery is not lost.
|
|
data[field] = canonical
|
|
if type(data["pipeline_version"]) is not int or data["pipeline_version"] < 1:
|
|
raise ValueError("invalid_job")
|
|
if type(data["attempt"]) is not int or data["attempt"] < 1:
|
|
raise ValueError("invalid_job")
|
|
created = data["created_at"]
|
|
if (
|
|
not isinstance(created, str)
|
|
or len(created) > 64
|
|
or not re.fullmatch(
|
|
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z", created
|
|
)
|
|
):
|
|
raise ValueError("invalid_job")
|
|
datetime.fromisoformat(created[:-1] + "+00:00")
|
|
if "traceparent" in data and (
|
|
not isinstance(data["traceparent"], str)
|
|
or not re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"])
|
|
):
|
|
raise ValueError("invalid_job")
|
|
if "tracestate" in data and (
|
|
not isinstance(data["tracestate"], str) or len(data["tracestate"]) > 512
|
|
):
|
|
raise ValueError("invalid_job")
|
|
return data
|
|
|
|
|
|
def consume_one(store, jobs, storage, broker, entry):
|
|
entry_id, fields = entry
|
|
try:
|
|
body = validate_delivery(fields.get("body", fields.get(b"body")))
|
|
except (KeyError, ValueError, TypeError):
|
|
logger.error(json.dumps({"event": "invalid_delivery"}))
|
|
broker.xack(STREAM, GROUP, entry_id)
|
|
broker.xdel(STREAM, entry_id)
|
|
return
|
|
with telemetry.operation("attachments consume", context=propagate.extract(body), kind=SpanKind.CONSUMER):
|
|
consume_job(store, jobs, storage, broker, body)
|
|
# Durable state owns recovery even when notification acknowledgements repeat.
|
|
broker.xack(STREAM, GROUP, entry_id)
|
|
broker.xdel(STREAM, entry_id)
|
|
|
|
|
|
def consume_job(store, jobs, storage, broker, body):
|
|
job = jobs.claim(body["job_id"], body["attachment_id"], body["pipeline_version"])
|
|
t = telemetry.current()
|
|
started = time.perf_counter()
|
|
outcome = "duplicate"
|
|
if job:
|
|
try:
|
|
if t:
|
|
t.busy = 1
|
|
outcome = "success" if process(job, jobs, storage) else "superseded"
|
|
except ValueError:
|
|
jobs.fail(job, "invalid_image", permanent=True)
|
|
outcome = "failed"
|
|
except (Failure, subprocess.TimeoutExpired, OSError, RuntimeError) as exc:
|
|
logger.warning(
|
|
json.dumps(
|
|
{"event": "job_retry", "job_id": str(job["id"]), "kind": type(exc).__name__}
|
|
)
|
|
)
|
|
jobs.fail(job, "processing_failed")
|
|
outcome = "retry" if job["attempt"] < job["attempt_limit"] else "failed"
|
|
finally:
|
|
if t:
|
|
t.busy = 0
|
|
attrs = {"job_type": "attachment.process.v1", "outcome": outcome}
|
|
t.jobs.add(1, attrs)
|
|
t.job_duration.record(time.perf_counter() - started, attrs)
|
|
span = trace.get_current_span()
|
|
span.set_attributes({"job.type": "attachment.process.v1", "job.outcome": outcome})
|
|
if outcome in {"failed", "retry"}:
|
|
span.set_status(Status(StatusCode.ERROR, "processing_failed"))
|
|
logger.info(json.dumps({"event": "job_finished", "outcome": outcome}))
|
|
else:
|
|
trace.get_current_span().set_attributes({"job.type": "attachment.process.v1", "job.outcome": "duplicate"})
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--dispatcher", action="store_true")
|
|
args = parser.parse_args()
|
|
signal.signal(signal.SIGTERM, stop)
|
|
signal.signal(signal.SIGINT, stop)
|
|
t = telemetry.setup("messenger-dispatcher" if args.dispatcher else "messenger-worker")
|
|
store = PostgresStore()
|
|
t.start_background(store)
|
|
jobs = Jobs(store)
|
|
broker = broker_client()
|
|
storage = None if args.dispatcher else ObjectStorage()
|
|
consumer = "worker-" + str(uuid4())
|
|
last_claim = 0
|
|
try:
|
|
while not stopping:
|
|
try:
|
|
if args.dispatcher:
|
|
jobs.recover_one()
|
|
if not jobs.publish_one(broker):
|
|
time.sleep(0.2)
|
|
continue
|
|
try:
|
|
broker.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
|
except redis.ResponseError as exc:
|
|
if "BUSYGROUP" not in str(exc):
|
|
raise
|
|
entries = []
|
|
if time.monotonic() - last_claim >= 5:
|
|
claimed = broker.xautoclaim(
|
|
STREAM,
|
|
GROUP,
|
|
consumer,
|
|
min_idle_time=jobs.lease_seconds * 1000,
|
|
start_id="0-0",
|
|
count=1,
|
|
)
|
|
entries = claimed[1]
|
|
last_claim = time.monotonic()
|
|
if not entries:
|
|
batches = broker.xreadgroup(GROUP, consumer, {STREAM: ">"}, count=1, block=500)
|
|
entries = batches[0][1] if batches else []
|
|
for entry in entries:
|
|
consume_one(store, jobs, storage, broker, entry)
|
|
except (redis.RedisError, Failure) as exc:
|
|
logger.warning(
|
|
json.dumps({"event": "dependency_retry", "kind": type(exc).__name__})
|
|
)
|
|
time.sleep(0.5)
|
|
finally:
|
|
broker.close()
|
|
if storage is not None:
|
|
storage.close()
|
|
t.close()
|
|
store.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|