Files
mtusi2026lab5/src/worker.py
T

258 lines
8.6 KiB
Python

#!/usr/bin/env python3
"""Отдельный процесс обработки. API этот файл не импортирует и сам не делает resize."""
from __future__ import annotations
import hashlib
import json
import os
import sys
import threading
import time
import uuid
from datetime import datetime, timedelta, timezone
import broker
import db
import relay
import settings
import storage
import variants
from variants import PermanentJobError, TransientJobError
_stop = threading.Event()
def _validate(payload) -> dict:
if not isinstance(payload, dict):
raise ValueError("payload")
allowed = {
"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
"attempt", "created_at", "traceparent", "tracestate",
}
if set(payload) - allowed:
raise ValueError("extra")
if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1":
raise ValueError("type")
for field in ("job_id", "attachment_id"):
uuid.UUID(str(payload.get(field)))
if int(payload.get("pipeline_version")) < 1 or int(payload.get("attempt")) < 1:
raise ValueError("attempt")
if not isinstance(payload.get("created_at"), str):
raise ValueError("created_at")
return payload
def _lab_fault(source: dict, attempt: int) -> None:
if os.environ.get("APP_ENV", "lab") != "lab":
return
name = source["filename"]
if name.startswith("__lab_fault_permanent"):
raise PermanentJobError("invalid_image")
if name.startswith("__lab_fault_transient") and attempt < 3:
raise TransientJobError("unavailable")
def _produce(source: dict, attempt: int) -> list:
_lab_fault(source, attempt)
if source["size_bytes"] > settings.max_upload_bytes():
raise PermanentJobError("payload_too_large")
if source["width"] * source["height"] > settings.max_image_pixels():
raise PermanentJobError("payload_too_large")
staging_key = source["staging_key"]
if not staging_key:
if storage.exists(source["object_key"]):
original = storage.get_bytes(source["object_key"])
else:
raise TransientJobError("unavailable")
else:
original = storage.get_bytes(staging_key)
if len(original) != source["size_bytes"] or hashlib.sha256(original).hexdigest() != source["sha256"]:
raise TransientJobError("unavailable")
rendered = variants.render(original)
storage.put_bytes(source["object_key"], original, source["content_type"])
stored = []
for name, (data, content_type, size) in rendered.items():
key = "variants/%s/%s" % (source["attachment_id"], name)
storage.put_bytes(key, data, content_type)
stored.append({
"name": name,
"content_type": content_type,
"size_bytes": len(data),
"width": size[0],
"height": size[1],
"sha256": hashlib.sha256(data).hexdigest(),
"object_key": key,
})
if os.environ.get("CRASH_AFTER_S3") == "1" or source["filename"].startswith("__lab_fault_crash"):
sys.stderr.write("crash after object write before ack attachment=%s\n" % source["attachment_id"])
os._exit(86)
return stored
def _retry(payload: dict, claim: dict, code: str) -> str:
delay = settings.backoff_delay(claim["attempt"])
available = datetime.now(timezone.utc) + timedelta(seconds=delay)
sys.stderr.write(
"retry job=%s attempt=%s delay=%.3f code=%s\n"
% (payload["job_id"], claim["attempt"], delay, code)
)
return db.retry_job(
payload["job_id"],
claim["lease_token"],
code,
payload,
available,
settings.max_attempts(),
)
def _dlq(channel, payload: dict) -> None:
try:
broker.publish_on(channel, payload, broker.FAILED_KEY)
except Exception:
sys.stderr.write("dlq publish failed job=%s\n" % payload.get("job_id"))
def _cleanup(source: dict) -> None:
key = source.get("staging_key")
if not key:
return
storage.delete_quiet(key)
db.clear_staging_key(source["attachment_id"])
def _work(channel, payload: dict) -> None:
claim = db.claim_job(payload["job_id"], settings.job_timeout())
if claim["action"] == "ack":
return
if claim["action"] != "work":
time.sleep(0.4)
raise TransientJobError("lease_held")
source = db.load_job_source(payload["attachment_id"])
if source is None or source["job_status"] == "ready" or source["attachment_status"] == "ready":
return
if int(payload["pipeline_version"]) != int(source["pipeline_version"]):
return
try:
produced = _produce(source, claim["attempt"])
outcome = db.finish_job(
payload["job_id"], claim["lease_token"], produced, source["pipeline_version"],
)
except PermanentJobError as exc:
outcome = db.fail_job(payload["job_id"], claim["lease_token"], exc.code, claim["attempt"])
if outcome == "failed":
_dlq(channel, payload)
return
except TransientJobError as exc:
if exc.code == "lease_held":
raise
outcome = _retry(payload, claim, exc.code)
if outcome == "failed":
_dlq(channel, payload)
return
except Exception as exc:
sys.stderr.write("transient processing error %s\n" % type(exc).__name__)
outcome = _retry(payload, claim, "unavailable")
if outcome == "failed":
_dlq(channel, payload)
return
if outcome == "lost":
raise TransientJobError("lease_lost")
if outcome == "ready":
_cleanup(source)
def _on_message(channel, method, _properties, body: bytes) -> None:
try:
payload = _validate(json.loads(body.decode("utf-8")))
except Exception:
sys.stderr.write("dropping invalid job message\n")
channel.basic_ack(method.delivery_tag)
return
try:
_work(channel, payload)
except TransientJobError:
channel.basic_nack(method.delivery_tag, requeue=True)
return
except Exception:
sys.stderr.write("worker failed closed for job %s\n" % payload.get("job_id"))
channel.basic_nack(method.delivery_tag, requeue=True)
return
channel.basic_ack(method.delivery_tag)
def _consume() -> None:
connection, channel = broker.connect()
channel.basic_qos(prefetch_count=1)
channel.basic_consume(broker.PROCESS_QUEUE, _on_message, auto_ack=False)
channel.start_consuming()
def _consume_loop() -> None:
while not _stop.is_set():
try:
_consume()
except Exception:
if _stop.is_set():
return
sys.stderr.write("consumer reconnect\n")
time.sleep(1)
def _sweep() -> None:
while not _stop.is_set():
try:
expired = db.recover_expired_leases(settings.max_attempts())
restored = db.reconcile_queued(settings.job_timeout())
if expired or restored:
sys.stderr.write("sweep expired=%s restored=%s\n" % (expired, restored))
for attachment_id, key in db.staging_ready_for_cleanup(40):
storage.delete_quiet(key)
db.clear_staging_key(attachment_id)
for attachment_id, key in db.expired_failed_staging(settings.staging_retention(), 40):
storage.delete_quiet(key)
db.clear_staging_key(attachment_id)
relay.publish_batch()
except Exception:
sys.stderr.write("sweeper failed\n")
_stop.wait(2)
def _memory_limit() -> str:
for path in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"):
try:
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
except OSError:
continue
return "unknown"
def main() -> None:
db.init()
storage.init()
sys.stderr.write(
"worker concurrency=%s prefetch_per_channel=1 max_attempts=%s timeout=%ss memory_limit=%s\n"
% (
settings.worker_concurrency(),
settings.max_attempts(),
settings.job_timeout(),
_memory_limit(),
)
)
threading.Thread(target=_sweep, name="sweep", daemon=True).start()
threads = []
for index in range(settings.worker_concurrency()):
thread = threading.Thread(target=_consume_loop, name="consumer-%s" % index, daemon=True)
thread.start()
threads.append(thread)
try:
while any(thread.is_alive() for thread in threads):
time.sleep(0.5)
except KeyboardInterrupt:
_stop.set()
if __name__ == "__main__":
main()