#!/usr/bin/env python3 """Отдельный процесс обработки. API этот файл не импортирует и сам не делает resize.""" from __future__ import annotations import hashlib import json import os import sys import threading import time import uuid from datetime import datetime, timedelta, timezone import broker import db import relay import settings import storage import variants from variants import PermanentJobError, TransientJobError _stop = threading.Event() def _validate(payload) -> dict: if not isinstance(payload, dict): raise ValueError("payload") allowed = { "schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at", "traceparent", "tracestate", } if set(payload) - allowed: raise ValueError("extra") if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1": raise ValueError("type") for field in ("job_id", "attachment_id"): uuid.UUID(str(payload.get(field))) if int(payload.get("pipeline_version")) < 1 or int(payload.get("attempt")) < 1: raise ValueError("attempt") if not isinstance(payload.get("created_at"), str): raise ValueError("created_at") return payload def _lab_fault(source: dict, attempt: int) -> None: if os.environ.get("APP_ENV", "lab") != "lab": return name = source["filename"] if name.startswith("__lab_fault_permanent"): raise PermanentJobError("invalid_image") if name.startswith("__lab_fault_transient") and attempt < 3: raise TransientJobError("unavailable") def _produce(source: dict, attempt: int) -> list: _lab_fault(source, attempt) if source["size_bytes"] > settings.max_upload_bytes(): raise PermanentJobError("payload_too_large") if source["width"] * source["height"] > settings.max_image_pixels(): raise PermanentJobError("payload_too_large") staging_key = source["staging_key"] if not staging_key: if storage.exists(source["object_key"]): original = storage.get_bytes(source["object_key"]) else: raise TransientJobError("unavailable") else: original = storage.get_bytes(staging_key) if len(original) != source["size_bytes"] or hashlib.sha256(original).hexdigest() != source["sha256"]: raise TransientJobError("unavailable") rendered = variants.render(original) storage.put_bytes(source["object_key"], original, source["content_type"]) stored = [] for name, (data, content_type, size) in rendered.items(): key = "variants/%s/%s" % (source["attachment_id"], name) storage.put_bytes(key, data, content_type) stored.append({ "name": name, "content_type": content_type, "size_bytes": len(data), "width": size[0], "height": size[1], "sha256": hashlib.sha256(data).hexdigest(), "object_key": key, }) if os.environ.get("CRASH_AFTER_S3") == "1" or source["filename"].startswith("__lab_fault_crash"): sys.stderr.write("crash after object write before ack attachment=%s\n" % source["attachment_id"]) os._exit(86) return stored def _retry(payload: dict, claim: dict, code: str) -> str: delay = settings.backoff_delay(claim["attempt"]) available = datetime.now(timezone.utc) + timedelta(seconds=delay) sys.stderr.write( "retry job=%s attempt=%s delay=%.3f code=%s\n" % (payload["job_id"], claim["attempt"], delay, code) ) return db.retry_job( payload["job_id"], claim["lease_token"], code, payload, available, settings.max_attempts(), ) def _dlq(channel, payload: dict) -> None: try: broker.publish_on(channel, payload, broker.FAILED_KEY) except Exception: sys.stderr.write("dlq publish failed job=%s\n" % payload.get("job_id")) def _cleanup(source: dict) -> None: key = source.get("staging_key") if not key: return storage.delete_quiet(key) db.clear_staging_key(source["attachment_id"]) def _work(channel, payload: dict) -> None: claim = db.claim_job(payload["job_id"], settings.job_timeout()) if claim["action"] == "ack": return if claim["action"] != "work": time.sleep(0.4) raise TransientJobError("lease_held") source = db.load_job_source(payload["attachment_id"]) if source is None or source["job_status"] == "ready" or source["attachment_status"] == "ready": return if int(payload["pipeline_version"]) != int(source["pipeline_version"]): return try: produced = _produce(source, claim["attempt"]) outcome = db.finish_job( payload["job_id"], claim["lease_token"], produced, source["pipeline_version"], ) except PermanentJobError as exc: outcome = db.fail_job(payload["job_id"], claim["lease_token"], exc.code, claim["attempt"]) if outcome == "failed": _dlq(channel, payload) return except TransientJobError as exc: if exc.code == "lease_held": raise outcome = _retry(payload, claim, exc.code) if outcome == "failed": _dlq(channel, payload) return except Exception as exc: sys.stderr.write("transient processing error %s\n" % type(exc).__name__) outcome = _retry(payload, claim, "unavailable") if outcome == "failed": _dlq(channel, payload) return if outcome == "lost": raise TransientJobError("lease_lost") if outcome == "ready": _cleanup(source) def _on_message(channel, method, _properties, body: bytes) -> None: try: payload = _validate(json.loads(body.decode("utf-8"))) except Exception: sys.stderr.write("dropping invalid job message\n") channel.basic_ack(method.delivery_tag) return try: _work(channel, payload) except TransientJobError: channel.basic_nack(method.delivery_tag, requeue=True) return except Exception: sys.stderr.write("worker failed closed for job %s\n" % payload.get("job_id")) channel.basic_nack(method.delivery_tag, requeue=True) return channel.basic_ack(method.delivery_tag) def _consume() -> None: connection, channel = broker.connect() channel.basic_qos(prefetch_count=1) channel.basic_consume(broker.PROCESS_QUEUE, _on_message, auto_ack=False) channel.start_consuming() def _consume_loop() -> None: while not _stop.is_set(): try: _consume() except Exception: if _stop.is_set(): return sys.stderr.write("consumer reconnect\n") time.sleep(1) def _sweep() -> None: while not _stop.is_set(): try: expired = db.recover_expired_leases(settings.max_attempts()) restored = db.reconcile_queued(settings.job_timeout()) if expired or restored: sys.stderr.write("sweep expired=%s restored=%s\n" % (expired, restored)) for attachment_id, key in db.staging_ready_for_cleanup(40): storage.delete_quiet(key) db.clear_staging_key(attachment_id) for attachment_id, key in db.expired_failed_staging(settings.staging_retention(), 40): storage.delete_quiet(key) db.clear_staging_key(attachment_id) relay.publish_batch() except Exception: sys.stderr.write("sweeper failed\n") _stop.wait(2) def _memory_limit() -> str: for path in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"): try: with open(path, encoding="utf-8") as handle: return handle.read().strip() except OSError: continue return "unknown" def main() -> None: db.init() storage.init() sys.stderr.write( "worker concurrency=%s prefetch_per_channel=1 max_attempts=%s timeout=%ss memory_limit=%s\n" % ( settings.worker_concurrency(), settings.max_attempts(), settings.job_timeout(), _memory_limit(), ) ) threading.Thread(target=_sweep, name="sweep", daemon=True).start() threads = [] for index in range(settings.worker_concurrency()): thread = threading.Thread(target=_consume_loop, name="consumer-%s" % index, daemon=True) thread.start() threads.append(thread) try: while any(thread.is_alive() for thread in threads): time.sleep(0.5) except KeyboardInterrupt: _stop.set() if __name__ == "__main__": main()