258 lines
8.6 KiB
Python
258 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Отдельный процесс обработки. API этот файл не импортирует и сам не делает resize."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import broker
|
|
import db
|
|
import relay
|
|
import settings
|
|
import storage
|
|
import variants
|
|
from variants import PermanentJobError, TransientJobError
|
|
|
|
_stop = threading.Event()
|
|
|
|
|
|
def _validate(payload) -> dict:
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("payload")
|
|
allowed = {
|
|
"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
|
|
"attempt", "created_at", "traceparent", "tracestate",
|
|
}
|
|
if set(payload) - allowed:
|
|
raise ValueError("extra")
|
|
if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1":
|
|
raise ValueError("type")
|
|
for field in ("job_id", "attachment_id"):
|
|
uuid.UUID(str(payload.get(field)))
|
|
if int(payload.get("pipeline_version")) < 1 or int(payload.get("attempt")) < 1:
|
|
raise ValueError("attempt")
|
|
if not isinstance(payload.get("created_at"), str):
|
|
raise ValueError("created_at")
|
|
return payload
|
|
|
|
|
|
def _lab_fault(source: dict, attempt: int) -> None:
|
|
if os.environ.get("APP_ENV", "lab") != "lab":
|
|
return
|
|
name = source["filename"]
|
|
if name.startswith("__lab_fault_permanent"):
|
|
raise PermanentJobError("invalid_image")
|
|
if name.startswith("__lab_fault_transient") and attempt < 3:
|
|
raise TransientJobError("unavailable")
|
|
|
|
|
|
def _produce(source: dict, attempt: int) -> list:
|
|
_lab_fault(source, attempt)
|
|
if source["size_bytes"] > settings.max_upload_bytes():
|
|
raise PermanentJobError("payload_too_large")
|
|
if source["width"] * source["height"] > settings.max_image_pixels():
|
|
raise PermanentJobError("payload_too_large")
|
|
staging_key = source["staging_key"]
|
|
if not staging_key:
|
|
if storage.exists(source["object_key"]):
|
|
original = storage.get_bytes(source["object_key"])
|
|
else:
|
|
raise TransientJobError("unavailable")
|
|
else:
|
|
original = storage.get_bytes(staging_key)
|
|
if len(original) != source["size_bytes"] or hashlib.sha256(original).hexdigest() != source["sha256"]:
|
|
raise TransientJobError("unavailable")
|
|
rendered = variants.render(original)
|
|
storage.put_bytes(source["object_key"], original, source["content_type"])
|
|
stored = []
|
|
for name, (data, content_type, size) in rendered.items():
|
|
key = "variants/%s/%s" % (source["attachment_id"], name)
|
|
storage.put_bytes(key, data, content_type)
|
|
stored.append({
|
|
"name": name,
|
|
"content_type": content_type,
|
|
"size_bytes": len(data),
|
|
"width": size[0],
|
|
"height": size[1],
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
"object_key": key,
|
|
})
|
|
if os.environ.get("CRASH_AFTER_S3") == "1" or source["filename"].startswith("__lab_fault_crash"):
|
|
sys.stderr.write("crash after object write before ack attachment=%s\n" % source["attachment_id"])
|
|
os._exit(86)
|
|
return stored
|
|
|
|
|
|
def _retry(payload: dict, claim: dict, code: str) -> str:
|
|
delay = settings.backoff_delay(claim["attempt"])
|
|
available = datetime.now(timezone.utc) + timedelta(seconds=delay)
|
|
sys.stderr.write(
|
|
"retry job=%s attempt=%s delay=%.3f code=%s\n"
|
|
% (payload["job_id"], claim["attempt"], delay, code)
|
|
)
|
|
return db.retry_job(
|
|
payload["job_id"],
|
|
claim["lease_token"],
|
|
code,
|
|
payload,
|
|
available,
|
|
settings.max_attempts(),
|
|
)
|
|
|
|
|
|
def _dlq(channel, payload: dict) -> None:
|
|
try:
|
|
broker.publish_on(channel, payload, broker.FAILED_KEY)
|
|
except Exception:
|
|
sys.stderr.write("dlq publish failed job=%s\n" % payload.get("job_id"))
|
|
|
|
|
|
def _cleanup(source: dict) -> None:
|
|
key = source.get("staging_key")
|
|
if not key:
|
|
return
|
|
storage.delete_quiet(key)
|
|
db.clear_staging_key(source["attachment_id"])
|
|
|
|
|
|
def _work(channel, payload: dict) -> None:
|
|
claim = db.claim_job(payload["job_id"], settings.job_timeout())
|
|
if claim["action"] == "ack":
|
|
return
|
|
if claim["action"] != "work":
|
|
time.sleep(0.4)
|
|
raise TransientJobError("lease_held")
|
|
source = db.load_job_source(payload["attachment_id"])
|
|
if source is None or source["job_status"] == "ready" or source["attachment_status"] == "ready":
|
|
return
|
|
if int(payload["pipeline_version"]) != int(source["pipeline_version"]):
|
|
return
|
|
try:
|
|
produced = _produce(source, claim["attempt"])
|
|
outcome = db.finish_job(
|
|
payload["job_id"], claim["lease_token"], produced, source["pipeline_version"],
|
|
)
|
|
except PermanentJobError as exc:
|
|
outcome = db.fail_job(payload["job_id"], claim["lease_token"], exc.code, claim["attempt"])
|
|
if outcome == "failed":
|
|
_dlq(channel, payload)
|
|
return
|
|
except TransientJobError as exc:
|
|
if exc.code == "lease_held":
|
|
raise
|
|
outcome = _retry(payload, claim, exc.code)
|
|
if outcome == "failed":
|
|
_dlq(channel, payload)
|
|
return
|
|
except Exception as exc:
|
|
sys.stderr.write("transient processing error %s\n" % type(exc).__name__)
|
|
outcome = _retry(payload, claim, "unavailable")
|
|
if outcome == "failed":
|
|
_dlq(channel, payload)
|
|
return
|
|
if outcome == "lost":
|
|
raise TransientJobError("lease_lost")
|
|
if outcome == "ready":
|
|
_cleanup(source)
|
|
|
|
|
|
def _on_message(channel, method, _properties, body: bytes) -> None:
|
|
try:
|
|
payload = _validate(json.loads(body.decode("utf-8")))
|
|
except Exception:
|
|
sys.stderr.write("dropping invalid job message\n")
|
|
channel.basic_ack(method.delivery_tag)
|
|
return
|
|
try:
|
|
_work(channel, payload)
|
|
except TransientJobError:
|
|
channel.basic_nack(method.delivery_tag, requeue=True)
|
|
return
|
|
except Exception:
|
|
sys.stderr.write("worker failed closed for job %s\n" % payload.get("job_id"))
|
|
channel.basic_nack(method.delivery_tag, requeue=True)
|
|
return
|
|
channel.basic_ack(method.delivery_tag)
|
|
|
|
|
|
def _consume() -> None:
|
|
connection, channel = broker.connect()
|
|
channel.basic_qos(prefetch_count=1)
|
|
channel.basic_consume(broker.PROCESS_QUEUE, _on_message, auto_ack=False)
|
|
channel.start_consuming()
|
|
|
|
|
|
def _consume_loop() -> None:
|
|
while not _stop.is_set():
|
|
try:
|
|
_consume()
|
|
except Exception:
|
|
if _stop.is_set():
|
|
return
|
|
sys.stderr.write("consumer reconnect\n")
|
|
time.sleep(1)
|
|
|
|
|
|
def _sweep() -> None:
|
|
while not _stop.is_set():
|
|
try:
|
|
expired = db.recover_expired_leases(settings.max_attempts())
|
|
restored = db.reconcile_queued(settings.job_timeout())
|
|
if expired or restored:
|
|
sys.stderr.write("sweep expired=%s restored=%s\n" % (expired, restored))
|
|
for attachment_id, key in db.staging_ready_for_cleanup(40):
|
|
storage.delete_quiet(key)
|
|
db.clear_staging_key(attachment_id)
|
|
for attachment_id, key in db.expired_failed_staging(settings.staging_retention(), 40):
|
|
storage.delete_quiet(key)
|
|
db.clear_staging_key(attachment_id)
|
|
relay.publish_batch()
|
|
except Exception:
|
|
sys.stderr.write("sweeper failed\n")
|
|
_stop.wait(2)
|
|
|
|
|
|
def _memory_limit() -> str:
|
|
for path in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"):
|
|
try:
|
|
with open(path, encoding="utf-8") as handle:
|
|
return handle.read().strip()
|
|
except OSError:
|
|
continue
|
|
return "unknown"
|
|
|
|
|
|
def main() -> None:
|
|
db.init()
|
|
storage.init()
|
|
sys.stderr.write(
|
|
"worker concurrency=%s prefetch_per_channel=1 max_attempts=%s timeout=%ss memory_limit=%s\n"
|
|
% (
|
|
settings.worker_concurrency(),
|
|
settings.max_attempts(),
|
|
settings.job_timeout(),
|
|
_memory_limit(),
|
|
)
|
|
)
|
|
threading.Thread(target=_sweep, name="sweep", daemon=True).start()
|
|
threads = []
|
|
for index in range(settings.worker_concurrency()):
|
|
thread = threading.Thread(target=_consume_loop, name="consumer-%s" % index, daemon=True)
|
|
thread.start()
|
|
threads.append(thread)
|
|
try:
|
|
while any(thread.is_alive() for thread in threads):
|
|
time.sleep(0.5)
|
|
except KeyboardInterrupt:
|
|
_stop.set()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|