Skeleton contracts / contracts (pull_request) Canceled after 0s
Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
349 lines
13 KiB
Python
349 lines
13 KiB
Python
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import signal
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import contextmanager
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
import pika
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from opentelemetry.trace import SpanKind
|
|
|
|
from app import broker, db, storage, telemetry
|
|
from app.broker import QUEUE, _params
|
|
from app.codec import utcnow
|
|
from app.errors import AppError
|
|
from app.faults import consume_process_failure
|
|
from app.images import thumbnail_jpeg
|
|
from app.poller import start
|
|
from app.settings import settings
|
|
|
|
_inflight = 0
|
|
_inflight_lock = threading.Lock()
|
|
_stop = threading.Event()
|
|
|
|
|
|
class Transient(Exception):
|
|
pass
|
|
|
|
|
|
class Permanent(Exception):
|
|
pass
|
|
|
|
|
|
@contextmanager
|
|
def _busy():
|
|
global _inflight
|
|
with _inflight_lock:
|
|
_inflight += 1
|
|
current = _inflight
|
|
telemetry.note_worker(current, settings.worker_concurrency)
|
|
try:
|
|
yield
|
|
finally:
|
|
with _inflight_lock:
|
|
_inflight -= 1
|
|
current = _inflight
|
|
telemetry.note_worker(current, settings.worker_concurrency)
|
|
|
|
|
|
def _mark_failed(attachment_id, attempt, code):
|
|
with db.connection() as conn:
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'failed', error_code = %s, lease_until = NULL, worker_id = NULL, updated_at = %s
|
|
WHERE id = %s AND status = 'processing' AND attempt = %s
|
|
""",
|
|
(code, utcnow(), attachment_id, attempt),
|
|
)
|
|
telemetry.log_event("job.failed", attachment_id=str(attachment_id), attempt=attempt, error_code=code)
|
|
|
|
|
|
def _release(attachment_id, attempt):
|
|
with db.connection() as conn:
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'queued', lease_until = NULL, worker_id = NULL, updated_at = %s
|
|
WHERE id = %s AND status = 'processing' AND attempt = %s
|
|
""",
|
|
(utcnow(), attachment_id, attempt),
|
|
)
|
|
|
|
|
|
def process_delivery(body, headers):
|
|
started = time.perf_counter()
|
|
try:
|
|
payload = json.loads(body.decode("utf-8"))
|
|
attachment_id = uuid.UUID(str(payload["attachment_id"]))
|
|
pipeline = int(payload["pipeline_version"])
|
|
except (ValueError, UnicodeError, KeyError, TypeError):
|
|
telemetry.log_event("job.poison", severity="ERROR")
|
|
return "ack"
|
|
carrier = {}
|
|
for key, value in (headers or {}).items():
|
|
if isinstance(value, bytes):
|
|
value = value.decode("utf-8", "replace")
|
|
if isinstance(value, str):
|
|
carrier[str(key).lower()] = value
|
|
if payload.get("traceparent"):
|
|
carrier.setdefault("traceparent", payload["traceparent"])
|
|
if payload.get("tracestate"):
|
|
carrier.setdefault("tracestate", payload["tracestate"])
|
|
parent = telemetry.extract_context(carrier)
|
|
outcome = None
|
|
with telemetry.tracer().start_as_current_span(
|
|
"attachment.process consume",
|
|
context=parent,
|
|
kind=SpanKind.CONSUMER,
|
|
attributes={
|
|
"messaging.system": "rabbitmq",
|
|
"messaging.destination.name": QUEUE,
|
|
"messaging.operation.type": "consume",
|
|
},
|
|
) as span:
|
|
try:
|
|
action, outcome, attempt = _consume(payload, attachment_id, pipeline, span)
|
|
except Exception:
|
|
telemetry.mark_error(span)
|
|
telemetry.log_event("job.failed", severity="ERROR", error_type="unexpected", attachment_id=str(attachment_id))
|
|
action, outcome = "requeue", "retry"
|
|
attempt = None
|
|
if outcome:
|
|
telemetry.record_job(outcome, time.perf_counter() - started)
|
|
if attempt is not None:
|
|
span.set_attribute("attempt", attempt)
|
|
if outcome in {"failed", "retry"}:
|
|
telemetry.mark_error(span)
|
|
return action
|
|
|
|
|
|
def _consume(payload, attachment_id, pipeline, span):
|
|
now = utcnow()
|
|
with db.connection() as conn:
|
|
row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
|
|
if not row or int(row["pipeline_version"]) != pipeline:
|
|
return "ack", None, None
|
|
if row["status"] == "ready":
|
|
return "ack", None, int(row["attempt"])
|
|
if row["status"] == "failed":
|
|
return "ack", None, int(row["attempt"])
|
|
if row["status"] == "processing" and row["lease_until"] and row["lease_until"] > now:
|
|
held_attempt = int(row["attempt"])
|
|
staging_key = None
|
|
else:
|
|
held_attempt = None
|
|
attempt = int(row["attempt"]) + 1
|
|
if attempt > settings.job_max_attempts:
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'failed', error_code = 'retry_exhausted', lease_until = NULL, updated_at = %s
|
|
WHERE id = %s
|
|
""",
|
|
(now, attachment_id),
|
|
)
|
|
return "ack", "failed", attempt
|
|
lease = now + dt.timedelta(seconds=settings.job_timeout)
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'processing', attempt = %s, lease_until = %s, worker_id = %s, updated_at = %s
|
|
WHERE id = %s
|
|
""",
|
|
(attempt, lease, settings.instance_id, now, attachment_id),
|
|
)
|
|
staging_key = row["staging_key"]
|
|
content_type = row["content_type"]
|
|
expected_sha = row["sha256"]
|
|
created_at = row["created_at"]
|
|
if staging_key is None:
|
|
span.add_event("lease_held")
|
|
time.sleep(1)
|
|
return "requeue", None, held_attempt
|
|
span.set_attribute("attempt", attempt)
|
|
telemetry.log_event("job.consumed", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
|
|
try:
|
|
if consume_process_failure():
|
|
span.add_event("retry", {"reason": "injected"})
|
|
raise Transient()
|
|
data = storage.get(staging_key, "staging")
|
|
if hashlib.sha256(data).hexdigest() != expected_sha:
|
|
raise Permanent("invalid_image")
|
|
thumb, width, height = thumbnail_jpeg(data)
|
|
original_key = f"originals/{attachment_id}"
|
|
thumb_key = f"variants/{attachment_id}/thumbnail"
|
|
storage.put(original_key, data, content_type, "original")
|
|
storage.put(thumb_key, thumb, "image/jpeg", "thumbnail")
|
|
except Permanent as exc:
|
|
_mark_failed(attachment_id, attempt, str(exc) or "processing_failed")
|
|
return "ack", "failed", attempt
|
|
except AppError as exc:
|
|
code = "invalid_image" if exc.status in {413, 415} else "processing_failed"
|
|
_mark_failed(attachment_id, attempt, code)
|
|
return "ack", "failed", attempt
|
|
except Transient:
|
|
_release(attachment_id, attempt)
|
|
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="transient")
|
|
time.sleep(0.5)
|
|
return "requeue", "retry", attempt
|
|
except (BotoCoreError, ClientError, OSError, TimeoutError):
|
|
_release(attachment_id, attempt)
|
|
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="storage")
|
|
time.sleep(min(2 ** max(attempt - 1, 0), 8))
|
|
return "requeue", "retry", attempt
|
|
thumb_sha = hashlib.sha256(thumb).hexdigest()
|
|
ready_at = utcnow()
|
|
with db.connection() as conn:
|
|
current = conn.execute("SELECT status, attempt FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
|
|
if not current or current["status"] != "processing" or int(current["attempt"]) != attempt:
|
|
return "ack", None, attempt
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO attachment_variants (
|
|
attachment_id, name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version
|
|
) VALUES (%s, 'thumbnail', 'image/jpeg', %s, %s, %s, %s, %s, %s)
|
|
ON CONFLICT (attachment_id, name, pipeline_version) DO UPDATE
|
|
SET content_type = EXCLUDED.content_type,
|
|
size_bytes = EXCLUDED.size_bytes,
|
|
width = EXCLUDED.width,
|
|
height = EXCLUDED.height,
|
|
sha256 = EXCLUDED.sha256,
|
|
object_key = EXCLUDED.object_key
|
|
""",
|
|
(attachment_id, len(thumb), width, height, thumb_sha, thumb_key, pipeline),
|
|
)
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'ready', error_code = NULL, original_key = %s, ready_at = %s,
|
|
lease_until = NULL, updated_at = %s
|
|
WHERE id = %s AND status = 'processing' AND attempt = %s
|
|
""",
|
|
(original_key, ready_at, ready_at, attachment_id, attempt),
|
|
)
|
|
try:
|
|
storage.delete(staging_key)
|
|
except (BotoCoreError, ClientError, OSError):
|
|
telemetry.log_event("staging.cleanup_deferred", severity="WARNING", attachment_id=str(attachment_id))
|
|
telemetry.record_ready((ready_at - created_at).total_seconds())
|
|
telemetry.log_event("job.ready", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
|
|
return "ack", "success", attempt
|
|
|
|
|
|
def _serve(connection):
|
|
channel = connection.channel()
|
|
channel.queue_declare(queue=QUEUE, durable=True)
|
|
channel.basic_qos(prefetch_count=settings.worker_concurrency)
|
|
executor = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job")
|
|
|
|
def on_message(ch, method, properties, body):
|
|
def work():
|
|
with _busy():
|
|
try:
|
|
action = process_delivery(body, properties.headers if properties else None)
|
|
except Exception:
|
|
action = "requeue"
|
|
|
|
def finish():
|
|
try:
|
|
if action == "ack":
|
|
ch.basic_ack(method.delivery_tag)
|
|
else:
|
|
ch.basic_nack(method.delivery_tag, requeue=True)
|
|
except Exception:
|
|
telemetry.log_event("job.ack_failed", severity="ERROR")
|
|
|
|
connection.add_callback_threadsafe(finish)
|
|
|
|
executor.submit(work)
|
|
|
|
channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False)
|
|
|
|
def watch():
|
|
_stop.wait()
|
|
connection.add_callback_threadsafe(channel.stop_consuming)
|
|
|
|
threading.Thread(target=watch, name="stop-watch", daemon=True).start()
|
|
channel.start_consuming()
|
|
executor.shutdown(wait=True, cancel_futures=False)
|
|
|
|
|
|
class _Health(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
if self.path.startswith("/health/live"):
|
|
self._send(200)
|
|
return
|
|
if self.path.startswith("/health/ready"):
|
|
try:
|
|
from app.secrets_loader import healthy
|
|
|
|
with db.connection() as conn:
|
|
conn.execute("SELECT 1")
|
|
ok = healthy() and storage.ready() and broker.ready()
|
|
except Exception:
|
|
ok = False
|
|
self._send(200 if ok else 503)
|
|
return
|
|
self._send(404)
|
|
|
|
def _send(self, status):
|
|
body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}'
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def log_message(self, fmt, *args):
|
|
return
|
|
|
|
|
|
def _health_server():
|
|
server = ThreadingHTTPServer(("127.0.0.1", 8081), _Health)
|
|
server.serve_forever()
|
|
|
|
|
|
def _request_stop(signum, _frame):
|
|
_stop.set()
|
|
|
|
|
|
def main():
|
|
signal.signal(signal.SIGTERM, _request_stop)
|
|
signal.signal(signal.SIGINT, _request_stop)
|
|
telemetry.init()
|
|
from app.main import _retry
|
|
|
|
_retry("postgres", db.init)
|
|
_retry("s3", storage.init)
|
|
_retry("broker", broker.init)
|
|
telemetry.note_worker(0, settings.worker_concurrency)
|
|
start(publish=False)
|
|
threading.Thread(target=_health_server, name="health", daemon=True).start()
|
|
while not _stop.is_set():
|
|
connection = None
|
|
try:
|
|
connection = pika.BlockingConnection(_params())
|
|
telemetry.update_runtime(worker_consumers=settings.worker_concurrency)
|
|
_serve(connection)
|
|
except Exception:
|
|
if _stop.is_set():
|
|
break
|
|
telemetry.log_event("broker.reconnect", severity="WARNING", error_type="connection")
|
|
time.sleep(1)
|
|
finally:
|
|
if connection is not None and connection.is_open:
|
|
try:
|
|
connection.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|