import datetime as dt import hashlib import json import signal import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import pika from botocore.exceptions import BotoCoreError, ClientError from opentelemetry.trace import SpanKind from app import broker, db, storage, telemetry from app.broker import QUEUE, _params from app.codec import utcnow from app.errors import AppError from app.faults import consume_process_failure from app.images import thumbnail_jpeg from app.poller import start from app.settings import settings _inflight = 0 _inflight_lock = threading.Lock() _stop = threading.Event() class Transient(Exception): pass class Permanent(Exception): pass @contextmanager def _busy(): global _inflight with _inflight_lock: _inflight += 1 current = _inflight telemetry.note_worker(current, settings.worker_concurrency) try: yield finally: with _inflight_lock: _inflight -= 1 current = _inflight telemetry.note_worker(current, settings.worker_concurrency) def _mark_failed(attachment_id, attempt, code): with db.connection() as conn: conn.execute( """ UPDATE attachments SET status = 'failed', error_code = %s, lease_until = NULL, worker_id = NULL, updated_at = %s WHERE id = %s AND status = 'processing' AND attempt = %s """, (code, utcnow(), attachment_id, attempt), ) telemetry.log_event("job.failed", attachment_id=str(attachment_id), attempt=attempt, error_code=code) def _release(attachment_id, attempt): with db.connection() as conn: conn.execute( """ UPDATE attachments SET status = 'queued', lease_until = NULL, worker_id = NULL, updated_at = %s WHERE id = %s AND status = 'processing' AND attempt = %s """, (utcnow(), attachment_id, attempt), ) def process_delivery(body, headers): started = time.perf_counter() try: payload = json.loads(body.decode("utf-8")) attachment_id = uuid.UUID(str(payload["attachment_id"])) pipeline = int(payload["pipeline_version"]) except (ValueError, UnicodeError, KeyError, TypeError): telemetry.log_event("job.poison", severity="ERROR") return "ack" carrier = {} for key, value in (headers or {}).items(): if isinstance(value, bytes): value = value.decode("utf-8", "replace") if isinstance(value, str): carrier[str(key).lower()] = value if payload.get("traceparent"): carrier.setdefault("traceparent", payload["traceparent"]) if payload.get("tracestate"): carrier.setdefault("tracestate", payload["tracestate"]) parent = telemetry.extract_context(carrier) outcome = None with telemetry.tracer().start_as_current_span( "attachment.process consume", context=parent, kind=SpanKind.CONSUMER, attributes={ "messaging.system": "rabbitmq", "messaging.destination.name": QUEUE, "messaging.operation.type": "consume", }, ) as span: try: action, outcome, attempt = _consume(payload, attachment_id, pipeline, span) except Exception: telemetry.mark_error(span) telemetry.log_event("job.failed", severity="ERROR", error_type="unexpected", attachment_id=str(attachment_id)) action, outcome = "requeue", "retry" attempt = None if outcome: telemetry.record_job(outcome, time.perf_counter() - started) if attempt is not None: span.set_attribute("attempt", attempt) if outcome in {"failed", "retry"}: telemetry.mark_error(span) return action def _consume(payload, attachment_id, pipeline, span): now = utcnow() with db.connection() as conn: row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone() if not row or int(row["pipeline_version"]) != pipeline: return "ack", None, None if row["status"] == "ready": return "ack", None, int(row["attempt"]) if row["status"] == "failed": return "ack", None, int(row["attempt"]) if row["status"] == "processing" and row["lease_until"] and row["lease_until"] > now: held_attempt = int(row["attempt"]) staging_key = None else: held_attempt = None attempt = int(row["attempt"]) + 1 if attempt > settings.job_max_attempts: conn.execute( """ UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', lease_until = NULL, updated_at = %s WHERE id = %s """, (now, attachment_id), ) return "ack", "failed", attempt lease = now + dt.timedelta(seconds=settings.job_timeout) conn.execute( """ UPDATE attachments SET status = 'processing', attempt = %s, lease_until = %s, worker_id = %s, updated_at = %s WHERE id = %s """, (attempt, lease, settings.instance_id, now, attachment_id), ) staging_key = row["staging_key"] content_type = row["content_type"] expected_sha = row["sha256"] created_at = row["created_at"] if staging_key is None: span.add_event("lease_held") time.sleep(1) return "requeue", None, held_attempt span.set_attribute("attempt", attempt) telemetry.log_event("job.consumed", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id"))) try: if consume_process_failure(): span.add_event("retry", {"reason": "injected"}) raise Transient() data = storage.get(staging_key, "staging") if hashlib.sha256(data).hexdigest() != expected_sha: raise Permanent("invalid_image") thumb, width, height = thumbnail_jpeg(data) original_key = f"originals/{attachment_id}" thumb_key = f"variants/{attachment_id}/thumbnail" storage.put(original_key, data, content_type, "original") storage.put(thumb_key, thumb, "image/jpeg", "thumbnail") except Permanent as exc: _mark_failed(attachment_id, attempt, str(exc) or "processing_failed") return "ack", "failed", attempt except AppError as exc: code = "invalid_image" if exc.status in {413, 415} else "processing_failed" _mark_failed(attachment_id, attempt, code) return "ack", "failed", attempt except Transient: _release(attachment_id, attempt) telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="transient") time.sleep(0.5) return "requeue", "retry", attempt except (BotoCoreError, ClientError, OSError, TimeoutError): _release(attachment_id, attempt) telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="storage") time.sleep(min(2 ** max(attempt - 1, 0), 8)) return "requeue", "retry", attempt thumb_sha = hashlib.sha256(thumb).hexdigest() ready_at = utcnow() with db.connection() as conn: current = conn.execute("SELECT status, attempt FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone() if not current or current["status"] != "processing" or int(current["attempt"]) != attempt: return "ack", None, attempt conn.execute( """ INSERT INTO attachment_variants ( attachment_id, name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version ) VALUES (%s, 'thumbnail', 'image/jpeg', %s, %s, %s, %s, %s, %s) ON CONFLICT (attachment_id, name, pipeline_version) DO UPDATE SET content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256, object_key = EXCLUDED.object_key """, (attachment_id, len(thumb), width, height, thumb_sha, thumb_key, pipeline), ) conn.execute( """ UPDATE attachments SET status = 'ready', error_code = NULL, original_key = %s, ready_at = %s, lease_until = NULL, updated_at = %s WHERE id = %s AND status = 'processing' AND attempt = %s """, (original_key, ready_at, ready_at, attachment_id, attempt), ) try: storage.delete(staging_key) except (BotoCoreError, ClientError, OSError): telemetry.log_event("staging.cleanup_deferred", severity="WARNING", attachment_id=str(attachment_id)) telemetry.record_ready((ready_at - created_at).total_seconds()) telemetry.log_event("job.ready", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id"))) return "ack", "success", attempt def _serve(connection): channel = connection.channel() channel.queue_declare(queue=QUEUE, durable=True) channel.basic_qos(prefetch_count=settings.worker_concurrency) executor = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job") def on_message(ch, method, properties, body): def work(): with _busy(): try: action = process_delivery(body, properties.headers if properties else None) except Exception: action = "requeue" def finish(): try: if action == "ack": ch.basic_ack(method.delivery_tag) else: ch.basic_nack(method.delivery_tag, requeue=True) except Exception: telemetry.log_event("job.ack_failed", severity="ERROR") connection.add_callback_threadsafe(finish) executor.submit(work) channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False) def watch(): _stop.wait() connection.add_callback_threadsafe(channel.stop_consuming) threading.Thread(target=watch, name="stop-watch", daemon=True).start() channel.start_consuming() executor.shutdown(wait=True, cancel_futures=False) class _Health(BaseHTTPRequestHandler): def do_GET(self): if self.path.startswith("/health/live"): self._send(200) return if self.path.startswith("/health/ready"): try: with db.connection() as conn: conn.execute("SELECT 1") ok = storage.ready() and broker.ready() except Exception: ok = False self._send(200 if ok else 503) return self._send(404) def _send(self, status): body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}' self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, fmt, *args): return def _health_server(): server = ThreadingHTTPServer(("0.0.0.0", 8081), _Health) server.serve_forever() def _request_stop(signum, _frame): _stop.set() def main(): signal.signal(signal.SIGTERM, _request_stop) signal.signal(signal.SIGINT, _request_stop) telemetry.init() from app.main import _retry _retry("postgres", db.init) db.migrate() _retry("s3", storage.init) _retry("broker", broker.init) telemetry.note_worker(0, settings.worker_concurrency) start() threading.Thread(target=_health_server, name="health", daemon=True).start() while not _stop.is_set(): connection = None try: connection = pika.BlockingConnection(_params()) telemetry.update_runtime(worker_consumers=settings.worker_concurrency) _serve(connection) except Exception: if _stop.is_set(): break telemetry.log_event("broker.reconnect", severity="WARNING", error_type="connection") time.sleep(1) finally: if connection is not None and connection.is_open: try: connection.close() except Exception: pass if __name__ == "__main__": main()