Files
mtusi2026lab7/app/worker.py
T
vika baafe2a3ae
Skeleton contracts / contracts (pull_request) Canceled after 0s
Добавить стенд лабораторной 7 с mTLS, OpenBao и least privilege worker.
Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
2026-11-07 20:46:02 +03:00

349 lines
13 KiB
Python

import datetime as dt
import hashlib
import json
import signal
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pika
from botocore.exceptions import BotoCoreError, ClientError
from opentelemetry.trace import SpanKind
from app import broker, db, storage, telemetry
from app.broker import QUEUE, _params
from app.codec import utcnow
from app.errors import AppError
from app.faults import consume_process_failure
from app.images import thumbnail_jpeg
from app.poller import start
from app.settings import settings
_inflight = 0
_inflight_lock = threading.Lock()
_stop = threading.Event()
class Transient(Exception):
pass
class Permanent(Exception):
pass
@contextmanager
def _busy():
global _inflight
with _inflight_lock:
_inflight += 1
current = _inflight
telemetry.note_worker(current, settings.worker_concurrency)
try:
yield
finally:
with _inflight_lock:
_inflight -= 1
current = _inflight
telemetry.note_worker(current, settings.worker_concurrency)
def _mark_failed(attachment_id, attempt, code):
with db.connection() as conn:
conn.execute(
"""
UPDATE attachments
SET status = 'failed', error_code = %s, lease_until = NULL, worker_id = NULL, updated_at = %s
WHERE id = %s AND status = 'processing' AND attempt = %s
""",
(code, utcnow(), attachment_id, attempt),
)
telemetry.log_event("job.failed", attachment_id=str(attachment_id), attempt=attempt, error_code=code)
def _release(attachment_id, attempt):
with db.connection() as conn:
conn.execute(
"""
UPDATE attachments
SET status = 'queued', lease_until = NULL, worker_id = NULL, updated_at = %s
WHERE id = %s AND status = 'processing' AND attempt = %s
""",
(utcnow(), attachment_id, attempt),
)
def process_delivery(body, headers):
started = time.perf_counter()
try:
payload = json.loads(body.decode("utf-8"))
attachment_id = uuid.UUID(str(payload["attachment_id"]))
pipeline = int(payload["pipeline_version"])
except (ValueError, UnicodeError, KeyError, TypeError):
telemetry.log_event("job.poison", severity="ERROR")
return "ack"
carrier = {}
for key, value in (headers or {}).items():
if isinstance(value, bytes):
value = value.decode("utf-8", "replace")
if isinstance(value, str):
carrier[str(key).lower()] = value
if payload.get("traceparent"):
carrier.setdefault("traceparent", payload["traceparent"])
if payload.get("tracestate"):
carrier.setdefault("tracestate", payload["tracestate"])
parent = telemetry.extract_context(carrier)
outcome = None
with telemetry.tracer().start_as_current_span(
"attachment.process consume",
context=parent,
kind=SpanKind.CONSUMER,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": QUEUE,
"messaging.operation.type": "consume",
},
) as span:
try:
action, outcome, attempt = _consume(payload, attachment_id, pipeline, span)
except Exception:
telemetry.mark_error(span)
telemetry.log_event("job.failed", severity="ERROR", error_type="unexpected", attachment_id=str(attachment_id))
action, outcome = "requeue", "retry"
attempt = None
if outcome:
telemetry.record_job(outcome, time.perf_counter() - started)
if attempt is not None:
span.set_attribute("attempt", attempt)
if outcome in {"failed", "retry"}:
telemetry.mark_error(span)
return action
def _consume(payload, attachment_id, pipeline, span):
now = utcnow()
with db.connection() as conn:
row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
if not row or int(row["pipeline_version"]) != pipeline:
return "ack", None, None
if row["status"] == "ready":
return "ack", None, int(row["attempt"])
if row["status"] == "failed":
return "ack", None, int(row["attempt"])
if row["status"] == "processing" and row["lease_until"] and row["lease_until"] > now:
held_attempt = int(row["attempt"])
staging_key = None
else:
held_attempt = None
attempt = int(row["attempt"]) + 1
if attempt > settings.job_max_attempts:
conn.execute(
"""
UPDATE attachments
SET status = 'failed', error_code = 'retry_exhausted', lease_until = NULL, updated_at = %s
WHERE id = %s
""",
(now, attachment_id),
)
return "ack", "failed", attempt
lease = now + dt.timedelta(seconds=settings.job_timeout)
conn.execute(
"""
UPDATE attachments
SET status = 'processing', attempt = %s, lease_until = %s, worker_id = %s, updated_at = %s
WHERE id = %s
""",
(attempt, lease, settings.instance_id, now, attachment_id),
)
staging_key = row["staging_key"]
content_type = row["content_type"]
expected_sha = row["sha256"]
created_at = row["created_at"]
if staging_key is None:
span.add_event("lease_held")
time.sleep(1)
return "requeue", None, held_attempt
span.set_attribute("attempt", attempt)
telemetry.log_event("job.consumed", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
try:
if consume_process_failure():
span.add_event("retry", {"reason": "injected"})
raise Transient()
data = storage.get(staging_key, "staging")
if hashlib.sha256(data).hexdigest() != expected_sha:
raise Permanent("invalid_image")
thumb, width, height = thumbnail_jpeg(data)
original_key = f"originals/{attachment_id}"
thumb_key = f"variants/{attachment_id}/thumbnail"
storage.put(original_key, data, content_type, "original")
storage.put(thumb_key, thumb, "image/jpeg", "thumbnail")
except Permanent as exc:
_mark_failed(attachment_id, attempt, str(exc) or "processing_failed")
return "ack", "failed", attempt
except AppError as exc:
code = "invalid_image" if exc.status in {413, 415} else "processing_failed"
_mark_failed(attachment_id, attempt, code)
return "ack", "failed", attempt
except Transient:
_release(attachment_id, attempt)
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="transient")
time.sleep(0.5)
return "requeue", "retry", attempt
except (BotoCoreError, ClientError, OSError, TimeoutError):
_release(attachment_id, attempt)
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="storage")
time.sleep(min(2 ** max(attempt - 1, 0), 8))
return "requeue", "retry", attempt
thumb_sha = hashlib.sha256(thumb).hexdigest()
ready_at = utcnow()
with db.connection() as conn:
current = conn.execute("SELECT status, attempt FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
if not current or current["status"] != "processing" or int(current["attempt"]) != attempt:
return "ack", None, attempt
conn.execute(
"""
INSERT INTO attachment_variants (
attachment_id, name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version
) VALUES (%s, 'thumbnail', 'image/jpeg', %s, %s, %s, %s, %s, %s)
ON CONFLICT (attachment_id, name, pipeline_version) DO UPDATE
SET content_type = EXCLUDED.content_type,
size_bytes = EXCLUDED.size_bytes,
width = EXCLUDED.width,
height = EXCLUDED.height,
sha256 = EXCLUDED.sha256,
object_key = EXCLUDED.object_key
""",
(attachment_id, len(thumb), width, height, thumb_sha, thumb_key, pipeline),
)
conn.execute(
"""
UPDATE attachments
SET status = 'ready', error_code = NULL, original_key = %s, ready_at = %s,
lease_until = NULL, updated_at = %s
WHERE id = %s AND status = 'processing' AND attempt = %s
""",
(original_key, ready_at, ready_at, attachment_id, attempt),
)
try:
storage.delete(staging_key)
except (BotoCoreError, ClientError, OSError):
telemetry.log_event("staging.cleanup_deferred", severity="WARNING", attachment_id=str(attachment_id))
telemetry.record_ready((ready_at - created_at).total_seconds())
telemetry.log_event("job.ready", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
return "ack", "success", attempt
def _serve(connection):
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.basic_qos(prefetch_count=settings.worker_concurrency)
executor = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job")
def on_message(ch, method, properties, body):
def work():
with _busy():
try:
action = process_delivery(body, properties.headers if properties else None)
except Exception:
action = "requeue"
def finish():
try:
if action == "ack":
ch.basic_ack(method.delivery_tag)
else:
ch.basic_nack(method.delivery_tag, requeue=True)
except Exception:
telemetry.log_event("job.ack_failed", severity="ERROR")
connection.add_callback_threadsafe(finish)
executor.submit(work)
channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False)
def watch():
_stop.wait()
connection.add_callback_threadsafe(channel.stop_consuming)
threading.Thread(target=watch, name="stop-watch", daemon=True).start()
channel.start_consuming()
executor.shutdown(wait=True, cancel_futures=False)
class _Health(BaseHTTPRequestHandler):
def do_GET(self):
if self.path.startswith("/health/live"):
self._send(200)
return
if self.path.startswith("/health/ready"):
try:
from app.secrets_loader import healthy
with db.connection() as conn:
conn.execute("SELECT 1")
ok = healthy() and storage.ready() and broker.ready()
except Exception:
ok = False
self._send(200 if ok else 503)
return
self._send(404)
def _send(self, status):
body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}'
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, fmt, *args):
return
def _health_server():
server = ThreadingHTTPServer(("127.0.0.1", 8081), _Health)
server.serve_forever()
def _request_stop(signum, _frame):
_stop.set()
def main():
signal.signal(signal.SIGTERM, _request_stop)
signal.signal(signal.SIGINT, _request_stop)
telemetry.init()
from app.main import _retry
_retry("postgres", db.init)
_retry("s3", storage.init)
_retry("broker", broker.init)
telemetry.note_worker(0, settings.worker_concurrency)
start(publish=False)
threading.Thread(target=_health_server, name="health", daemon=True).start()
while not _stop.is_set():
connection = None
try:
connection = pika.BlockingConnection(_params())
telemetry.update_runtime(worker_consumers=settings.worker_concurrency)
_serve(connection)
except Exception:
if _stop.is_set():
break
telemetry.log_event("broker.reconnect", severity="WARNING", error_type="connection")
time.sleep(1)
finally:
if connection is not None and connection.is_open:
try:
connection.close()
except Exception:
pass
if __name__ == "__main__":
main()