Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия: - внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial; - TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL), Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA; - OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь; - роли БД, S3-политики и права брокера по принципу минимальных прав; - Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS; - hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия; - отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao.
374 lines
18 KiB
Python
374 lines
18 KiB
Python
"""Attachment worker: a separate process consuming attachments.process (python -m app.worker).
|
|
|
|
Per delivery:
|
|
1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ
|
|
2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op
|
|
3. download staging object, verify size+sha256 against DB metadata
|
|
4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants
|
|
5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes)
|
|
6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner
|
|
7. delete staging (best effort, reconciler finishes it), then ack
|
|
Ack happens only after the durable effect (6) or after the failure/retry decision is committed.
|
|
A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases.
|
|
"""
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import resource
|
|
import signal
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
from opentelemetry.trace import SpanKind
|
|
|
|
from app import config, faults, jobs, openbao, telemetry
|
|
from app.broker import connect, get_queue
|
|
from app.db import Database
|
|
from app.s3 import S3Storage
|
|
|
|
log = logging.getLogger("messenger.worker")
|
|
|
|
PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"}
|
|
|
|
|
|
class Permanent(Exception):
|
|
def __init__(self, code: str, detail: str = ""):
|
|
super().__init__(code)
|
|
self.code, self.detail = code, detail or code
|
|
|
|
|
|
class Transient(Exception):
|
|
def __init__(self, code: str, detail: str = ""):
|
|
super().__init__(code)
|
|
self.code, self.detail = code, detail or code
|
|
|
|
|
|
class LeaseLost(Exception):
|
|
pass
|
|
|
|
|
|
def child_limits() -> None:
|
|
memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024
|
|
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
|
cpu = config.JOB_TIMEOUT_SECONDS
|
|
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
|
|
os.nice(5)
|
|
|
|
|
|
def sha256_file(path: str):
|
|
digest, size = hashlib.sha256(), 0
|
|
with open(path, "rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
|
digest.update(chunk)
|
|
size += len(chunk)
|
|
return digest.hexdigest(), size
|
|
|
|
|
|
class Worker:
|
|
def __init__(self) -> None:
|
|
self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8])
|
|
self.db = Database()
|
|
self.s3 = S3Storage()
|
|
self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY)
|
|
self.stopping = asyncio.Event()
|
|
self.inflight = 0
|
|
self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0}
|
|
|
|
# ------------------------------------------------------------------ lifecycle
|
|
|
|
async def run(self) -> None:
|
|
loop = asyncio.get_running_loop()
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
loop.add_signal_handler(sig, self.stopping.set)
|
|
await self.db.open()
|
|
telemetry.register_pool(self.db)
|
|
telemetry.register_worker(self)
|
|
await asyncio.to_thread(self.s3.open)
|
|
bao = openbao.client()
|
|
if bao is not None:
|
|
# Rotation of the worker's DB password / S3 key in OpenBao is picked up without a restart.
|
|
bao.on_change(lambda data: self.db.rotate(data.get("database_url", "")))
|
|
bao.on_change(self.s3.rotate)
|
|
bao.start_refresher(config.BAO_REFRESH_SECONDS)
|
|
connection = await connect()
|
|
channel = await connection.channel()
|
|
await channel.set_qos(prefetch_count=config.WORKER_PREFETCH)
|
|
queue = await get_queue(channel)
|
|
tag = await queue.consume(self.on_message, no_ack=False)
|
|
background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())]
|
|
log.info("worker_started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB",
|
|
self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS,
|
|
config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB)
|
|
await self.stopping.wait()
|
|
log.info("worker_stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight)
|
|
try:
|
|
await queue.cancel(tag)
|
|
except Exception:
|
|
pass
|
|
deadline = time.monotonic() + config.WORKER_GRACE_SECONDS
|
|
while self.inflight and time.monotonic() < deadline:
|
|
await asyncio.sleep(0.2)
|
|
for task in background:
|
|
task.cancel()
|
|
# Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed.
|
|
await connection.close()
|
|
await self.db.close()
|
|
log.info("worker_stopped stats=%s", self.stats)
|
|
|
|
async def alive(self) -> None:
|
|
while True:
|
|
try:
|
|
with open(config.WORKER_HEARTBEAT_FILE, "w") as handle:
|
|
handle.write(str(time.time()))
|
|
except OSError:
|
|
pass
|
|
await asyncio.sleep(2)
|
|
|
|
async def sweeper(self) -> None:
|
|
while True:
|
|
await asyncio.sleep(config.JOB_SWEEP_SECONDS)
|
|
try:
|
|
swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner))
|
|
if swept:
|
|
log.warning("lease_expired recovered=%s", swept)
|
|
except Exception:
|
|
log.warning("lease_sweep_skipped: database unavailable")
|
|
|
|
# ------------------------------------------------------------------ delivery
|
|
|
|
async def on_message(self, message) -> None:
|
|
async with self.slots:
|
|
self.inflight += 1
|
|
started = time.monotonic()
|
|
# CONSUMER span: child of the PRODUCER span whose traceparent is in the job message (W3C carrier).
|
|
try:
|
|
carrier = json.loads(message.body.decode("utf-8"))
|
|
except (UnicodeDecodeError, ValueError):
|
|
carrier = None
|
|
if not isinstance(carrier, dict) or "traceparent" not in carrier:
|
|
carrier = dict(message.headers or {})
|
|
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "process",
|
|
"messaging.destination.name": config.BROKER_QUEUE,
|
|
"messaging.message.id": str(message.message_id or ""),
|
|
"messaging.rabbitmq.message.redelivered": bool(message.redelivered)}
|
|
outcome = "requeued"
|
|
with telemetry.span(config.BROKER_QUEUE + " process", kind=SpanKind.CONSUMER,
|
|
context=telemetry.extract(carrier), attributes=attrs) as current:
|
|
try:
|
|
outcome = await self.handle(message)
|
|
except Exception:
|
|
log.exception("unexpected_error, message returned to queue")
|
|
await asyncio.sleep(1)
|
|
await self.settle(message.nack(requeue=True))
|
|
finally:
|
|
self.inflight -= 1
|
|
current.set_attribute("messenger.job.outcome", outcome)
|
|
if outcome in ("retry", "failed", "dlq", "requeued", "lost"):
|
|
telemetry.mark_error(current, outcome)
|
|
labels = {"job.type": telemetry.JOB_TYPE, "outcome": outcome}
|
|
telemetry.jobs_processed.add(1, labels)
|
|
telemetry.job_duration.record(time.monotonic() - started, labels)
|
|
|
|
@staticmethod
|
|
async def settle(awaitable) -> None:
|
|
try:
|
|
await awaitable
|
|
except Exception:
|
|
log.warning("could not settle delivery (connection lost); broker will redeliver")
|
|
|
|
async def handle(self, message) -> str:
|
|
"""Returns the delivery outcome: succeeded | retry | failed | lost | noop | dlq | requeued."""
|
|
msg = jobs.parse_message(message.body)
|
|
if msg is None:
|
|
self.stats["dlq"] += 1
|
|
log.error("job_invalid_message -> DLQ message_id=%s", message.message_id)
|
|
await self.settle(message.reject(requeue=False))
|
|
return "dlq"
|
|
telemetry.set_attributes(messenger__attachment_id=msg["attachment_id"], messenger__job__attempt=msg["attempt"])
|
|
try:
|
|
decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner))
|
|
except Exception:
|
|
log.warning("job_claim_failed: database unavailable, requeue job=%s", msg["job_id"])
|
|
await asyncio.sleep(2)
|
|
await self.settle(message.nack(requeue=True))
|
|
return "requeued"
|
|
if decision == "done" and job["state"] == "succeeded":
|
|
await self.cleanup_staging(job["attachment_id"])
|
|
if decision != "claimed":
|
|
self.stats["noop"] += 1
|
|
telemetry.set_attributes(messenger__job__decision=decision)
|
|
log.info("job_noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision,
|
|
message.redelivered)
|
|
await self.settle(message.ack())
|
|
return "noop"
|
|
started = time.monotonic()
|
|
log.info("job_claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"],
|
|
job["max_attempts"])
|
|
lease_lost = asyncio.Event()
|
|
keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost))
|
|
outcome = None
|
|
work = asyncio.create_task(self.process(job, message))
|
|
lost = asyncio.create_task(lease_lost.wait())
|
|
try:
|
|
done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2,
|
|
return_when=asyncio.FIRST_COMPLETED)
|
|
if work not in done:
|
|
work.cancel()
|
|
raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout")
|
|
outcome = "succeeded" if work.result() else "lost"
|
|
except LeaseLost:
|
|
outcome = "lost"
|
|
except Permanent as exc:
|
|
outcome = await self.record_failure(job, exc.code, exc.detail, transient=False)
|
|
except Transient as exc:
|
|
outcome = await self.record_failure(job, exc.code, exc.detail, transient=True)
|
|
except Exception as exc: # S3/DB/network errors
|
|
outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True)
|
|
finally:
|
|
keeper.cancel()
|
|
lost.cancel()
|
|
if outcome is None:
|
|
# Could not record the decision: keep the message; the lease expires and the sweeper takes over.
|
|
await asyncio.sleep(2)
|
|
await self.settle(message.nack(requeue=True))
|
|
return "requeued"
|
|
self.stats[outcome] = self.stats.get(outcome, 0) + 1
|
|
log.info("job_finished job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome,
|
|
time.monotonic() - started)
|
|
await self.settle(message.ack())
|
|
return outcome
|
|
|
|
async def cleanup_staging(self, attachment_id) -> None:
|
|
"""Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup."""
|
|
try:
|
|
async with self.db.connection() as conn:
|
|
src = await jobs.attachment_source(conn, attachment_id)
|
|
if src and src["status"] == "ready" and src["staging_key"]:
|
|
await asyncio.to_thread(self.s3.delete, src["staging_key"])
|
|
except Exception:
|
|
log.warning("staging_cleanup_deferred attachment=%s", attachment_id)
|
|
|
|
async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None:
|
|
interval = max(1.0, config.JOB_LEASE_SECONDS / 3)
|
|
while True:
|
|
await asyncio.sleep(interval)
|
|
try:
|
|
alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner))
|
|
except Exception:
|
|
continue # DB blip: keep trying until the lease really expires
|
|
if not alive:
|
|
log.warning("lease_lost job=%s: another worker owns it now", job_id)
|
|
lease_lost.set()
|
|
return
|
|
|
|
async def record_failure(self, job, code: str, detail: str, transient: bool):
|
|
log.warning("job_attempt_failed job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code,
|
|
transient)
|
|
try:
|
|
return await self.db.in_transaction(
|
|
lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient))
|
|
except Exception:
|
|
log.warning("job_failure_not_recorded job=%s: database unavailable", job["id"])
|
|
return None
|
|
|
|
# ------------------------------------------------------------------ the actual work
|
|
|
|
async def process(self, job, message) -> bool:
|
|
async with self.db.connection() as conn:
|
|
src = await jobs.attachment_source(conn, job["attachment_id"])
|
|
if src is None or not src["staging_key"]:
|
|
raise Permanent("staging_missing", "attachment has no staging object")
|
|
filename = src["filename"]
|
|
first_delivery = job["attempt"] == 1 and not message.redelivered
|
|
if faults.active(filename, "worker_hang") and job["attempt"] == 1:
|
|
log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5))
|
|
time.sleep(config.JOB_LEASE_SECONDS * 2.5)
|
|
if faults.active(filename, "permanent"):
|
|
raise Permanent("invalid_image", "injected permanent failure")
|
|
if job["attempt"] <= faults.transient_until(filename):
|
|
raise Transient("dependency_unavailable", "injected transient failure")
|
|
|
|
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
|
|
source = os.path.join(tmp, "source")
|
|
if not await asyncio.to_thread(self.s3.download, src["staging_key"], source):
|
|
raise Permanent("staging_missing", "staging object not found")
|
|
digest, size = await asyncio.to_thread(sha256_file, source)
|
|
if digest != src["sha256"] or size != src["size_bytes"]:
|
|
raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload")
|
|
out_dir = os.path.join(tmp, "out")
|
|
os.mkdir(out_dir)
|
|
with telemetry.span("imageproc", attributes={"messenger.pipeline_version": job["pipeline_version"]}):
|
|
produced = await self.run_child(source, out_dir)
|
|
|
|
variants = []
|
|
prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"])
|
|
await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"])
|
|
for item in produced:
|
|
digest, size = await asyncio.to_thread(sha256_file, item["file"])
|
|
key = prefix + os.path.basename(item["file"])
|
|
await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"])
|
|
variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"],
|
|
"size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest})
|
|
if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS):
|
|
raise Permanent("processing_failed", "not all variants were produced")
|
|
|
|
if first_delivery and faults.active(filename, "worker_crash_after_put"):
|
|
faults.crash("after S3 put, before result COMMIT")
|
|
committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants))
|
|
if not committed:
|
|
log.warning("job_result_not_committed job=%s: lease lost, effect belongs to the new owner", job["id"])
|
|
return False
|
|
telemetry.time_to_ready.record(max(0.0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()),
|
|
{"job.type": telemetry.JOB_TYPE})
|
|
if first_delivery and faults.active(filename, "worker_crash_after_commit"):
|
|
faults.crash("after result COMMIT, before ack")
|
|
try:
|
|
await asyncio.to_thread(self.s3.delete, src["staging_key"])
|
|
except Exception:
|
|
log.warning("staging_cleanup_deferred attachment=%s", job["attachment_id"])
|
|
return True
|
|
|
|
async def run_child(self, source: str, out_dir: str) -> list:
|
|
env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app",
|
|
"JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)}
|
|
proc = await asyncio.create_subprocess_exec(
|
|
sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS),
|
|
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits,
|
|
)
|
|
try:
|
|
stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS)
|
|
except asyncio.TimeoutError:
|
|
proc.kill()
|
|
await proc.wait()
|
|
raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None
|
|
except asyncio.CancelledError:
|
|
proc.kill()
|
|
raise
|
|
code = proc.returncode
|
|
if code == 0:
|
|
return json.loads(stdout)
|
|
if code in PERMANENT_EXIT:
|
|
raise Permanent(PERMANENT_EXIT[code])
|
|
if code == -signal.SIGXCPU:
|
|
raise Permanent("image_too_complex", "CPU limit exceeded")
|
|
if code == -signal.SIGKILL:
|
|
raise Transient("worker_resource_exhausted", "child killed (OOM)")
|
|
raise Permanent("processing_failed", "child exit %s" % code)
|
|
|
|
|
|
def main() -> None:
|
|
telemetry.setup("messenger-worker")
|
|
try:
|
|
asyncio.run(Worker().run())
|
|
finally:
|
|
telemetry.shutdown()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|