Files
mtusi2026lab7/app/worker.py
T
alenchha 861e7722b5 ЛР 7 — БВТ2301 — Курлова — на 4
Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия:
- внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial;
- TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL),
  Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA;
- OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь;
- роли БД, S3-политики и права брокера по принципу минимальных прав;
- Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS;
- hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия;
- отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao.
2026-11-16 10:49:13 +03:00

374 lines
18 KiB
Python

"""Attachment worker: a separate process consuming attachments.process (python -m app.worker).
Per delivery:
1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ
2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op
3. download staging object, verify size+sha256 against DB metadata
4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants
5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes)
6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner
7. delete staging (best effort, reconciler finishes it), then ack
Ack happens only after the durable effect (6) or after the failure/retry decision is committed.
A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases.
"""
import asyncio
import hashlib
import json
import logging
import os
import resource
import signal
import sys
import tempfile
import time
import uuid
from datetime import datetime, timezone
from opentelemetry.trace import SpanKind
from app import config, faults, jobs, openbao, telemetry
from app.broker import connect, get_queue
from app.db import Database
from app.s3 import S3Storage
log = logging.getLogger("messenger.worker")
PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"}
class Permanent(Exception):
def __init__(self, code: str, detail: str = ""):
super().__init__(code)
self.code, self.detail = code, detail or code
class Transient(Exception):
def __init__(self, code: str, detail: str = ""):
super().__init__(code)
self.code, self.detail = code, detail or code
class LeaseLost(Exception):
pass
def child_limits() -> None:
memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
cpu = config.JOB_TIMEOUT_SECONDS
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
os.nice(5)
def sha256_file(path: str):
digest, size = hashlib.sha256(), 0
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1 << 20), b""):
digest.update(chunk)
size += len(chunk)
return digest.hexdigest(), size
class Worker:
def __init__(self) -> None:
self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8])
self.db = Database()
self.s3 = S3Storage()
self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY)
self.stopping = asyncio.Event()
self.inflight = 0
self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0}
# ------------------------------------------------------------------ lifecycle
async def run(self) -> None:
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(sig, self.stopping.set)
await self.db.open()
telemetry.register_pool(self.db)
telemetry.register_worker(self)
await asyncio.to_thread(self.s3.open)
bao = openbao.client()
if bao is not None:
# Rotation of the worker's DB password / S3 key in OpenBao is picked up without a restart.
bao.on_change(lambda data: self.db.rotate(data.get("database_url", "")))
bao.on_change(self.s3.rotate)
bao.start_refresher(config.BAO_REFRESH_SECONDS)
connection = await connect()
channel = await connection.channel()
await channel.set_qos(prefetch_count=config.WORKER_PREFETCH)
queue = await get_queue(channel)
tag = await queue.consume(self.on_message, no_ack=False)
background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())]
log.info("worker_started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB",
self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS,
config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB)
await self.stopping.wait()
log.info("worker_stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight)
try:
await queue.cancel(tag)
except Exception:
pass
deadline = time.monotonic() + config.WORKER_GRACE_SECONDS
while self.inflight and time.monotonic() < deadline:
await asyncio.sleep(0.2)
for task in background:
task.cancel()
# Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed.
await connection.close()
await self.db.close()
log.info("worker_stopped stats=%s", self.stats)
async def alive(self) -> None:
while True:
try:
with open(config.WORKER_HEARTBEAT_FILE, "w") as handle:
handle.write(str(time.time()))
except OSError:
pass
await asyncio.sleep(2)
async def sweeper(self) -> None:
while True:
await asyncio.sleep(config.JOB_SWEEP_SECONDS)
try:
swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner))
if swept:
log.warning("lease_expired recovered=%s", swept)
except Exception:
log.warning("lease_sweep_skipped: database unavailable")
# ------------------------------------------------------------------ delivery
async def on_message(self, message) -> None:
async with self.slots:
self.inflight += 1
started = time.monotonic()
# CONSUMER span: child of the PRODUCER span whose traceparent is in the job message (W3C carrier).
try:
carrier = json.loads(message.body.decode("utf-8"))
except (UnicodeDecodeError, ValueError):
carrier = None
if not isinstance(carrier, dict) or "traceparent" not in carrier:
carrier = dict(message.headers or {})
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "process",
"messaging.destination.name": config.BROKER_QUEUE,
"messaging.message.id": str(message.message_id or ""),
"messaging.rabbitmq.message.redelivered": bool(message.redelivered)}
outcome = "requeued"
with telemetry.span(config.BROKER_QUEUE + " process", kind=SpanKind.CONSUMER,
context=telemetry.extract(carrier), attributes=attrs) as current:
try:
outcome = await self.handle(message)
except Exception:
log.exception("unexpected_error, message returned to queue")
await asyncio.sleep(1)
await self.settle(message.nack(requeue=True))
finally:
self.inflight -= 1
current.set_attribute("messenger.job.outcome", outcome)
if outcome in ("retry", "failed", "dlq", "requeued", "lost"):
telemetry.mark_error(current, outcome)
labels = {"job.type": telemetry.JOB_TYPE, "outcome": outcome}
telemetry.jobs_processed.add(1, labels)
telemetry.job_duration.record(time.monotonic() - started, labels)
@staticmethod
async def settle(awaitable) -> None:
try:
await awaitable
except Exception:
log.warning("could not settle delivery (connection lost); broker will redeliver")
async def handle(self, message) -> str:
"""Returns the delivery outcome: succeeded | retry | failed | lost | noop | dlq | requeued."""
msg = jobs.parse_message(message.body)
if msg is None:
self.stats["dlq"] += 1
log.error("job_invalid_message -> DLQ message_id=%s", message.message_id)
await self.settle(message.reject(requeue=False))
return "dlq"
telemetry.set_attributes(messenger__attachment_id=msg["attachment_id"], messenger__job__attempt=msg["attempt"])
try:
decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner))
except Exception:
log.warning("job_claim_failed: database unavailable, requeue job=%s", msg["job_id"])
await asyncio.sleep(2)
await self.settle(message.nack(requeue=True))
return "requeued"
if decision == "done" and job["state"] == "succeeded":
await self.cleanup_staging(job["attachment_id"])
if decision != "claimed":
self.stats["noop"] += 1
telemetry.set_attributes(messenger__job__decision=decision)
log.info("job_noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision,
message.redelivered)
await self.settle(message.ack())
return "noop"
started = time.monotonic()
log.info("job_claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"],
job["max_attempts"])
lease_lost = asyncio.Event()
keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost))
outcome = None
work = asyncio.create_task(self.process(job, message))
lost = asyncio.create_task(lease_lost.wait())
try:
done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2,
return_when=asyncio.FIRST_COMPLETED)
if work not in done:
work.cancel()
raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout")
outcome = "succeeded" if work.result() else "lost"
except LeaseLost:
outcome = "lost"
except Permanent as exc:
outcome = await self.record_failure(job, exc.code, exc.detail, transient=False)
except Transient as exc:
outcome = await self.record_failure(job, exc.code, exc.detail, transient=True)
except Exception as exc: # S3/DB/network errors
outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True)
finally:
keeper.cancel()
lost.cancel()
if outcome is None:
# Could not record the decision: keep the message; the lease expires and the sweeper takes over.
await asyncio.sleep(2)
await self.settle(message.nack(requeue=True))
return "requeued"
self.stats[outcome] = self.stats.get(outcome, 0) + 1
log.info("job_finished job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome,
time.monotonic() - started)
await self.settle(message.ack())
return outcome
async def cleanup_staging(self, attachment_id) -> None:
"""Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup."""
try:
async with self.db.connection() as conn:
src = await jobs.attachment_source(conn, attachment_id)
if src and src["status"] == "ready" and src["staging_key"]:
await asyncio.to_thread(self.s3.delete, src["staging_key"])
except Exception:
log.warning("staging_cleanup_deferred attachment=%s", attachment_id)
async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None:
interval = max(1.0, config.JOB_LEASE_SECONDS / 3)
while True:
await asyncio.sleep(interval)
try:
alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner))
except Exception:
continue # DB blip: keep trying until the lease really expires
if not alive:
log.warning("lease_lost job=%s: another worker owns it now", job_id)
lease_lost.set()
return
async def record_failure(self, job, code: str, detail: str, transient: bool):
log.warning("job_attempt_failed job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code,
transient)
try:
return await self.db.in_transaction(
lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient))
except Exception:
log.warning("job_failure_not_recorded job=%s: database unavailable", job["id"])
return None
# ------------------------------------------------------------------ the actual work
async def process(self, job, message) -> bool:
async with self.db.connection() as conn:
src = await jobs.attachment_source(conn, job["attachment_id"])
if src is None or not src["staging_key"]:
raise Permanent("staging_missing", "attachment has no staging object")
filename = src["filename"]
first_delivery = job["attempt"] == 1 and not message.redelivered
if faults.active(filename, "worker_hang") and job["attempt"] == 1:
log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5))
time.sleep(config.JOB_LEASE_SECONDS * 2.5)
if faults.active(filename, "permanent"):
raise Permanent("invalid_image", "injected permanent failure")
if job["attempt"] <= faults.transient_until(filename):
raise Transient("dependency_unavailable", "injected transient failure")
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
source = os.path.join(tmp, "source")
if not await asyncio.to_thread(self.s3.download, src["staging_key"], source):
raise Permanent("staging_missing", "staging object not found")
digest, size = await asyncio.to_thread(sha256_file, source)
if digest != src["sha256"] or size != src["size_bytes"]:
raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload")
out_dir = os.path.join(tmp, "out")
os.mkdir(out_dir)
with telemetry.span("imageproc", attributes={"messenger.pipeline_version": job["pipeline_version"]}):
produced = await self.run_child(source, out_dir)
variants = []
prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"])
await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"])
for item in produced:
digest, size = await asyncio.to_thread(sha256_file, item["file"])
key = prefix + os.path.basename(item["file"])
await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"])
variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"],
"size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest})
if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS):
raise Permanent("processing_failed", "not all variants were produced")
if first_delivery and faults.active(filename, "worker_crash_after_put"):
faults.crash("after S3 put, before result COMMIT")
committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants))
if not committed:
log.warning("job_result_not_committed job=%s: lease lost, effect belongs to the new owner", job["id"])
return False
telemetry.time_to_ready.record(max(0.0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()),
{"job.type": telemetry.JOB_TYPE})
if first_delivery and faults.active(filename, "worker_crash_after_commit"):
faults.crash("after result COMMIT, before ack")
try:
await asyncio.to_thread(self.s3.delete, src["staging_key"])
except Exception:
log.warning("staging_cleanup_deferred attachment=%s", job["attachment_id"])
return True
async def run_child(self, source: str, out_dir: str) -> list:
env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app",
"JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)}
proc = await asyncio.create_subprocess_exec(
sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS),
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits,
)
try:
stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None
except asyncio.CancelledError:
proc.kill()
raise
code = proc.returncode
if code == 0:
return json.loads(stdout)
if code in PERMANENT_EXIT:
raise Permanent(PERMANENT_EXIT[code])
if code == -signal.SIGXCPU:
raise Permanent("image_too_complex", "CPU limit exceeded")
if code == -signal.SIGKILL:
raise Transient("worker_resource_exhausted", "child killed (OOM)")
raise Permanent("processing_failed", "child exit %s" % code)
def main() -> None:
telemetry.setup("messenger-worker")
try:
asyncio.run(Worker().run())
finally:
telemetry.shutdown()
if __name__ == "__main__":
main()