forked from mtusi2026/mtusi2026lab5
RabbitMQ + transactional outbox, отдельный worker (thumbnail/crop/WebP), lease и heartbeat, retry с backoff+jitter, DLQ и replay, admission control; тесты отказов, трек 1 и отчёт.
330 lines
15 KiB
Python
330 lines
15 KiB
Python
"""Attachment worker: a separate process consuming attachments.process (python -m app.worker).
|
|
|
|
Per delivery:
|
|
1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ
|
|
2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op
|
|
3. download staging object, verify size+sha256 against DB metadata
|
|
4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants
|
|
5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes)
|
|
6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner
|
|
7. delete staging (best effort, reconciler finishes it), then ack
|
|
Ack happens only after the durable effect (6) or after the failure/retry decision is committed.
|
|
A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases.
|
|
"""
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import resource
|
|
import signal
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
|
|
from app import config, faults, jobs
|
|
from app.broker import connect, declare
|
|
from app.db import Database
|
|
from app.s3 import S3Storage
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
|
log = logging.getLogger("messenger.worker")
|
|
|
|
PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"}
|
|
|
|
|
|
class Permanent(Exception):
|
|
def __init__(self, code: str, detail: str = ""):
|
|
super().__init__(code)
|
|
self.code, self.detail = code, detail or code
|
|
|
|
|
|
class Transient(Exception):
|
|
def __init__(self, code: str, detail: str = ""):
|
|
super().__init__(code)
|
|
self.code, self.detail = code, detail or code
|
|
|
|
|
|
class LeaseLost(Exception):
|
|
pass
|
|
|
|
|
|
def child_limits() -> None:
|
|
memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024
|
|
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
|
cpu = config.JOB_TIMEOUT_SECONDS
|
|
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
|
|
os.nice(5)
|
|
|
|
|
|
def sha256_file(path: str):
|
|
digest, size = hashlib.sha256(), 0
|
|
with open(path, "rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
|
digest.update(chunk)
|
|
size += len(chunk)
|
|
return digest.hexdigest(), size
|
|
|
|
|
|
class Worker:
|
|
def __init__(self) -> None:
|
|
self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8])
|
|
self.db = Database()
|
|
self.s3 = S3Storage()
|
|
self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY)
|
|
self.stopping = asyncio.Event()
|
|
self.inflight = 0
|
|
self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0}
|
|
|
|
# ------------------------------------------------------------------ lifecycle
|
|
|
|
async def run(self) -> None:
|
|
loop = asyncio.get_running_loop()
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
loop.add_signal_handler(sig, self.stopping.set)
|
|
await self.db.open()
|
|
await asyncio.to_thread(self.s3.open)
|
|
connection = await connect()
|
|
channel = await connection.channel()
|
|
await channel.set_qos(prefetch_count=config.WORKER_PREFETCH)
|
|
_, queue = await declare(channel)
|
|
tag = await queue.consume(self.on_message, no_ack=False)
|
|
background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())]
|
|
log.info("worker started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB",
|
|
self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS,
|
|
config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB)
|
|
await self.stopping.wait()
|
|
log.info("stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight)
|
|
try:
|
|
await queue.cancel(tag)
|
|
except Exception:
|
|
pass
|
|
deadline = time.monotonic() + config.WORKER_GRACE_SECONDS
|
|
while self.inflight and time.monotonic() < deadline:
|
|
await asyncio.sleep(0.2)
|
|
for task in background:
|
|
task.cancel()
|
|
# Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed.
|
|
await connection.close()
|
|
await self.db.close()
|
|
log.info("stopped stats=%s", self.stats)
|
|
|
|
async def alive(self) -> None:
|
|
while True:
|
|
try:
|
|
with open(config.WORKER_HEARTBEAT_FILE, "w") as handle:
|
|
handle.write(str(time.time()))
|
|
except OSError:
|
|
pass
|
|
await asyncio.sleep(2)
|
|
|
|
async def sweeper(self) -> None:
|
|
while True:
|
|
await asyncio.sleep(config.JOB_SWEEP_SECONDS)
|
|
try:
|
|
swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner))
|
|
if swept:
|
|
log.warning("lease_expired recovered=%s", swept)
|
|
except Exception:
|
|
log.warning("lease sweep skipped: database unavailable")
|
|
|
|
# ------------------------------------------------------------------ delivery
|
|
|
|
async def on_message(self, message) -> None:
|
|
async with self.slots:
|
|
self.inflight += 1
|
|
try:
|
|
await self.handle(message)
|
|
except Exception:
|
|
log.exception("unexpected error, message returned to queue")
|
|
await asyncio.sleep(1)
|
|
await self.settle(message.nack(requeue=True))
|
|
finally:
|
|
self.inflight -= 1
|
|
|
|
@staticmethod
|
|
async def settle(awaitable) -> None:
|
|
try:
|
|
await awaitable
|
|
except Exception:
|
|
log.warning("could not settle delivery (connection lost); broker will redeliver")
|
|
|
|
async def handle(self, message) -> None:
|
|
msg = jobs.parse_message(message.body)
|
|
if msg is None:
|
|
self.stats["dlq"] += 1
|
|
log.error("invalid message -> DLQ message_id=%s", message.message_id)
|
|
await self.settle(message.reject(requeue=False))
|
|
return
|
|
try:
|
|
decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner))
|
|
except Exception:
|
|
log.warning("claim failed: database unavailable, requeue job=%s", msg["job_id"])
|
|
await asyncio.sleep(2)
|
|
await self.settle(message.nack(requeue=True))
|
|
return
|
|
if decision == "done" and job["state"] == "succeeded":
|
|
await self.cleanup_staging(job["attachment_id"])
|
|
if decision != "claimed":
|
|
self.stats["noop"] += 1
|
|
log.info("noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision,
|
|
message.redelivered)
|
|
await self.settle(message.ack())
|
|
return
|
|
started = time.monotonic()
|
|
log.info("claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"],
|
|
job["max_attempts"])
|
|
lease_lost = asyncio.Event()
|
|
keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost))
|
|
outcome = None
|
|
work = asyncio.create_task(self.process(job, message))
|
|
lost = asyncio.create_task(lease_lost.wait())
|
|
try:
|
|
done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2,
|
|
return_when=asyncio.FIRST_COMPLETED)
|
|
if work not in done:
|
|
work.cancel()
|
|
raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout")
|
|
outcome = "succeeded" if work.result() else "lost"
|
|
except LeaseLost:
|
|
outcome = "lost"
|
|
except Permanent as exc:
|
|
outcome = await self.record_failure(job, exc.code, exc.detail, transient=False)
|
|
except Transient as exc:
|
|
outcome = await self.record_failure(job, exc.code, exc.detail, transient=True)
|
|
except Exception as exc: # S3/DB/network errors
|
|
outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True)
|
|
finally:
|
|
keeper.cancel()
|
|
lost.cancel()
|
|
if outcome is None:
|
|
# Could not record the decision: keep the message; the lease expires and the sweeper takes over.
|
|
await asyncio.sleep(2)
|
|
await self.settle(message.nack(requeue=True))
|
|
return
|
|
self.stats[outcome] = self.stats.get(outcome, 0) + 1
|
|
log.info("job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome, time.monotonic() - started)
|
|
await self.settle(message.ack())
|
|
|
|
async def cleanup_staging(self, attachment_id) -> None:
|
|
"""Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup."""
|
|
try:
|
|
async with self.db.connection() as conn:
|
|
src = await jobs.attachment_source(conn, attachment_id)
|
|
if src and src["status"] == "ready" and src["staging_key"]:
|
|
await asyncio.to_thread(self.s3.delete, src["staging_key"])
|
|
except Exception:
|
|
log.warning("staging cleanup deferred to reconciler attachment=%s", attachment_id)
|
|
|
|
async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None:
|
|
interval = max(1.0, config.JOB_LEASE_SECONDS / 3)
|
|
while True:
|
|
await asyncio.sleep(interval)
|
|
try:
|
|
alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner))
|
|
except Exception:
|
|
continue # DB blip: keep trying until the lease really expires
|
|
if not alive:
|
|
log.warning("lease lost job=%s: another worker owns it now", job_id)
|
|
lease_lost.set()
|
|
return
|
|
|
|
async def record_failure(self, job, code: str, detail: str, transient: bool):
|
|
log.warning("job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code, transient)
|
|
try:
|
|
return await self.db.in_transaction(
|
|
lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient))
|
|
except Exception:
|
|
log.warning("could not record failure job=%s: database unavailable", job["id"])
|
|
return None
|
|
|
|
# ------------------------------------------------------------------ the actual work
|
|
|
|
async def process(self, job, message) -> bool:
|
|
async with self.db.connection() as conn:
|
|
src = await jobs.attachment_source(conn, job["attachment_id"])
|
|
if src is None or not src["staging_key"]:
|
|
raise Permanent("staging_missing", "attachment has no staging object")
|
|
filename = src["filename"]
|
|
first_delivery = job["attempt"] == 1 and not message.redelivered
|
|
if faults.active(filename, "worker_hang") and job["attempt"] == 1:
|
|
log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5))
|
|
time.sleep(config.JOB_LEASE_SECONDS * 2.5)
|
|
if faults.active(filename, "permanent"):
|
|
raise Permanent("invalid_image", "injected permanent failure")
|
|
if job["attempt"] <= faults.transient_until(filename):
|
|
raise Transient("dependency_unavailable", "injected transient failure")
|
|
|
|
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
|
|
source = os.path.join(tmp, "source")
|
|
if not await asyncio.to_thread(self.s3.download, src["staging_key"], source):
|
|
raise Permanent("staging_missing", "staging object not found")
|
|
digest, size = await asyncio.to_thread(sha256_file, source)
|
|
if digest != src["sha256"] or size != src["size_bytes"]:
|
|
raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload")
|
|
out_dir = os.path.join(tmp, "out")
|
|
os.mkdir(out_dir)
|
|
produced = await self.run_child(source, out_dir)
|
|
|
|
variants = []
|
|
prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"])
|
|
await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"])
|
|
for item in produced:
|
|
digest, size = await asyncio.to_thread(sha256_file, item["file"])
|
|
key = prefix + os.path.basename(item["file"])
|
|
await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"])
|
|
variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"],
|
|
"size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest})
|
|
if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS):
|
|
raise Permanent("processing_failed", "not all variants were produced")
|
|
|
|
if first_delivery and faults.active(filename, "worker_crash_after_put"):
|
|
faults.crash("after S3 put, before result COMMIT")
|
|
committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants))
|
|
if not committed:
|
|
log.warning("result not committed job=%s: lease lost, effect belongs to the new owner", job["id"])
|
|
return False
|
|
if first_delivery and faults.active(filename, "worker_crash_after_commit"):
|
|
faults.crash("after result COMMIT, before ack")
|
|
try:
|
|
await asyncio.to_thread(self.s3.delete, src["staging_key"])
|
|
except Exception:
|
|
log.warning("staging cleanup deferred to reconciler attachment=%s", job["attachment_id"])
|
|
return True
|
|
|
|
async def run_child(self, source: str, out_dir: str) -> list:
|
|
env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app",
|
|
"JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)}
|
|
proc = await asyncio.create_subprocess_exec(
|
|
sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS),
|
|
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits,
|
|
)
|
|
try:
|
|
stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS)
|
|
except asyncio.TimeoutError:
|
|
proc.kill()
|
|
await proc.wait()
|
|
raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None
|
|
except asyncio.CancelledError:
|
|
proc.kill()
|
|
raise
|
|
code = proc.returncode
|
|
if code == 0:
|
|
return json.loads(stdout)
|
|
if code in PERMANENT_EXIT:
|
|
raise Permanent(PERMANENT_EXIT[code])
|
|
if code == -signal.SIGXCPU:
|
|
raise Permanent("image_too_complex", "CPU limit exceeded")
|
|
if code == -signal.SIGKILL:
|
|
raise Transient("worker_resource_exhausted", "child killed (OOM)")
|
|
raise Permanent("processing_failed", "child exit %s" % code)
|
|
|
|
|
|
def main() -> None:
|
|
asyncio.run(Worker().run())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|