Files
mtusi2026lab5/app/worker.py

378 lines
20 KiB
Python

"""Attachment worker: consumes attachments.process, publishes the original and variants, commits the result.
Delivery is at-least-once; the effect is idempotent because
* PostgreSQL (jobs) decides whether a delivery may start an attempt: claim = queued -> processing with
attempt+1 and a lease (lease_owner, lease_until). Duplicates/stale/reordered messages are acked as no-ops.
* (job_id, attempt) is a fencing token: the result COMMIT succeeds only if this worker still owns the lease.
* S3 keys are the effect key (attachment, pipeline_version, variant): a retry overwrites identical bytes.
* attachment_variants has PRIMARY KEY (attachment_id, pipeline_version, name): one row per variant.
* A result of an older pipeline_version never replaces a newer published one (superseded).
The message is acked only after the result (or the decision) is durably committed in PostgreSQL.
"""
from __future__ import annotations
import asyncio
import hashlib
import logging
import os
import secrets
import signal
import tempfile
import time
import uuid
from typing import Optional
import asyncpg
from aio_pika.abc import AbstractIncomingMessage
from app import config, faults, jobs
from app.broker import Broker, InvalidJob, parse_job
from app.imaging import ImageRejected, TransientFailure, run_isolated
from app.objects import ObjectStore, S3NotFound, S3Unavailable, original_key, variant_key
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
logger = logging.getLogger("worker")
HEARTBEAT_FILE = "/tmp/worker-heartbeat"
DB_ERRORS = (OSError, asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.TooManyConnectionsError,
asyncpg.exceptions.OperatorInterventionError, asyncio.TimeoutError)
class Worker:
def __init__(self):
self.id = "worker-" + secrets.token_hex(4)
self.pool: Optional[asyncpg.Pool] = None
self.objects = ObjectStore()
self.broker = Broker()
self.inflight = set()
self.stopping = asyncio.Event()
# ------------------------------------------------------------------ message entry point
async def on_message(self, message: AbstractIncomingMessage) -> None:
task = asyncio.current_task()
self.inflight.add(task)
try:
await self._handle(message)
finally:
self.inflight.discard(task)
async def _handle(self, message: AbstractIncomingMessage) -> None:
try:
job = parse_job(message.body)
except InvalidJob as exc:
logger.error("rejecting malformed job message to DLQ: %s", exc)
await message.reject(requeue=False) # -> attachments.dlx -> attachments.dlq
return
try:
outcome = await self.process(job, message.redelivered)
except DB_ERRORS as exc:
# Nothing was decided durably. Give the message back after a pause (no hot loop); if the DB stays down
# the lease expires and the reaper re-enqueues the job through the outbox.
logger.warning("db unavailable while handling job %s: %s; requeue", job["job_id"], type(exc).__name__)
await asyncio.sleep(2)
await message.nack(requeue=True)
return
await faults.hit(self.pool, "worker.after_commit_before_ack", job["attachment_id"], self.id)
await message.ack()
logger.info("job %s attempt %s -> %s (redelivered=%s)", job["job_id"], job["attempt"], outcome, message.redelivered)
# ------------------------------------------------------------------ claim
async def claim(self, job: dict, redelivered: bool):
async with self.pool.acquire() as conn, conn.transaction():
row = await conn.fetchrow(
"""
SELECT j.*, a.status AS a_status, a.content_type, a.size_bytes, a.sha256, a.staging_key,
a.staging_deleted_at, a.object_key, a.pipeline_version AS a_version,
(j.lease_until > clock_timestamp()) AS lease_active,
(j.next_attempt_at <= clock_timestamp() + interval '1 second') AS due
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.job_id = $1
FOR UPDATE OF j
""",
job["job_id"],
)
reason = None
if row is None:
reason = "unknown_job_noop"
elif row["attachment_id"] != job["attachment_id"] or row["pipeline_version"] != job["pipeline_version"]:
reason = "mismatched_message_noop"
elif row["status"] == "done":
reason = "duplicate_noop" # result already committed: ack after checking state
elif row["status"] in ("failed", "superseded"):
reason = row["status"] + "_noop"
elif row["status"] == "processing":
reason = "lease_held_noop" if row["lease_active"] else "lease_expired_noop"
elif job["attempt"] != row["attempt"] + 1:
reason = "stale_attempt_noop" # reordered/old message of an earlier attempt
elif not row["due"]:
reason = "early_noop"
elif row["a_version"] is not None and row["a_version"] > row["pipeline_version"]:
await conn.execute(
"UPDATE jobs SET status = 'superseded', updated_at = clock_timestamp() WHERE job_id = $1", row["job_id"])
reason = "superseded_before_start"
if reason:
await jobs.event(conn, job["job_id"], job["attachment_id"] if row is None else row["attachment_id"],
reason, self.id, job["attempt"], "redelivered=%s" % redelivered)
return None, reason
claimed = await conn.fetchrow(
"""
UPDATE jobs SET status = 'processing', attempt = attempt + 1, lease_owner = $2,
lease_until = clock_timestamp() + make_interval(secs => $3), updated_at = clock_timestamp()
WHERE job_id = $1
RETURNING *
""",
row["job_id"], self.id, config.LEASE_SECONDS,
)
await conn.execute(
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() WHERE id = $1 AND status = 'queued'",
row["attachment_id"],
)
await jobs.event(conn, row["job_id"], row["attachment_id"], "claimed", self.id, claimed["attempt"],
"redelivered=%s" % redelivered)
info = dict(row)
info.update(dict(claimed))
return info, "claimed"
async def heartbeat(self, job: dict, lost: asyncio.Event) -> None:
"""Renew the lease while the attempt is alive; a dead worker stops renewing and the lease expires."""
while True:
await asyncio.sleep(config.HEARTBEAT_SECONDS)
try:
async with self.pool.acquire() as conn:
renewed = await conn.fetchval(
"""
UPDATE jobs SET lease_until = clock_timestamp() + make_interval(secs => $4)
WHERE job_id = $1 AND status = 'processing' AND lease_owner = $2 AND attempt = $3
RETURNING 1
""",
job["job_id"], self.id, job["attempt"], config.LEASE_SECONDS,
)
except DB_ERRORS:
continue
if not renewed:
lost.set()
return
# ------------------------------------------------------------------ process one delivery
async def process(self, job: dict, redelivered: bool) -> str:
claimed, reason = await self.claim(job, redelivered)
if claimed is None:
return reason
await faults.hit(self.pool, "worker.after_claim", claimed["attachment_id"], self.id)
lost = asyncio.Event()
hb = asyncio.create_task(self.heartbeat(claimed, lost))
try:
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
try:
variants = await self.execute(claimed, tmp)
await faults.hit(self.pool, "worker.after_s3_before_commit", claimed["attachment_id"], self.id)
except ImageRejected as exc:
return await self.finish_failure(claimed, exc.code, permanent=True)
except (TransientFailure, S3Unavailable) as exc:
code = exc.code if isinstance(exc, TransientFailure) else "s3_unavailable"
return await self.finish_failure(claimed, code, permanent=False)
return await self.commit(claimed, variants)
finally:
hb.cancel()
async def execute(self, job: dict, tmp: str) -> list:
"""Side effects only in S3 at deterministic keys; nothing visible to clients until commit()."""
src = os.path.join(tmp, "source")
republish = job["a_status"] == "ready" # newer pipeline for an already published attachment
if not republish and job["staging_deleted_at"] is not None:
raise ImageRejected("staging_missing")
# Source: staging, or the already published original (newer pipeline / a concurrent version committed
# and removed staging in between). Both hold the same bytes, verified against the DB checksum below.
keys = [job["object_key"]] if republish else [job["staging_key"], job["object_key"]]
size = None
for key in keys:
try:
with open(src, "wb") as fh:
size = await asyncio.to_thread(self.objects.download_sync, key, fh, config.MAX_UPLOAD_BYTES)
break
except S3NotFound:
continue
if size is None:
raise ImageRejected("staging_missing")
digest = hashlib.sha256()
with open(src, "rb") as fh:
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
digest.update(chunk)
if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]:
raise ImageRejected("staging_corrupt") # trusted metadata from the DB disagrees with the bytes
await faults.hit(self.pool, "worker.before_transform", job["attachment_id"], self.id)
out = os.path.join(tmp, "out")
os.mkdir(out)
started = time.monotonic()
variants = await asyncio.to_thread(run_isolated, src, job["content_type"], out, job["pipeline_version"],
config.JOB_TIMEOUT_SECONDS, config.WORKER_MEMORY_LIMIT_MB)
logger.info("transformed %s v%d in %.0f ms", job["attachment_id"], job["pipeline_version"],
1000 * (time.monotonic() - started))
if not republish:
with open(src, "rb") as fh:
await asyncio.to_thread(self.objects.put_sync, original_key(job["attachment_id"]), fh, size,
job["content_type"], job["sha256"])
for v in variants:
v["object_key"] = variant_key(job["attachment_id"], job["pipeline_version"], v["name"], v["content_type"])
with open(v["path"], "rb") as fh:
await asyncio.to_thread(self.objects.put_sync, v["object_key"], fh, v["size_bytes"],
v["content_type"], v["sha256"])
return variants
async def fenced(self, conn: asyncpg.Connection, job: dict):
row = await conn.fetchrow("SELECT * FROM jobs WHERE job_id = $1 FOR UPDATE", job["job_id"])
if row is None or row["status"] != "processing" or row["lease_owner"] != self.id or row["attempt"] != job["attempt"]:
await jobs.event(conn, job["job_id"], job["attachment_id"], "fenced_out", self.id, job["attempt"],
"lease lost; result of this attempt discarded")
return None
return row
async def commit(self, job: dict, variants: list) -> str:
async with self.pool.acquire() as conn:
async with conn.transaction():
row = await self.fenced(conn, job)
if row is None:
return "fenced_out"
current = await conn.fetchval("SELECT pipeline_version FROM attachments WHERE id = $1 FOR UPDATE",
job["attachment_id"])
if current is not None and current > job["pipeline_version"]:
await conn.execute(
"""
UPDATE jobs SET status = 'superseded', lease_owner = NULL, lease_until = NULL,
updated_at = clock_timestamp() WHERE job_id = $1
""", job["job_id"])
await jobs.event(conn, job["job_id"], job["attachment_id"], "superseded", self.id, job["attempt"],
"published v%d is newer than v%d" % (current, job["pipeline_version"]))
return "superseded"
for v in variants:
await conn.execute(
"""
INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type,
size_bytes, width, height, sha256, job_id, attempt)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
""",
job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"],
v["size_bytes"], v["width"], v["height"], v["sha256"], job["job_id"], job["attempt"],
)
await conn.execute(
"""
UPDATE attachments SET status = 'ready', pipeline_version = $2, error_code = NULL,
updated_at = clock_timestamp()
WHERE id = $1
""",
job["attachment_id"], job["pipeline_version"],
)
await conn.execute(
"""
UPDATE jobs SET status = 'done', lease_owner = NULL, lease_until = NULL, error_code = NULL,
last_error = NULL, updated_at = clock_timestamp()
WHERE job_id = $1
""", job["job_id"])
await jobs.event(conn, job["job_id"], job["attachment_id"], "ready", self.id, job["attempt"],
"v%d: %s" % (job["pipeline_version"], ",".join(v["name"] for v in variants)))
# Staging is removed only after the result is committed; a failure here is repaired by the sweeper.
if job["a_status"] != "ready" and job["staging_deleted_at"] is None:
try:
await asyncio.to_thread(self.objects.delete_sync, job["staging_key"])
await conn.execute(
"UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = $1 AND status = 'ready'",
job["attachment_id"])
except (S3Unavailable, *DB_ERRORS):
logger.warning("staging cleanup deferred to sweeper for %s", job["attachment_id"])
return "ready"
async def finish_failure(self, job: dict, code: str, permanent: bool) -> str:
async with self.pool.acquire() as conn, conn.transaction():
row = await self.fenced(conn, job)
if row is None:
return "fenced_out"
if permanent:
await jobs.fail(conn, row, code, code, self.id)
return "failed"
return await jobs.retry_or_fail(conn, row, code, self.id)
# ------------------------------------------------------------------ reaper: expired leases, lost messages
async def reap_once(self) -> None:
async with self.pool.acquire() as conn:
async with conn.transaction():
expired = await conn.fetch(
"""
SELECT * FROM jobs WHERE status = 'processing' AND lease_until < clock_timestamp()
ORDER BY lease_until LIMIT 50 FOR UPDATE SKIP LOCKED
""")
for row in expired:
await jobs.event(conn, row["job_id"], row["attachment_id"], "lease_expired", self.id, row["attempt"],
"owner %s stopped renewing" % row["lease_owner"])
await jobs.retry_or_fail(conn, row, "lease_expired", self.id)
async with conn.transaction():
# Safety net for a message lost after the outbox marked it published (e.g. broker volume lost).
stuck = await conn.fetch(
"""
SELECT j.* FROM jobs j
WHERE j.status = 'queued' AND j.next_attempt_at < clock_timestamp() - make_interval(secs => $1)
AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id
AND (o.published_at IS NULL OR o.published_at > clock_timestamp() - make_interval(secs => $1)))
LIMIT 50 FOR UPDATE OF j SKIP LOCKED
""", config.STUCK_QUEUED_SECONDS)
for row in stuck:
await jobs.enqueue(conn, row["job_id"], row["attachment_id"], row["pipeline_version"], row["attempt"] + 1)
await jobs.event(conn, row["job_id"], row["attachment_id"], "republished_stuck", self.id, row["attempt"])
with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh:
fh.write(str(time.time()))
async def reaper(self) -> None:
while not self.stopping.is_set():
try:
await self.reap_once()
except DB_ERRORS as exc:
logger.warning("reaper: db unavailable (%s)", type(exc).__name__)
except Exception:
logger.exception("reaper failed")
try:
await asyncio.wait_for(self.stopping.wait(), config.REAPER_INTERVAL_SECONDS)
except asyncio.TimeoutError:
pass
# ------------------------------------------------------------------ lifecycle
async def run(self) -> None:
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(sig, self.stopping.set)
for attempt in range(1, 61):
try:
self.pool = await asyncpg.create_pool(config.DATABASE_URL, min_size=1,
max_size=config.WORKER_CONCURRENCY + 4,
timeout=config.DB_CONNECT_TIMEOUT,
command_timeout=config.DB_COMMAND_TIMEOUT)
await self.broker.connect(prefetch=config.WORKER_CONCURRENCY)
break
except Exception as exc:
logger.warning("dependencies not ready (attempt %d): %s", attempt, type(exc).__name__)
await asyncio.sleep(1)
else:
raise SystemExit("cannot connect to PostgreSQL/RabbitMQ")
logger.info("%s consuming %s, prefetch=%d, pipeline=%d, max_attempts=%d, lease=%.0fs, fault_injection=%s",
self.id, config.BROKER_QUEUE, config.WORKER_CONCURRENCY, config.PIPELINE_VERSION,
config.JOB_MAX_ATTEMPTS, config.LEASE_SECONDS, config.FAULT_INJECTION)
reaper = asyncio.create_task(self.reaper())
tag = await self.broker.queue.consume(self.on_message, no_ack=False)
await self.stopping.wait()
logger.info("stopping: no new deliveries, waiting for %d in-flight jobs", len(self.inflight))
await self.broker.queue.cancel(tag)
if self.inflight:
await asyncio.wait(list(self.inflight), timeout=25)
reaper.cancel()
await self.broker.close() # unacked deliveries (if any) return to the queue
await self.pool.close()
def main() -> None:
if not config.DATABASE_URL:
raise SystemExit("DATABASE_URL(_FILE) is required")
asyncio.run(Worker().run())
if __name__ == "__main__":
main()