forked from mtusi2026/mtusi2026lab5
378 lines
20 KiB
Python
378 lines
20 KiB
Python
"""Attachment worker: consumes attachments.process, publishes the original and variants, commits the result.
|
|
|
|
Delivery is at-least-once; the effect is idempotent because
|
|
* PostgreSQL (jobs) decides whether a delivery may start an attempt: claim = queued -> processing with
|
|
attempt+1 and a lease (lease_owner, lease_until). Duplicates/stale/reordered messages are acked as no-ops.
|
|
* (job_id, attempt) is a fencing token: the result COMMIT succeeds only if this worker still owns the lease.
|
|
* S3 keys are the effect key (attachment, pipeline_version, variant): a retry overwrites identical bytes.
|
|
* attachment_variants has PRIMARY KEY (attachment_id, pipeline_version, name): one row per variant.
|
|
* A result of an older pipeline_version never replaces a newer published one (superseded).
|
|
The message is acked only after the result (or the decision) is durably committed in PostgreSQL.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import secrets
|
|
import signal
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
from typing import Optional
|
|
|
|
import asyncpg
|
|
from aio_pika.abc import AbstractIncomingMessage
|
|
|
|
from app import config, faults, jobs
|
|
from app.broker import Broker, InvalidJob, parse_job
|
|
from app.imaging import ImageRejected, TransientFailure, run_isolated
|
|
from app.objects import ObjectStore, S3NotFound, S3Unavailable, original_key, variant_key
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
logger = logging.getLogger("worker")
|
|
|
|
HEARTBEAT_FILE = "/tmp/worker-heartbeat"
|
|
DB_ERRORS = (OSError, asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.TooManyConnectionsError,
|
|
asyncpg.exceptions.OperatorInterventionError, asyncio.TimeoutError)
|
|
|
|
|
|
class Worker:
|
|
def __init__(self):
|
|
self.id = "worker-" + secrets.token_hex(4)
|
|
self.pool: Optional[asyncpg.Pool] = None
|
|
self.objects = ObjectStore()
|
|
self.broker = Broker()
|
|
self.inflight = set()
|
|
self.stopping = asyncio.Event()
|
|
|
|
# ------------------------------------------------------------------ message entry point
|
|
async def on_message(self, message: AbstractIncomingMessage) -> None:
|
|
task = asyncio.current_task()
|
|
self.inflight.add(task)
|
|
try:
|
|
await self._handle(message)
|
|
finally:
|
|
self.inflight.discard(task)
|
|
|
|
async def _handle(self, message: AbstractIncomingMessage) -> None:
|
|
try:
|
|
job = parse_job(message.body)
|
|
except InvalidJob as exc:
|
|
logger.error("rejecting malformed job message to DLQ: %s", exc)
|
|
await message.reject(requeue=False) # -> attachments.dlx -> attachments.dlq
|
|
return
|
|
try:
|
|
outcome = await self.process(job, message.redelivered)
|
|
except DB_ERRORS as exc:
|
|
# Nothing was decided durably. Give the message back after a pause (no hot loop); if the DB stays down
|
|
# the lease expires and the reaper re-enqueues the job through the outbox.
|
|
logger.warning("db unavailable while handling job %s: %s; requeue", job["job_id"], type(exc).__name__)
|
|
await asyncio.sleep(2)
|
|
await message.nack(requeue=True)
|
|
return
|
|
await faults.hit(self.pool, "worker.after_commit_before_ack", job["attachment_id"], self.id)
|
|
await message.ack()
|
|
logger.info("job %s attempt %s -> %s (redelivered=%s)", job["job_id"], job["attempt"], outcome, message.redelivered)
|
|
|
|
# ------------------------------------------------------------------ claim
|
|
async def claim(self, job: dict, redelivered: bool):
|
|
async with self.pool.acquire() as conn, conn.transaction():
|
|
row = await conn.fetchrow(
|
|
"""
|
|
SELECT j.*, a.status AS a_status, a.content_type, a.size_bytes, a.sha256, a.staging_key,
|
|
a.staging_deleted_at, a.object_key, a.pipeline_version AS a_version,
|
|
(j.lease_until > clock_timestamp()) AS lease_active,
|
|
(j.next_attempt_at <= clock_timestamp() + interval '1 second') AS due
|
|
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
|
WHERE j.job_id = $1
|
|
FOR UPDATE OF j
|
|
""",
|
|
job["job_id"],
|
|
)
|
|
reason = None
|
|
if row is None:
|
|
reason = "unknown_job_noop"
|
|
elif row["attachment_id"] != job["attachment_id"] or row["pipeline_version"] != job["pipeline_version"]:
|
|
reason = "mismatched_message_noop"
|
|
elif row["status"] == "done":
|
|
reason = "duplicate_noop" # result already committed: ack after checking state
|
|
elif row["status"] in ("failed", "superseded"):
|
|
reason = row["status"] + "_noop"
|
|
elif row["status"] == "processing":
|
|
reason = "lease_held_noop" if row["lease_active"] else "lease_expired_noop"
|
|
elif job["attempt"] != row["attempt"] + 1:
|
|
reason = "stale_attempt_noop" # reordered/old message of an earlier attempt
|
|
elif not row["due"]:
|
|
reason = "early_noop"
|
|
elif row["a_version"] is not None and row["a_version"] > row["pipeline_version"]:
|
|
await conn.execute(
|
|
"UPDATE jobs SET status = 'superseded', updated_at = clock_timestamp() WHERE job_id = $1", row["job_id"])
|
|
reason = "superseded_before_start"
|
|
if reason:
|
|
await jobs.event(conn, job["job_id"], job["attachment_id"] if row is None else row["attachment_id"],
|
|
reason, self.id, job["attempt"], "redelivered=%s" % redelivered)
|
|
return None, reason
|
|
claimed = await conn.fetchrow(
|
|
"""
|
|
UPDATE jobs SET status = 'processing', attempt = attempt + 1, lease_owner = $2,
|
|
lease_until = clock_timestamp() + make_interval(secs => $3), updated_at = clock_timestamp()
|
|
WHERE job_id = $1
|
|
RETURNING *
|
|
""",
|
|
row["job_id"], self.id, config.LEASE_SECONDS,
|
|
)
|
|
await conn.execute(
|
|
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() WHERE id = $1 AND status = 'queued'",
|
|
row["attachment_id"],
|
|
)
|
|
await jobs.event(conn, row["job_id"], row["attachment_id"], "claimed", self.id, claimed["attempt"],
|
|
"redelivered=%s" % redelivered)
|
|
info = dict(row)
|
|
info.update(dict(claimed))
|
|
return info, "claimed"
|
|
|
|
async def heartbeat(self, job: dict, lost: asyncio.Event) -> None:
|
|
"""Renew the lease while the attempt is alive; a dead worker stops renewing and the lease expires."""
|
|
while True:
|
|
await asyncio.sleep(config.HEARTBEAT_SECONDS)
|
|
try:
|
|
async with self.pool.acquire() as conn:
|
|
renewed = await conn.fetchval(
|
|
"""
|
|
UPDATE jobs SET lease_until = clock_timestamp() + make_interval(secs => $4)
|
|
WHERE job_id = $1 AND status = 'processing' AND lease_owner = $2 AND attempt = $3
|
|
RETURNING 1
|
|
""",
|
|
job["job_id"], self.id, job["attempt"], config.LEASE_SECONDS,
|
|
)
|
|
except DB_ERRORS:
|
|
continue
|
|
if not renewed:
|
|
lost.set()
|
|
return
|
|
|
|
# ------------------------------------------------------------------ process one delivery
|
|
async def process(self, job: dict, redelivered: bool) -> str:
|
|
claimed, reason = await self.claim(job, redelivered)
|
|
if claimed is None:
|
|
return reason
|
|
await faults.hit(self.pool, "worker.after_claim", claimed["attachment_id"], self.id)
|
|
lost = asyncio.Event()
|
|
hb = asyncio.create_task(self.heartbeat(claimed, lost))
|
|
try:
|
|
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
|
|
try:
|
|
variants = await self.execute(claimed, tmp)
|
|
await faults.hit(self.pool, "worker.after_s3_before_commit", claimed["attachment_id"], self.id)
|
|
except ImageRejected as exc:
|
|
return await self.finish_failure(claimed, exc.code, permanent=True)
|
|
except (TransientFailure, S3Unavailable) as exc:
|
|
code = exc.code if isinstance(exc, TransientFailure) else "s3_unavailable"
|
|
return await self.finish_failure(claimed, code, permanent=False)
|
|
return await self.commit(claimed, variants)
|
|
finally:
|
|
hb.cancel()
|
|
|
|
async def execute(self, job: dict, tmp: str) -> list:
|
|
"""Side effects only in S3 at deterministic keys; nothing visible to clients until commit()."""
|
|
src = os.path.join(tmp, "source")
|
|
republish = job["a_status"] == "ready" # newer pipeline for an already published attachment
|
|
if not republish and job["staging_deleted_at"] is not None:
|
|
raise ImageRejected("staging_missing")
|
|
# Source: staging, or the already published original (newer pipeline / a concurrent version committed
|
|
# and removed staging in between). Both hold the same bytes, verified against the DB checksum below.
|
|
keys = [job["object_key"]] if republish else [job["staging_key"], job["object_key"]]
|
|
size = None
|
|
for key in keys:
|
|
try:
|
|
with open(src, "wb") as fh:
|
|
size = await asyncio.to_thread(self.objects.download_sync, key, fh, config.MAX_UPLOAD_BYTES)
|
|
break
|
|
except S3NotFound:
|
|
continue
|
|
if size is None:
|
|
raise ImageRejected("staging_missing")
|
|
digest = hashlib.sha256()
|
|
with open(src, "rb") as fh:
|
|
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]:
|
|
raise ImageRejected("staging_corrupt") # trusted metadata from the DB disagrees with the bytes
|
|
await faults.hit(self.pool, "worker.before_transform", job["attachment_id"], self.id)
|
|
out = os.path.join(tmp, "out")
|
|
os.mkdir(out)
|
|
started = time.monotonic()
|
|
variants = await asyncio.to_thread(run_isolated, src, job["content_type"], out, job["pipeline_version"],
|
|
config.JOB_TIMEOUT_SECONDS, config.WORKER_MEMORY_LIMIT_MB)
|
|
logger.info("transformed %s v%d in %.0f ms", job["attachment_id"], job["pipeline_version"],
|
|
1000 * (time.monotonic() - started))
|
|
if not republish:
|
|
with open(src, "rb") as fh:
|
|
await asyncio.to_thread(self.objects.put_sync, original_key(job["attachment_id"]), fh, size,
|
|
job["content_type"], job["sha256"])
|
|
for v in variants:
|
|
v["object_key"] = variant_key(job["attachment_id"], job["pipeline_version"], v["name"], v["content_type"])
|
|
with open(v["path"], "rb") as fh:
|
|
await asyncio.to_thread(self.objects.put_sync, v["object_key"], fh, v["size_bytes"],
|
|
v["content_type"], v["sha256"])
|
|
return variants
|
|
|
|
async def fenced(self, conn: asyncpg.Connection, job: dict):
|
|
row = await conn.fetchrow("SELECT * FROM jobs WHERE job_id = $1 FOR UPDATE", job["job_id"])
|
|
if row is None or row["status"] != "processing" or row["lease_owner"] != self.id or row["attempt"] != job["attempt"]:
|
|
await jobs.event(conn, job["job_id"], job["attachment_id"], "fenced_out", self.id, job["attempt"],
|
|
"lease lost; result of this attempt discarded")
|
|
return None
|
|
return row
|
|
|
|
async def commit(self, job: dict, variants: list) -> str:
|
|
async with self.pool.acquire() as conn:
|
|
async with conn.transaction():
|
|
row = await self.fenced(conn, job)
|
|
if row is None:
|
|
return "fenced_out"
|
|
current = await conn.fetchval("SELECT pipeline_version FROM attachments WHERE id = $1 FOR UPDATE",
|
|
job["attachment_id"])
|
|
if current is not None and current > job["pipeline_version"]:
|
|
await conn.execute(
|
|
"""
|
|
UPDATE jobs SET status = 'superseded', lease_owner = NULL, lease_until = NULL,
|
|
updated_at = clock_timestamp() WHERE job_id = $1
|
|
""", job["job_id"])
|
|
await jobs.event(conn, job["job_id"], job["attachment_id"], "superseded", self.id, job["attempt"],
|
|
"published v%d is newer than v%d" % (current, job["pipeline_version"]))
|
|
return "superseded"
|
|
for v in variants:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type,
|
|
size_bytes, width, height, sha256, job_id, attempt)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
|
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
|
|
""",
|
|
job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"],
|
|
v["size_bytes"], v["width"], v["height"], v["sha256"], job["job_id"], job["attempt"],
|
|
)
|
|
await conn.execute(
|
|
"""
|
|
UPDATE attachments SET status = 'ready', pipeline_version = $2, error_code = NULL,
|
|
updated_at = clock_timestamp()
|
|
WHERE id = $1
|
|
""",
|
|
job["attachment_id"], job["pipeline_version"],
|
|
)
|
|
await conn.execute(
|
|
"""
|
|
UPDATE jobs SET status = 'done', lease_owner = NULL, lease_until = NULL, error_code = NULL,
|
|
last_error = NULL, updated_at = clock_timestamp()
|
|
WHERE job_id = $1
|
|
""", job["job_id"])
|
|
await jobs.event(conn, job["job_id"], job["attachment_id"], "ready", self.id, job["attempt"],
|
|
"v%d: %s" % (job["pipeline_version"], ",".join(v["name"] for v in variants)))
|
|
# Staging is removed only after the result is committed; a failure here is repaired by the sweeper.
|
|
if job["a_status"] != "ready" and job["staging_deleted_at"] is None:
|
|
try:
|
|
await asyncio.to_thread(self.objects.delete_sync, job["staging_key"])
|
|
await conn.execute(
|
|
"UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = $1 AND status = 'ready'",
|
|
job["attachment_id"])
|
|
except (S3Unavailable, *DB_ERRORS):
|
|
logger.warning("staging cleanup deferred to sweeper for %s", job["attachment_id"])
|
|
return "ready"
|
|
|
|
async def finish_failure(self, job: dict, code: str, permanent: bool) -> str:
|
|
async with self.pool.acquire() as conn, conn.transaction():
|
|
row = await self.fenced(conn, job)
|
|
if row is None:
|
|
return "fenced_out"
|
|
if permanent:
|
|
await jobs.fail(conn, row, code, code, self.id)
|
|
return "failed"
|
|
return await jobs.retry_or_fail(conn, row, code, self.id)
|
|
|
|
# ------------------------------------------------------------------ reaper: expired leases, lost messages
|
|
async def reap_once(self) -> None:
|
|
async with self.pool.acquire() as conn:
|
|
async with conn.transaction():
|
|
expired = await conn.fetch(
|
|
"""
|
|
SELECT * FROM jobs WHERE status = 'processing' AND lease_until < clock_timestamp()
|
|
ORDER BY lease_until LIMIT 50 FOR UPDATE SKIP LOCKED
|
|
""")
|
|
for row in expired:
|
|
await jobs.event(conn, row["job_id"], row["attachment_id"], "lease_expired", self.id, row["attempt"],
|
|
"owner %s stopped renewing" % row["lease_owner"])
|
|
await jobs.retry_or_fail(conn, row, "lease_expired", self.id)
|
|
async with conn.transaction():
|
|
# Safety net for a message lost after the outbox marked it published (e.g. broker volume lost).
|
|
stuck = await conn.fetch(
|
|
"""
|
|
SELECT j.* FROM jobs j
|
|
WHERE j.status = 'queued' AND j.next_attempt_at < clock_timestamp() - make_interval(secs => $1)
|
|
AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id
|
|
AND (o.published_at IS NULL OR o.published_at > clock_timestamp() - make_interval(secs => $1)))
|
|
LIMIT 50 FOR UPDATE OF j SKIP LOCKED
|
|
""", config.STUCK_QUEUED_SECONDS)
|
|
for row in stuck:
|
|
await jobs.enqueue(conn, row["job_id"], row["attachment_id"], row["pipeline_version"], row["attempt"] + 1)
|
|
await jobs.event(conn, row["job_id"], row["attachment_id"], "republished_stuck", self.id, row["attempt"])
|
|
with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh:
|
|
fh.write(str(time.time()))
|
|
|
|
async def reaper(self) -> None:
|
|
while not self.stopping.is_set():
|
|
try:
|
|
await self.reap_once()
|
|
except DB_ERRORS as exc:
|
|
logger.warning("reaper: db unavailable (%s)", type(exc).__name__)
|
|
except Exception:
|
|
logger.exception("reaper failed")
|
|
try:
|
|
await asyncio.wait_for(self.stopping.wait(), config.REAPER_INTERVAL_SECONDS)
|
|
except asyncio.TimeoutError:
|
|
pass
|
|
|
|
# ------------------------------------------------------------------ lifecycle
|
|
async def run(self) -> None:
|
|
loop = asyncio.get_running_loop()
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
loop.add_signal_handler(sig, self.stopping.set)
|
|
for attempt in range(1, 61):
|
|
try:
|
|
self.pool = await asyncpg.create_pool(config.DATABASE_URL, min_size=1,
|
|
max_size=config.WORKER_CONCURRENCY + 4,
|
|
timeout=config.DB_CONNECT_TIMEOUT,
|
|
command_timeout=config.DB_COMMAND_TIMEOUT)
|
|
await self.broker.connect(prefetch=config.WORKER_CONCURRENCY)
|
|
break
|
|
except Exception as exc:
|
|
logger.warning("dependencies not ready (attempt %d): %s", attempt, type(exc).__name__)
|
|
await asyncio.sleep(1)
|
|
else:
|
|
raise SystemExit("cannot connect to PostgreSQL/RabbitMQ")
|
|
logger.info("%s consuming %s, prefetch=%d, pipeline=%d, max_attempts=%d, lease=%.0fs, fault_injection=%s",
|
|
self.id, config.BROKER_QUEUE, config.WORKER_CONCURRENCY, config.PIPELINE_VERSION,
|
|
config.JOB_MAX_ATTEMPTS, config.LEASE_SECONDS, config.FAULT_INJECTION)
|
|
reaper = asyncio.create_task(self.reaper())
|
|
tag = await self.broker.queue.consume(self.on_message, no_ack=False)
|
|
await self.stopping.wait()
|
|
logger.info("stopping: no new deliveries, waiting for %d in-flight jobs", len(self.inflight))
|
|
await self.broker.queue.cancel(tag)
|
|
if self.inflight:
|
|
await asyncio.wait(list(self.inflight), timeout=25)
|
|
reaper.cancel()
|
|
await self.broker.close() # unacked deliveries (if any) return to the queue
|
|
await self.pool.close()
|
|
|
|
|
|
def main() -> None:
|
|
if not config.DATABASE_URL:
|
|
raise SystemExit("DATABASE_URL(_FILE) is required")
|
|
asyncio.run(Worker().run())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|