Files
mtusi2026lab5/src/worker.py
T

454 lines
17 KiB
Python

"""Two bounded consumers; every decode runs in a killable isolated process."""
import hashlib
import io
import json
import multiprocessing
import resource
import threading
import time
from pathlib import Path
from uuid import uuid4
import redis
from PIL import Image, ImageOps, UnidentifiedImageError
from src.images import MAX_BYTES, MAX_PIXELS
from src.object_store import ObjectStore
from src.pipeline import (
GROUP,
STREAM,
PermanentFailure,
broker,
database,
event,
fault,
retry,
validate_job,
)
from src.settings import integer
def sizes(width, height, bound):
longest = max(width, height)
return (
(width, height)
if longest <= bound
else (
max(1, width * bound // longest),
max(1, height * bound // longest),
)
)
def make_variants(raw, row):
if (
len(raw) != row["size_bytes"]
or hashlib.sha256(raw).hexdigest() != row["sha256"]
):
raise PermanentFailure("source_checksum_mismatch")
try:
with Image.open(io.BytesIO(raw)) as source:
if source.width * source.height > MAX_PIXELS:
raise PermanentFailure("pixel_limit")
if source.format not in {"PNG", "JPEG"}:
raise PermanentFailure("invalid_image")
source.load()
oriented = ImageOps.exif_transpose(source)
rgba = oriented.convert("RGBA")
rgb = Image.new("RGB", rgba.size, "white")
rgb.paste(rgba, mask=rgba.getchannel("A"))
rgb.info.clear()
yield "original", raw, row["content_type"], source.size
thumbnail = rgb.resize(
sizes(*rgb.size, 256), Image.Resampling.LANCZOS
)
crop = ImageOps.fit(
rgb, (128, 128), Image.Resampling.LANCZOS, centering=(0.5, 0.5)
)
optimized = rgb.resize(
sizes(*rgb.size, 1280), Image.Resampling.LANCZOS
)
for name, image, fmt, mime, options in [
(
"thumbnail",
thumbnail,
"JPEG",
"image/jpeg",
{"quality": 85, "subsampling": 0},
),
(
"crop",
crop,
"JPEG",
"image/jpeg",
{"quality": 85, "subsampling": 0},
),
(
"optimized",
optimized,
"WEBP",
"image/webp",
{
"quality": 82 if row["pipeline_version"] == 1 else 76,
"method": 4,
},
),
]:
output = io.BytesIO()
image.info.clear()
image.save(output, format=fmt, **options)
yield name, output.getvalue(), mime, image.size
except (
UnidentifiedImageError,
OSError,
ValueError,
Image.DecompressionBombError,
) as exc:
raise PermanentFailure("invalid_image") from exc
def process(job_id, fence, sender):
db = store = None
started = time.monotonic()
cpu = time.process_time()
try:
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
db, store = database(), ObjectStore()
with db.connection() as conn:
row = conn.execute(
"SELECT j.*,a.size_bytes,a.sha256,a.content_type FROM jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s AND j.fence=%s",
(job_id, fence),
).fetchone()
if not row:
sender.send({"result": "stale"})
return
# Record limits before fault hooks: a slow external inspector must not
# consume the child timeout while attempting to observe the limit.
cgroup = Path("/sys/fs/cgroup")
limits = {
name: (cgroup / name).read_text().strip()
if (cgroup / name).exists()
else "unavailable"
for name in ["memory.max", "cpu.max"]
}
limits["address_space_bytes"] = list(
resource.getrlimit(resource.RLIMIT_AS)
)
with db.connection() as conn:
event(conn, job_id, "worker_resource_limits", limits)
fault(db, job_id, "before_effect")
upstream = store.get(key=row["source_key"])
try:
raw = upstream.read(MAX_BYTES + 1)
finally:
upstream.close()
upstream.release_conn()
results = []
for name, data, mime, (width, height) in make_variants(raw, row):
key = f"objects/{row['attachment_id']}/v{row['pipeline_version']}/{name}"
digest = hashlib.sha256(data).hexdigest()
store.put(
key=key,
stream=io.BytesIO(data),
size=len(data),
content_type=mime,
sha256=digest,
)
results.append((name, key, mime, len(data), digest, width, height))
fault(db, job_id, "after_effect")
with db.connection() as conn, conn.transaction():
current = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
allowed = conn.execute(
"SELECT pipeline_version FROM attachments WHERE id=%s FOR UPDATE",
(row["attachment_id"],),
).fetchone()
valid_lease = conn.execute(
"SELECT %s::timestamptz>clock_timestamp() live",
(current["lease_until"],),
).fetchone()["live"]
if (
current["state"] != "processing"
or current["fence"] != fence
or not valid_lease
or allowed["pipeline_version"] != row["pipeline_version"]
):
event(
conn,
job_id,
"stale_completion_rejected",
{"fence": fence, "version": row["pipeline_version"]},
)
sender.send({"result": "stale"})
return
for result in results:
conn.execute(
"INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT(attachment_id,pipeline_version,name) DO NOTHING",
(row["attachment_id"], row["pipeline_version"], *result),
)
conn.execute(
"INSERT INTO effect_publications(attachment_id,pipeline_version,job_id,fence) VALUES(%s,%s,%s,%s)",
(row["attachment_id"], row["pipeline_version"], job_id, fence),
)
conn.execute(
"UPDATE attachments SET state='ready',object_key=%s,published_version=pipeline_version,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(results[0][1], row["attachment_id"]),
)
conn.execute(
"UPDATE jobs SET state='ready',error_code=NULL,lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
(job_id,),
)
event(
conn,
job_id,
"effect_committed",
{
"fence": fence,
"version": row["pipeline_version"],
"variants": 4,
},
)
fault(db, job_id, "after_commit")
# Cleanup is independently retryable; it cannot undo the committed effect.
if row["source_key"].startswith("objects/staging/"):
try:
store.delete(key=row["source_key"])
except Exception:
pass
sender.send(
{
"result": "ready",
"seconds": time.monotonic() - started,
"cpu_seconds": time.process_time() - cpu,
"rss_peak_mib": resource.getrusage(
resource.RUSAGE_SELF
).ru_maxrss
/ 1024,
}
)
except PermanentFailure as exc:
sender.send({"result": "permanent", "code": str(exc)})
except MemoryError:
sender.send({"result": "permanent", "code": "memory_limit"})
except Exception:
sender.send({"result": "transient", "code": "processing_unavailable"})
finally:
if store:
store.close()
if db:
db.close()
sender.close()
def claim(db, data, consumer):
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (data["job_id"],)
).fetchone()
if (
not row
or str(row["attachment_id"]) != data["attachment_id"]
or row["pipeline_version"] != data["pipeline_version"]
):
return "obsolete", None
if row["state"] in {"ready", "failed", "obsolete"}:
return row["state"], row
if data["attempt"] < row["attempt"] or (
row["state"] == "queued" and data["attempt"] <= row["attempt"]
):
return "obsolete", row
if row["state"] == "processing":
return "busy", row
if data["attempt"] != row["attempt"] + 1:
return "obsolete", row
due = conn.execute(
"SELECT %s::timestamptz<=clock_timestamp() due", (row["next_at"],)
).fetchone()["due"]
if not due:
return "busy", row
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
updated = conn.execute(
"UPDATE jobs SET state='processing',attempt=attempt+1,fence=fence+1,lease_owner=%s,lease_until=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s RETURNING *",
(consumer, lease, row["id"]),
).fetchone()
conn.execute(
"UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
(row["attachment_id"], row["pipeline_version"]),
)
event(
conn,
row["id"],
"claimed",
{
"attempt": updated["attempt"],
"fence": updated["fence"],
"version": updated["pipeline_version"],
},
)
return "claimed", updated
def ready_effect(db, store, row):
with db.connection() as conn:
variants = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s",
(row["attachment_id"], row["pipeline_version"]),
).fetchall()
if len(variants) != 4:
return False
for variant in variants:
head = store.head(key=variant["object_key"])
if (
head is None
or head.size != variant["size_bytes"]
or head.metadata.get("x-amz-meta-sha256") != variant["sha256"]
):
return False
return True
def consume(slot):
db, queue, store = database(), broker(), ObjectStore()
consumer = str(uuid4()) + f"-{slot}"
context = multiprocessing.get_context("spawn")
cursor = "0-0"
timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117)
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
if lease <= timeout:
raise ValueError("Lease must exceed child timeout")
try:
while True:
try:
try:
queue.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
reclaimed = queue.xautoclaim(
STREAM,
GROUP,
consumer,
# Do not steal an entry from a consumer inside its live budget.
min_idle_time=lease * 1000,
start_id=cursor,
count=1,
)
cursor, messages = reclaimed[0], reclaimed[1]
if not messages:
batch = queue.xreadgroup(
GROUP, consumer, {STREAM: ">"}, count=1, block=500
)
messages = batch[0][1] if batch else []
for entry, fields in messages:
try:
data = validate_job(
json.loads(fields.get("job", "null"))
)
except ValueError, TypeError, KeyError:
# Malformed notifications cannot become jobs; preserve diagnostic count.
with db.connection() as conn:
event(conn, None, "invalid_envelope")
queue.xack(STREAM, GROUP, entry)
queue.xdel(STREAM, entry)
continue
state, row = claim(db, data, consumer)
if state == "busy":
continue
if state == "ready" and not ready_effect(db, store, row):
with db.connection() as conn:
event(conn, row["id"], "ready_effect_missing")
continue
if state in {"ready", "obsolete", "failed"}:
with db.connection() as conn:
event(
conn,
data["job_id"],
"duplicate_decision",
{"state": state, "entry": entry},
)
if state == "claimed":
receiver, sender = context.Pipe(duplex=False)
child = context.Process(
target=process,
args=(str(row["id"]), row["fence"], sender),
)
child.start()
sender.close()
child.join(timeout)
if child.is_alive():
child.kill()
child.join(2)
result = {
"result": "transient",
"code": "worker_timeout",
}
else:
try:
result = (
receiver.recv()
if receiver.poll()
else {
"result": "transient",
"code": "worker_crashed",
}
)
except EOFError:
result = {
"result": "transient",
"code": "worker_crashed",
}
receiver.close()
child.close()
with db.connection() as conn, conn.transaction():
current = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE",
(row["id"],),
).fetchone()
event(conn, row["id"], "child_finished", result)
if (
current["state"] == "processing"
and current["fence"] == row["fence"]
):
retry(
conn,
current,
result.get("code", "processing_failed"),
permanent=result["result"] == "permanent",
)
# Commit result or a retry/terminal decision precedes ack.
queue.xack(STREAM, GROUP, entry)
queue.xdel(STREAM, entry)
except Exception:
print(
json.dumps(
{
"event": "worker_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
time.sleep(0.5)
finally:
queue.close()
store.close()
db.close()
def main():
count = integer("WORKER_CONCURRENCY", 2, 1, 4)
threads = [
threading.Thread(target=consume, args=(i,), daemon=False)
for i in range(count)
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
if __name__ == "__main__":
main()