forked from mtusi2026/mtusi2026lab5
454 lines
17 KiB
Python
454 lines
17 KiB
Python
"""Two bounded consumers; every decode runs in a killable isolated process."""
|
|
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import multiprocessing
|
|
import resource
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import redis
|
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
|
|
|
from src.images import MAX_BYTES, MAX_PIXELS
|
|
from src.object_store import ObjectStore
|
|
from src.pipeline import (
|
|
GROUP,
|
|
STREAM,
|
|
PermanentFailure,
|
|
broker,
|
|
database,
|
|
event,
|
|
fault,
|
|
retry,
|
|
validate_job,
|
|
)
|
|
from src.settings import integer
|
|
|
|
|
|
def sizes(width, height, bound):
|
|
longest = max(width, height)
|
|
return (
|
|
(width, height)
|
|
if longest <= bound
|
|
else (
|
|
max(1, width * bound // longest),
|
|
max(1, height * bound // longest),
|
|
)
|
|
)
|
|
|
|
|
|
def make_variants(raw, row):
|
|
if (
|
|
len(raw) != row["size_bytes"]
|
|
or hashlib.sha256(raw).hexdigest() != row["sha256"]
|
|
):
|
|
raise PermanentFailure("source_checksum_mismatch")
|
|
try:
|
|
with Image.open(io.BytesIO(raw)) as source:
|
|
if source.width * source.height > MAX_PIXELS:
|
|
raise PermanentFailure("pixel_limit")
|
|
if source.format not in {"PNG", "JPEG"}:
|
|
raise PermanentFailure("invalid_image")
|
|
source.load()
|
|
oriented = ImageOps.exif_transpose(source)
|
|
rgba = oriented.convert("RGBA")
|
|
rgb = Image.new("RGB", rgba.size, "white")
|
|
rgb.paste(rgba, mask=rgba.getchannel("A"))
|
|
rgb.info.clear()
|
|
yield "original", raw, row["content_type"], source.size
|
|
thumbnail = rgb.resize(
|
|
sizes(*rgb.size, 256), Image.Resampling.LANCZOS
|
|
)
|
|
crop = ImageOps.fit(
|
|
rgb, (128, 128), Image.Resampling.LANCZOS, centering=(0.5, 0.5)
|
|
)
|
|
optimized = rgb.resize(
|
|
sizes(*rgb.size, 1280), Image.Resampling.LANCZOS
|
|
)
|
|
for name, image, fmt, mime, options in [
|
|
(
|
|
"thumbnail",
|
|
thumbnail,
|
|
"JPEG",
|
|
"image/jpeg",
|
|
{"quality": 85, "subsampling": 0},
|
|
),
|
|
(
|
|
"crop",
|
|
crop,
|
|
"JPEG",
|
|
"image/jpeg",
|
|
{"quality": 85, "subsampling": 0},
|
|
),
|
|
(
|
|
"optimized",
|
|
optimized,
|
|
"WEBP",
|
|
"image/webp",
|
|
{
|
|
"quality": 82 if row["pipeline_version"] == 1 else 76,
|
|
"method": 4,
|
|
},
|
|
),
|
|
]:
|
|
output = io.BytesIO()
|
|
image.info.clear()
|
|
image.save(output, format=fmt, **options)
|
|
yield name, output.getvalue(), mime, image.size
|
|
except (
|
|
UnidentifiedImageError,
|
|
OSError,
|
|
ValueError,
|
|
Image.DecompressionBombError,
|
|
) as exc:
|
|
raise PermanentFailure("invalid_image") from exc
|
|
|
|
|
|
def process(job_id, fence, sender):
|
|
db = store = None
|
|
started = time.monotonic()
|
|
cpu = time.process_time()
|
|
try:
|
|
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
|
|
db, store = database(), ObjectStore()
|
|
with db.connection() as conn:
|
|
row = conn.execute(
|
|
"SELECT j.*,a.size_bytes,a.sha256,a.content_type FROM jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s AND j.fence=%s",
|
|
(job_id, fence),
|
|
).fetchone()
|
|
if not row:
|
|
sender.send({"result": "stale"})
|
|
return
|
|
# Record limits before fault hooks: a slow external inspector must not
|
|
# consume the child timeout while attempting to observe the limit.
|
|
cgroup = Path("/sys/fs/cgroup")
|
|
limits = {
|
|
name: (cgroup / name).read_text().strip()
|
|
if (cgroup / name).exists()
|
|
else "unavailable"
|
|
for name in ["memory.max", "cpu.max"]
|
|
}
|
|
limits["address_space_bytes"] = list(
|
|
resource.getrlimit(resource.RLIMIT_AS)
|
|
)
|
|
with db.connection() as conn:
|
|
event(conn, job_id, "worker_resource_limits", limits)
|
|
fault(db, job_id, "before_effect")
|
|
upstream = store.get(key=row["source_key"])
|
|
try:
|
|
raw = upstream.read(MAX_BYTES + 1)
|
|
finally:
|
|
upstream.close()
|
|
upstream.release_conn()
|
|
results = []
|
|
for name, data, mime, (width, height) in make_variants(raw, row):
|
|
key = f"objects/{row['attachment_id']}/v{row['pipeline_version']}/{name}"
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
store.put(
|
|
key=key,
|
|
stream=io.BytesIO(data),
|
|
size=len(data),
|
|
content_type=mime,
|
|
sha256=digest,
|
|
)
|
|
results.append((name, key, mime, len(data), digest, width, height))
|
|
fault(db, job_id, "after_effect")
|
|
with db.connection() as conn, conn.transaction():
|
|
current = conn.execute(
|
|
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
|
|
).fetchone()
|
|
allowed = conn.execute(
|
|
"SELECT pipeline_version FROM attachments WHERE id=%s FOR UPDATE",
|
|
(row["attachment_id"],),
|
|
).fetchone()
|
|
valid_lease = conn.execute(
|
|
"SELECT %s::timestamptz>clock_timestamp() live",
|
|
(current["lease_until"],),
|
|
).fetchone()["live"]
|
|
if (
|
|
current["state"] != "processing"
|
|
or current["fence"] != fence
|
|
or not valid_lease
|
|
or allowed["pipeline_version"] != row["pipeline_version"]
|
|
):
|
|
event(
|
|
conn,
|
|
job_id,
|
|
"stale_completion_rejected",
|
|
{"fence": fence, "version": row["pipeline_version"]},
|
|
)
|
|
sender.send({"result": "stale"})
|
|
return
|
|
for result in results:
|
|
conn.execute(
|
|
"INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT(attachment_id,pipeline_version,name) DO NOTHING",
|
|
(row["attachment_id"], row["pipeline_version"], *result),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO effect_publications(attachment_id,pipeline_version,job_id,fence) VALUES(%s,%s,%s,%s)",
|
|
(row["attachment_id"], row["pipeline_version"], job_id, fence),
|
|
)
|
|
conn.execute(
|
|
"UPDATE attachments SET state='ready',object_key=%s,published_version=pipeline_version,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
|
(results[0][1], row["attachment_id"]),
|
|
)
|
|
conn.execute(
|
|
"UPDATE jobs SET state='ready',error_code=NULL,lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
|
(job_id,),
|
|
)
|
|
event(
|
|
conn,
|
|
job_id,
|
|
"effect_committed",
|
|
{
|
|
"fence": fence,
|
|
"version": row["pipeline_version"],
|
|
"variants": 4,
|
|
},
|
|
)
|
|
fault(db, job_id, "after_commit")
|
|
# Cleanup is independently retryable; it cannot undo the committed effect.
|
|
if row["source_key"].startswith("objects/staging/"):
|
|
try:
|
|
store.delete(key=row["source_key"])
|
|
except Exception:
|
|
pass
|
|
sender.send(
|
|
{
|
|
"result": "ready",
|
|
"seconds": time.monotonic() - started,
|
|
"cpu_seconds": time.process_time() - cpu,
|
|
"rss_peak_mib": resource.getrusage(
|
|
resource.RUSAGE_SELF
|
|
).ru_maxrss
|
|
/ 1024,
|
|
}
|
|
)
|
|
except PermanentFailure as exc:
|
|
sender.send({"result": "permanent", "code": str(exc)})
|
|
except MemoryError:
|
|
sender.send({"result": "permanent", "code": "memory_limit"})
|
|
except Exception:
|
|
sender.send({"result": "transient", "code": "processing_unavailable"})
|
|
finally:
|
|
if store:
|
|
store.close()
|
|
if db:
|
|
db.close()
|
|
sender.close()
|
|
|
|
|
|
def claim(db, data, consumer):
|
|
with db.connection() as conn, conn.transaction():
|
|
row = conn.execute(
|
|
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (data["job_id"],)
|
|
).fetchone()
|
|
if (
|
|
not row
|
|
or str(row["attachment_id"]) != data["attachment_id"]
|
|
or row["pipeline_version"] != data["pipeline_version"]
|
|
):
|
|
return "obsolete", None
|
|
if row["state"] in {"ready", "failed", "obsolete"}:
|
|
return row["state"], row
|
|
if data["attempt"] < row["attempt"] or (
|
|
row["state"] == "queued" and data["attempt"] <= row["attempt"]
|
|
):
|
|
return "obsolete", row
|
|
if row["state"] == "processing":
|
|
return "busy", row
|
|
if data["attempt"] != row["attempt"] + 1:
|
|
return "obsolete", row
|
|
due = conn.execute(
|
|
"SELECT %s::timestamptz<=clock_timestamp() due", (row["next_at"],)
|
|
).fetchone()["due"]
|
|
if not due:
|
|
return "busy", row
|
|
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
|
|
updated = conn.execute(
|
|
"UPDATE jobs SET state='processing',attempt=attempt+1,fence=fence+1,lease_owner=%s,lease_until=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s RETURNING *",
|
|
(consumer, lease, row["id"]),
|
|
).fetchone()
|
|
conn.execute(
|
|
"UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
|
|
(row["attachment_id"], row["pipeline_version"]),
|
|
)
|
|
event(
|
|
conn,
|
|
row["id"],
|
|
"claimed",
|
|
{
|
|
"attempt": updated["attempt"],
|
|
"fence": updated["fence"],
|
|
"version": updated["pipeline_version"],
|
|
},
|
|
)
|
|
return "claimed", updated
|
|
|
|
|
|
def ready_effect(db, store, row):
|
|
with db.connection() as conn:
|
|
variants = conn.execute(
|
|
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s",
|
|
(row["attachment_id"], row["pipeline_version"]),
|
|
).fetchall()
|
|
if len(variants) != 4:
|
|
return False
|
|
for variant in variants:
|
|
head = store.head(key=variant["object_key"])
|
|
if (
|
|
head is None
|
|
or head.size != variant["size_bytes"]
|
|
or head.metadata.get("x-amz-meta-sha256") != variant["sha256"]
|
|
):
|
|
return False
|
|
return True
|
|
|
|
|
|
def consume(slot):
|
|
db, queue, store = database(), broker(), ObjectStore()
|
|
consumer = str(uuid4()) + f"-{slot}"
|
|
context = multiprocessing.get_context("spawn")
|
|
cursor = "0-0"
|
|
timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117)
|
|
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
|
|
if lease <= timeout:
|
|
raise ValueError("Lease must exceed child timeout")
|
|
try:
|
|
while True:
|
|
try:
|
|
try:
|
|
queue.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
|
except redis.ResponseError as exc:
|
|
if "BUSYGROUP" not in str(exc):
|
|
raise
|
|
reclaimed = queue.xautoclaim(
|
|
STREAM,
|
|
GROUP,
|
|
consumer,
|
|
# Do not steal an entry from a consumer inside its live budget.
|
|
min_idle_time=lease * 1000,
|
|
start_id=cursor,
|
|
count=1,
|
|
)
|
|
cursor, messages = reclaimed[0], reclaimed[1]
|
|
if not messages:
|
|
batch = queue.xreadgroup(
|
|
GROUP, consumer, {STREAM: ">"}, count=1, block=500
|
|
)
|
|
messages = batch[0][1] if batch else []
|
|
for entry, fields in messages:
|
|
try:
|
|
data = validate_job(
|
|
json.loads(fields.get("job", "null"))
|
|
)
|
|
except ValueError, TypeError, KeyError:
|
|
# Malformed notifications cannot become jobs; preserve diagnostic count.
|
|
with db.connection() as conn:
|
|
event(conn, None, "invalid_envelope")
|
|
queue.xack(STREAM, GROUP, entry)
|
|
queue.xdel(STREAM, entry)
|
|
continue
|
|
state, row = claim(db, data, consumer)
|
|
if state == "busy":
|
|
continue
|
|
if state == "ready" and not ready_effect(db, store, row):
|
|
with db.connection() as conn:
|
|
event(conn, row["id"], "ready_effect_missing")
|
|
continue
|
|
if state in {"ready", "obsolete", "failed"}:
|
|
with db.connection() as conn:
|
|
event(
|
|
conn,
|
|
data["job_id"],
|
|
"duplicate_decision",
|
|
{"state": state, "entry": entry},
|
|
)
|
|
if state == "claimed":
|
|
receiver, sender = context.Pipe(duplex=False)
|
|
child = context.Process(
|
|
target=process,
|
|
args=(str(row["id"]), row["fence"], sender),
|
|
)
|
|
child.start()
|
|
sender.close()
|
|
child.join(timeout)
|
|
if child.is_alive():
|
|
child.kill()
|
|
child.join(2)
|
|
result = {
|
|
"result": "transient",
|
|
"code": "worker_timeout",
|
|
}
|
|
else:
|
|
try:
|
|
result = (
|
|
receiver.recv()
|
|
if receiver.poll()
|
|
else {
|
|
"result": "transient",
|
|
"code": "worker_crashed",
|
|
}
|
|
)
|
|
except EOFError:
|
|
result = {
|
|
"result": "transient",
|
|
"code": "worker_crashed",
|
|
}
|
|
receiver.close()
|
|
child.close()
|
|
with db.connection() as conn, conn.transaction():
|
|
current = conn.execute(
|
|
"SELECT * FROM jobs WHERE id=%s FOR UPDATE",
|
|
(row["id"],),
|
|
).fetchone()
|
|
event(conn, row["id"], "child_finished", result)
|
|
if (
|
|
current["state"] == "processing"
|
|
and current["fence"] == row["fence"]
|
|
):
|
|
retry(
|
|
conn,
|
|
current,
|
|
result.get("code", "processing_failed"),
|
|
permanent=result["result"] == "permanent",
|
|
)
|
|
# Commit result or a retry/terminal decision precedes ack.
|
|
queue.xack(STREAM, GROUP, entry)
|
|
queue.xdel(STREAM, entry)
|
|
except Exception:
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"event": "worker_retry",
|
|
"code": "dependency_unavailable",
|
|
}
|
|
),
|
|
flush=True,
|
|
)
|
|
time.sleep(0.5)
|
|
finally:
|
|
queue.close()
|
|
store.close()
|
|
db.close()
|
|
|
|
|
|
def main():
|
|
count = integer("WORKER_CONCURRENCY", 2, 1, 4)
|
|
threads = [
|
|
threading.Thread(target=consume, args=(i,), daemon=False)
|
|
for i in range(count)
|
|
]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|