forked from mtusi2026/mtusi2026lab5
ЛР 5: асинхронная обработка изображений и идемпотентность pipeline
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
"""Check image edge cases through the public API using the worker's pinned Pillow."""
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def pillow(code, data=""):
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "exec", "-T", "worker", "python3", "-c", code],
|
||||
cwd=ROOT, input=data, capture_output=True, text=True, encoding="utf-8", timeout=30, check=True,
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def main():
|
||||
client = Client()
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=synthetic"
|
||||
fixtures = pillow("""
|
||||
import base64, io, json
|
||||
from PIL import Image
|
||||
jpeg = Image.new("RGB", (300, 100), "red")
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
exif[271] = "Synthetic EXIF"
|
||||
png = Image.new("RGBA", (8, 4), (255, 0, 0, 0))
|
||||
def encode(image, kind, **options):
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, kind, **options)
|
||||
return base64.b64encode(buffer.getvalue()).decode()
|
||||
print(json.dumps([encode(jpeg, "JPEG", exif=exif), encode(png, "PNG")]))
|
||||
""")
|
||||
downloads = {}
|
||||
for fixture, mime, dimensions in zip(fixtures, ("image/jpeg", "image/png"),
|
||||
([(85, 256), (128, 128), (100, 300)],
|
||||
[(8, 4), (128, 128), (8, 4)])):
|
||||
raw = base64.b64decode(fixture)
|
||||
att = client.call("POST", path, 202, actor=alice, raw=raw,
|
||||
headers={"Content-Type": mime}, schema="Attachment")["json"]
|
||||
ready = client.wait_ready(bob, att)
|
||||
content = f"/api/v1/attachments/{att['id']}/content"
|
||||
check(client.call("GET", content, actor=bob)["bytes"] == raw, "original changed")
|
||||
for (width, height), name in zip(dimensions, ("thumbnail", "crop", "optimized")):
|
||||
download = client.call("GET", content + "?variant=" + name, actor=bob)
|
||||
downloads[mime + "/" + name] = [base64.b64encode(download["bytes"]).decode(), width, height]
|
||||
print(f"{mime} attachment={att['id']} dimensions={dimensions}")
|
||||
probes = pillow("""
|
||||
import base64, io, json, sys
|
||||
from PIL import Image
|
||||
results = {}
|
||||
for key, (body, width, height) in json.load(sys.stdin).items():
|
||||
with Image.open(io.BytesIO(base64.b64decode(body))) as image:
|
||||
image.load()
|
||||
white = all(channel >= 250 for channel in image.convert("RGB").getpixel((0, 0)))
|
||||
results[key] = image.size == (width, height) and not image.getexif() and (white or not key.startswith("image/png"))
|
||||
print(json.dumps(results))
|
||||
""", json.dumps(downloads))
|
||||
check(all(probes.values()), "wrong actual dimensions, leaked EXIF or invalid alpha composition")
|
||||
|
||||
fixture = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
ihdr = struct.pack(">II", 10_000, 4_001) + fixture[24:29]
|
||||
too_many_pixels = fixture[:16] + ihdr + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr)) + fixture[33:]
|
||||
for body, mime, status in (
|
||||
(too_many_pixels, "image/png", 413),
|
||||
(b"0" * (10 * 1024 * 1024 + 1), "image/png", 413),
|
||||
(fixture, "image/jpeg", 415),
|
||||
(fixture[:33], "image/png", 415),
|
||||
):
|
||||
client.call("POST", path, status, actor=alice, raw=body, headers={"Content-Type": mime})
|
||||
print("PASS EXIF, fit, crop, WebP, transparency, metadata removal, byte/pixel/format limits")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,320 @@
|
||||
"""Fault experiment on a dedicated lab5 Compose stand with synthetic data."""
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check
|
||||
|
||||
|
||||
def run(*args, expected=0, timeout=120):
|
||||
proc = subprocess.run(
|
||||
args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace", timeout=timeout,
|
||||
)
|
||||
if proc.returncode != expected:
|
||||
raise Failure(f"{args[:5]} exit={proc.returncode}, expected={expected}")
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return run("docker", "compose", "--ansi", "never", *args, **kwargs)
|
||||
|
||||
|
||||
def sql(query):
|
||||
return compose("exec", "-T", "db", "psql", "-XAt", "-v", "ON_ERROR_STOP=1",
|
||||
"-U", "messenger", "-d", "messenger", "-c", query)
|
||||
|
||||
|
||||
def broker(*args):
|
||||
output = compose(
|
||||
"exec", "-T", "broker", "/bin/sh", "-ec",
|
||||
'export REDISCLI_AUTH="$(cat /run/secrets/broker_password)"; exec valkey-cli --json "$@"',
|
||||
"sh", *args,
|
||||
)
|
||||
return json.loads(output)
|
||||
|
||||
|
||||
def until(fn, timeout=90):
|
||||
start = time.monotonic()
|
||||
while time.monotonic() - start < timeout:
|
||||
try:
|
||||
result = fn()
|
||||
except Failure:
|
||||
result = None
|
||||
if result:
|
||||
return result
|
||||
time.sleep(0.3)
|
||||
raise Failure(f"condition timed out after {timeout}s")
|
||||
|
||||
|
||||
def job(att, version=1):
|
||||
return json.loads(sql(f"""
|
||||
SELECT json_build_object('id',id,'state',state,'attempt',attempt,'error',error_code,
|
||||
'published',published,'created',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"'),
|
||||
'updated',updated_at,'delay_ms',round(extract(epoch FROM (next_at-updated_at))*1000))
|
||||
FROM image_jobs WHERE attachment_id='{att['id']}' AND version={version}
|
||||
"""))
|
||||
|
||||
|
||||
def carrier(att, version):
|
||||
row = job(att, version)
|
||||
return {
|
||||
"schema_version": 1, "job_id": row["id"], "type": "attachment.process.v1",
|
||||
"attachment_id": att["id"], "pipeline_version": version,
|
||||
"attempt": row["attempt"], "created_at": row["created"],
|
||||
}
|
||||
|
||||
|
||||
def entries(att):
|
||||
return [json.loads(fields[1]) for _, fields in broker("XRANGE", "attachment-jobs", "-", "+")
|
||||
if json.loads(fields[1])["attachment_id"] == att["id"]]
|
||||
|
||||
|
||||
def publish(payload):
|
||||
return broker("XADD", "attachment-jobs", "*", "job", json.dumps(payload, separators=(",", ":")))
|
||||
|
||||
|
||||
def s3stat(key):
|
||||
setup = ('mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" '
|
||||
'"$(cat /run/secrets/s3_root_password)" >/dev/null; '
|
||||
'mc stat --json "local/messenger-attachments/$1"')
|
||||
# mc reports an absent object in JSON on stdout.
|
||||
args = ("docker", "compose", "--ansi", "never", "run", "--rm", "--no-deps",
|
||||
"--entrypoint", "/bin/sh", "s3-init", "-ec", setup, "sh", key)
|
||||
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace", timeout=60)
|
||||
result = json.loads(proc.stdout.strip().splitlines()[-1])
|
||||
if proc.returncode:
|
||||
check("does not exist" in json.dumps(result).lower(), "unexpected object-store error")
|
||||
return None
|
||||
return result
|
||||
|
||||
|
||||
def fault(service, phase, att):
|
||||
out = compose("run", "--rm", "--no-deps", "-e", f"LAB5_FAULT={phase}",
|
||||
"-e", f"LAB5_FAULT_ATTACHMENT={att['id']}", service, expected=86)
|
||||
check('"test crash"' in out and f'"phase":"{phase}"' in out, "fault marker missing")
|
||||
|
||||
|
||||
def generations(*services):
|
||||
ids = compose("ps", "-q", *services).splitlines()
|
||||
return dict(line.split("|", 1) for line in run(
|
||||
"docker", "inspect", "--format", "{{.Id}}|{{.State.StartedAt}}", *ids,
|
||||
).splitlines())
|
||||
|
||||
|
||||
class Experiment:
|
||||
def __init__(self):
|
||||
self.client = Client()
|
||||
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
|
||||
self.results = []
|
||||
|
||||
def record(self, scenario, att, started, **data):
|
||||
row = {"scenario": scenario, "attachment_id": att["id"],
|
||||
"seconds": round(time.monotonic() - started, 3), **data}
|
||||
self.results.append(row)
|
||||
print(json.dumps(row, ensure_ascii=False), flush=True)
|
||||
|
||||
def metadata(self, att):
|
||||
return self.client.call("GET", "/api/v1/attachments/" + att["id"],
|
||||
actor=self.bob, schema="Attachment")["json"]
|
||||
|
||||
def content(self, att, variant, status=200, actor=None):
|
||||
return self.client.call("GET", f"/api/v1/attachments/{att['id']}/content?variant={variant}",
|
||||
status, actor=actor or self.bob)
|
||||
|
||||
def ready(self, att, raw):
|
||||
result = self.client.wait_ready(self.bob, att, wait_seconds=155)
|
||||
check(self.content(att, "original")["bytes"] == raw, "original changed")
|
||||
variants = {v["name"]: v for v in result["variants"]}
|
||||
check(len(result["variants"]) == 3 and set(variants) == {"thumbnail", "crop", "optimized"},
|
||||
"expected exactly one of each variant")
|
||||
for name, (width, height, mime) in {
|
||||
"thumbnail": (256, 160, "image/jpeg"), "crop": (128, 128, "image/jpeg"),
|
||||
"optimized": (320, 200, "image/webp"),
|
||||
}.items():
|
||||
variant = variants[name]
|
||||
download = self.content(att, name)
|
||||
body = download["bytes"]
|
||||
check((variant["width"], variant["height"], download["headers"].get_content_type())
|
||||
== (width, height, mime), "wrong dimensions or MIME")
|
||||
check(variant["size_bytes"] == len(body) and variant["sha256"] == hashlib.sha256(body).hexdigest(),
|
||||
"wrong variant size or checksum")
|
||||
check(body.startswith(b"\xff\xd8") if mime == "image/jpeg" else
|
||||
body[:4] == b"RIFF" and body[8:12] == b"WEBP", "wrong encoding")
|
||||
return result
|
||||
|
||||
def restart(self):
|
||||
compose("stop", "worker")
|
||||
att, raw = self.client.attachment(self.alice, self.chat)
|
||||
started = time.monotonic()
|
||||
check(self.metadata(att)["status"] == "queued", "job ran while worker stopped")
|
||||
self.content(att, "original", 409)
|
||||
self.content(att, "original", 404, self.eve)
|
||||
message = self.client.message(self.alice, self.chat, "", attachments=[att["id"]])["json"]
|
||||
before = until(lambda: entries(att))
|
||||
services = ("api", "broker")
|
||||
old = generations(*services)
|
||||
check(len(old) >= 3, "expected two API replicas and broker")
|
||||
compose("restart", *services)
|
||||
new = generations(*services)
|
||||
check(old.keys() == new.keys() and all(old[key] != new[key] for key in old), "restart not observed")
|
||||
check(until(lambda: entries(att)) == before, "durable broker entry changed")
|
||||
until(lambda: self.client.call("GET", "/health/ready", schema="Health"))
|
||||
compose("start", "worker")
|
||||
self.ready(att, raw)
|
||||
until(lambda: not entries(att))
|
||||
self.record("api_broker_restart_with_worker_stopped", att, started,
|
||||
message_id=message["id"], job_id=before[0]["job_id"], processes_restarted=len(old))
|
||||
|
||||
def publish_crash(self, phase):
|
||||
compose("stop", "dispatcher", "worker")
|
||||
att, raw = self.client.attachment(self.alice, self.chat)
|
||||
started = time.monotonic()
|
||||
fault("dispatcher", phase, att)
|
||||
old = job(att)
|
||||
check(old["state"] == "queued" and not old["published"], "outbox lost pending intent")
|
||||
observed = entries(att)
|
||||
check(len(observed) == (phase == "after_publish"), "incorrect broker state at crash")
|
||||
# Keep the worker stopped until both XADDs are visible in the ambiguous case.
|
||||
compose("start", "dispatcher")
|
||||
count = 2 if phase == "after_publish" else 1
|
||||
until(lambda: len(entries(att)) == count)
|
||||
check(all(entry == carrier(att, 1) for entry in entries(att)), "unexpected carrier")
|
||||
compose("start", "worker")
|
||||
self.ready(att, raw)
|
||||
until(lambda: not entries(att))
|
||||
check(job(att)["attempt"] == 1, "duplicate delivery consumed an attempt")
|
||||
self.record(phase, att, started, copies=count, attempt=1)
|
||||
|
||||
def worker_crash(self, phase):
|
||||
compose("stop", "worker")
|
||||
att, raw = self.client.attachment(self.alice, self.chat)
|
||||
started = time.monotonic()
|
||||
fault("worker", phase, att)
|
||||
old = job(att)
|
||||
path = f"original/{att['id']}/v1"
|
||||
written = s3stat(path)
|
||||
check((written is not None) == (phase != "before_effect"), "incorrect S3 state at crash")
|
||||
check(old["state"] == ("ready" if phase == "after_commit" else "processing"), "incorrect job state at crash")
|
||||
check(broker("XPENDING", "attachment-jobs", "image-workers")[0] > 0, "missing unacked delivery")
|
||||
compose("start", "worker")
|
||||
self.ready(att, raw)
|
||||
until(lambda: not entries(att), timeout=155)
|
||||
if written:
|
||||
final = s3stat(path)
|
||||
check(final["lastModified"] == written["lastModified"] and final["etag"] == written["etag"],
|
||||
"retry rewrote an existing effect")
|
||||
check(job(att)["attempt"] == (1 if phase == "after_commit" else 2), "wrong attempt after recovery")
|
||||
until(lambda: s3stat(f"staging/{att['id']}") is None, timeout=50)
|
||||
self.record(phase, att, started, state_at_crash=old["state"], object_at_crash=written is not None,
|
||||
final_attempt=job(att)["attempt"], staging_removed=True)
|
||||
|
||||
def replay(self, phase):
|
||||
compose("stop", "worker")
|
||||
att, raw = self.client.attachment(self.alice, self.chat)
|
||||
started = time.monotonic()
|
||||
name = f"lab5-fault-{uuid.uuid4().hex[:12]}"
|
||||
compose("run", "-d", "--no-deps", "--name", name, "-e", f"LAB5_FAULT={phase}",
|
||||
"-e", f"LAB5_FAULT_ATTACHMENT={att['id']}", "worker")
|
||||
delays = {}
|
||||
try:
|
||||
def failure():
|
||||
row = job(att)
|
||||
if row["attempt"] > 1 and row["delay_ms"] > 0:
|
||||
delays.setdefault(row["attempt"], row["delay_ms"])
|
||||
return row if row["state"] == "failed" else None
|
||||
|
||||
failed = until(failure, timeout=155)
|
||||
finally:
|
||||
run("docker", "stop", name)
|
||||
run("docker", "rm", name)
|
||||
code = "invalid_image" if phase == "invalid_image" else "retry_exhausted"
|
||||
meta = self.metadata(att)
|
||||
check(meta["status"] == "failed" and meta["error_code"] == code, "incorrect final failure")
|
||||
if phase == "invalid_image":
|
||||
check(failed["attempt"] == 1, "permanent error was retried")
|
||||
else:
|
||||
config = json.loads(compose("config", "--format", "json"))
|
||||
max_attempts = int(config["services"]["worker"]["environment"]["JOB_MAX_ATTEMPTS"])
|
||||
check(failed["attempt"] == max_attempts, "incorrect retry limit")
|
||||
check(set(delays) == set(range(2, max_attempts + 1)), "missed retry schedule")
|
||||
for attempt, delay in delays.items():
|
||||
base = 1000 << min(attempt - 2, 5)
|
||||
check(base - 5 <= delay < 2 * base, "retry delay outside jitter window")
|
||||
self.content(att, "original", 409)
|
||||
self.client.call("POST", f"/api/v1/chats/{self.chat['id']}/messages", 409, actor=self.alice,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex}, body={"text": "", "attachment_ids": [att["id"]]})
|
||||
def operator(*args, **kwargs):
|
||||
return compose("run", "--rm", "--no-deps", "dispatcher", "jobs", *args, **kwargs)
|
||||
check(failed["id"] in operator("failed"), "failed registry did not contain job")
|
||||
operator("replay", failed["id"])
|
||||
check(self.metadata(att)["status"] == "queued", "replay failed to queue job")
|
||||
compose("start", "worker")
|
||||
self.ready(att, raw)
|
||||
operator("replay", failed["id"], expected=1)
|
||||
audit = sql(f"SELECT count(*) FROM image_job_replays WHERE job_id='{failed['id']}'")
|
||||
check(audit == "1" and job(att)["attempt"] == failed["attempt"] + 1, "replay audit/attempt mismatch")
|
||||
self.record(phase + "_replay", att, started, failed_attempt=failed["attempt"],
|
||||
error_code=code, retry_delays_ms=delays, replay_audit=int(audit))
|
||||
|
||||
def version_order(self):
|
||||
att, raw = self.client.attachment(self.alice, self.chat)
|
||||
started = time.monotonic()
|
||||
v1 = self.ready(att, raw)
|
||||
old_carrier = carrier(att, 1)
|
||||
old_object = s3stat(f"optimized/{att['id']}/v1")
|
||||
until(lambda: not entries(att))
|
||||
compose("stop", "worker")
|
||||
compose("run", "--rm", "--no-deps", "dispatcher", "jobs", "upgrade", att["id"])
|
||||
until(lambda: job(att, 2)["published"])
|
||||
new_carrier = carrier(att, 2)
|
||||
publish(old_carrier)
|
||||
publish(new_carrier)
|
||||
check([entry["pipeline_version"] for entry in entries(att)] == [2, 1, 2], "reorder not injected")
|
||||
check(self.metadata(att) == v1, "upgrade hid the previous result")
|
||||
compose("start", "worker")
|
||||
until(lambda: sql(f"SELECT result_version FROM attachments WHERE id='{att['id']}'") == "2")
|
||||
v2 = self.ready(att, raw)
|
||||
until(lambda: not entries(att))
|
||||
check(v1["variants"] != v2["variants"], "v2 was not distinct")
|
||||
new_object = s3stat(f"optimized/{att['id']}/v2")
|
||||
publish(old_carrier)
|
||||
publish(new_carrier)
|
||||
until(lambda: not entries(att))
|
||||
check(self.ready(att, raw) == v2 and job(att, 2)["attempt"] == 1, "stale job changed v2")
|
||||
for version, obj in ((1, old_object), (2, new_object)):
|
||||
current = s3stat(f"optimized/{att['id']}/v{version}")
|
||||
check(current["lastModified"] == obj["lastModified"] and current["etag"] == obj["etag"],
|
||||
"duplicate or reordered job rewrote an object")
|
||||
self.record("reorder_and_stale_version", att, started, delivery_versions=[2, 1, 2, 1, 2],
|
||||
result_version=2, attempts=[job(att)["attempt"], job(att, 2)["attempt"]])
|
||||
|
||||
|
||||
def main():
|
||||
experiment = Experiment()
|
||||
try:
|
||||
experiment.restart()
|
||||
for phase in ("before_publish", "after_publish"):
|
||||
experiment.publish_crash(phase)
|
||||
for phase in ("before_effect", "after_effect", "after_commit"):
|
||||
experiment.worker_crash(phase)
|
||||
for phase in ("invalid_image", "temporary"):
|
||||
experiment.replay(phase)
|
||||
experiment.version_order()
|
||||
finally:
|
||||
compose("start", "dispatcher", "worker")
|
||||
(ROOT / "evidence" / "pipeline.json").write_text(
|
||||
json.dumps(experiment.results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8",
|
||||
)
|
||||
print("PASS pipeline fault experiment")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user