Files
mtusi2026lab6/student-tests/worker_probe.py

149 lines
7.4 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Пробник, который выполняется ВНУТРИ контейнера worker (там Pillow, модули приложения, доступ к БД и S3).
Тест передаёт этот файл через stdin: `docker compose exec -T worker python - <команда> [аргументы]`.
Секреты остаются в контейнере; наружу выходит одна строка JSON (без ключей доступа, только факты).
make <kind> — тестовое изображение, base64 (kind: thumb320, exif6, alpha, tiny, large, strip, big40)
inspect <attachment_id> — метаданные из БД + декодированные объекты из S3: размеры, формат, EXIF,
цвета контрольных точек, SHA-256, список ключей под attachments/<id>/
exists <key> — есть ли объект в bucket
imaging-limits — imaging.py на 40-Мп PNG: пик RSS дочернего процесса и отказ по лимиту пикселей
"""
import base64
import hashlib
import io
import json
import os
import resource
import subprocess
import sys
import tempfile
from uuid import UUID
from PIL import Image
import psycopg
from psycopg.rows import dict_row
import settings
import storage
def make(kind: str) -> bytes:
out = io.BytesIO()
if kind == "thumb320": # 320x200 -> thumbnail 256x160 (контракт)
img = Image.new("RGB", (320, 200), (200, 30, 30))
img.paste((30, 30, 200), (0, 0, 160, 200))
img.save(out, "PNG")
elif kind == "exif6": # JPEG 400x300 с EXIF Orientation=6 и Make/Model/GPS
img = Image.new("RGB", (400, 300), (0, 160, 0))
img.paste((255, 0, 0), (0, 0, 100, 300)) # левая полоса красная: после поворота на 90° по часовой — сверху
exif = Image.Exif()
exif[0x0112] = 6
exif[0x010F] = "ProbeCamera"
exif[0x0110] = "Model-X"
exif[0x8825] = {1: "N", 2: (55.0, 45.0, 0.0)}
img.save(out, "JPEG", quality=95, exif=exif.tobytes())
elif kind == "alpha": # RGBA 300x200: левая половина прозрачная
img = Image.new("RGBA", (300, 200), (0, 0, 255, 255))
img.paste((0, 0, 0, 0), (0, 0, 150, 200))
img.save(out, "PNG")
elif kind == "tiny": # 40x30: без увеличения для thumbnail/optimized
Image.new("RGB", (40, 30), (10, 120, 220)).save(out, "PNG")
elif kind == "large": # 3000x2000 JPEG: optimized 1280x853, thumbnail 256x170
img = Image.radial_gradient("L").resize((3000, 2000)).convert("RGB")
img.save(out, "JPEG", quality=90)
elif kind == "strip": # 5000x2: thumbnail 256x1 (минимум 1 px)
Image.new("RGB", (5000, 2), (90, 90, 90)).save(out, "PNG")
elif kind == "big40": # 8000x5000 = 40 Мп ровно на лимите
Image.new("RGB", (8000, 5000), (120, 60, 30)).save(out, "PNG")
else:
raise SystemExit("unknown kind")
return out.getvalue()
def connect():
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row)
def read(key: str) -> bytes:
stream = storage.open_read(key)
try:
return b"".join(storage.iter_chunks(stream))
finally:
stream.close()
def inspect(attachment_id: str) -> dict:
with connect() as conn:
a = conn.execute("SELECT state, object_key, source_key, pipeline_version, sha256, size_bytes "
"FROM attachments WHERE id = %s", (UUID(attachment_id),)).fetchone()
variants = conn.execute("SELECT name, content_type, width, height, sha256, size_bytes, object_key "
"FROM attachment_variants WHERE attachment_id = %s ORDER BY name",
(UUID(attachment_id),)).fetchall()
result = {"state": a["state"], "staging_present": a["source_key"] is not None and storage.exists(a["source_key"]),
"variant_rows": len(variants), "variants": {}}
if a["object_key"]:
original = read(a["object_key"])
result["original_sha_ok"] = hashlib.sha256(original).hexdigest() == a["sha256"]
result["original_size_ok"] = len(original) == a["size_bytes"]
for v in variants:
data = read(v["object_key"])
img = Image.open(io.BytesIO(data))
info = {"format": img.format, "size": list(img.size), "mode": img.mode,
"meta_size_ok": [v["width"], v["height"]] == list(img.size),
"sha_ok": hashlib.sha256(data).hexdigest() == v["sha256"] and len(data) == v["size_bytes"],
"exif_tags": len(img.getexif()), "has_xmp": "xmp" in img.info or b"http://ns.adobe.com/xap" in data,
"quality_hint": None}
rgb = img.convert("RGBA")
w, h = rgb.size
info["pixel_top_center"] = list(rgb.getpixel((w // 2, 0)))
info["pixel_left_center"] = list(rgb.getpixel((0, h // 2)))
info["pixel_right_center"] = list(rgb.getpixel((w - 1, h // 2)))
result["variants"][v["name"]] = info
prefix = f"attachments/{attachment_id}/"
result["final_keys"] = sorted(key[len(prefix):] for key, _m, _s in storage.list_objects(prefix))
return result
def imaging_limits() -> dict:
"""Пик памяти процесса обработки на худшем допустимом входе и отказ при превышении лимита пикселей."""
here = os.path.dirname(os.path.abspath(storage.__file__))
report = {}
with tempfile.TemporaryDirectory(dir=settings.WORKER_TMP_DIR) as tmp:
source = os.path.join(tmp, "big.png")
with open(source, "wb") as f:
f.write(make("big40"))
env = {"PATH": os.environ.get("PATH", ""), "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB),
"IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)}
for label, max_pixels in (("at_limit_40mpx", 40_000_000), ("over_limit_max_10mpx", 10_000_000)):
out = os.path.join(tmp, label)
os.mkdir(out)
before = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
proc = subprocess.run([sys.executable, os.path.join(here, "imaging.py"), source, out, "image/png",
str(max_pixels)], env=env, capture_output=True, timeout=120)
peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
report[label] = {"exit_code": proc.returncode, "peak_rss_mib_max_children": round(peak / 1024, 1),
"rss_grew": peak > before}
report["memory_limit_mib"] = settings.WORKER_MEMORY_LIMIT_MB
report["timeout_s"] = settings.JOB_TIMEOUT_SECONDS
return report
def main():
command = sys.argv[1]
if command == "make":
print(json.dumps({"b64": base64.b64encode(make(sys.argv[2])).decode("ascii")}))
elif command == "inspect":
print(json.dumps(inspect(sys.argv[2])))
elif command == "exists":
print(json.dumps({"exists": storage.exists(sys.argv[2])}))
elif command == "imaging-limits":
print(json.dumps(imaging_limits()))
else:
raise SystemExit("unknown command")
if __name__ == "__main__":
main()