diff --git a/student-tests/conftest.py b/student-tests/conftest.py index e1efdbb..daf4abe 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -120,21 +120,33 @@ def service_down(service: str, how: str = "stop"): wait_stand() +def recreate(service: str, env: dict, scale: int | None = None) -> None: + """`docker compose up` of ONE service with the given environment (no dependency restarts).""" + cmd = ["docker", "compose", "up", "-d", "--no-deps", "--wait", service] + if scale is not None: + cmd[4:4] = ["--scale", f"{service}={scale}"] + subprocess.run(cmd, cwd=ROOT, env=env, capture_output=True, text=True, check=True, timeout=180) + + @contextlib.contextmanager -def api_with_env(**overrides: str): - """Recreate the api container with environment overrides (e.g. SESSION_TTL_SECONDS=2), - then ALWAYS recreate it with the normal configuration.""" - env = {**os.environ, **overrides} - subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env, - capture_output=True, text=True, check=True, timeout=180) - wait_stand() +def service_with_env(service: str, scale: int | None = None, **overrides: str): + """Recreate a service (api, relay, worker) with environment overrides (e.g. FAULT_POINTS of the + worker = WORKER_FAULT_POINTS), then ALWAYS recreate it with the normal configuration.""" + clean = {k: v for k, v in os.environ.items() if k not in overrides} + recreate(service, {**os.environ, **overrides}, scale) + if service == "api": + wait_stand() try: yield finally: - clean = {k: v for k, v in os.environ.items() if k not in overrides} - subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean, - capture_output=True, text=True, check=True, timeout=180) - wait_stand() + recreate(service, clean, 1 if scale is not None and service != "api" else None) + wait_healthy(service) + if service == "api": + wait_stand() + + +def api_with_env(**overrides: str): + return service_with_env("api", **overrides) def running_replicas() -> list[dict]: @@ -235,7 +247,7 @@ class Api: def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict: return self.post_message(actor, chat, {"text": text}, key=key).json() - def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png", + def upload(self, actor: dict, chat: dict, data, expected=202, *, content_type: str | None = "image/png", filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None): """POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string (to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked).""" @@ -247,12 +259,31 @@ class Api: path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "") if isinstance(data, (bytes, bytearray)): return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h, - schema="Attachment" if expected == 201 else None) + schema="Attachment" if expected == 202 else None) h["Authorization"] = "Bearer " + actor["token"] response = self.client.post(path, content=data, headers=h) assert response.status_code == expected, (response.status_code, response.text[:300]) return response + def attachment(self, actor: dict, attachment_id: str, expected=200) -> dict: + return self.call("GET", f"/api/v1/attachments/{attachment_id}", expected, token=actor["token"], + schema="Attachment" if expected == 200 else None).json() + + def wait_status(self, actor: dict, attachment_id: str, *statuses: str, timeout: float = 60) -> dict: + """Poll the metadata until the status is one of `statuses` (default: a final one).""" + wanted = statuses or ("ready", "failed") + deadline = time.monotonic() + timeout + while True: + att = self.attachment(actor, attachment_id) + if att["status"] in wanted: + return att + assert time.monotonic() < deadline, f"attachment stayed {att['status']}, wanted {wanted}" + time.sleep(0.3) + + def content(self, actor: dict, attachment_id: str, variant: str | None = None, expected=200): + query = f"?variant={variant}" if variant else "" + return self.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"]) + def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]: """Walk all pages; returns (items, number_of_pages).""" items, pages, cursor = [], 0, None @@ -300,73 +331,101 @@ def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.Complete timeout=timeout, env={**os.environ, **env}) -def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str: - """Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3 - credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is - not published on the host; this is how the tests inject faults directly into the bucket.""" +def s3_exec(body: str, bucket: str = "final", service: str = "worker", stdin: str | None = None) -> str: + """Run `body` (async Python, `s` = messenger.storage.Storage of the final or the staging bucket) + INSIDE a container that holds S3 credentials: `worker` (Read+Write both buckets, no List) or + `auditor` (Read+List, no Write). The S3 port is not published on the host; this is how the + tests look into the buckets and inject faults.""" script = ("import asyncio, base64, json, sys\n" "from messenger.config import Settings, s3_settings\n" "from messenger.storage import Storage, ObjectMissing\n" "async def main():\n" " st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n" - " s = Storage(st)\n" + f" s = Storage(st, bucket=st.s3_staging_bucket if {bucket == 'staging'} else None)\n" " await s.open()\n" " try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) + " finally:\n" " await s.close()\n" "asyncio.run(main())\n") - if service == "api": - cmd = ["exec", "-T", "api", "python", "-c", script] - else: + if service == "auditor": cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script] + else: + cmd = ["exec", "-T", service, "python", "-c", script] return compose(*cmd, timeout=120, stdin=stdin).stdout.strip() -def s3_put(key: str, data: bytes) -> None: +def s3_put(key: str, data: bytes, bucket: str = "final") -> None: """Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB).""" - s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})", + s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})", bucket, stdin=base64.b64encode(data).decode()) -def s3_delete(key: str) -> None: - s3_exec(f"await s.delete({key!r})") +def s3_delete(key: str, bucket: str = "final") -> None: + s3_exec(f"await s.delete({key!r})", bucket) -def s3_size(key: str) -> int | None: - out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)") +def s3_size(key: str, bucket: str = "final") -> int | None: + out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)", bucket) size = int(out.splitlines()[-1]) return None if size < 0 else size -def s3_keys() -> set[str]: - """Every object key under o/ (needs List: runs with the reconciler's key pair).""" +def s3_keys(prefix: str, bucket: str = "final") -> set[str]: + """Every object key under `prefix` (needs List: runs with the auditor's key pair).""" out = s3_exec("keys, after = [], None\n" "while True:\n" - " page, more = await s.list_page('o/', after, 1000)\n" + f" page, more = await s.list_page({prefix!r}, after, 1000)\n" " keys += [o.key for o in page]\n" " if not more or not page:\n" " break\n" " after = page[-1].key\n" - "print(json.dumps(keys))", service="reconciler") + "print(json.dumps(keys))", bucket, service="auditor") return set(json.loads(out.splitlines()[-1])) -def reconcile(*args: str, expect_code: int = 0) -> dict: - """Run the reconciler CLI (compose service, profile tools) and return its JSON report.""" - # --no-deps: never let `compose run` start a store that a test has stopped on purpose. - result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False, - timeout=300) - assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:]) +def jobctl(*args: str, expect_code: int = 0) -> dict: + """Operator CLI (compose service jobctl, profile tools); returns its JSON output.""" + result = compose("run", "--rm", "--no-deps", "-T", "jobctl", *args, check=False, timeout=120) + assert result.returncode == expect_code, (result.returncode, result.stdout[-1000:], result.stderr[-1000:]) return json.loads(result.stdout) -def findings_by_key(report: dict) -> dict[str, dict]: - return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items} +def audit(*args: str) -> dict: + """Read-only invariant checker (compose service auditor); asserts that it found no violation.""" + result = compose("run", "--rm", "--no-deps", "-T", "auditor", *args, check=False, timeout=300) + report = json.loads(result.stdout) + assert result.returncode == 0 and report["ok"], json.dumps(report.get("violations"), ensure_ascii=False)[:3000] + return report + + +def job_rows(attachment_id: str) -> list[dict]: + out = psql("SELECT coalesce(json_agg(json_build_object('id', id, 'state', state, 'attempt', attempt, 'kind', kind, " + "'pipeline_version', pipeline_version, 'error_code', error_code) ORDER BY created_at), '[]') " + "FROM jobs WHERE attachment_id = '%s'" % attachment_id) + return json.loads(out) + + +def events_of(attachment_id: str) -> list[dict]: + out = psql("SELECT coalesce(json_agg(json_build_object('event', e.event, 'attempt', e.attempt, 'actor', e.actor, " + "'error_code', e.error_code, 'detail', e.detail, 'at', e.at) ORDER BY e.id), '[]') " + "FROM job_events e JOIN jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s'" % attachment_id) + return json.loads(out) + + +def queue_depth(queue: str = "messenger.attachment-jobs") -> int: + out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages").stdout + for line in out.splitlines(): + name, _, count = line.partition("\t") + if name == queue: + return int(count) + raise AssertionError("queue not found: " + queue) def attachment_row(attachment_id: str) -> dict | None: out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, " - "'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id) + "'lost_reason', lost_reason, 'staging_key', staging_key, 'staging_state', staging_state, " + "'published_version', published_version, 'error_code', error_code) " + "FROM attachments WHERE id = '%s'" % attachment_id) return json.loads(out) if out else None diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py index 0026fea..1f03da4 100644 --- a/student-tests/imagegen.py +++ b/student-tests/imagegen.py @@ -91,3 +91,27 @@ def jpeg_bomb_header() -> bytes: """SOF0 announcing 20000 x 20000 = 400 Mpx.""" return (b"\xff\xd8\xff\xc0\x00\x11\x08" + struct.pack(">HH", 20000, 20000) + b"\x03\x01\x22\x00\x02\x11\x01\x03\x11\x01" + b"\xff\xd9") + + +# ------------------------------------------------------------------ cases the reviewer used in lab 4/5 PRs +def png_bad_filter(filter_byte: int) -> bytes: + """PNG 1x1 RGB, correct CRCs and zlib stream, but the row filter byte is not 0..4.""" + raw = bytes([filter_byte]) + b"\x00\x00\x00" + return PNG_SIGNATURE + ihdr(1, 1) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") + + +def png_idat_not_zlib() -> bytes: + """Correct PNG structure, IDAT = b'not-a-zlib-stream' with a recomputed CRC.""" + return PNG_SIGNATURE + ihdr(64, 64) + chunk(b"IDAT", b"not-a-zlib-stream") + chunk(b"IEND", b"") + + +# SOI, SOF0 1x1 grey, SOS, EOI: no tables, no scan data (27 bytes). +JPEG_27_BYTES = bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9") + + +def jpeg_sos_unknown_component() -> bytes: + """A valid JPEG whose first SOS component id (byte sos+5) is 250, absent from SOF.""" + data = bytearray(jpeg(64, 48)) + sos = data.index(b"\xff\xda") + data[sos + 5] = 250 + return bytes(data) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index 8aaa015..7bd6113 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -1,10 +1,14 @@ -"""Attachments through the load balancer (lab 4, levels 3 and 4): upload/download, ACL, limits, -content checks, filename rules and messages with attachment_ids. +"""Attachments through the load balancer (lab 4, levels 3 and 4; lab 5: upload is 202 queued and +the worker publishes the original + variants): upload/download, ACL, limits, content checks, +filename rules and messages with attachment_ids. Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000 header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace -filename, text=null / missing text / attachment_ids=null in a message.""" +filename, text=null / missing text / attachment_ids=null in a message; and the lab 4/5 review +cases: PNG row filter 5 and 0xff, IDAT that is not a zlib stream, the 27-byte JPEG without tables +and scan data, a JPEG whose SOS names a component missing from SOF - all 415 AT UPLOAD, never +202 followed by failed/invalid_image.""" from __future__ import annotations import hashlib @@ -42,12 +46,14 @@ def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio): jpg = imagegen.jpeg(321, 123, noise=True) for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")): att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json() - assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None + assert att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"] assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest() assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin" - meta = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"], schema="Attachment").json() - assert meta == att + meta = api.wait_status(bob, att["id"], "ready") + assert {k: v for k, v in meta.items() if k not in ("status", "variants", "updated_at")} == \ + {k: v for k, v in att.items() if k not in ("status", "variants", "updated_at")} + assert [v["name"] for v in meta["variants"]] == ["thumbnail", "crop", "optimized"] got = content(api, bob, att["id"]) assert got.content == data and got.headers["content-type"] == ctype assert got.headers["content-length"] == str(len(data)) @@ -59,13 +65,16 @@ def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio): def test_metadata_reveals_no_storage_details(api, trio): alice, _, _, chat, _ = trio att = api.upload(alice, chat, SAMPLE).json() + api.wait_status(alice, att["id"], "ready") row = attachment_row(att["id"]) - assert row["status"] == "ready" and row["object_key"].startswith("o/") + assert row["status"] == "ready" and row["object_key"].startswith(f"a/{att['id']}/") + assert row["staging_key"].startswith("s/") text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text - assert row["object_key"] not in text and "messenger-attachments" not in text and "s3" not in text.lower() - # The key is random: neither the id nor the filename is part of it. - assert att["id"].replace("-", "") not in row["object_key"] and "photo" not in row["object_key"] - assert s3_size(row["object_key"]) == len(SAMPLE) # the bytes ARE in the bucket + for secret in (row["object_key"], row["staging_key"], "messenger-attachments", "messenger-staging", "a/", "s3"): + assert secret not in text.lower(), secret + # The staging key is random: neither the id nor the filename is part of it. + assert att["id"].replace("-", "") not in row["staging_key"] and "photo" not in row["staging_key"] + assert s3_size(row["object_key"]) == len(SAMPLE) # the published original IS in the final bucket def test_original_is_not_stored_on_api_containers(api, trio): @@ -174,6 +183,7 @@ def test_size_limit_through_the_load_balancer(api, trio): exact = imagegen.png_exactly(10 * MiB) assert len(exact) == 10 * MiB att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB + api.wait_status(bob, att["id"], "ready") assert content(api, bob, att["id"]).content == exact started = time.monotonic() api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit @@ -201,6 +211,10 @@ def test_exactly_40_megapixels_is_accepted(api, trio): data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file att = api.upload(alice, chat, data).json() assert att["size_bytes"] == len(data) + # Lab 5: the worker processes the biggest allowed picture within its memory limit. + ready = api.wait_status(alice, att["id"], "ready", timeout=120) + assert {v["name"]: (v["width"], v["height"]) for v in ready["variants"]} == \ + {"thumbnail": (256, 160), "crop": (128, 128), "optimized": (1280, 800)} # ------------------------------------------------------------------ content checks @@ -213,6 +227,11 @@ BROKEN = { "png: bytes after IEND": (SAMPLE + b"", "image/png"), "jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"), "jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"), + "review: png row filter 5": (imagegen.png_bad_filter(5), "image/png"), + "review: png row filter 0xff": (imagegen.png_bad_filter(0xFF), "image/png"), + "review: png IDAT not-a-zlib-stream": (imagegen.png_idat_not_zlib(), "image/png"), + "review: jpeg 27 bytes, no tables/scan": (imagegen.JPEG_27_BYTES, "image/jpeg"), + "review: jpeg SOS component not in SOF": (imagegen.jpeg_sos_unknown_component(), "image/jpeg"), "mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"), "mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"), "mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"), @@ -253,6 +272,7 @@ def test_unicode_filename_is_kept_and_safely_exposed(api, trio): name = 'отчёт "Q3"; ok.png' att = api.upload(alice, chat, SAMPLE, filename=name).json() assert att["filename"] == name + api.wait_status(alice, att["id"], "ready") disposition = content(api, alice, att["id"]).headers["content-disposition"] assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback assert "filename*=UTF-8''" in disposition @@ -262,7 +282,10 @@ def test_unicode_filename_is_kept_and_safely_exposed(api, trio): def test_variant_parameter(api, trio): alice, _, eve, chat, _ = trio att = api.upload(alice, chat, SAMPLE).json() + api.wait_status(alice, att["id"], "ready") assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE - content(api, alice, att["id"], 404, query="?variant=thumbnail") # schema name, not produced in lab 4 + assert content(api, alice, att["id"], query="?variant=thumbnail").headers["content-type"] == "image/jpeg" content(api, alice, att["id"], 400, query="?variant=huge") + content(api, alice, att["id"], 400, query="?variant=thumbnail&variant=crop") content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed + content(api, eve, att["id"], 404, query="?variant=huge") # outsider first, then the value diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py index 01f3d6e..6a1ba86 100644 --- a/student-tests/test_balancing.py +++ b/student-tests/test_balancing.py @@ -278,15 +278,15 @@ def test_scale_up_and_down_is_picked_up_without_errors(api): def test_compose_policy_limits_ports_secrets_images(): config = json.loads(compose("config", "--format", "json").stdout) services = config["services"] - # The reconciler (profile "tools") is started on demand and not part of the running stand. - assert set(services) == {"postgres", "valkey", "s3", "migrate", "s3-init", "api", "lb"} + # auditor and jobctl (profile "tools") are started on demand and not part of the running stand. + assert set(services) == {"postgres", "valkey", "s3", "broker", "migrate", "s3-init", "api", "lb", "relay", "worker"} for name, svc in services.items(): limits = svc.get("deploy", {}).get("resources", {}).get("limits", {}) assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits" assert "container_name" not in svc, f"{name}: container_name prevents scaling" if name in ("api", "migrate"): assert "build" in svc # built from this repository's source - elif name == "s3-init": + elif name in ("s3-init", "relay", "worker"): assert svc["image"] == services["api"]["image"] # the same locally built image else: assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest" @@ -294,21 +294,28 @@ def test_compose_policy_limits_ports_secrets_images(): published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")} assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"]) secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()} - s3_keys = ['s3_admin_access_key_id', 's3_admin_secret_access_key', 's3_api_access_key_id', 's3_api_secret_access_key', 's3_reconciler_access_key_id', 's3_reconciler_secret_access_key'] + s3_keys = sorted(f"s3_{who}_{part}" for who in ("admin", "api", "worker", "auditor") + for part in ("access_key_id", "secret_access_key")) assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys, "migrate": ["database_url"], "s3-init": s3_keys, "lb": [], - # the API: its own S3 key pair only (no admin, no reconciler keys) - "api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"]} + "broker": ["rabbitmq_users"], + # the API: its own S3 key pair only (no admin/worker keys, no broker credentials) + "api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"], + # relay: database + its broker user only; worker: + its own S3 key pair + "relay": ["broker_url_relay", "database_url"], + "worker": ["broker_url_worker", "database_url", "s3_worker_access_key_id", + "s3_worker_secret_access_key"]} volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"] for name, svc in services.items()} assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [], - "s3-init": [], "api": [], "lb": []} + "s3-init": [], "api": [], "lb": [], "broker": ["rabbitmqdata"], "relay": [], "worker": []} networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()} assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"], - "s3-init": ["storage"], "api": ["data", "edge", "storage"], - "lb": ["edge"]}, networks # the LB cannot reach the stores at all - assert config["networks"]["data"].get("internal") is True - assert config["networks"]["storage"].get("internal") is True + "s3-init": ["storage"], "api": ["data", "edge", "storage"], "broker": ["queue"], + "relay": ["data", "queue"], "worker": ["data", "queue", "storage"], + "lb": ["edge"]}, networks # the LB cannot reach the stores; the API not the broker + for network in ("data", "storage", "queue"): + assert config["networks"][network].get("internal") is True, network def test_running_containers_have_limits_and_no_bypass_ports(): @@ -326,7 +333,8 @@ def test_running_containers_have_limits_and_no_bypass_ports(): def test_migrations_ran_once_by_the_migrate_service(): applied = psql("SELECT version FROM schema_migrations ORDER BY version").split() - assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments"] + assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments", + "0005_pipeline"] migrate = compose("ps", "-a", "--format", "json", "migrate").stdout rows = [json.loads(line) for line in migrate.splitlines() if line.strip()] assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows) diff --git a/student-tests/test_consistency.py b/student-tests/test_consistency.py index 9e375a9..2ddab1d 100644 --- a/student-tests/test_consistency.py +++ b/student-tests/test_consistency.py @@ -1,13 +1,15 @@ -"""Consistency of PostgreSQL and S3 around one upload (lab 4, level 4). +"""Consistency of PostgreSQL and S3 around one upload (lab 4 level 4, lab 5 failure windows of the API). -The two-store write is: intent row 'pending' -> S3 PUT -> row 'ready' -> 201. Failures between -the steps are injected with FAULT_POINTS (compose variable, empty in normal runs; the API -raises 503 at that exact point, as if the process or the network failed there): +Lab 5 upload: intent row 'pending' -> staging PUT -> ONE transaction 'queued' + job + outbox -> 202. +Failures between the steps are injected with FAULT_POINTS of the api service (compose variable +API_FAULT_POINTS, empty in normal runs; the API raises 503 at that exact point, as if the process +or the network failed there): -* after_s3_put object written, row still pending, client got 503 -> the janitor removes - the object and the row after PENDING_GRACE_SECONDS (here 3 s); -* after_ready_commit row ready and object written, but the answer is lost (client got 503) - -> nothing may be deleted: the attachment stays intact and downloadable. +* after_s3_put "after staging, before the job is written": staging object exists, row + still pending, NO job and NO outbox row; the client got 503 -> invisible, + the janitor removes the staging object and the row after PENDING_GRACE_SECONDS; +* after_enqueue_commit job committed, the answer is lost (client got 503) -> nothing may be + deleted: the worker processes the job and the attachment becomes ready. Plus: an upload interrupted by the client and a stalled body leave nothing behind.""" from __future__ import annotations @@ -28,10 +30,18 @@ SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]: - out = psql(f"SELECT id || ' ' || status || ' ' || object_key FROM attachments WHERE chat_id = '{chat_id}'") + out = psql(f"SELECT id || ' ' || status || ' ' || staging_key FROM attachments WHERE chat_id = '{chat_id}'") return [tuple(line.split()) for line in out.splitlines() if line] +def jobs_of(attachment_id: str) -> tuple[int, int]: + """(jobs rows, outbox rows) of an attachment.""" + out = psql(f"SELECT (SELECT count(*) FROM jobs WHERE attachment_id = '{attachment_id}') || ' ' || " + f"(SELECT count(*) FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = '{attachment_id}')") + jobs, outbox = out.split() + return int(jobs), int(outbox) + + def wait_for(condition, timeout: float, what: str) -> None: deadline = time.monotonic() + timeout while not condition(): @@ -68,44 +78,48 @@ def test_interrupted_and_stalled_uploads_leave_nothing(api): assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write -def test_crash_between_s3_put_and_commit_is_cleaned_by_janitor(api): +def test_crash_after_staging_before_job_is_cleaned_by_janitor(api): + """Failure window 1 of contracts/attachment-states.md: the original is in staging, the job is + not written. No 202 was sent, so nothing was promised: the janitor removes the staging copy + (object first) and then the intent row; no job, no message ever existed.""" alice = api.user("alice") chat = api.chat(alice) - with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"): + with api_with_env(API_FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"): assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable" [(attachment_id, status, key)] = rows_of_chat(chat["id"]) - assert status == "pending" and s3_size(key) == len(SAMPLE) # the orphan-to-be exists + assert status == "pending" and s3_size(key, "staging") == len(SAMPLE) # the orphan-to-be exists + assert jobs_of(attachment_id) == (0, 0) # no job, no outbox row api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent") - assert s3_size(key) is None # object removed first + assert s3_size(key, "staging") is None # object removed first # Normal configuration again: an upload in the same chat works and is not touched. att = api.upload(alice, chat, SAMPLE).json() - assert attachment_row(att["id"])["status"] == "ready" + assert api.wait_status(alice, att["id"], "ready")["status"] == "ready" -def test_lost_answer_after_commit_never_deletes_the_object(api): - """The reviewer found this in another PR: COMMIT succeeded, the answer was lost, the API - 'compensated' by deleting the object -> ready row without bytes. Here nothing compensates - in the request path, and the janitor only touches 'pending' rows.""" +def test_lost_answer_after_job_commit_still_becomes_ready(api): + """The job COMMIT succeeded, the answer was lost (503). Nothing compensates in the request path + (the janitor only touches 'pending' rows); the worker processes the job as usual.""" alice, bob = api.user("alice"), api.user("bob") chat = api.chat(alice, bob) - with api_with_env(FAULT_POINTS="after_ready_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"): + with api_with_env(API_FAULT_POINTS="after_enqueue_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"): api.upload(alice, chat, SAMPLE, 503) [(attachment_id, status, key)] = rows_of_chat(chat["id"]) - assert status == "ready" - time.sleep(6) # several janitor rounds - assert rows_of_chat(chat["id"]) == [(attachment_id, "ready", key)] and s3_size(key) == len(SAMPLE) + assert status in ("queued", "processing", "ready") and jobs_of(attachment_id)[0] == 1 + att = api.wait_status(bob, attachment_id, "ready") + time.sleep(4) # several janitor rounds + assert attachment_row(attachment_id)["status"] == "ready" got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"]) - assert got.content == SAMPLE + assert got.content == SAMPLE and att["sha256"] == __import__("hashlib").sha256(SAMPLE).hexdigest() def test_fresh_pending_upload_is_not_touched_by_janitor(api): """An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed.""" alice = api.user("alice") chat = api.chat(alice) - with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"): + with api_with_env(API_FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"): api.upload(alice, chat, SAMPLE, 503) time.sleep(4) [(_, status, key)] = rows_of_chat(chat["id"]) - assert status == "pending" and s3_size(key) == len(SAMPLE) + assert status == "pending" and s3_size(key, "staging") == len(SAMPLE) diff --git a/student-tests/test_db.py b/student-tests/test_db.py index 544d378..641bab2 100644 --- a/student-tests/test_db.py +++ b/student-tests/test_db.py @@ -79,7 +79,8 @@ def test_migrations_rerun_is_a_noop(): assert "0002_idempotency_keys: already applied, skipped" in result.stdout assert "0003_post_message_function: already applied, skipped" in result.stdout assert "0004_attachments: already applied, skipped" in result.stdout - assert psql("SELECT count(*) FROM schema_migrations") == "4" + assert "0005_pipeline: already applied, skipped" in result.stdout + assert psql("SELECT count(*) FROM schema_migrations") == "5" assert psql("SELECT count(*) FROM users") == users_before diff --git a/student-tests/test_storage_failures.py b/student-tests/test_storage_failures.py index 412b210..2b07503 100644 --- a/student-tests/test_storage_failures.py +++ b/student-tests/test_storage_failures.py @@ -1,4 +1,5 @@ -"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4). +"""S3 outages, private buckets and failover of attachments (lab 4, levels 3 and 4; lab 5: two +buckets, upload 202 + worker). Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the stand is waited for until HAProxy sees all replicas UP again.""" @@ -33,6 +34,7 @@ def test_s3_outage_is_503_and_text_keeps_working(api, pair, how): * after recovery the earlier file is intact.""" alice, bob, chat = pair att = api.upload(alice, chat, SAMPLE).json() + api.wait_status(alice, att["id"], "ready") with service_down("s3", how): deadline = time.monotonic() + 15 while api.client.get("/health/ready").status_code != 503: @@ -63,6 +65,7 @@ def test_upload_survives_failover_of_the_serving_replica(api, pair): response = api.upload(alice, chat, data) att, served_by = response.json(), response.headers["x-instance-id"] replica = next(r for r in running_replicas() if r["hostname"] == served_by) + api.wait_status(bob, att["id"], "ready") with replica_stopped(replica, "kill"): for _ in range(5): got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) @@ -75,6 +78,7 @@ def test_bucket_is_private_outside_the_api(api, pair): SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable.""" alice, _, chat = pair att = api.upload(alice, chat, SAMPLE).json() + api.wait_status(alice, att["id"], "ready") key = attachment_row(att["id"])["object_key"] probe = ( "import urllib.request as u, urllib.error as e, socket\n" @@ -102,6 +106,7 @@ def test_s3_restart_keeps_objects(api, pair): alice, bob, chat = pair data = imagegen.png_of_size(200_000) att = api.upload(alice, chat, data).json() + api.wait_status(bob, att["id"], "ready") with service_down("s3", "stop"): pass assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data diff --git a/student-tests/unit/test_pipeline.py b/student-tests/unit/test_pipeline.py new file mode 100644 index 0000000..64e79fa --- /dev/null +++ b/student-tests/unit/test_pipeline.py @@ -0,0 +1,224 @@ +"""Unit tests of lab 5 without services: variant geometry and encoding (imaging.build, the same +code the worker runs in its child process), the job message schema, backoff with jitter and the +pipeline configuration.""" +from __future__ import annotations + +import io +import json +import sys +import uuid +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator +from PIL import Image + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "student-tests")) + +import imagegen # noqa: E402 +from messenger import broker, imaging, jobs # noqa: E402 +from messenger.config import ConfigError, Settings # noqa: E402 +from messenger.pipeline import PIPELINES, center_square, fit # noqa: E402 + +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +SCHEMA = json.loads((ROOT / "contracts" / "jobs.schema.json").read_text(encoding="utf-8")) +LIMIT = 40_000_000 + + +def variants(data: bytes, content_type: str = "image/png", version: int = 1) -> dict[str, imaging.Rendered]: + return {v.name: v for v in imaging.build(data, content_type, version, LIMIT)} + + +def decoded(v: imaging.Rendered) -> Image.Image: + img = Image.open(io.BytesIO(v.data)) + img.load() + return img + + +# ------------------------------------------------------------------ geometry (contract rules) +@pytest.mark.parametrize("size, box, expected", [ + ((320, 200), 256, (256, 160)), # public fixture + ((200, 320), 256, (160, 256)), + ((100, 50), 256, (100, 50)), # never upscale + ((256, 256), 256, (256, 256)), + ((1000, 1), 256, (256, 1)), # 0.256 px -> minimum 1 px + ((1, 1000), 256, (1, 256)), + ((257, 255), 256, (256, 254)), # 255 * 256 / 257 = 254.0 (floor) + ((3000, 2000), 1280, (1280, 853)), # 853.33 -> 853 +]) +def test_fit_rounds_down_without_upscaling(size, box, expected): + assert fit(*size, box) == expected + + +def test_center_square(): + assert center_square(320, 200) == (60, 0, 260, 200) + assert center_square(7, 10) == (0, 1, 7, 8) + + +def test_public_fixture_variants_match_the_contract(): + out = variants(SAMPLE) + assert list(out) == ["thumbnail", "crop", "optimized"] + assert (out["thumbnail"].content_type, decoded(out["thumbnail"]).format, decoded(out["thumbnail"]).size) == \ + ("image/jpeg", "JPEG", (256, 160)) + assert (out["crop"].content_type, decoded(out["crop"]).size) == ("image/jpeg", (128, 128)) + assert (out["optimized"].content_type, decoded(out["optimized"]).format, decoded(out["optimized"]).size) == \ + ("image/webp", "WEBP", (320, 200)) # fits 1280 already: no upscale + for v in out.values(): + assert (v.width, v.height) == decoded(v).size + + +def test_small_image_is_not_upscaled_but_crop_is(): + out = variants(imagegen.png(40, 30)) + assert decoded(out["thumbnail"]).size == (40, 30) + assert decoded(out["optimized"]).size == (40, 30) + assert decoded(out["crop"]).size == (128, 128) + + +def test_big_image_is_bounded_by_1280(): + out = variants(imagegen.png(3000, 2000)) + assert decoded(out["optimized"]).size == (1280, 853) + assert decoded(out["thumbnail"]).size == (256, 170) + + +def test_exif_orientation_is_applied_and_dropped(): + # 60x20 stored landscape with Orientation=6 (rotate 90 CW): displayed portrait 20x60. + img = Image.new("RGB", (60, 20), (200, 30, 30)) + exif = Image.Exif() + exif[0x0112] = 6 + buf = io.BytesIO() + img.save(buf, "JPEG", exif=exif.tobytes(), quality=95) + out = variants(buf.getvalue(), "image/jpeg") + assert decoded(out["thumbnail"]).size == (20, 60) + assert decoded(out["optimized"]).size == (20, 60) + for v in out.values(): + assert not decoded(v).getexif(), v.name + + +def test_alpha_is_composited_on_white_for_jpeg_and_kept_for_webp(): + img = Image.new("RGBA", (64, 64), (255, 0, 0, 0)) # fully transparent red + buf = io.BytesIO() + img.save(buf, "PNG") + out = variants(buf.getvalue()) + thumb = decoded(out["thumbnail"]).convert("RGB") + assert min(thumb.getpixel((32, 32))) >= 250, "transparent pixels must become white, not red/black" + webp = decoded(out["optimized"]) + assert webp.mode == "RGBA" and webp.getpixel((10, 10))[3] == 0 + + +def test_metadata_is_not_copied(): + img = Image.new("RGB", (300, 300), (10, 200, 10)) + exif = Image.Exif() + exif[0x010F] = "SecretCamera" # Make + buf = io.BytesIO() + img.save(buf, "JPEG", exif=exif.tobytes(), icc_profile=b"\0" * 128, comment=b"private") + out = variants(buf.getvalue(), "image/jpeg") + for name in ("thumbnail", "crop"): + data = out[name].data + assert b"Exif" not in data and b"ICC_PROFILE" not in data and b"SecretCamera" not in data, name + assert b"EXIF" not in out["optimized"].data and b"ICCP" not in out["optimized"].data + + +@pytest.mark.parametrize("mode", ["RGBA", "P", "1", "L", "LA", "RGB"]) +def test_png_modes_are_processed(mode): + out = variants(imagegen.png(33, 17, mode)) + assert decoded(out["thumbnail"]).size == (33, 17) + + +def test_16_bit_grey_is_scaled_not_clipped(): + img = Image.new("I;16", (8, 8), 32768) # 50 % grey in 16 bits + buf = io.BytesIO() + img.save(buf, "PNG") + out = variants(buf.getvalue()) + value = decoded(out["thumbnail"]).convert("L").getpixel((4, 4)) + assert 120 <= value <= 135 + + +def test_pipeline_versions_differ_in_bytes_not_in_geometry(): + v1, v2 = variants(SAMPLE, version=1), variants(SAMPLE, version=2) + for name in v1: + assert (v1[name].width, v1[name].height) == (v2[name].width, v2[name].height) + assert v1[name].sha256 != v2[name].sha256 + assert variants(SAMPLE, version=1)["thumbnail"].sha256 == v1["thumbnail"].sha256 # deterministic + + +def test_pixel_limit_is_checked_before_decoding(): + # 8001 x 5000 = 40.005 Mpx with only a few rows of data: refused from the header (TooLarge); + # 20000 x 10000 is refused even earlier by Pillow's own bomb check (> 2 x MAX_IMAGE_PIXELS). + with pytest.raises(imaging.TooLarge): + imaging.build(imagegen.png_bomb_header(8001, 5000), "image/png", 1, LIMIT) + with pytest.raises((imaging.TooLarge, Image.DecompressionBombError)): + imaging.build(imagegen.png_bomb_header(), "image/png", 1, LIMIT) + + +def test_child_process_protocol_round_trip(): + out = list(variants(SAMPLE).values()) + raw = json.dumps({"variants": [{"name": v.name, "content_type": v.content_type, "width": v.width, + "height": v.height, "size": len(v.data)} for v in out]}).encode() + b"\n" + raw += b"".join(v.data for v in out) + parsed = imaging.parse_output(raw) + assert [(p.name, p.sha256) for p in parsed] == [(v.name, v.sha256) for v in out] + with pytest.raises(imaging.ProcessingError): + imaging.parse_output(raw[:-1]) + + +# ------------------------------------------------------------------ message format +def test_encoded_message_matches_jobs_schema_and_carries_ids_only(): + import datetime as dt + raw = broker.encode(str(uuid.uuid4()), str(uuid.uuid4()), 1, 2, dt.datetime.now(dt.timezone.utc)) + body = json.loads(raw) + assert not list(Draft202012Validator(SCHEMA, format_checker=Draft202012Validator.FORMAT_CHECKER).iter_errors(body)) + assert set(body) == {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + assert broker.decode(raw) == body + + +@pytest.mark.parametrize("change", [ + {"schema_version": 2}, {"type": "other"}, {"job_id": "not-a-uuid"}, {"attempt": 0}, {"attempt": True}, + {"pipeline_version": "1"}, {"created_at": "yesterday"}, {"url": "http://evil"}, {"object_key": "a/b"}, + {"traceparent": "bad"}, +]) +def test_invalid_messages_are_rejected(change): + body = json.loads(broker.encode(str(uuid.uuid4()), str(uuid.uuid4()), 1, 1, + __import__("datetime").datetime(2026, 1, 1, tzinfo=__import__("datetime").timezone.utc))) + body.update(change) + with pytest.raises(broker.BadMessage): + broker.decode(json.dumps(body).encode()) + # The contract schema refuses the same messages (date-time needs an optional jsonschema extra). + checker = Draft202012Validator.FORMAT_CHECKER + assert list(Draft202012Validator(SCHEMA, format_checker=checker).iter_errors(body)) or "created_at" in change + + +def test_garbage_is_rejected(): + for raw in (b"", b"[]", b"{", b"\xff", b"x" * 5000): + with pytest.raises(broker.BadMessage): + broker.decode(raw) + + +# ------------------------------------------------------------------ backoff and configuration +def test_backoff_is_full_jitter_and_capped(): + assert jobs.backoff_delay(1, 2, 60, rnd=lambda: 1.0) == 2 + assert jobs.backoff_delay(3, 2, 60, rnd=lambda: 1.0) == 8 + assert jobs.backoff_delay(10, 2, 60, rnd=lambda: 1.0) == 60 + assert jobs.backoff_delay(4, 2, 60, rnd=lambda: 0.0) == 0 + samples = [jobs.backoff_delay(3, 2, 60) for _ in range(500)] + assert 0 <= min(samples) < 2 and 6 < max(samples) <= 8 # spread over the whole window + + +def test_pipeline_settings(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + monkeypatch.setenv("S3_ACCESS_KEY_ID", "AKTEST") + monkeypatch.setenv("S3_SECRET_ACCESS_KEY", "secret-for-unit-tests") + s = Settings.from_env() + assert (s.job_max_attempts, s.job_timeout_seconds, s.worker_concurrency, s.staging_retention_seconds) == (5, 60, 2, 86400) + assert s.s3_staging_bucket == "messenger-staging" and s.pipeline_version == 1 + monkeypatch.setenv("FAULT_POINTS", "transient_error=3,slow") + s = Settings.from_env() + assert s.fault_points == {"transient_error", "slow"} and s.fault_value("transient_error") == 3 + for name, value in (("PIPELINE_VERSION", "9"), ("JOB_MAX_ATTEMPTS", "0"), ("FAULT_POINTS", "slow=x"), + ("S3_STAGING_BUCKET", "messenger-attachments"), ("JOB_RETRY_BASE_SECONDS", "100")): + monkeypatch.setenv(name, value) + with pytest.raises(ConfigError): + Settings.from_env() + monkeypatch.delenv(name) + assert set(PIPELINES) == {1, 2, 3}